From e19437c09af0ba8eb83b282ffd4b76a2444f01d3 Mon Sep 17 00:00:00 2001 From: Shanqing Cai Date: Thu, 30 Jun 2016 08:53:19 -0800 Subject: [PATCH 01/36] Add --no-install-recommends to apt-get In response to GitHub issue: https://github.com/tensorflow/tensorflow/issues/1145 Avoids unnecessary installs and reduces docker image sizes. Change: 126312137 --- .../ci_build/install/install_bootstrap_deb_packages.sh | 2 +- tensorflow/tools/ci_build/install/install_deb_packages.sh | 3 ++- tensorflow/tools/ci_build/install/install_pip_packages.sh | 3 +++ .../ci_build/install/install_python3.5_pip_packages.sh | 4 ++-- .../tools/ci_build/install/install_tensorboard_packages.sh | 2 +- tensorflow/tools/dist_test/Dockerfile | 2 +- tensorflow/tools/dist_test/Dockerfile.local | 2 +- tensorflow/tools/dist_test/local/Dockerfile | 2 +- tensorflow/tools/docker/Dockerfile | 7 ++++++- tensorflow/tools/docker/Dockerfile.devel | 6 ++++-- tensorflow/tools/docker/Dockerfile.devel-gpu | 7 +++++-- tensorflow/tools/docker/Dockerfile.gpu | 7 ++++++- 12 files changed, 33 insertions(+), 14 deletions(-) diff --git a/tensorflow/tools/ci_build/install/install_bootstrap_deb_packages.sh b/tensorflow/tools/ci_build/install/install_bootstrap_deb_packages.sh index b56225c0b84..15526ef4f8c 100755 --- a/tensorflow/tools/ci_build/install/install_bootstrap_deb_packages.sh +++ b/tensorflow/tools/ci_build/install/install_bootstrap_deb_packages.sh @@ -18,7 +18,7 @@ set -e # Install bootstrap dependencies from ubuntu deb repository. apt-get update -apt-get install -y \ +apt-get install -y --no-install-recommends \ software-properties-common apt-get clean rm -rf /var/lib/apt/lists/* diff --git a/tensorflow/tools/ci_build/install/install_deb_packages.sh b/tensorflow/tools/ci_build/install/install_deb_packages.sh index 9b5f1418b14..4dc58c8ce45 100755 --- a/tensorflow/tools/ci_build/install/install_deb_packages.sh +++ b/tensorflow/tools/ci_build/install/install_deb_packages.sh @@ -19,7 +19,7 @@ set -e # Install dependencies from ubuntu deb repository. apt-get update -apt-get install -y \ +apt-get install -y --no-install-recommends \ autoconf \ automake \ build-essential \ @@ -37,6 +37,7 @@ apt-get install -y \ python-virtualenv \ python3-dev \ python3-pip \ + rsync \ sudo \ swig \ unzip \ diff --git a/tensorflow/tools/ci_build/install/install_pip_packages.sh b/tensorflow/tools/ci_build/install/install_pip_packages.sh index 1e667690d20..d244a249504 100755 --- a/tensorflow/tools/ci_build/install/install_pip_packages.sh +++ b/tensorflow/tools/ci_build/install/install_pip_packages.sh @@ -19,6 +19,9 @@ set -e # Install pip packages from whl files to avoid the time-consuming process of # building from source. +pip install wheel +pip3 install wheel + # Use pip to install numpy to the latest version, instead of 1.8.2 through # apt-get wget -q https://pypi.python.org/packages/17/f3/404bc85be67150663024d2bb5af654c7d16cf678077690dda27b91be14eb/numpy-1.8.2-cp27-cp27mu-manylinux1_x86_64.whl#md5=3ccf5c004fc99bd06dd443de80d622e6 diff --git a/tensorflow/tools/ci_build/install/install_python3.5_pip_packages.sh b/tensorflow/tools/ci_build/install/install_python3.5_pip_packages.sh index e781361627f..ba4293cb276 100755 --- a/tensorflow/tools/ci_build/install/install_python3.5_pip_packages.sh +++ b/tensorflow/tools/ci_build/install/install_python3.5_pip_packages.sh @@ -30,7 +30,7 @@ tar xzf swig-3.0.8.tar.gz pushd /swig-3.0.8 -apt-get install -y libpcre3-dev +apt-get install -y --no-install-recommends libpcre3-dev ./configure make make install @@ -43,7 +43,7 @@ rm -rf swig-3.0.8 rm -f swig-3.0.8.tar.gz # Install Python 3.5 and dev library -apt-get install -y python3.5 libpython3.5-dev +apt-get install -y --no-install-recommends python3.5 libpython3.5-dev # Install pip3.4 and numpy for Python 3.4 # This strange-looking install step is a stopgap measure to make the genrule diff --git a/tensorflow/tools/ci_build/install/install_tensorboard_packages.sh b/tensorflow/tools/ci_build/install/install_tensorboard_packages.sh index 95b8314f4c6..ca5092cd475 100755 --- a/tensorflow/tools/ci_build/install/install_tensorboard_packages.sh +++ b/tensorflow/tools/ci_build/install/install_tensorboard_packages.sh @@ -18,7 +18,7 @@ set -e # Install dependencies from ubuntu deb repository. apt-get update -apt-get install -y \ +apt-get install -y --no-install-recommends \ chromium-browser \ nodejs \ nodejs-legacy \ diff --git a/tensorflow/tools/dist_test/Dockerfile b/tensorflow/tools/dist_test/Dockerfile index b55433b3108..5705767c7da 100644 --- a/tensorflow/tools/dist_test/Dockerfile +++ b/tensorflow/tools/dist_test/Dockerfile @@ -3,7 +3,7 @@ FROM ubuntu:14.04 MAINTAINER Shanqing Cai RUN apt-get update -RUN apt-get install -y \ +RUN apt-get install -y --no-install-recommends \ curl \ python \ python-numpy \ diff --git a/tensorflow/tools/dist_test/Dockerfile.local b/tensorflow/tools/dist_test/Dockerfile.local index fae7ddb14a4..e23fa034a3d 100644 --- a/tensorflow/tools/dist_test/Dockerfile.local +++ b/tensorflow/tools/dist_test/Dockerfile.local @@ -4,7 +4,7 @@ MAINTAINER Shanqing Cai RUN apt-get update -RUN apt-get install -y \ +RUN apt-get install -y --no-install-recommends \ build-essential \ dbus \ git \ diff --git a/tensorflow/tools/dist_test/local/Dockerfile b/tensorflow/tools/dist_test/local/Dockerfile index dece508c0df..96846f65648 100644 --- a/tensorflow/tools/dist_test/local/Dockerfile +++ b/tensorflow/tools/dist_test/local/Dockerfile @@ -4,7 +4,7 @@ MAINTAINER Shanqing Cai RUN apt-get update -RUN apt-get install -y \ +RUN apt-get install -y --no-install-recommends \ build-essential \ git \ software-properties-common diff --git a/tensorflow/tools/docker/Dockerfile b/tensorflow/tools/docker/Dockerfile index 19d27a50c9c..a5927e79a50 100644 --- a/tensorflow/tools/docker/Dockerfile +++ b/tensorflow/tools/docker/Dockerfile @@ -3,15 +3,20 @@ FROM ubuntu:14.04 MAINTAINER Craig Citro # Pick up some TF dependencies -RUN apt-get update && apt-get install -y \ +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ curl \ libfreetype6-dev \ libpng12-dev \ libzmq3-dev \ pkg-config \ + python \ + python-dev \ python-numpy \ python-pip \ python-scipy \ + rsync \ + unzip \ && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* diff --git a/tensorflow/tools/docker/Dockerfile.devel b/tensorflow/tools/docker/Dockerfile.devel index 4ed80b12ad8..5e8693525be 100644 --- a/tensorflow/tools/docker/Dockerfile.devel +++ b/tensorflow/tools/docker/Dockerfile.devel @@ -2,7 +2,7 @@ FROM ubuntu:14.04 MAINTAINER Craig Citro -RUN apt-get update && apt-get install -y \ +RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ curl \ git \ @@ -13,8 +13,10 @@ RUN apt-get update && apt-get install -y \ python-dev \ python-numpy \ python-pip \ + rsync \ software-properties-common \ swig \ + unzip \ zip \ zlib1g-dev \ && \ @@ -50,7 +52,7 @@ COPY run_jupyter.sh / # https://bugs.launchpad.net/trusty-backports/+bug/1368094 RUN add-apt-repository -y ppa:openjdk-r/ppa && \ apt-get update && \ - apt-get install -y openjdk-8-jdk openjdk-8-jre-headless && \ + apt-get install -y --no-install-recommends openjdk-8-jdk openjdk-8-jre-headless && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* diff --git a/tensorflow/tools/docker/Dockerfile.devel-gpu b/tensorflow/tools/docker/Dockerfile.devel-gpu index b3db52e081d..74f41ca746f 100644 --- a/tensorflow/tools/docker/Dockerfile.devel-gpu +++ b/tensorflow/tools/docker/Dockerfile.devel-gpu @@ -2,7 +2,7 @@ FROM nvidia/cuda:7.5-cudnn4-devel MAINTAINER Craig Citro -RUN apt-get update && apt-get install -y \ +RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ curl \ git \ @@ -10,11 +10,14 @@ RUN apt-get update && apt-get install -y \ libpng12-dev \ libzmq3-dev \ pkg-config \ + python \ python-dev \ python-numpy \ python-pip \ + rsync \ software-properties-common \ swig \ + unzip \ zip \ zlib1g-dev \ && \ @@ -50,7 +53,7 @@ COPY run_jupyter.sh / # https://bugs.launchpad.net/trusty-backports/+bug/1368094 RUN add-apt-repository -y ppa:openjdk-r/ppa && \ apt-get update && \ - apt-get install -y openjdk-8-jdk openjdk-8-jre-headless && \ + apt-get install -y --no-install-recommends openjdk-8-jdk openjdk-8-jre-headless && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* diff --git a/tensorflow/tools/docker/Dockerfile.gpu b/tensorflow/tools/docker/Dockerfile.gpu index c6507c5e87f..4dd97a6f20b 100644 --- a/tensorflow/tools/docker/Dockerfile.gpu +++ b/tensorflow/tools/docker/Dockerfile.gpu @@ -3,15 +3,20 @@ FROM nvidia/cuda:7.5-cudnn4-devel MAINTAINER Craig Citro # Pick up some TF dependencies -RUN apt-get update && apt-get install -y \ +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ curl \ libfreetype6-dev \ libpng12-dev \ libzmq3-dev \ pkg-config \ + python \ + python-dev \ python-numpy \ python-pip \ python-scipy \ + rsync \ + unzip \ && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* From cfbb4beda154c72be09df5a5f89452e7ce1c13c7 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 30 Jun 2016 09:12:17 -0800 Subject: [PATCH 02/36] Update generated Python Op docs. Change: 126314412 --- tensorflow/g3doc/api_docs/python/contrib.learn.md | 3 +-- .../shard4/tf.contrib.learn.read_batch_features.md | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tensorflow/g3doc/api_docs/python/contrib.learn.md b/tensorflow/g3doc/api_docs/python/contrib.learn.md index b573b02bd71..25bfe07e7a0 100644 --- a/tensorflow/g3doc/api_docs/python/contrib.learn.md +++ b/tensorflow/g3doc/api_docs/python/contrib.learn.md @@ -5083,7 +5083,7 @@ Use `parse_fn` if you need to do parsing / processing on single examples. - - - -### `tf.contrib.learn.read_batch_features(file_pattern, batch_size, features, reader, randomize_input=True, num_epochs=None, queue_capacity=10000, reader_num_threads=1, parser_num_threads=1, read_batch_size=1, name=None)` {#read_batch_features} +### `tf.contrib.learn.read_batch_features(file_pattern, batch_size, features, reader, randomize_input=True, num_epochs=None, queue_capacity=10000, reader_num_threads=1, parser_num_threads=1, name=None)` {#read_batch_features} Adds operations to read, queue, batch and parse `Example` protos. @@ -5115,7 +5115,6 @@ All ops are added to the default graph. * `queue_capacity`: Capacity for input queue. * `reader_num_threads`: The number of threads to read examples. * `parser_num_threads`: The number of threads to parse examples. -* `read_batch_size`: An int or scalar `Tensor` specifying the number of records to read at once * `name`: Name of resulting op. diff --git a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard4/tf.contrib.learn.read_batch_features.md b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard4/tf.contrib.learn.read_batch_features.md index 6327760ca08..d18c316080e 100644 --- a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard4/tf.contrib.learn.read_batch_features.md +++ b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard4/tf.contrib.learn.read_batch_features.md @@ -1,4 +1,4 @@ -### `tf.contrib.learn.read_batch_features(file_pattern, batch_size, features, reader, randomize_input=True, num_epochs=None, queue_capacity=10000, reader_num_threads=1, parser_num_threads=1, read_batch_size=1, name=None)` {#read_batch_features} +### `tf.contrib.learn.read_batch_features(file_pattern, batch_size, features, reader, randomize_input=True, num_epochs=None, queue_capacity=10000, reader_num_threads=1, parser_num_threads=1, name=None)` {#read_batch_features} Adds operations to read, queue, batch and parse `Example` protos. @@ -30,7 +30,6 @@ All ops are added to the default graph. * `queue_capacity`: Capacity for input queue. * `reader_num_threads`: The number of threads to read examples. * `parser_num_threads`: The number of threads to parse examples. -* `read_batch_size`: An int or scalar `Tensor` specifying the number of records to read at once * `name`: Name of resulting op. From 4d9cb631194d03e27f9142b1e38a5a1a6aa29b56 Mon Sep 17 00:00:00 2001 From: Martin Wicke Date: Thu, 30 Jun 2016 10:06:01 -0800 Subject: [PATCH 03/36] Remove svg files from repo. Change: 126321208 --- tensorflow/g3doc/images/wide_n_deep.svg | 1540 ----------------------- 1 file changed, 1540 deletions(-) delete mode 100644 tensorflow/g3doc/images/wide_n_deep.svg diff --git a/tensorflow/g3doc/images/wide_n_deep.svg b/tensorflow/g3doc/images/wide_n_deep.svg deleted file mode 100644 index 6dfe9e7f102..00000000000 --- a/tensorflow/g3doc/images/wide_n_deep.svg +++ /dev/null @@ -1,1540 +0,0 @@ - - - - - - image/svg+xml - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - From 6062f26f626555ca980c716d52c6204e17745503 Mon Sep 17 00:00:00 2001 From: Derek Murray Date: Thu, 30 Jun 2016 10:32:46 -0800 Subject: [PATCH 04/36] Adapt the shape function for `tf.fill()` to handle partial shapes. Fixes #3102. Change: 126324523 --- .../python/kernel_tests/constant_op_test.py | 3 +++ tensorflow/python/ops/array_ops.py | 16 ++++++++-------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/tensorflow/python/kernel_tests/constant_op_test.py b/tensorflow/python/kernel_tests/constant_op_test.py index cc769ec2748..adfedbed64b 100644 --- a/tensorflow/python/kernel_tests/constant_op_test.py +++ b/tensorflow/python/kernel_tests/constant_op_test.py @@ -516,6 +516,9 @@ class FillTest(tf.test.TestCase): tf.placeholder(tf.int32, shape=(4,)), 3.0) self.assertEqual([None, None, None, None], f.get_shape().as_list()) + f = tf.fill([tf.placeholder(tf.int32, shape=()), 17], 1.0) + self.assertEqual([None, 17], f.get_shape().as_list()) + def testGradient(self): with self.test_session(): in_v = tf.constant(5.0) diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py index 509f6271700..658844dfb3f 100644 --- a/tensorflow/python/ops/array_ops.py +++ b/tensorflow/python/ops/array_ops.py @@ -1835,16 +1835,16 @@ def _FillShape(op): Returns: A single-element list containing the shape of the output. + + Raises: + ValueError: If the shapes or arguments are known to be invalid. """ - dimensions_shape = op.inputs[0].get_shape().with_rank(1) - op.inputs[1].get_shape().assert_is_compatible_with(tensor_shape.scalar()) + op.inputs[0].get_shape().assert_has_rank(1) + op.inputs[1].get_shape().assert_has_rank(0) fill_dims = tensor_util.constant_value(op.inputs[0]) - if fill_dims is None: - # Attempt to infer the rank of the output from the length of - # dimensions. - return [tensor_shape.unknown_shape(ndims=dimensions_shape[0].value)] - else: - return [tensor_shape.TensorShape(fill_dims.tolist())] + if fill_dims is not None and any(d < 0 for d in fill_dims): + raise ValueError("Fill dimensions must be >= 0") + return [tensor_util.constant_value_as_shape(op.inputs[0])] @ops.RegisterShape("InvertPermutation") From 533d891cf64f1ad4b8470dd31141b1b1f642bae4 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 30 Jun 2016 12:10:50 -0800 Subject: [PATCH 05/36] Merge changes from github. Change: 126335170 --- README.md | 6 +- RELEASE.md | 30 +- configure | 13 + .../framework/python/framework/tensor_util.py | 16 +- .../simple/RunModelViewController.mm | 2 + .../learn/python/learn/io/graph_io_test.py | 8 +- tensorflow/contrib/makefile/Makefile | 2 +- tensorflow/contrib/makefile/build_all_ios.sh | 12 +- .../contrib/makefile/compile_ios_protobuf.sh | 21 +- .../makefile/compile_ios_tensorflow.sh | 20 + tensorflow/contrib/makefile/tf_cc_files.txt | 2 + .../contrib/session_bundle/example/BUILD | 30 +- tensorflow/core/framework/op_kernel.cc | 7 + tensorflow/core/framework/op_kernel.h | 4 + tensorflow/core/public/version.h | 2 +- tensorflow/examples/skflow/examples_test.sh | 2 +- tensorflow/examples/skflow/iris_run_config.py | 2 +- .../text_classification_character_cnn.py | 2 +- .../skflow/text_classification_cnn.py | 2 +- .../examples/udacity/2_fullyconnected.ipynb | 2 +- .../api_docs/python/contrib.framework.md | 15 +- .../shard0/tf.nn.ctc_loss.md | 2 +- .../shard5/tf.image.pad_to_bounding_box.md | 3 +- .../tf.image.resize_image_with_crop_or_pad.md | 2 +- .../shard8/tf.image.crop_to_bounding_box.md | 3 +- .../shard9/tf.contrib.framework.is_tensor.md | 15 +- .../functions_and_classes/shard9/tf.tanh.md | 8 +- tensorflow/g3doc/api_docs/python/image.md | 8 +- tensorflow/g3doc/api_docs/python/nn.md | 10 +- tensorflow/g3doc/get_started/os_setup.md | 194 +++--- tensorflow/g3doc/resources/roadmap.md | 14 +- tensorflow/models/embedding/word2vec_test.py | 4 +- .../python/kernel_tests/cwise_ops_test.py | 6 + tensorflow/python/kernel_tests/rnn_test.py | 109 ++++ tensorflow/python/ops/ctc_ops.py | 2 +- tensorflow/python/ops/data_flow_ops.py | 2 +- tensorflow/python/ops/image_ops.py | 226 +++++-- tensorflow/python/ops/image_ops_test.py | 613 +++++++++++++----- tensorflow/python/ops/math_ops.py | 15 +- tensorflow/python/ops/nn.py | 2 +- tensorflow/python/ops/nn_conv_test.py | 189 ++++++ tensorflow/python/ops/nn_grad.py | 28 + tensorflow/python/ops/nn_ops.py | 67 ++ tensorflow/python/ops/rnn.py | 119 ++++ tensorflow/python/training/saver_test.py | 2 +- tensorflow/tensorboard/README.md | 10 +- .../ci_build/builds/test_installation.sh | 1 + tensorflow/tools/dist_test/Dockerfile | 2 +- tensorflow/tools/dist_test/server/Dockerfile | 2 +- .../tools/dist_test/server/Dockerfile.test | 2 +- tensorflow/tools/docker/Dockerfile | 2 +- tensorflow/tools/docker/Dockerfile.gpu | 2 +- tensorflow/tools/docker/README.md | 8 +- tensorflow/tools/docker/docker_run_gpu.sh | 29 - .../docker/parameterized_docker_build.sh | 3 + tensorflow/tools/pip_package/BUILD | 6 +- tensorflow/tools/pip_package/setup.py | 2 +- tensorflow/tools/proto_text/BUILD | 7 + tensorflow/workspace.bzl | 4 +- .../bin/crosstool_wrapper_driver_is_not_gcc | 6 +- 60 files changed, 1485 insertions(+), 444 deletions(-) delete mode 100755 tensorflow/tools/docker/docker_run_gpu.sh mode change 100644 => 100755 tensorflow/tools/docker/parameterized_docker_build.sh diff --git a/README.md b/README.md index 2327f8e92cb..ae8fe81370c 100644 --- a/README.md +++ b/README.md @@ -33,9 +33,9 @@ and discussion.** People who are a little bit adventurous can also try our nightly binaries: -* Linux CPU only: [Python 2](http://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_CONTAINER_TYPE=CPU,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.8.0-cp27-none-linux_x86_64.whl) ([build history](http://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_CONTAINER_TYPE=CPU,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave/)) / [Python 3.4](http://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_CONTAINER_TYPE=CPU,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.8.0-cp34-cp34m-linux_x86_64.whl) ([build history](http://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_CONTAINER_TYPE=CPU,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=cpu-slave/)) / [Python 3.5](http://ci.tensorflow.org/view/Nightly/job/nightly-python35-linux-cpu/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.8.0-cp35-cp35m-linux_x86_64.whl) ([build history](http://ci.tensorflow.org/view/Nightly/job/nightly-python35-linux-cpu/)) -* Linux GPU: [Python 2](http://ci.tensorflow.org/view/Nightly/job/nigntly-matrix-linux-gpu/TF_BUILD_CONTAINER_TYPE=GPU,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=gpu-linux/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.8.0-cp27-none-linux_x86_64.whl) ([build history](http://ci.tensorflow.org/view/Nightly/job/nigntly-matrix-linux-gpu/TF_BUILD_CONTAINER_TYPE=GPU,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=gpu-linux/)) / [Python 3.4](http://ci.tensorflow.org/view/Nightly/job/nigntly-matrix-linux-gpu/TF_BUILD_CONTAINER_TYPE=GPU,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=gpu-linux/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.8.0-cp34-cp34m-linux_x86_64.whl) ([build history](http://ci.tensorflow.org/view/Nightly/job/nigntly-matrix-linux-gpu/TF_BUILD_CONTAINER_TYPE=GPU,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=gpu-linux/)) / [Python 3.5](http://ci.tensorflow.org/view/Nightly/job/nigntly-matrix-linux-gpu/TF_BUILD_CONTAINER_TYPE=GPU,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3.5,label=gpu-linux/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.8.0-cp35-cp35m-linux_x86_64.whl) ([build history](http://ci.tensorflow.org/view/Nightly/job/nigntly-matrix-linux-gpu/TF_BUILD_CONTAINER_TYPE=GPU,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3.5,label=gpu-linux/)) -* Mac CPU only: [Python 2](http://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_CONTAINER_TYPE=CPU,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=mac-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.8.0-py2-none-any.whl) ([build history](http://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_CONTAINER_TYPE=CPU,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=mac-slave/)) / [Python 3](http://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_CONTAINER_TYPE=CPU,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=mac-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.8.0-py3-none-any.whl) ([build history](http://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_CONTAINER_TYPE=CPU,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=mac-slave/)) +* Linux CPU only: [Python 2](http://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_CONTAINER_TYPE=CPU,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.9.0-cp27-none-linux_x86_64.whl) ([build history](http://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_CONTAINER_TYPE=CPU,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave/)) / [Python 3.4](http://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_CONTAINER_TYPE=CPU,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.9.0-cp34-cp34m-linux_x86_64.whl) ([build history](http://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_CONTAINER_TYPE=CPU,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=cpu-slave/)) / [Python 3.5](http://ci.tensorflow.org/view/Nightly/job/nightly-python35-linux-cpu/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.9.0-cp35-cp35m-linux_x86_64.whl) ([build history](http://ci.tensorflow.org/view/Nightly/job/nightly-python35-linux-cpu/)) +* Linux GPU: [Python 2](http://ci.tensorflow.org/view/Nightly/job/nigntly-matrix-linux-gpu/TF_BUILD_CONTAINER_TYPE=GPU,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=gpu-linux/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.9.0-cp27-none-linux_x86_64.whl) ([build history](http://ci.tensorflow.org/view/Nightly/job/nigntly-matrix-linux-gpu/TF_BUILD_CONTAINER_TYPE=GPU,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=gpu-linux/)) / [Python 3.4](http://ci.tensorflow.org/view/Nightly/job/nigntly-matrix-linux-gpu/TF_BUILD_CONTAINER_TYPE=GPU,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=gpu-linux/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.9.0-cp34-cp34m-linux_x86_64.whl) ([build history](http://ci.tensorflow.org/view/Nightly/job/nigntly-matrix-linux-gpu/TF_BUILD_CONTAINER_TYPE=GPU,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=gpu-linux/)) / [Python 3.5](http://ci.tensorflow.org/view/Nightly/job/nigntly-matrix-linux-gpu/TF_BUILD_CONTAINER_TYPE=GPU,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3.5,label=gpu-linux/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.9.0-cp35-cp35m-linux_x86_64.whl) ([build history](http://ci.tensorflow.org/view/Nightly/job/nigntly-matrix-linux-gpu/TF_BUILD_CONTAINER_TYPE=GPU,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3.5,label=gpu-linux/)) +* Mac CPU only: [Python 2](http://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_CONTAINER_TYPE=CPU,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=mac-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.9.0-py2-none-any.whl) ([build history](http://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_CONTAINER_TYPE=CPU,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=mac-slave/)) / [Python 3](http://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_CONTAINER_TYPE=CPU,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=mac-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.9.0-py3-none-any.whl) ([build history](http://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_CONTAINER_TYPE=CPU,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=mac-slave/)) * [Android](http://ci.tensorflow.org/view/Nightly/job/nightly-matrix-android/TF_BUILD_CONTAINER_TYPE=ANDROID,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=NO_PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=android-slave/lastSuccessfulBuild/artifact/bazel-out/local_linux/bin/tensorflow/examples/android/tensorflow_demo.apk) ([build history](http://ci.tensorflow.org/view/Nightly/job/nightly-matrix-android/TF_BUILD_CONTAINER_TYPE=ANDROID,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=NO_PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=android-slave/)) #### *Try your first TensorFlow program* diff --git a/RELEASE.md b/RELEASE.md index ea368e037ab..3843d543e93 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,8 +1,12 @@ # Changes Since Last Release +## Features and Improvements +* Connectionist Temporal Classification ops are now "official" (see, e.g., + `tf.nn.ctc_loss`) + ## Breaking Changes to the API -* env.h replaces use of New*File() functions to use std::unique_ptr return - arguments, removing the old raw pointer returns. +* `env.h` replaces use of `New*File()` functions to use `std::unique_ptr` + return arguments, removing the old raw pointer returns. # Release 0.9.0 @@ -52,28 +56,6 @@ Aaron Schumacher, Aidan Dang, Akihiko ITOH, Aki Sukegawa, Arbit Chen, Aziz Alto, We are also grateful to all who filed issues or helped resolve them, asked and answered questions, and were part of inspiring discussions. - -## Features & Improvements -* Connectionist Temporal Classification ops are now "official" (see, e.g., - `tf.nn.ctc_loss`) -* The RNN api is finally "official" (see, e.g., `tf.nn.dynamic_rnn`, - `tf.nn.rnn`, and the classes in `tf.nn.rnn_cell`). -* TensorBoard now has an Audio Dashboard, with associated audio summaries. -* TensorBoard now has a reload button, and supports auto-reloading -* TensorBoard scalar charts now show tooltips with more information -* TensorBoard now supports run filtering -* TensorBoard has color changes: the same run always gets the same hue -* Tensorboard graph visualizer now supports run metadata. Clicking on nodes - while viewing a stats for a particular run will show runtime statistics, such - as memory or compute usage. Unused nodes will be faded out. - -## Bug Fixes and Other Changes -* TensorBoard now displays graphs with only one data point -* TensorBoard now visually displays NaN values -* `tf.nn.moments()` now accepts a `shift` argument. Shifting by a good estimate - of the mean improves numerical stability. Also changes the behavior of the - `shift` argument to `tf.nn.sufficient_statistics()`. - # Release 0.8.0 ## Major Features and Improvements diff --git a/configure b/configure index 98048ba91db..20bbf123abc 100755 --- a/configure +++ b/configure @@ -174,6 +174,19 @@ while true; do if [[ -z "$TF_CUDNN_VERSION" ]]; then TF_CUDNN_EXT="" + # Resolve to the SONAME of the symlink. Use readlink without -f + # to resolve exactly once to the SONAME. E.g, libcudnn.so -> + # libcudnn.so.4 + REALVAL=`readlink ${CUDNN_INSTALL_PATH}/lib64/libcudnn.so` + + # Extract the version of the SONAME, if it was indeed symlinked to + # the SONAME version of the file. + if [[ "$REALVAL" =~ .so[.]+([0-9]*) ]]; + then + TF_CUDNN_EXT="."${BASH_REMATCH[1]} + TF_CUDNN_VERSION=${BASH_REMATCH[1]} + echo "libcudnn.so resolves to libcudnn${TF_CUDNN_EXT}" + fi else TF_CUDNN_EXT=".$TF_CUDNN_VERSION" fi diff --git a/tensorflow/contrib/framework/python/framework/tensor_util.py b/tensorflow/contrib/framework/python/framework/tensor_util.py index 466ff38aa01..d46e00cf966 100644 --- a/tensorflow/contrib/framework/python/framework/tensor_util.py +++ b/tensorflow/contrib/framework/python/framework/tensor_util.py @@ -216,9 +216,19 @@ def with_same_shape(expected_tensor, tensor): return with_shape(expected_shape, tensor) -def is_tensor(t): - """Check if `t` is a tensor: `Tensor`, `SparseTensor`, or `Variable`.""" - return isinstance(t, (ops.Tensor, ops.SparseTensor, variables.Variable)) +def is_tensor(x): + """Check for tensor types. + Check whether an object is a tensor. Equivalent to + `isinstance(x, [tf.Tensor, tf.SparseTensor, tf.Variable])`. + + Args: + x: An python object to check. + + Returns: + `True` if `x` is a tensor, `False` if not. + """ + tensor_types = (ops.Tensor, ops.SparseTensor, variables.Variable) + return isinstance(x, tensor_types) def with_shape(expected_shape, tensor): diff --git a/tensorflow/contrib/ios_examples/simple/RunModelViewController.mm b/tensorflow/contrib/ios_examples/simple/RunModelViewController.mm index 2e389b39d4b..2ebb197399f 100644 --- a/tensorflow/contrib/ios_examples/simple/RunModelViewController.mm +++ b/tensorflow/contrib/ios_examples/simple/RunModelViewController.mm @@ -25,6 +25,7 @@ #include "google/protobuf/io/zero_copy_stream_impl.h" #include "google/protobuf/io/zero_copy_stream_impl_lite.h" #include "google/protobuf/message_lite.h" +#include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/types.pb.h" #include "tensorflow/core/platform/env.h" @@ -218,6 +219,7 @@ NSString* RunInferenceOnImage() { {output_layer}, {}, &outputs); if (!run_status.ok()) { LOG(ERROR) << "Running model failed: " << run_status; + tensorflow::LogAllRegisteredKernels(); result = @"Error running model"; return result; } diff --git a/tensorflow/contrib/learn/python/learn/io/graph_io_test.py b/tensorflow/contrib/learn/python/learn/io/graph_io_test.py index a991344b914..334ef425efc 100644 --- a/tensorflow/contrib/learn/python/learn/io/graph_io_test.py +++ b/tensorflow/contrib/learn/python/learn/io/graph_io_test.py @@ -124,8 +124,8 @@ class GraphIOTest(tf.test.TestCase): _VALID_FILE_PATTERN, batch_size, features, randomize_input=False, queue_capacity=queue_capacity, reader_num_threads=2, parser_num_threads=2, name=name) - self.assertEquals("%s/parse_example_batch_join:1" % name, - features["feature"].name) + self.assertEqual("%s/parse_example_batch_join:1" % name, + features["feature"].name) file_name_queue_name = "%s/file_name_queue" % name file_names_name = "%s/input" % file_name_queue_name example_queue_name = "%s/fifo_queue" % name @@ -153,7 +153,7 @@ class GraphIOTest(tf.test.TestCase): reader=tf.TFRecordReader, randomize_input=True, num_epochs=1, queue_capacity=queue_capacity, name=name) - self.assertEquals("%s:1" % name, inputs.name) + self.assertEqual("%s:1" % name, inputs.name) file_name_queue_name = "%s/file_name_queue" % name file_name_queue_limit_name = ( "%s/limit_epochs/epochs" % file_name_queue_name) @@ -182,7 +182,7 @@ class GraphIOTest(tf.test.TestCase): _VALID_FILE_PATTERN, batch_size, reader=tf.TFRecordReader, randomize_input=True, queue_capacity=queue_capacity, name=name) - self.assertEquals("%s:1" % name, inputs.name) + self.assertEqual("%s:1" % name, inputs.name) file_name_queue_name = "%s/file_name_queue" % name file_names_name = "%s/input" % file_name_queue_name example_queue_name = "%s/random_shuffle_queue" % name diff --git a/tensorflow/contrib/makefile/Makefile b/tensorflow/contrib/makefile/Makefile index c9b4641afa2..6bb16551862 100644 --- a/tensorflow/contrib/makefile/Makefile +++ b/tensorflow/contrib/makefile/Makefile @@ -400,7 +400,7 @@ $(BENCHMARK_NAME): $(BENCHMARK_OBJS) $(LIB_PATH) $(LIBFLAGS) $(LIB_PATH) $(LDFLAGS) $(LIBS) # Matches on the normal hand-written TensorFlow C++ source files. -$(OBJDIR)%.o: %.cc +$(OBJDIR)%.o: %.cc | $(PBT_GEN_FILES) @mkdir -p $(dir $@) $(CXX) $(CXXFLAGS) $(INCLUDES) -c $< -o $@ diff --git a/tensorflow/contrib/makefile/build_all_ios.sh b/tensorflow/contrib/makefile/build_all_ios.sh index 39d859aad66..6b6ed389fc8 100755 --- a/tensorflow/contrib/makefile/build_all_ios.sh +++ b/tensorflow/contrib/makefile/build_all_ios.sh @@ -27,6 +27,14 @@ fi SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" cd ${SCRIPT_DIR}/../../../ +# You can set the parallelism of the make process with the first argument, with +# a default of four if nothing is supplied. +if [ "$#" -gt 1 ]; then + JOBS_COUNT=$1 +else + JOBS_COUNT=4 +fi + # Remove any old files first. make -f tensorflow/contrib/makefile/Makefile clean rm -rf tensorflow/contrib/makefile/downloads @@ -35,10 +43,10 @@ rm -rf tensorflow/contrib/makefile/downloads tensorflow/contrib/makefile/download_dependencies.sh # Compile protobuf for the target iOS device architectures. -tensorflow/contrib/makefile/compile_ios_protobuf.sh +tensorflow/contrib/makefile/compile_ios_protobuf.sh ${JOBS_COUNT} # Build the iOS TensorFlow libraries. -tensorflow/contrib/makefile/compile_ios_tensorflow.sh +tensorflow/contrib/makefile/compile_ios_tensorflow.sh "-O3" -j ${JOBS_COUNT} # Creates a static universal library in # tensorflow/contrib/makefile/gen/lib/libtensorflow-core.a diff --git a/tensorflow/contrib/makefile/compile_ios_protobuf.sh b/tensorflow/contrib/makefile/compile_ios_protobuf.sh index 9a452023b7b..d2b1d4e846f 100755 --- a/tensorflow/contrib/makefile/compile_ios_protobuf.sh +++ b/tensorflow/contrib/makefile/compile_ios_protobuf.sh @@ -1,4 +1,4 @@ -#!/bin/bash -x +#!/bin/bash -x -e # Copyright 2015 The TensorFlow Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,6 +15,9 @@ # ============================================================================== # Builds protobuf 3 for iOS. +SCRIPT_DIR=$(dirname $0) +source "${SCRIPT_DIR}/build_helper.subr" + cd tensorflow/contrib/makefile HOST_GENDIR="$(pwd)/gen/protobuf-host" @@ -25,6 +28,12 @@ if [[ ! -f "./downloads/protobuf/autogen.sh" ]]; then exit 1 fi +if [ "$#" -gt 1 ]; then + JOBS_COUNT=$1 +else + JOBS_COUNT=4 +fi + GENDIR=`pwd`/gen/protobuf_ios/ LIBDIR=${GENDIR}lib mkdir -p ${LIBDIR} @@ -85,7 +94,7 @@ ${LDFLAGS} \ -L${IPHONESIMULATOR_SYSROOT}/usr/lib/ \ -L${IPHONESIMULATOR_SYSROOT}/usr/lib/system" \ "LIBS=${LIBS}" -make +make -j ${JOBS_COUNT} make install make distclean @@ -113,7 +122,7 @@ ${LDFLAGS} \ -L${IPHONESIMULATOR_SYSROOT}/usr/lib/ \ -L${IPHONESIMULATOR_SYSROOT}/usr/lib/system" \ "LIBS=${LIBS}" -make +make -j ${JOBS_COUNT} make install make distclean @@ -137,7 +146,7 @@ LDFLAGS="-arch armv7 \ -miphoneos-version-min=${MIN_SDK_VERSION} \ ${LDFLAGS}" \ "LIBS=${LIBS}" -make +make -j ${JOBS_COUNT} make install make distclean @@ -161,7 +170,7 @@ LDFLAGS="-arch armv7s \ -miphoneos-version-min=${MIN_SDK_VERSION} \ ${LDFLAGS}" \ "LIBS=${LIBS}" -make +make -j ${JOBS_COUNT} make install make distclean @@ -184,7 +193,7 @@ LDFLAGS="-arch arm64 \ -miphoneos-version-min=${MIN_SDK_VERSION} \ ${LDFLAGS}" \ "LIBS=${LIBS}" -make +make -j ${JOBS_COUNT} make install lipo \ diff --git a/tensorflow/contrib/makefile/compile_ios_tensorflow.sh b/tensorflow/contrib/makefile/compile_ios_tensorflow.sh index 2efc4bbe7f2..be1a1d3ec54 100755 --- a/tensorflow/contrib/makefile/compile_ios_tensorflow.sh +++ b/tensorflow/contrib/makefile/compile_ios_tensorflow.sh @@ -16,10 +16,30 @@ # Builds the TensorFlow core library with ARM and x86 architectures for iOS, and # packs them into a fat file. +ACTUAL_XCODE_VERSION=`xcodebuild -version | head -n 1 | sed 's/Xcode //'` +REQUIRED_XCODE_VERSION=7.3.0 +if [ ${ACTUAL_XCODE_VERSION//.} -lt ${REQUIRED_XCODE_VERSION//.} ] +then + echo "error: Xcode ${REQUIRED_XCODE_VERSION} or later is required." + exit 1 +fi + GENDIR=tensorflow/contrib/makefile/gen/ LIBDIR=${GENDIR}lib LIB_PREFIX=libtensorflow-core +# TODO(petewarden) - Some new code in Eigen triggers a clang bug, so work +# around it by patching the source. +sed -e 's#static uint32x4_t p4ui_CONJ_XOR = vld1q_u32( conj_XOR_DATA );#static uint32x4_t p4ui_CONJ_XOR; // = vld1q_u32( conj_XOR_DATA ); - Removed by script#' \ +-i '' \ +tensorflow/contrib/makefile/downloads/eigen-latest/eigen/src/Core/arch/NEON/Complex.h +sed -e 's#static uint32x2_t p2ui_CONJ_XOR = vld1_u32( conj_XOR_DATA );#static uint32x2_t p2ui_CONJ_XOR;// = vld1_u32( conj_XOR_DATA ); - Removed by scripts#' \ +-i '' \ +tensorflow/contrib/makefile/downloads/eigen-latest/eigen/src/Core/arch/NEON/Complex.h +sed -e 's#static uint64x2_t p2ul_CONJ_XOR = vld1q_u64( p2ul_conj_XOR_DATA );#static uint64x2_t p2ul_CONJ_XOR;// = vld1q_u64( p2ul_conj_XOR_DATA ); - Removed by script#' \ +-i '' \ +tensorflow/contrib/makefile/downloads/eigen-latest/eigen/src/Core/arch/NEON/Complex.h + make -f tensorflow/contrib/makefile/Makefile cleantarget make -f tensorflow/contrib/makefile/Makefile \ TARGET=IOS IOS_ARCH=ARMV7 LIB_NAME=${LIB_PREFIX}-armv7.a OPTFLAGS="$1" $2 $3 diff --git a/tensorflow/contrib/makefile/tf_cc_files.txt b/tensorflow/contrib/makefile/tf_cc_files.txt index 5402642f5b0..16420744841 100644 --- a/tensorflow/contrib/makefile/tf_cc_files.txt +++ b/tensorflow/contrib/makefile/tf_cc_files.txt @@ -47,6 +47,7 @@ tensorflow/core/kernels/in_topk_op.cc tensorflow/core/kernels/immutable_constant_op.cc tensorflow/core/kernels/identity_op.cc tensorflow/core/kernels/gather_op.cc +tensorflow/core/kernels/fill_functor.cc tensorflow/core/kernels/example_parsing_ops.cc tensorflow/core/kernels/dynamic_stitch_op.cc tensorflow/core/kernels/dynamic_partition_op.cc @@ -227,6 +228,7 @@ tensorflow/core/framework/graph_def_util.cc tensorflow/core/framework/function.cc tensorflow/core/framework/fake_input.cc tensorflow/core/framework/device_base.cc +tensorflow/core/framework/common_shape_fns.cc tensorflow/core/framework/cancellation.cc tensorflow/core/framework/bfloat16.cc tensorflow/core/framework/attr_value_util.cc diff --git a/tensorflow/contrib/session_bundle/example/BUILD b/tensorflow/contrib/session_bundle/example/BUILD index 8fa0c0b020b..80e22e5d44b 100644 --- a/tensorflow/contrib/session_bundle/example/BUILD +++ b/tensorflow/contrib/session_bundle/example/BUILD @@ -13,12 +13,12 @@ exports_files(["LICENSE"]) filegroup( name = "all_files", srcs = glob( - ["**/*"], - exclude = [ - "**/METADATA", - "**/OWNERS", - "g3doc/sitemap.md", - ], + ["**/*"], + exclude = [ + "**/METADATA", + "**/OWNERS", + "g3doc/sitemap.md", + ], ), visibility = ["//visibility:public"], ) @@ -26,27 +26,27 @@ filegroup( py_binary( name = "export_half_plus_two", srcs = [ - "export_half_plus_two.py", + "export_half_plus_two.py", ], srcs_version = "PY2AND3", deps = [ - "//tensorflow:tensorflow_py", - "//tensorflow/contrib/session_bundle:exporter", + "//tensorflow:tensorflow_py", + "//tensorflow/contrib/session_bundle:exporter", ], ) genrule( name = "half_plus_two", outs = [ - "half_plus_two/00000123/export.meta", - "half_plus_two/00000123/export-00000-of-00001", + "half_plus_two/00000123/export.meta", + "half_plus_two/00000123/export-00000-of-00001", ], cmd = - "rm -rf /tmp/half_plus_two; " + - "$(PYTHON_BIN_PATH) $(locations :export_half_plus_two); " + - "cp -r /tmp/half_plus_two/* $(@D)/half_plus_two", + "rm -rf /tmp/half_plus_two; " + + "$(PYTHON_BIN_PATH) $(locations :export_half_plus_two); " + + "cp -r /tmp/half_plus_two/* $(@D)/half_plus_two", tools = [ - ":export_half_plus_two", + ":export_half_plus_two", ], visibility = ["//visibility:public"], ) diff --git a/tensorflow/core/framework/op_kernel.cc b/tensorflow/core/framework/op_kernel.cc index 66a281e29ff..6bfc55df41f 100644 --- a/tensorflow/core/framework/op_kernel.cc +++ b/tensorflow/core/framework/op_kernel.cc @@ -787,6 +787,13 @@ Status SupportedDeviceTypesForNode( return Status::OK(); } +void LogAllRegisteredKernels() { + for (const auto& key_registration : *GlobalKernelRegistryTyped()) { + const KernelDef& kernel_def(key_registration.second.def); + LOG(INFO) << "OpKernel ('" << ProtoShortDebugString(kernel_def) << "')"; + } +} + std::unique_ptr CreateOpKernel( DeviceType device_type, DeviceBase* device, Allocator* allocator, const NodeDef& node_def, int graph_def_version, Status* status) { diff --git a/tensorflow/core/framework/op_kernel.h b/tensorflow/core/framework/op_kernel.h index 0b5309d4f66..0092c6286f8 100644 --- a/tensorflow/core/framework/op_kernel.h +++ b/tensorflow/core/framework/op_kernel.h @@ -1078,6 +1078,10 @@ Status FindKernelDef(DeviceType device_type, const NodeDef& node_def, // calling GlobalKernelRegistry()), inserts 'k' into registry_ptr. extern "C" void RegisterKernels(void* registry_ptr); +// Writes a list of all registered kernels to LOG(INFO), to help users debug +// missing kernel errors. +void LogAllRegisteredKernels(); + namespace kernel_factory { class OpKernelRegistrar { diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h index 78117bb48fb..443eabaee02 100644 --- a/tensorflow/core/public/version.h +++ b/tensorflow/core/public/version.h @@ -19,7 +19,7 @@ limitations under the License. // TensorFlow uses semantic versioning, see http://semver.org/. #define TF_MAJOR_VERSION 0 -#define TF_MINOR_VERSION 8 +#define TF_MINOR_VERSION 9 #define TF_PATCH_VERSION 0 // TF_VERSION_SUFFIX is non-empty for pre-releases (e.g. "-alpha", "-alpha.1", diff --git a/tensorflow/examples/skflow/examples_test.sh b/tensorflow/examples/skflow/examples_test.sh index 9ce8b690e96..da6b35c9bb3 100755 --- a/tensorflow/examples/skflow/examples_test.sh +++ b/tensorflow/examples/skflow/examples_test.sh @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# This script excercises the examples of using SkFlow. +# This script exercises the examples of using SkFlow. DIR="$TEST_SRCDIR" diff --git a/tensorflow/examples/skflow/iris_run_config.py b/tensorflow/examples/skflow/iris_run_config.py index 4f057f817de..dff0daf9e8c 100644 --- a/tensorflow/examples/skflow/iris_run_config.py +++ b/tensorflow/examples/skflow/iris_run_config.py @@ -29,7 +29,7 @@ X_train, X_test, y_train, y_test = cross_validation.train_test_split(iris.data, # estimator to control session configurations, e.g. num_cores and gpu_memory_fraction run_config = learn.estimators.RunConfig(num_cores=3, gpu_memory_fraction=0.6) -# Build 3 layer DNN with 10, 20, 10 units respecitvely. +# Build 3 layer DNN with 10, 20, 10 units respectively. classifier = learn.TensorFlowDNNClassifier(hidden_units=[10, 20, 10], n_classes=3, steps=200, config=run_config) diff --git a/tensorflow/examples/skflow/text_classification_character_cnn.py b/tensorflow/examples/skflow/text_classification_character_cnn.py index 7eb185d77a0..998ed308078 100644 --- a/tensorflow/examples/skflow/text_classification_character_cnn.py +++ b/tensorflow/examples/skflow/text_classification_character_cnn.py @@ -58,7 +58,7 @@ def char_cnn_model(x, y): FILTER_SHAPE1, padding='VALID') # Add a RELU for non linearity. conv1 = tf.nn.relu(conv1) - # Max pooling across output of Convlution+Relu. + # Max pooling across output of Convolution+Relu. pool1 = tf.nn.max_pool(conv1, ksize=[1, POOLING_WINDOW, 1, 1], strides=[1, POOLING_STRIDE, 1, 1], padding='SAME') # Transpose matrix so that n_filters from convolution becomes width. diff --git a/tensorflow/examples/skflow/text_classification_cnn.py b/tensorflow/examples/skflow/text_classification_cnn.py index 0d3c8684b54..0cbed33ef13 100644 --- a/tensorflow/examples/skflow/text_classification_cnn.py +++ b/tensorflow/examples/skflow/text_classification_cnn.py @@ -56,7 +56,7 @@ def cnn_model(x, y): FILTER_SHAPE1, padding='VALID') # Add a RELU for non linearity. conv1 = tf.nn.relu(conv1) - # Max pooling across output of Convlution+Relu. + # Max pooling across output of Convolution+Relu. pool1 = tf.nn.max_pool(conv1, ksize=[1, POOLING_WINDOW, 1, 1], strides=[1, POOLING_STRIDE, 1, 1], padding='SAME') # Transpose matrix so that n_filters from convolution becomes width. diff --git a/tensorflow/examples/udacity/2_fullyconnected.ipynb b/tensorflow/examples/udacity/2_fullyconnected.ipynb index 588b581a69b..2d1dfa2fba6 100644 --- a/tensorflow/examples/udacity/2_fullyconnected.ipynb +++ b/tensorflow/examples/udacity/2_fullyconnected.ipynb @@ -258,7 +258,7 @@ " \n", " # Variables.\n", " # These are the parameters that we are going to be training. The weight\n", - " # matrix will be initialized using random valued following a (truncated)\n", + " # matrix will be initialized using random values following a (truncated)\n", " # normal distribution. The biases get initialized to zero.\n", " weights = tf.Variable(\n", " tf.truncated_normal([image_size * image_size, num_labels]))\n", diff --git a/tensorflow/g3doc/api_docs/python/contrib.framework.md b/tensorflow/g3doc/api_docs/python/contrib.framework.md index 9500f189aa1..4c234555975 100644 --- a/tensorflow/g3doc/api_docs/python/contrib.framework.md +++ b/tensorflow/g3doc/api_docs/python/contrib.framework.md @@ -184,9 +184,20 @@ See also: `is_non_decreasing` - - - -### `tf.contrib.framework.is_tensor(t)` {#is_tensor} +### `tf.contrib.framework.is_tensor(x)` {#is_tensor} -Check if `t` is a tensor: `Tensor`, `SparseTensor`, or `Variable`. +Check for tensor types. +Check whether an object is a tensor. Equivalent to +`isinstance(x, [tf.Tensor, tf.SparseTensor, tf.Variable])`. + +##### Args: + + +* `x`: An python object to check. + +##### Returns: + + `True` if `x` is a tensor, `False` if not. - - - diff --git a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard0/tf.nn.ctc_loss.md b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard0/tf.nn.ctc_loss.md index a99c0b478b8..229df8bdbd9 100644 --- a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard0/tf.nn.ctc_loss.md +++ b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard0/tf.nn.ctc_loss.md @@ -69,7 +69,7 @@ Here is a table of the (roughly) expected first order behavior: ##### Returns: - A 1-D `float` `Tensor`, size `[batch]`, containing logits. + A 1-D `float` `Tensor`, size `[batch]`, containing the negative log probabilities. ##### Raises: diff --git a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard5/tf.image.pad_to_bounding_box.md b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard5/tf.image.pad_to_bounding_box.md index 04c155c03c1..c731fb2d2ae 100644 --- a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard5/tf.image.pad_to_bounding_box.md +++ b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard5/tf.image.pad_to_bounding_box.md @@ -26,5 +26,6 @@ This op does nothing if `offset_*` is zero and the image already has size * `ValueError`: If the shape of `image` is incompatible with the `offset_*` or - `target_*` arguments + `target_*` arguments, or either `offset_height` or `offset_width` is + negative. diff --git a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard5/tf.image.resize_image_with_crop_or_pad.md b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard5/tf.image.resize_image_with_crop_or_pad.md index c93111bd992..24104b647c1 100644 --- a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard5/tf.image.resize_image_with_crop_or_pad.md +++ b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard5/tf.image.resize_image_with_crop_or_pad.md @@ -14,7 +14,7 @@ dimension. ##### Args: -* `image`: 3-D tensor of shape [height, width, channels] +* `image`: 3-D tensor of shape `[height, width, channels]` * `target_height`: Target height. * `target_width`: Target width. diff --git a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard8/tf.image.crop_to_bounding_box.md b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard8/tf.image.crop_to_bounding_box.md index 4724ff5eb93..1ca4247a9b2 100644 --- a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard8/tf.image.crop_to_bounding_box.md +++ b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard8/tf.image.crop_to_bounding_box.md @@ -26,5 +26,6 @@ lower-right corner is at * `ValueError`: If the shape of `image` is incompatible with the `offset_*` or - `target_*` arguments + `target_*` arguments, or either `offset_height` or `offset_width` is + negative, or either `target_height` or `target_width` is not positive. diff --git a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard9/tf.contrib.framework.is_tensor.md b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard9/tf.contrib.framework.is_tensor.md index eaab61a99d8..3d8b9b56046 100644 --- a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard9/tf.contrib.framework.is_tensor.md +++ b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard9/tf.contrib.framework.is_tensor.md @@ -1,4 +1,15 @@ -### `tf.contrib.framework.is_tensor(t)` {#is_tensor} +### `tf.contrib.framework.is_tensor(x)` {#is_tensor} -Check if `t` is a tensor: `Tensor`, `SparseTensor`, or `Variable`. +Check for tensor types. +Check whether an object is a tensor. Equivalent to +`isinstance(x, [tf.Tensor, tf.SparseTensor, tf.Variable])`. + +##### Args: + + +* `x`: An python object to check. + +##### Returns: + + `True` if `x` is a tensor, `False` if not. diff --git a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard9/tf.tanh.md b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard9/tf.tanh.md index 6f42cfdc2f0..154a13059ce 100644 --- a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard9/tf.tanh.md +++ b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard9/tf.tanh.md @@ -5,12 +5,12 @@ Computes hyperbolic tangent of `x` element-wise. ##### Args: -* `x`: A Tensor with type `float32`, `float64`, `int32`, `complex64`, `int64`, - or `qint32`. +* `x`: A Tensor or SparseTensor with type `float`, `double`, `int32`, + `complex64`, `int64`, or `qint32`. * `name`: A name for the operation (optional). ##### Returns: - A Tensor with the same type as `x` if `x.dtype != qint32` otherwise - the return type is `quint8`. + A Tensor or SparseTensor respectively with the same type as `x` if + `x.dtype != qint32` otherwise the return type is `quint8`. diff --git a/tensorflow/g3doc/api_docs/python/image.md b/tensorflow/g3doc/api_docs/python/image.md index 5c017ec042c..505cc9259bc 100644 --- a/tensorflow/g3doc/api_docs/python/image.md +++ b/tensorflow/g3doc/api_docs/python/image.md @@ -382,7 +382,7 @@ dimension. ##### Args: -* `image`: 3-D tensor of shape [height, width, channels] +* `image`: 3-D tensor of shape `[height, width, channels]` * `target_height`: Target height. * `target_width`: Target width. @@ -461,7 +461,8 @@ This op does nothing if `offset_*` is zero and the image already has size * `ValueError`: If the shape of `image` is incompatible with the `offset_*` or - `target_*` arguments + `target_*` arguments, or either `offset_height` or `offset_width` is + negative. - - - @@ -494,7 +495,8 @@ lower-right corner is at * `ValueError`: If the shape of `image` is incompatible with the `offset_*` or - `target_*` arguments + `target_*` arguments, or either `offset_height` or `offset_width` is + negative, or either `target_height` or `target_width` is not positive. - - - diff --git a/tensorflow/g3doc/api_docs/python/nn.md b/tensorflow/g3doc/api_docs/python/nn.md index a7069e2732b..372b1f8aeaf 100644 --- a/tensorflow/g3doc/api_docs/python/nn.md +++ b/tensorflow/g3doc/api_docs/python/nn.md @@ -205,14 +205,14 @@ Computes hyperbolic tangent of `x` element-wise. ##### Args: -* `x`: A Tensor with type `float32`, `float64`, `int32`, `complex64`, `int64`, - or `qint32`. +* `x`: A Tensor or SparseTensor with type `float`, `double`, `int32`, + `complex64`, `int64`, or `qint32`. * `name`: A name for the operation (optional). ##### Returns: - A Tensor with the same type as `x` if `x.dtype != qint32` otherwise - the return type is `quint8`. + A Tensor or SparseTensor respectively with the same type as `x` if + `x.dtype != qint32` otherwise the return type is `quint8`. @@ -1770,7 +1770,7 @@ Here is a table of the (roughly) expected first order behavior: ##### Returns: - A 1-D `float` `Tensor`, size `[batch]`, containing logits. + A 1-D `float` `Tensor`, size `[batch]`, containing the negative log probabilities. ##### Raises: diff --git a/tensorflow/g3doc/get_started/os_setup.md b/tensorflow/g3doc/get_started/os_setup.md index 97b7a5ed7b9..158c84b4ef0 100644 --- a/tensorflow/g3doc/get_started/os_setup.md +++ b/tensorflow/g3doc/get_started/os_setup.md @@ -54,36 +54,48 @@ $ sudo apt-get install python-pip python-dev # Mac OS X $ sudo easy_install pip +$ sudo easy_install --upgrade six +``` + +Then, select the correct binary to install: + +```bash +# Ubuntu/Linux 64-bit, CPU only, Python 2.7 +$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.9.0-cp27-none-linux_x86_64.whl + +# Ubuntu/Linux 64-bit, GPU enabled, Python 2.7 +# Requires CUDA toolkit 7.5 and CuDNN v4. For other versions, see "Install from sources" below. +$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.9.0-cp27-none-linux_x86_64.whl + +# Mac OS X, CPU only, Python 2.7: +$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/tensorflow-0.9.0-py2-none-any.whl + +# Ubuntu/Linux 64-bit, CPU only, Python 3.4 +$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.9.0-cp34-cp34m-linux_x86_64.whl + +# Ubuntu/Linux 64-bit, GPU enabled, Python 3.4 +# Requires CUDA toolkit 7.5 and CuDNN v4. For other versions, see "Install from sources" below. +$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.9.0-cp34-cp34m-linux_x86_64.whl + +# Ubuntu/Linux 64-bit, CPU only, Python 3.5 +$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.9.0-cp35-cp35m-linux_x86_64.whl + +# Ubuntu/Linux 64-bit, GPU enabled, Python 3.5 +# Requires CUDA toolkit 7.5 and CuDNN v4. For other versions, see "Install from sources" below. +$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.9.0-cp35-cp35m-linux_x86_64.whl + +# Mac OS X, CPU only, Python 3.4 or 3.5: +$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/tensorflow-0.9.0-py3-none-any.whl ``` Install TensorFlow: ```bash -# Ubuntu/Linux 64-bit, CPU only, Python 2.7: -$ sudo pip install --upgrade https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.8.0-cp27-none-linux_x86_64.whl +# Python 2 +$ sudo pip install --upgrade $TF_BINARY_URL -# Ubuntu/Linux 64-bit, GPU enabled, Python 2.7. Requires CUDA toolkit 7.5 and cuDNN v4. -# For other versions, see "Install from sources" below. -$ sudo pip install --upgrade https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.8.0-cp27-none-linux_x86_64.whl - -# Mac OS X, CPU only: -$ sudo easy_install --upgrade six -$ sudo pip install --upgrade https://storage.googleapis.com/tensorflow/mac/tensorflow-0.8.0-py2-none-any.whl -``` - -For python3: - -```bash -# Ubuntu/Linux 64-bit, CPU only, Python 3.4: -$ sudo pip3 install --upgrade https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.8.0-cp34-cp34m-linux_x86_64.whl - -# Ubuntu/Linux 64-bit, GPU enabled, Python 3.4. Requires CUDA toolkit 7.5 and cuDNN v4. -# For other versions, see "Install from sources" below. -$ sudo pip3 install --upgrade https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.8.0-cp34-cp34m-linux_x86_64.whl - -# Mac OS X, CPU only: -$ sudo easy_install --upgrade six -$ sudo pip3 install --upgrade https://storage.googleapis.com/tensorflow/mac/tensorflow-0.8.0-py3-none-any.whl +# Python 3 +$ sudo pip3 install --upgrade $TF_BINARY_URL ``` NOTE: If you are upgrading from a previous installation of TensorFlow < 0.7.1, @@ -127,40 +139,53 @@ Create a Virtualenv environment in the directory `~/tensorflow`: $ virtualenv --system-site-packages ~/tensorflow ``` -Activate the environment and use pip to install TensorFlow inside it: +Activate the environment: ```bash $ source ~/tensorflow/bin/activate # If using bash $ source ~/tensorflow/bin/activate.csh # If using csh (tensorflow)$ # Your prompt should change - -# Ubuntu/Linux 64-bit, CPU only, Python 2.7: -(tensorflow)$ pip install --upgrade https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.8.0-cp27-none-linux_x86_64.whl - -# Ubuntu/Linux 64-bit, GPU enabled, Python 2.7. Requires CUDA toolkit 7.5 and cuDNN v4. -# For other versions, see "Install from sources" below. -(tensorflow)$ pip install --upgrade https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.8.0-cp27-none-linux_x86_64.whl - -# Mac OS X, CPU only: -(tensorflow)$ pip install --upgrade https://storage.googleapis.com/tensorflow/mac/tensorflow-0.8.0-py2-none-any.whl ``` -and again for python3: +Now, install TensorFlow just as you would for a regular Pip installation. First select the correct binary to install: ```bash -$ source ~/tensorflow/bin/activate # If using bash -$ source ~/tensorflow/bin/activate.csh # If using csh -(tensorflow)$ # Your prompt should change +# Ubuntu/Linux 64-bit, CPU only, Python 2.7 +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.9.0-cp27-none-linux_x86_64.whl -# Ubuntu/Linux 64-bit, CPU only, Python 3.4: -(tensorflow)$ pip3 install --upgrade https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.8.0-cp34-cp34m-linux_x86_64.whl +# Ubuntu/Linux 64-bit, GPU enabled, Python 2.7 +# Requires CUDA toolkit 7.5 and CuDNN v4. For other versions, see "Install from sources" below. +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.9.0-cp27-none-linux_x86_64.whl -# Ubuntu/Linux 64-bit, GPU enabled, Python 3.4. Requires CUDA toolkit 7.5 and cuDNN v4. -# For other versions, see "Install from sources" below. -(tensorflow)$ pip3 install --upgrade https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.8.0-cp34-cp34m-linux_x86_64.whl +# Mac OS X, CPU only, Python 2.7: +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/tensorflow-0.9.0-py2-none-any.whl -# Mac OS X, CPU only: -(tensorflow)$ pip3 install --upgrade https://storage.googleapis.com/tensorflow/mac/tensorflow-0.8.0-py3-none-any.whl +# Ubuntu/Linux 64-bit, CPU only, Python 3.4 +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.9.0-cp34-cp34m-linux_x86_64.whl + +# Ubuntu/Linux 64-bit, GPU enabled, Python 3.4 +# Requires CUDA toolkit 7.5 and CuDNN v4. For other versions, see "Install from sources" below. +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.9.0-cp34-cp34m-linux_x86_64.whl + +# Ubuntu/Linux 64-bit, CPU only, Python 3.5 +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.9.0-cp35-cp35m-linux_x86_64.whl + +# Ubuntu/Linux 64-bit, GPU enabled, Python 3.5 +# Requires CUDA toolkit 7.5 and CuDNN v4. For other versions, see "Install from sources" below. +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.9.0-cp35-cp35m-linux_x86_64.whl + +# Mac OS X, CPU only, Python 3.4 or 3.5: +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/tensorflow-0.9.0-py3-none-any.whl +``` + +Finally install TensorFlow: + +```bash +# Python 2 +(tensorflow)$ pip install --upgrade $TF_BINARY_URL + +# Python 3 +(tensorflow)$ pip3 install --upgrade $TF_BINARY_URL ``` With the Virtualenv environment activated, you can now @@ -216,6 +241,9 @@ $ conda create -n tensorflow python=2.7 # Python 3.4 $ conda create -n tensorflow python=3.4 + +# Python 3.5 +$ conda create -n tensorflow python=3.5 ``` Activate the environment and use conda or pip to install TensorFlow inside it. @@ -241,33 +269,47 @@ If using pip make sure to use the `--ignore-installed` flag to prevent errors ab ```bash $ source activate tensorflow (tensorflow)$ # Your prompt should change - -# Ubuntu/Linux 64-bit, CPU only, Python 2.7: -(tensorflow)$ pip install --ignore-installed --upgrade https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.8.0-cp27-none-linux_x86_64.whl - -# Ubuntu/Linux 64-bit, GPU enabled, Python 2.7. Requires CUDA toolkit 7.5 and cuDNN v4. -# For other versions, see "Install from sources" below. -(tensorflow)$ pip install --ignore-installed --upgrade https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.8.0-cp27-none-linux_x86_64.whl - -# Mac OS X, CPU only: -(tensorflow)$ pip install --ignore-installed --upgrade https://storage.googleapis.com/tensorflow/mac/tensorflow-0.8.0-py2-none-any.whl ``` -and again for Python 3: +Now, install TensorFlow just as you would for a regular Pip installation. First select the correct binary to install: ```bash -$ source activate tensorflow -(tensorflow)$ # Your prompt should change +# Ubuntu/Linux 64-bit, CPU only, Python 2.7 +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.9.0-cp27-none-linux_x86_64.whl -# Ubuntu/Linux 64-bit, CPU only, Python 3.4: -(tensorflow)$ pip install --ignore-installed --upgrade https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.8.0-cp34-cp34m-linux_x86_64.whl +# Ubuntu/Linux 64-bit, GPU enabled, Python 2.7 +# Requires CUDA toolkit 7.5 and CuDNN v4. For other versions, see "Install from sources" below. +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.9.0-cp27-none-linux_x86_64.whl -# Ubuntu/Linux 64-bit, GPU enabled, Python 3.4. Requires CUDA toolkit 7.5 and cuDNN v4. -# For other versions, see "Install from sources" below. -(tensorflow)$ pip install --ignore-installed --upgrade https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.8.0-cp34-cp34m-linux_x86_64.whl +# Mac OS X, CPU only, Python 2.7: +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/tensorflow-0.9.0-py2-none-any.whl -# Mac OS X, CPU only: -(tensorflow)$ pip install --ignore-installed --upgrade https://storage.googleapis.com/tensorflow/mac/tensorflow-0.8.0-py3-none-any.whl +# Ubuntu/Linux 64-bit, CPU only, Python 3.4 +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.9.0-cp34-cp34m-linux_x86_64.whl + +# Ubuntu/Linux 64-bit, GPU enabled, Python 3.4 +# Requires CUDA toolkit 7.5 and CuDNN v4. For other versions, see "Install from sources" below. +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.9.0-cp34-cp34m-linux_x86_64.whl + +# Ubuntu/Linux 64-bit, CPU only, Python 3.5 +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.9.0-cp35-cp35m-linux_x86_64.whl + +# Ubuntu/Linux 64-bit, GPU enabled, Python 3.5 +# Requires CUDA toolkit 7.5 and CuDNN v4. For other versions, see "Install from sources" below. +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.9.0-cp35-cp35m-linux_x86_64.whl + +# Mac OS X, CPU only, Python 3.4 or 3.5: +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/tensorflow-0.9.0-py3-none-any.whl +``` + +Finally install TensorFlow: + +```bash +# Python 2 +(tensorflow)$ pip install --ignore-installed --upgrade $TF_BINARY_URL + +# Python 3 +(tensorflow)$ pip3 install --ignore-installed --upgrade $TF_BINARY_URL ``` ### Usage @@ -310,7 +352,7 @@ code. * `gcr.io/tensorflow/tensorflow:latest-devel-gpu`: GPU Binary image plus source code. -We also have tags with `latest` replaced by a released version (e.g., `0.8.0-gpu`). +We also have tags with `latest` replaced by a released version (e.g., `0.9.0-gpu`). With Docker the installation is as follows: @@ -335,16 +377,16 @@ The option `-p 8888:8888` is used to publish the Docker container᾿s internal p The format of the port mapping is `hostPort:containerPort`. You can specify any valid port number for the host port but have to use `8888` for the container port portion. -If you're using a container with GPU support, some additional flags must be -passed to expose the GPU device to the container. For the default config, we -include a -[script](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/tools/docker/docker_run_gpu.sh) -in the repo with these flags, so the command-line would look like +For NVidia GPU support install latest NVidia drivers and +[nvidia-docker](https://github.com/NVIDIA/nvidia-docker). +Run with ```bash -$ path/to/repo/tensorflow/tools/docker/docker_run_gpu.sh gcr.io/tensorflow/tensorflow:gpu +$ nvidia-docker run -it -p 8888:8888 gcr.io/tensorflow/tensorflow:latest-gpu ``` +For more details see (TensorFlow docker readme)[https://github.com/tensorflow/tensorflow/tree/master/tensorflow/tools/docker]. + You can now [test your installation](#test-the-tensorflow-installation) within the Docker container. ## Test the TensorFlow installation @@ -517,8 +559,8 @@ to reflect the cuDNN version you downloaded): ``` bash tar xvzf cudnn-7.5-linux-x64-v4.tgz -sudo cp cudnn-7.5-linux-x64-v4/cudnn.h /usr/local/cuda/include -sudo cp cudnn-7.5-linux-x64-v4/libcudnn* /usr/local/cuda/lib64 +sudo cp cuda/include/cudnn.h /usr/local/cuda/include +sudo cp cuda/lib64/libcudnn* /usr/local/cuda/lib64 sudo chmod a+r /usr/local/cuda/include/cudnn.h /usr/local/cuda/lib64/libcudnn* ``` @@ -719,7 +761,7 @@ $ bazel build -c opt --config=cuda //tensorflow/tools/pip_package:build_pip_pack $ bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_pkg # The name of the .whl file will depend on your platform. -$ sudo pip install /tmp/tensorflow_pkg/tensorflow-0.8.0-py2-none-any.whl +$ sudo pip install /tmp/tensorflow_pkg/tensorflow-0.9.0-py2-none-any.whl ``` ## Setting up TensorFlow for Development diff --git a/tensorflow/g3doc/resources/roadmap.md b/tensorflow/g3doc/resources/roadmap.md index be7ac5e778b..99724815150 100644 --- a/tensorflow/g3doc/resources/roadmap.md +++ b/tensorflow/g3doc/resources/roadmap.md @@ -1,5 +1,5 @@ # Roadmap -**Last updated: April 12, 2016** +**Last updated: June 3, 2016** TensorFlow is a fast moving project. In order for the community to better understand what the near future will bring, this document shares what we are @@ -16,7 +16,7 @@ we do not have timelines for these features. * Shape Inference ### Making TensorFlow easier to use -* Higher level APIs (for instance, layers) +* Easier setup for distributed training jobs ### Performance * Speed and memory benchmarks @@ -24,13 +24,15 @@ we do not have timelines for these features. ### Core Features * Repeated partial graph evaluation ([#672](https://github.com/tensorflow/tensorflow/issues/672)) +* Automatic op placement ([#2126](https://github.com/tensorflow/tensorflow/issues/2126)) ### Platforms -* iOS support ([#16](https://github.com/tensorflow/tensorflow/issues/16)) * OpenCL support ([#22](https://github.com/tensorflow/tensorflow/issues/22)) * Windows support ([#17](https://github.com/tensorflow/tensorflow/issues/17)) -* MacOS GPU support ### Community -* Integration with other machine learning frameworks -* Better installation support; support for package managers +* More educational resources +* Better integration of TensorFlow into the opensource big data ecosystem ([#1996](https://github.com/tensorflow/tensorflow/issues/1996), +[#2218](https://github.com/tensorflow/tensorflow/issues/2218), +[#2655](https://github.com/tensorflow/tensorflow/issues/2655)) +* Models benchmarking and comparison tooling diff --git a/tensorflow/models/embedding/word2vec_test.py b/tensorflow/models/embedding/word2vec_test.py index 34cb21d8215..3f6fbc78aa4 100644 --- a/tensorflow/models/embedding/word2vec_test.py +++ b/tensorflow/models/embedding/word2vec_test.py @@ -33,8 +33,8 @@ FLAGS = flags.FLAGS class Word2VecTest(tf.test.TestCase): def setUp(self): - FLAGS.train_data = os.path.join(self.get_temp_dir() + "test-text.txt") - FLAGS.eval_data = os.path.join(self.get_temp_dir() + "eval-text.txt") + FLAGS.train_data = os.path.join(self.get_temp_dir(), "test-text.txt") + FLAGS.eval_data = os.path.join(self.get_temp_dir(), "eval-text.txt") FLAGS.save_path = self.get_temp_dir() with open(FLAGS.train_data, "w") as f: f.write( diff --git a/tensorflow/python/kernel_tests/cwise_ops_test.py b/tensorflow/python/kernel_tests/cwise_ops_test.py index 4a21d1acc63..7f1be574bbf 100644 --- a/tensorflow/python/kernel_tests/cwise_ops_test.py +++ b/tensorflow/python/kernel_tests/cwise_ops_test.py @@ -213,6 +213,7 @@ class UnaryOpTest(tf.test.TestCase): self._compareBothSparse(x, np.negative, tf.neg) self._compareBothSparse(x, np.square, tf.square) self._compareBothSparse(z, np.sqrt, tf.sqrt, tol=1e-3) + self._compareBothSparse(x, np.tanh, tf.tanh) self._compareBothSparse(y, np.sign, tf.sign) def testFloatTanhEdge(self): @@ -251,6 +252,7 @@ class UnaryOpTest(tf.test.TestCase): self._compareBothSparse(x, np.negative, tf.neg) self._compareBothSparse(x, np.square, tf.square) self._compareBothSparse(x, np.sqrt, tf.sqrt, tol=1e-3) + self._compareBothSparse(x, np.tanh, tf.tanh) self._compareBothSparse(x, np.sign, tf.sign) def testDoubleBasic(self): @@ -288,6 +290,7 @@ class UnaryOpTest(tf.test.TestCase): self._compareBothSparse(x, np.negative, tf.neg) self._compareBothSparse(x, np.square, tf.square) self._compareBothSparse(z, np.sqrt, tf.sqrt, tol=1e-3) + self._compareBothSparse(x, np.tanh, tf.tanh) self._compareBothSparse(y, np.sign, tf.sign) def testHalfBasic(self): @@ -320,6 +323,7 @@ class UnaryOpTest(tf.test.TestCase): self._compareBothSparse(x, np.negative, tf.neg) self._compareBothSparse(x, np.square, tf.square) self._compareBothSparse(z, np.sqrt, tf.sqrt, tol=1e-3) + self._compareBothSparse(x, np.tanh, tf.tanh) self._compareBothSparse(y, np.sign, tf.sign) def testInt32Basic(self): @@ -374,6 +378,7 @@ class UnaryOpTest(tf.test.TestCase): self._compareBothSparse(x, np.negative, tf.neg) self._compareBothSparse(x, np.square, tf.square) self._compareBothSparse(x, np.sqrt, tf.sqrt, 1e-3) + self._compareBothSparse(x, np.tanh, tf.tanh) # Numpy uses an incorrect definition of sign; use the right one instead. def complex_sign(x): @@ -404,6 +409,7 @@ class UnaryOpTest(tf.test.TestCase): self._compareBothSparse(x, np.negative, tf.neg) self._compareBothSparse(x, np.square, tf.square) self._compareBothSparse(x, np.sqrt, tf.sqrt, 1e-3) + self._compareBothSparse(x, np.tanh, tf.tanh) # Numpy uses an incorrect definition of sign; use the right one instead. def complex_sign(x): diff --git a/tensorflow/python/kernel_tests/rnn_test.py b/tensorflow/python/kernel_tests/rnn_test.py index 8db74d56fd4..8ec29ea7966 100644 --- a/tensorflow/python/kernel_tests/rnn_test.py +++ b/tensorflow/python/kernel_tests/rnn_test.py @@ -19,6 +19,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import itertools import time import timeit @@ -1177,6 +1178,114 @@ class BidirectionalRNNTest(tf.test.TestCase): self._testBidirectionalRNNWithoutSequenceLength(use_gpu=True, use_shape=True) + def _createBidirectionalDynamicRNN(self, use_gpu, use_shape, + use_state_tuple, use_time_major): + num_units = 3 + input_size = 5 + batch_size = 2 + max_length = 8 + + initializer = tf.random_uniform_initializer(-0.01, 0.01, seed=self._seed) + sequence_length = tf.placeholder(tf.int64) + cell_fw = tf.nn.rnn_cell.LSTMCell(num_units, + initializer=initializer, + state_is_tuple=use_state_tuple) + cell_bw = tf.nn.rnn_cell.LSTMCell(num_units, + initializer=initializer, + state_is_tuple=use_state_tuple) + inputs = max_length * [ + tf.placeholder(tf.float32, + shape=(batch_size if use_shape else None, input_size))] + inputs_c = tf.pack(inputs) + if not use_time_major: + inputs_c = tf.transpose(inputs_c, [1, 0, 2]) + outputs, states = tf.nn.bidirectional_dynamic_rnn( + cell_fw, + cell_bw, + inputs_c, + sequence_length, + dtype=tf.float32, + time_major=use_time_major) + outputs = tf.concat(2, outputs) + state_fw, state_bw = states + outputs_shape = [None, max_length, 2 * num_units] + if use_shape: + outputs_shape[0] = batch_size + if use_time_major: + outputs_shape[0], outputs_shape[1] = outputs_shape[1], outputs_shape[0] + self.assertEqual( + outputs.get_shape().as_list(), + outputs_shape) + + input_value = np.random.randn(batch_size, input_size) + + return input_value, inputs, outputs, state_fw, state_bw, sequence_length + + def _testBidirectionalDynamicRNN(self, use_gpu, use_shape, + use_state_tuple, use_time_major): + with self.test_session(use_gpu=use_gpu, graph=tf.Graph()) as sess: + input_value, inputs, outputs, state_fw, state_bw, sequence_length = ( + self._createBidirectionalDynamicRNN( + use_gpu, use_shape, use_state_tuple, use_time_major)) + tf.initialize_all_variables().run() + # Run with pre-specified sequence length of 2, 3 + if use_state_tuple: + out, c_fw, m_fw, c_bw, m_bw = sess.run( + [outputs, state_fw[0], state_fw[1], state_bw[0], state_bw[1]], + feed_dict={inputs[0]: input_value, + sequence_length: [2, 3]}) + s_fw = (c_fw, m_fw) + s_bw = (c_bw, m_bw) + else: + out, s_fw, s_bw = sess.run([outputs, state_fw, state_bw], + feed_dict={inputs[0]: input_value, + sequence_length: [2, 3]}) + + # Since the forward and backward LSTM cells were initialized with the + # same parameters, the forward and backward output has to be the same, + # but reversed in time. The format is output[time][batch][depth], and + # due to depth concatenation (as num_units=3 for both RNNs): + # - forward output: out[][][depth] for 0 <= depth < 3 + # - backward output: out[][][depth] for 4 <= depth < 6 + # + # First sequence in batch is length=2 + # Check that the time=0 forward output is equal to time=1 backward output + if not use_time_major: + out = np.swapaxes(out, 0, 1) + self.assertEqual(out[0][0][0], out[1][0][3]) + self.assertEqual(out[0][0][1], out[1][0][4]) + self.assertEqual(out[0][0][2], out[1][0][5]) + # Check that the time=1 forward output is equal to time=0 backward output + self.assertEqual(out[1][0][0], out[0][0][3]) + self.assertEqual(out[1][0][1], out[0][0][4]) + self.assertEqual(out[1][0][2], out[0][0][5]) + + # Second sequence in batch is length=3 + # Check that the time=0 forward output is equal to time=2 backward output + self.assertEqual(out[0][1][0], out[2][1][3]) + self.assertEqual(out[0][1][1], out[2][1][4]) + self.assertEqual(out[0][1][2], out[2][1][5]) + # Check that the time=1 forward output is equal to time=1 backward output + self.assertEqual(out[1][1][0], out[1][1][3]) + self.assertEqual(out[1][1][1], out[1][1][4]) + self.assertEqual(out[1][1][2], out[1][1][5]) + # Check that the time=2 forward output is equal to time=0 backward output + self.assertEqual(out[2][1][0], out[0][1][3]) + self.assertEqual(out[2][1][1], out[0][1][4]) + self.assertEqual(out[2][1][2], out[0][1][5]) + # Via the reasoning above, the forward and backward final state should be + # exactly the same + self.assertAllClose(s_fw, s_bw) + + def testBidirectionalDynamicRNN(self): + # Generate 2^4 option values + # from [True, True, True, True] to [False, False, False, False] + options = itertools.product([True, False], repeat=4) + for option in options: + self._testBidirectionalDynamicRNN(use_gpu=option[0], use_shape=option[1], + use_state_tuple=option[2], + use_time_major=option[3]) + class MultiDimensionalLSTMTest(tf.test.TestCase): diff --git a/tensorflow/python/ops/ctc_ops.py b/tensorflow/python/ops/ctc_ops.py index 58c1c210100..bab9dc0ef5c 100644 --- a/tensorflow/python/ops/ctc_ops.py +++ b/tensorflow/python/ops/ctc_ops.py @@ -95,7 +95,7 @@ def ctc_loss(inputs, labels, sequence_length, ctc_merge_repeated: Boolean. Default: True. Returns: - A 1-D `float` `Tensor`, size `[batch]`, containing logits. + A 1-D `float` `Tensor`, size `[batch]`, containing the negative log probabilities. Raises: TypeError: if labels is not a `SparseTensor`. diff --git a/tensorflow/python/ops/data_flow_ops.py b/tensorflow/python/ops/data_flow_ops.py index 56dfde3bfb2..9b1381eebf2 100644 --- a/tensorflow/python/ops/data_flow_ops.py +++ b/tensorflow/python/ops/data_flow_ops.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# ============================================================================== +#============================================================================== """Data Flow Operations.""" # pylint: disable=g-bad-name diff --git a/tensorflow/python/ops/image_ops.py b/tensorflow/python/ops/image_ops.py index 7502c99e414..a72690c0f26 100644 --- a/tensorflow/python/ops/image_ops.py +++ b/tensorflow/python/ops/image_ops.py @@ -170,6 +170,9 @@ from tensorflow.python.ops import gen_image_ops from tensorflow.python.ops import gen_nn_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import random_ops +from tensorflow.python.ops import logging_ops +from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import check_ops # go/tf-wildcard-import # pylint: disable=wildcard-import @@ -177,6 +180,7 @@ from tensorflow.python.ops.gen_image_ops import * # pylint: enable=wildcard-import from tensorflow.python.util.all_util import make_all +from tensorflow.contrib.framework.python.framework import is_tensor @@ -189,19 +193,50 @@ ops.NoGradient('SampleDistortedBoundingBox') ops.NoGradient("ExtractGlimpse") -def _ImageDimensions(images): +def _assert(cond, ex_type, msg): + """A polymorphic assert, works with tensors and boolean expressions. + + If `cond` is not a tensor, behave like an ordinary assert statement, except + that a empty list is returned. If `cond` is a tensor, return a list + containing a single TensorFlow assert op. + + Args: + cond: Something evaluates to a boolean value. May be a tensor. + ex_type: The exception class to use. + msg: The error message. + + Returns: + A list, containing at most one assert op. + """ + if is_tensor(cond): + return [logging_ops.Assert(cond, [msg])] + else: + if not cond: + raise ex_type(msg) + else: + return [] + + +def _ImageDimensions(images, static_only=True): """Returns the dimensions of an image tensor. Args: - images: 4-D Tensor of shape [batch, height, width, channels] + images: 4-D Tensor of shape `[batch, height, width, channels]` + static_only: Boolean, whether to return only static shape. Returns: - list of integers [batch, height, width, channels] + list of integers `[batch, height, width, channels]`, when static shape is + fully defined or `static_only` is `True`. + list of integer scalar tensors `[batch, height, width, channels]`, when + static shape is not fully defined. """ # A simple abstraction to provide names for each dimension. This abstraction # should make it simpler to switch dimensions in the future (e.g. if we ever # want to switch height and width.) - return images.get_shape().as_list() + if static_only or images.get_shape().is_fully_defined(): + return images.get_shape().as_list() + else: + return array_ops.unpack(array_ops.shape(images)) def _Check3DImage(image, require_static=True): @@ -213,17 +248,27 @@ def _Check3DImage(image, require_static=True): known and non-zero. Raises: - ValueError: if image.shape is not a [3] vector. + ValueError: if `image.shape` is not a 3-vector. + + Returns: + An empty list, if `image` has fully defined dimensions. Otherwise, a list + containing an assert op is returned. """ try: image_shape = image.get_shape().with_rank(3) except ValueError: - raise ValueError('\'image\' must be three-dimensional.') + raise ValueError("'image' must be three-dimensional.") if require_static and not image_shape.is_fully_defined(): - raise ValueError('\'image\' must be fully defined.') + raise ValueError("'image' must be fully defined.") if any(x == 0 for x in image_shape): - raise ValueError('all dims of \'image.shape\' must be > 0: %s' % + raise ValueError("all dims of 'image.shape' must be > 0: %s" % image_shape) + if not image_shape.is_fully_defined(): + return [check_ops.assert_positive(array_ops.shape(image), + ["all dims of 'image.shape' " + "must be > 0."])] + else: + return [] def _CheckAtLeast3DImage(image): @@ -429,36 +474,39 @@ def pad_to_bounding_box(image, offset_height, offset_width, target_height, Raises: ValueError: If the shape of `image` is incompatible with the `offset_*` or - `target_*` arguments + `target_*` arguments, or either `offset_height` or `offset_width` is + negative. """ image = ops.convert_to_tensor(image, name='image') - _Check3DImage(image, require_static=True) - height, width, depth = _ImageDimensions(image) - if target_width < width: - raise ValueError('target_width must be >= width') - if target_height < height: - raise ValueError('target_height must be >= height') + assert_ops = [] + assert_ops += _Check3DImage(image, require_static=False) + height, width, depth = _ImageDimensions(image, static_only=False) after_padding_width = target_width - offset_width - width after_padding_height = target_height - offset_height - height - if after_padding_width < 0: - raise ValueError('target_width not possible given ' - 'offset_width and image width') - if after_padding_height < 0: - raise ValueError('target_height not possible given ' - 'offset_height and image height') + assert_ops += _assert(offset_height >= 0, ValueError, + 'offset_height must be >= 0') + assert_ops += _assert(offset_width >= 0, ValueError, + 'offset_width must be >= 0') + assert_ops += _assert(after_padding_width >= 0, ValueError, + 'width must be <= target - offset') + assert_ops += _assert(after_padding_height >= 0, ValueError, + 'height must be <= target - offset') + image = control_flow_ops.with_dependencies(assert_ops, image) # Do not pad on the depth dimensions. - if (offset_width or offset_height or after_padding_width or - after_padding_height): - paddings = [[offset_height, after_padding_height], - [offset_width, after_padding_width], [0, 0]] - padded = array_ops.pad(image, paddings) - padded.set_shape([target_height, target_width, depth]) - else: - padded = image + paddings = array_ops.reshape( + array_ops.pack([offset_height, after_padding_height, + offset_width, after_padding_width, + 0, 0]), + [3, 2]) + padded = array_ops.pad(image, paddings) + + padded_shape = [None if is_tensor(i) else i + for i in [target_height, target_width, depth]] + padded.set_shape(padded_shape) return padded @@ -486,24 +534,38 @@ def crop_to_bounding_box(image, offset_height, offset_width, target_height, Raises: ValueError: If the shape of `image` is incompatible with the `offset_*` or - `target_*` arguments + `target_*` arguments, or either `offset_height` or `offset_width` is + negative, or either `target_height` or `target_width` is not positive. """ image = ops.convert_to_tensor(image, name='image') - _Check3DImage(image, require_static=True) - height, width, _ = _ImageDimensions(image) - if offset_width < 0: - raise ValueError('offset_width must be >= 0.') - if offset_height < 0: - raise ValueError('offset_height must be >= 0.') + assert_ops = [] + assert_ops += _Check3DImage(image, require_static=False) - if width < (target_width + offset_width): - raise ValueError('width must be >= target + offset.') - if height < (target_height + offset_height): - raise ValueError('height must be >= target + offset.') + height, width, depth = _ImageDimensions(image, static_only=False) - cropped = array_ops.slice(image, [offset_height, offset_width, 0], - [target_height, target_width, -1]) + assert_ops += _assert(offset_width >= 0, ValueError, + 'offset_width must be >= 0.') + assert_ops += _assert(offset_height >= 0, ValueError, + 'offset_height must be >= 0.') + assert_ops += _assert(target_width > 0, ValueError, + 'target_width must be > 0.') + assert_ops += _assert(target_height > 0, ValueError, + 'target_height must be > 0.') + assert_ops += _assert(width >= (target_width + offset_width), ValueError, + 'width must be >= target + offset.') + assert_ops += _assert(height >= (target_height + offset_height), ValueError, + 'height must be >= target + offset.') + image = control_flow_ops.with_dependencies(assert_ops, image) + + cropped = array_ops.slice( + image, + array_ops.pack([offset_height, offset_width, 0]), + array_ops.pack([target_height, target_width, -1])) + + cropped_shape = [None if is_tensor(i) else i + for i in [target_height, target_width, depth]] + cropped.set_shape(cropped_shape) return cropped @@ -521,7 +583,7 @@ def resize_image_with_crop_or_pad(image, target_height, target_width): dimension. Args: - image: 3-D tensor of shape [height, width, channels] + image: 3-D tensor of shape `[height, width, channels]` target_height: Target height. target_width: Target width. @@ -533,43 +595,73 @@ def resize_image_with_crop_or_pad(image, target_height, target_width): `[target_height, target_width, channels]` """ image = ops.convert_to_tensor(image, name='image') - _Check3DImage(image, require_static=True) - original_height, original_width, _ = _ImageDimensions(image) - if target_width <= 0: - raise ValueError('target_width must be > 0.') - if target_height <= 0: - raise ValueError('target_height must be > 0.') + assert_ops = [] + assert_ops += _Check3DImage(image, require_static=False) + assert_ops += _assert(target_width > 0, ValueError, + 'target_width must be > 0.') + assert_ops += _assert(target_height > 0, ValueError, + 'target_height must be > 0.') - offset_crop_width = 0 - offset_pad_width = 0 - if target_width < original_width: - offset_crop_width = (original_width - target_width) // 2 - elif target_width > original_width: - offset_pad_width = (target_width - original_width) // 2 + image = control_flow_ops.with_dependencies(assert_ops, image) + # `crop_to_bounding_box` and `pad_to_bounding_box` have their own checks. + # Make sure our checks come first, so that error messages are clearer. + if is_tensor(target_height): + target_height = control_flow_ops.with_dependencies( + assert_ops, target_height) + if is_tensor(target_width): + target_width = control_flow_ops.with_dependencies(assert_ops, target_width) - offset_crop_height = 0 - offset_pad_height = 0 - if target_height < original_height: - offset_crop_height = (original_height - target_height) // 2 - elif target_height > original_height: - offset_pad_height = (target_height - original_height) // 2 + def max_(x, y): + if is_tensor(x) or is_tensor(y): + return math_ops.maximum(x, y) + else: + return max(x, y) + + def min_(x, y): + if is_tensor(x) or is_tensor(y): + return math_ops.minimum(x, y) + else: + return min(x, y) + + def equal_(x, y): + if is_tensor(x) or is_tensor(y): + return math_ops.equal(x, y) + else: + return x == y + + height, width, _ = _ImageDimensions(image, static_only=False) + width_diff = target_width - width + offset_crop_width = max_(-width_diff // 2, 0) + offset_pad_width = max_(width_diff // 2, 0) + + height_diff = target_height - height + offset_crop_height = max_(-height_diff // 2, 0) + offset_pad_height = max_(height_diff // 2, 0) # Maybe crop if needed. cropped = crop_to_bounding_box(image, offset_crop_height, offset_crop_width, - min(target_height, original_height), - min(target_width, original_width)) + min_(target_height, height), + min_(target_width, width)) # Maybe pad if needed. resized = pad_to_bounding_box(cropped, offset_pad_height, offset_pad_width, target_height, target_width) + # In theory all the checks below are redundant. if resized.get_shape().ndims is None: raise ValueError('resized contains no shape.') - if not resized.get_shape()[0].is_compatible_with(target_height): - raise ValueError('resized height is not correct.') - if not resized.get_shape()[1].is_compatible_with(target_width): - raise ValueError('resized width is not correct.') + + resized_height, resized_width, _ = \ + _ImageDimensions(resized, static_only=False) + + assert_ops = [] + assert_ops += _assert(equal_(resized_height, target_height), ValueError, + 'resized height is not correct.') + assert_ops += _assert(equal_(resized_width, target_width), ValueError, + 'resized width is not correct.') + + resized = control_flow_ops.with_dependencies(assert_ops, resized) return resized diff --git a/tensorflow/python/ops/image_ops_test.py b/tensorflow/python/ops/image_ops_test.py index db29f740458..7f5109d7df1 100644 --- a/tensorflow/python/ops/image_ops_test.py +++ b/tensorflow/python/ops/image_ops_test.py @@ -26,6 +26,7 @@ from six.moves import xrange # pylint: disable=redefined-builtin from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import image_ops @@ -478,39 +479,144 @@ class PerImageWhiteningTest(test_util.TensorFlowTestCase): class CropToBoundingBoxTest(test_util.TensorFlowTestCase): + def _CropToBoundingBox(self, x, offset_height, offset_width, + target_height, target_width, use_tensor_inputs): + if use_tensor_inputs: + offset_height = ops.convert_to_tensor(offset_height) + offset_width = ops.convert_to_tensor(offset_width) + target_height = ops.convert_to_tensor(target_height) + target_width = ops.convert_to_tensor(target_width) + x_tensor = array_ops.placeholder(x.dtype, shape=[None]*x.ndim) + feed_dict = {x_tensor: x} + else: + x_tensor = x + feed_dict = {} + + y = image_ops.crop_to_bounding_box(x_tensor, offset_height, offset_width, + target_height, target_width) + if not use_tensor_inputs: + self.assertTrue(y.get_shape().is_fully_defined()) + + with self.test_session(): + return y.eval(feed_dict=feed_dict) + + def _assertReturns(self, x, x_shape, offset_height, offset_width, + y, y_shape, use_tensor_inputs_options=None): + use_tensor_inputs_options = use_tensor_inputs_options or [False, True] + target_height, target_width, _ = y_shape + x = np.array(x).reshape(x_shape) + y = np.array(y).reshape(y_shape) + + for use_tensor_inputs in use_tensor_inputs_options: + y_tf = self._CropToBoundingBox(x, offset_height, offset_width, + target_height, target_width, + use_tensor_inputs) + self.assertAllClose(y, y_tf) + + def _assertRaises(self, x, x_shape, offset_height, offset_width, + target_height, target_width, err_msg, + use_tensor_inputs_options=None): + use_tensor_inputs_options = use_tensor_inputs_options or [False, True] + x = np.array(x).reshape(x_shape) + + for use_tensor_inputs in use_tensor_inputs_options: + try: + self._CropToBoundingBox(x, offset_height, offset_width, + target_height, target_width, + use_tensor_inputs) + except Exception as e: + if err_msg not in str(e): + raise + else: + raise AssertionError('Exception not raised: %s' % err_msg) + def testNoOp(self): - x_shape = [13, 9, 3] - x_np = np.ones(x_shape, dtype=np.float32) + x_shape = [10, 10, 10] + x = np.random.uniform(size=x_shape) + self._assertReturns(x, x_shape, 0, 0, x, x_shape) - with self.test_session(): - x = constant_op.constant(x_np, shape=x_shape) - target_height = x_shape[0] - target_width = x_shape[1] - y = image_ops.crop_to_bounding_box(x, 0, 0, target_height, target_width) - y_tf = y.eval() - self.assertAllEqual(y_tf, x_np) + def testCrop(self): + x = [1, 2, 3, + 4, 5, 6, + 7, 8, 9] + x_shape = [3, 3, 1] - def testCropping(self): - x_np = np.arange(0, 30, dtype=np.int32).reshape([6, 5, 1]) + offset_height, offset_width = [1, 0] + y_shape = [2, 3, 1] + y = [4, 5, 6, + 7, 8, 9] + self._assertReturns(x, x_shape, offset_height, offset_width, y, y_shape) - offset_height = 1 - after_height = 2 + offset_height, offset_width = [0, 1] + y_shape = [3, 2, 1] + y = [2, 3, + 5, 6, + 8, 9] + self._assertReturns(x, x_shape, offset_height, offset_width, y, y_shape) - offset_width = 0 - after_width = 3 + offset_height, offset_width = [0, 0] + y_shape = [2, 3, 1] + y = [1, 2, 3, + 4, 5, 6] + self._assertReturns(x, x_shape, offset_height, offset_width, y, y_shape) - target_height = x_np.shape[0] - offset_height - after_height - target_width = x_np.shape[1] - offset_width - after_width + offset_height, offset_width = [0, 0] + y_shape = [3, 2, 1] + y = [1, 2, + 4, 5, + 7, 8] + self._assertReturns(x, x_shape, offset_height, offset_width, y, y_shape) - y_np = x_np[offset_height:offset_height + target_height, - offset_width:offset_width + target_width, :] + def testNon3DInput(self): + # Input image is not 3D + x = [0] * 15 + offset_height, offset_width = [0, 0] + target_height, target_width = [2, 2] - with self.test_session(): - x = constant_op.constant(x_np, shape=x_np.shape) - y = image_ops.crop_to_bounding_box(x, offset_height, offset_width, - target_height, target_width) - y_tf = y.eval() - self.assertAllEqual(y_tf.flatten(), y_np.flatten()) + for x_shape in ([1, 3, 5, 1], [3, 5]): + self._assertRaises(x, x_shape, offset_height, offset_width, + target_height, target_width, + "'image' must be three-dimensional") + + def testZeroLengthInput(self): + # Input image has 0-length dimension(s). + # Each line is a test configuration: + # x_shape, target_height, target_width + test_config = (([0, 2, 2], 1, 1), + ([2, 0, 2], 1, 1), + ([2, 2, 0], 1, 1), + ([0, 2, 2], 0, 1), + ([2, 0, 2], 1, 0)) + offset_height, offset_width = [0, 0] + x = [] + + for x_shape, target_height, target_width in test_config: + self._assertRaises(x, x_shape, offset_height, offset_width, + target_height, target_width, + "all dims of 'image.shape' must be > 0", + use_tensor_inputs_options=[False]) + # Multiple assertion could fail, but the evaluation order is arbitrary. + # Match gainst generic pattern. + self._assertRaises(x, x_shape, offset_height, offset_width, + target_height, target_width, + "assertion failed:", + use_tensor_inputs_options=[True]) + + def testBadParams(self): + x_shape = [4, 4, 1] + x = np.zeros(x_shape) + + # Each line is a test configuration: + # (offset_height, offset_width, target_height, target_width), err_msg + test_config = (([-1, 0, 3, 3], 'offset_height must be >= 0'), + ([0, -1, 3, 3], 'offset_width must be >= 0'), + ([0, 0, 0, 3], 'target_height must be > 0'), + ([0, 0, 3, 0], 'target_width must be > 0'), + ([2, 0, 3, 3], 'height must be >= target + offset'), + ([0, 2, 3, 3], 'width must be >= target + offset')) + + for params, err_msg in test_config: + self._assertRaises(x, x_shape, *params, err_msg=err_msg) class CentralCropTest(test_util.TensorFlowTestCase): @@ -550,45 +656,148 @@ class CentralCropTest(test_util.TensorFlowTestCase): class PadToBoundingBoxTest(test_util.TensorFlowTestCase): - def testNoOp(self): - x_shape = [13, 9, 3] - x_np = np.ones(x_shape, dtype=np.float32) + def _PadToBoundingBox(self, x, offset_height, offset_width, + target_height, target_width, use_tensor_inputs): + if use_tensor_inputs: + offset_height = ops.convert_to_tensor(offset_height) + offset_width = ops.convert_to_tensor(offset_width) + target_height = ops.convert_to_tensor(target_height) + target_width = ops.convert_to_tensor(target_width) + x_tensor = array_ops.placeholder(x.dtype, shape=[None]*x.ndim) + feed_dict = {x_tensor: x} + else: + x_tensor = x + feed_dict = {} - target_height = x_shape[0] - target_width = x_shape[1] + y = image_ops.pad_to_bounding_box(x_tensor, offset_height, offset_width, + target_height, target_width) + if not use_tensor_inputs: + self.assertTrue(y.get_shape().is_fully_defined()) with self.test_session(): - x = constant_op.constant(x_np, shape=x_shape) - y = image_ops.pad_to_bounding_box(x, 0, 0, target_height, target_width) - y_tf = y.eval() - self.assertAllEqual(y_tf, x_np) + return y.eval(feed_dict=feed_dict) + + def _assertReturns(self, x, x_shape, offset_height, offset_width, + y, y_shape, use_tensor_inputs_options=None): + use_tensor_inputs_options = use_tensor_inputs_options or [False, True] + target_height, target_width, _ = y_shape + x = np.array(x).reshape(x_shape) + y = np.array(y).reshape(y_shape) + + for use_tensor_inputs in use_tensor_inputs_options: + y_tf = self._PadToBoundingBox(x, offset_height, offset_width, + target_height, target_width, + use_tensor_inputs) + self.assertAllClose(y, y_tf) + + def _assertRaises(self, x, x_shape, offset_height, offset_width, + target_height, target_width, err_msg, + use_tensor_inputs_options=None): + use_tensor_inputs_options = use_tensor_inputs_options or [False, True] + x = np.array(x).reshape(x_shape) + + for use_tensor_inputs in use_tensor_inputs_options: + try: + self._PadToBoundingBox(x, offset_height, offset_width, + target_height, target_width, + use_tensor_inputs) + except Exception as e: + if err_msg not in str(e): + raise + else: + raise AssertionError('Exception not raised: %s' % err_msg) + + def testNoOp(self): + x_shape = [10, 10, 10] + x = np.random.uniform(size=x_shape) + offset_height, offset_width = [0, 0] + + self._assertReturns(x, x_shape, offset_height, offset_width, x, x_shape) def testPadding(self): - x_shape = [3, 4, 1] - x_np = np.ones(x_shape, dtype=np.float32) + x = [1, 2, 3, + 4, 5, 6, + 7, 8, 9] + x_shape = [3, 3, 1] - offset_height = 2 - after_height = 3 + offset_height, offset_width = [1, 0] + y = [0, 0, 0, + 1, 2, 3, + 4, 5, 6, + 7, 8, 9] + y_shape = [4, 3, 1] + self._assertReturns(x, x_shape, offset_height, offset_width, y, y_shape) - offset_width = 1 - after_width = 4 + offset_height, offset_width = [0, 1] + y = [0, 1, 2, 3, + 0, 4, 5, 6, + 0, 7, 8, 9] + y_shape = [3, 4, 1] + self._assertReturns(x, x_shape, offset_height, offset_width, y, y_shape) - target_height = x_shape[0] + offset_height + after_height - target_width = x_shape[1] + offset_width + after_width + offset_height, offset_width = [0, 0] + y = [1, 2, 3, + 4, 5, 6, + 7, 8, 9, + 0, 0, 0] + y_shape = [4, 3, 1] + self._assertReturns(x, x_shape, offset_height, offset_width, y, y_shape) - # Note the padding are along batch, height, width and depth. - paddings = ((offset_height, after_height), - (offset_width, after_width), - (0, 0)) + offset_height, offset_width = [0, 0] + y = [1, 2, 3, 0, + 4, 5, 6, 0, + 7, 8, 9, 0] + y_shape = [3, 4, 1] + self._assertReturns(x, x_shape, offset_height, offset_width, y, y_shape) - y_np = np.pad(x_np, paddings, 'constant') + def testNon3DInput(self): + # Input image is not 3D + x = [0] * 15 + offset_height, offset_width = [0, 0] + target_height, target_width = [2, 2] - with self.test_session(): - x = constant_op.constant(x_np, shape=x_shape) - y = image_ops.pad_to_bounding_box(x, offset_height, offset_width, - target_height, target_width) - y_tf = y.eval() - self.assertAllEqual(y_tf, y_np) + for x_shape in ([1, 3, 5, 1], [3, 5]): + self._assertRaises(x, x_shape, offset_height, offset_width, + target_height, target_width, + "'image' must be three-dimensional") + + def testZeroLengthInput(self): + # Input image has 0-length dimension(s). + # Each line is a test configuration: + # x_shape, target_height, target_width + test_config = (([0, 2, 2], 2, 2), + ([2, 0, 2], 2, 2), + ([2, 2, 0], 2, 2)) + offset_height, offset_width = [0, 0] + x = [] + + for x_shape, target_height, target_width in test_config: + self._assertRaises(x, x_shape, offset_height, offset_width, + target_height, target_width, + "all dims of 'image.shape' must be > 0", + use_tensor_inputs_options=[False]) + + # The orignal error message does not contain back slashes. However, they + # are added by either the assert op or the runtime. If this behaviour + # changes in the future, the match string will also needs to be changed. + self._assertRaises(x, x_shape, offset_height, offset_width, + target_height, target_width, + "all dims of \\'image.shape\\' must be > 0", + use_tensor_inputs_options=[True]) + + def testBadParams(self): + x_shape = [3, 3, 1] + x = np.zeros(x_shape) + + # Each line is a test configuration: + # offset_height, offset_width, target_height, target_width, err_msg + test_config = ((-1, 0, 4, 4, 'offset_height must be >= 0'), + ( 0,-1, 4, 4, 'offset_width must be >= 0'), + ( 2, 0, 4, 4, 'height must be <= target - offset'), + ( 0, 2, 4, 4, 'width must be <= target - offset')) + + for config_item in test_config: + self._assertRaises(x, x_shape, *config_item) class SelectDistortedCropBoxTest(test_util.TensorFlowTestCase): @@ -998,128 +1207,230 @@ class ResizeImagesTest(test_util.TensorFlowTestCase): class ResizeImageWithCropOrPadTest(test_util.TensorFlowTestCase): - def _ResizeImageWithCropOrPad(self, original, original_shape, - expected, expected_shape): - x_np = np.array(original, dtype=np.uint8).reshape(original_shape) - y_np = np.array(expected).reshape(expected_shape) + def _ResizeImageWithCropOrPad(self, x, target_height, target_width, + use_tensor_inputs): + if use_tensor_inputs: + target_height = ops.convert_to_tensor(target_height) + target_width = ops.convert_to_tensor(target_width) + x_tensor = array_ops.placeholder(x.dtype, shape=[None]*x.ndim) + feed_dict = {x_tensor: x} + else: + x_tensor = x + feed_dict = {} - target_height = expected_shape[0] - target_width = expected_shape[1] + y = image_ops.resize_image_with_crop_or_pad( + x_tensor, target_height, target_width) + if not use_tensor_inputs: + self.assertTrue(y.get_shape().is_fully_defined()) with self.test_session(): - image = constant_op.constant(x_np, shape=original_shape) - y = image_ops.resize_image_with_crop_or_pad(image, - target_height, - target_width) - resized = y.eval() - self.assertAllClose(resized, y_np, atol=1e-5) + return y.eval(feed_dict=feed_dict) - def testBasic(self): - # Basic no-op. - original = [1, 2, 3, 4, - 5, 6, 7, 8] - self._ResizeImageWithCropOrPad(original, [2, 4, 1], - original, [2, 4, 1]) + def _assertReturns(self, x, x_shape, y, y_shape, + use_tensor_inputs_options=None): + use_tensor_inputs_options = use_tensor_inputs_options or [False, True] + target_height, target_width, _ = y_shape + x = np.array(x).reshape(x_shape) + y = np.array(y).reshape(y_shape) + + for use_tensor_inputs in use_tensor_inputs_options: + y_tf = self._ResizeImageWithCropOrPad(x, target_height, target_width, + use_tensor_inputs) + self.assertAllClose(y, y_tf) + + def _assertRaises(self, x, x_shape, target_height, target_width, err_msg, + use_tensor_inputs_options=None): + use_tensor_inputs_options = use_tensor_inputs_options or [False, True] + x = np.array(x).reshape(x_shape) + + for use_tensor_inputs in use_tensor_inputs_options: + try: + self._ResizeImageWithCropOrPad(x, target_height, target_width, + use_tensor_inputs) + except Exception as e: + if err_msg not in str(e): + raise + else: + raise AssertionError('Exception not raised: %s' % err_msg) + + def testNoOp(self): + x_shape = [10, 10, 10] + x = np.random.uniform(size=x_shape) + + self._assertReturns(x, x_shape, x, x_shape) def testPad(self): # Pad even along col. - original = [1, 2, 3, 4, 5, 6, 7, 8] - expected = [0, 1, 2, 3, 4, 0, - 0, 5, 6, 7, 8, 0] - self._ResizeImageWithCropOrPad(original, [2, 4, 1], - expected, [2, 6, 1]) + x = [1, 2, 3, 4, + 5, 6, 7, 8] + x_shape = [2, 4, 1] + + y = [0, 1, 2, 3, 4, 0, + 0, 5, 6, 7, 8, 0] + y_shape = [2, 6, 1] + + self._assertReturns(x, x_shape, y, y_shape) + # Pad odd along col. - original = [1, 2, 3, 4, - 5, 6, 7, 8] - expected = [0, 1, 2, 3, 4, 0, 0, - 0, 5, 6, 7, 8, 0, 0] - self._ResizeImageWithCropOrPad(original, [2, 4, 1], - expected, [2, 7, 1]) + x = [1, 2, 3, 4, + 5, 6, 7, 8] + x_shape = [2, 4, 1] + + y = [0, 1, 2, 3, 4, 0, 0, + 0, 5, 6, 7, 8, 0, 0] + y_shape = [2, 7, 1] + + self._assertReturns(x, x_shape, y, y_shape) # Pad even along row. - original = [1, 2, 3, 4, - 5, 6, 7, 8] - expected = [0, 0, 0, 0, - 1, 2, 3, 4, - 5, 6, 7, 8, - 0, 0, 0, 0] - self._ResizeImageWithCropOrPad(original, [2, 4, 1], - expected, [4, 4, 1]) + x = [1, 2, 3, 4, + 5, 6, 7, 8] + x_shape = [2, 4, 1] + + y = [0, 0, 0, 0, + 1, 2, 3, 4, + 5, 6, 7, 8, + 0, 0, 0, 0] + y_shape = [4, 4, 1] + + self._assertReturns(x, x_shape, y, y_shape) + # Pad odd along row. - original = [1, 2, 3, 4, - 5, 6, 7, 8] - expected = [0, 0, 0, 0, - 1, 2, 3, 4, - 5, 6, 7, 8, - 0, 0, 0, 0, - 0, 0, 0, 0] - self._ResizeImageWithCropOrPad(original, [2, 4, 1], - expected, [5, 4, 1]) + x = [1, 2, 3, 4, + 5, 6, 7, 8] + x_shape = [2, 4, 1] + + y = [0, 0, 0, 0, + 1, 2, 3, 4, + 5, 6, 7, 8, + 0, 0, 0, 0, + 0, 0, 0, 0] + y_shape = [5, 4, 1] + + self._assertReturns(x, x_shape, y, y_shape) def testCrop(self): # Crop even along col. - original = [1, 2, 3, 4, - 5, 6, 7, 8] - expected = [2, 3, - 6, 7] - self._ResizeImageWithCropOrPad(original, [2, 4, 1], - expected, [2, 2, 1]) - # Crop odd along col. + x = [1, 2, 3, 4, + 5, 6, 7, 8] + x_shape = [2, 4, 1] - original = [1, 2, 3, 4, 5, 6, - 7, 8, 9, 10, 11, 12] - expected = [2, 3, 4, - 8, 9, 10] - self._ResizeImageWithCropOrPad(original, [2, 6, 1], - expected, [2, 3, 1]) + y = [2, 3, + 6, 7] + y_shape = [2, 2, 1] + + self._assertReturns(x, x_shape, y, y_shape) + + # Crop odd along col. + x = [1, 2, 3, 4, 5, 6, + 7, 8, 9, 10, 11, 12] + x_shape = [2, 6, 1] + + y = [2, 3, 4, + 8, 9, 10] + y_shape = [2, 3, 1] + + self._assertReturns(x, x_shape, y, y_shape) # Crop even along row. - original = [1, 2, - 3, 4, - 5, 6, - 7, 8] - expected = [3, 4, - 5, 6] - self._ResizeImageWithCropOrPad(original, [4, 2, 1], - expected, [2, 2, 1]) + x = [1, 2, + 3, 4, + 5, 6, + 7, 8] + x_shape = [4, 2, 1] + + y = [3, 4, + 5, 6] + y_shape = [2, 2, 1] + + self._assertReturns(x, x_shape, y, y_shape) # Crop odd along row. - original = [1, 2, - 3, 4, - 5, 6, - 7, 8, - 9, 10, - 11, 12, - 13, 14, - 15, 16] - expected = [3, 4, - 5, 6, - 7, 8, - 9, 10, - 11, 12] - self._ResizeImageWithCropOrPad(original, [8, 2, 1], - expected, [5, 2, 1]) + x = [1, 2, + 3, 4, + 5, 6, + 7, 8, + 9, 10, + 11, 12, + 13, 14, + 15, 16] + x_shape = [8, 2, 1] + + y = [3, 4, + 5, 6, + 7, 8, + 9, 10, + 11, 12] + y_shape = [5, 2, 1] + + self._assertReturns(x, x_shape, y, y_shape) def testCropAndPad(self): # Pad along row but crop along col. - original = [1, 2, 3, 4, - 5, 6, 7, 8] - expected = [0, 0, - 2, 3, - 6, 7, - 0, 0] - self._ResizeImageWithCropOrPad(original, [2, 4, 1], - expected, [4, 2, 1]) + x = [1, 2, 3, 4, + 5, 6, 7, 8] + x_shape = [2, 4, 1] + + y = [0, 0, + 2, 3, + 6, 7, + 0, 0] + y_shape = [4, 2, 1] + + self._assertReturns(x, x_shape, y, y_shape) # Crop along row but pad along col. - original = [1, 2, - 3, 4, - 5, 6, - 7, 8] - expected = [0, 3, 4, 0, - 0, 5, 6, 0] - self._ResizeImageWithCropOrPad(original, [4, 2, 1], - expected, [2, 4, 1]) + x = [1, 2, + 3, 4, + 5, 6, + 7, 8] + x_shape = [4, 2, 1] + + y = [0, 3, 4, 0, + 0, 5, 6, 0] + y_shape = [2, 4, 1] + + self._assertReturns(x, x_shape, y, y_shape) + + def testNon3DInput(self): + # Input image is not 3D + x = [0] * 15 + target_height, target_width = [4, 4] + + for x_shape in ([1, 3, 5, 1], [3, 5]): + self._assertRaises(x, x_shape, target_height, target_width, + "'image' must be three-dimensional") + + def testZeroLengthInput(self): + # Input image has 0-length dimension(s). + target_height, target_width = [1, 1] + x = [] + + for x_shape in ([0, 2, 2], [2, 0, 2], [2, 2, 0]): + self._assertRaises(x, x_shape, target_height, target_width, + "all dims of 'image.shape' must be > 0", + use_tensor_inputs_options=[False]) + + # The orignal error message does not contain back slashes. However, they + # are added by either the assert op or the runtime. If this behaviour + # changes in the future, the match string will also needs to be changed. + self._assertRaises(x, x_shape, target_height, target_width, + "all dims of \\'image.shape\\' must be > 0", + use_tensor_inputs_options=[True]) + + def testBadParams(self): + x_shape = [4, 4, 1] + x = np.zeros(x_shape) + + # target_height <= 0 + target_height, target_width = [0, 5] + self._assertRaises(x, x_shape, target_height, target_width, + 'target_height must be > 0') + + # target_width <= 0 + target_height, target_width = [5, 0] + self._assertRaises(x, x_shape, target_height, target_width, + 'target_width must be > 0') def _SimpleColorRamp(): diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py index d27cefc61da..0a76450c5b1 100644 --- a/tensorflow/python/ops/math_ops.py +++ b/tensorflow/python/ops/math_ops.py @@ -1548,17 +1548,20 @@ def tanh(x, name=None): """Computes hyperbolic tangent of `x` element-wise. Args: - x: A Tensor with type `float32`, `float64`, `int32`, `complex64`, `int64`, - or `qint32`. + x: A Tensor or SparseTensor with type `float`, `double`, `int32`, + `complex64`, `int64`, or `qint32`. name: A name for the operation (optional). Returns: - A Tensor with the same type as `x` if `x.dtype != qint32` otherwise - the return type is `quint8`. + A Tensor or SparseTensor respectively with the same type as `x` if + `x.dtype != qint32` otherwise the return type is `quint8`. """ with ops.op_scope([x], name, "Tanh") as name: - x = ops.convert_to_tensor(x, name="x") - return gen_math_ops._tanh(x, name=name) + if isinstance(x, ops.SparseTensor): + x_tanh = gen_math_ops._tanh(x.values, name=name) + return ops.SparseTensor(indices=x.indices, values=x_tanh, shape=x.shape) + else: + return gen_math_ops._tanh(x, name=name) ops.RegisterShape("Abs")(common_shapes.unchanged_shape) diff --git a/tensorflow/python/ops/nn.py b/tensorflow/python/ops/nn.py index f7981601114..79030645f28 100644 --- a/tensorflow/python/ops/nn.py +++ b/tensorflow/python/ops/nn.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# ============================================================================== +# ============================================================================= # pylint: disable=unused-import,g-bad-import-order """## Activation Functions diff --git a/tensorflow/python/ops/nn_conv_test.py b/tensorflow/python/ops/nn_conv_test.py index 8c771fc2a3d..3d3425292c2 100644 --- a/tensorflow/python/ops/nn_conv_test.py +++ b/tensorflow/python/ops/nn_conv_test.py @@ -319,6 +319,195 @@ class Conv2DBackpropFilterGradTest(tf.test.TestCase): self.assertLess(err, err_tolerance) +class Conv3DTransposeTest(tf.test.TestCase): + + def testConv3DTransposeSingleStride(self): + with self.test_session(): + strides = [1, 1, 1, 1, 1] + + # Input, output: [batch, depth, height, width, channel] + x_shape = [2, 5, 6, 4, 3] + y_shape = [2, 5, 6, 4, 2] + + # Filter: [kernel_depth, kernel_height, kernel_width, out_depth, in_depth] + f_shape = [3, 3, 3, 2, 3] + + x = tf.constant(1.0, shape=x_shape, name="x", dtype=tf.float32) + f = tf.constant(1.0, shape=f_shape, name="filter", dtype=tf.float32) + output = tf.nn.conv3d_transpose(x, f, y_shape, strides=strides, + padding="SAME") + value = output.eval() + + # We count the number of cells being added at the locations in the output. + # At the center, #cells = kernel_depth * kernel_height * kernel_width + # At the corners, #cells = ceil(kernel_depth/2) * ceil(kernel_height/2) + # * ceil(kernel_width/2) + # At the edges, #cells = + # kernel_depth * ceil(kernel_height/2) * ceil(kernel_width/2) or + # ceil(kernel_depth/2) * kernel_height * ceil(kernel_width/2) or + # ceil(kernel_depth/2) * ceil(kernel_height/2) * kernel_width + # At the borders, #cells = + # ceil(kernel_depth/2) * kernel_height * kernel_width or + # kernel_depth * ceil(kernel_height/2) * kernel_width or + # kernel_depth * kernel_height * ceil(kernel_width/2) + + for n in xrange(x_shape[0]): + for k in xrange(f_shape[3]): + for w in xrange(y_shape[3]): + for h in xrange(y_shape[2]): + for d in xrange(y_shape[1]): + d_in = d > 0 and d < y_shape[1] - 1 + h_in = h > 0 and h < y_shape[2] - 1 + w_in = w > 0 and w < y_shape[3] - 1 + if d_in + h_in + w_in == 3: + target = 27 * 3.0 + elif d_in + h_in + w_in == 2: + target = 18 * 3.0 + elif d_in or h_in or w_in: + target = 12 * 3.0 + else: + target = 8 * 3.0 + self.assertAllClose(target, value[n, d, h, w, k]) + + def testConv3DTransposeSame(self): + with self.test_session(): + strides = [1, 2, 2, 2, 1] + + # Input, output: [batch, depth, height, width, depth] + x_shape = [2, 5, 6, 4, 3] + y_shape = [2, 10, 12, 8, 2] + + # Filter: [kernel_depth, kernel_height, kernel_width, out_depth, in_depth] + f_shape = [3, 3, 3, 2, 3] + + x = tf.constant(1.0, shape=x_shape, name="x", dtype=tf.float32) + f = tf.constant(1.0, shape=f_shape, name="filter", dtype=tf.float32) + output = tf.nn.conv3d_transpose(x, f, y_shape, strides=strides, + padding="SAME") + value = output.eval() + + for n in xrange(x_shape[0]): + for k in xrange(f_shape[3]): + for w in xrange(y_shape[3]): + for h in xrange(y_shape[2]): + for d in xrange(y_shape[1]): + # We add a case for locations divisible by the stride. + d_in = d % strides[1] == 0 and 0 < d < y_shape[1] - 1 + h_in = h % strides[2] == 0 and 0 < h < y_shape[2] - 1 + w_in = w % strides[3] == 0 and 0 < w < y_shape[3] - 1 + if d_in + h_in + w_in == 3: + target = 8 * 3.0 + elif d_in + h_in + w_in == 2: + target = 4 * 3.0 + elif d_in or h_in or w_in: + target = 2 * 3.0 + else: + target = 3.0 + self.assertAllClose(target, value[n, d, h, w, k]) + + def testConv3DTransposeValid(self): + with self.test_session(): + strides = [1, 2, 2, 2, 1] + + # Input, output: [batch, depth, height, width, depth] + x_shape = [2, 5, 6, 4, 3] + y_shape = [2, 11, 13, 9, 2] + + # Filter: [kernel_depth, kernel_height, kernel_width, out_depth, in_depth] + f_shape = [3, 3, 3, 2, 3] + + x = tf.constant(1.0, shape=x_shape, name="x", dtype=tf.float32) + f = tf.constant(1.0, shape=f_shape, name="filter", dtype=tf.float32) + output = tf.nn.conv3d_transpose(x, f, y_shape, strides=strides, + padding="VALID") + value = output.eval() + + cache_values = np.zeros(y_shape, dtype=np.float32) + + # The amount of padding added + pad = 1 + + for n in xrange(x_shape[0]): + for k in xrange(f_shape[3]): + for w in xrange(y_shape[3]): + for h in xrange(y_shape[2]): + for d in xrange(y_shape[1]): + # We add a case for locations divisible by the stride. + d_in = d % strides[1] == 0 and pad < d < y_shape[1] - 1 - pad + h_in = h % strides[2] == 0 and pad < h < y_shape[2] - 1 - pad + w_in = w % strides[3] == 0 and pad < w < y_shape[3] - 1 - pad + if d_in + h_in + w_in == 3: + target = 8 * 3.0 + elif d_in + h_in + w_in == 2: + target = 4 * 3.0 + elif d_in or h_in or w_in: + target = 2 * 3.0 + else: + target = 3.0 + cache_values[n, d, h, w, k] = target + + # copy values in the border + cache_values[n, :, :, 0, k] = cache_values[n, :, :, 1, k] + cache_values[n, :, :, -1, k] = cache_values[n, :, :, -2, k] + cache_values[n, :, 0, :, k] = cache_values[n, :, 1, :, k] + cache_values[n, :, -1, :, k] = cache_values[n, :, -2, :, k] + cache_values[n, 0, :, :, k] = cache_values[n, 1, :, :, k] + cache_values[n, -1, :, :, k] = cache_values[n, -2, :, :, k] + + self.assertAllClose(cache_values, value) + + def testGradient(self): + x_shape = [2, 3, 4, 3, 2] + f_shape = [3, 3, 3, 2, 2] + y_shape = [2, 6, 8, 6, 2] + strides = [1, 2, 2, 2, 1] + np.random.seed(1) # Make it reproducible. + x_val = np.random.random_sample(x_shape).astype(np.float64) + f_val = np.random.random_sample(f_shape).astype(np.float64) + with self.test_session(): + x = tf.constant(x_val, name="x", dtype=tf.float32) + f = tf.constant(f_val, name="f", dtype=tf.float32) + output = tf.nn.conv3d_transpose(x, f, y_shape, strides=strides, + padding="SAME") + err = tf.test.compute_gradient_error( + [x, f], [x_shape, f_shape], output, y_shape) + print("conv3d_transpose gradient err = %g " % err) + err_tolerance = 0.0005 + self.assertLess(err, err_tolerance) + + +class Conv3DBackpropFilterV2GradTest(tf.test.TestCase): + + def testGradient(self): + with self.test_session(): + for padding in ["SAME", "VALID"]: + for stride in [1, 2]: + np.random.seed(1) + in_shape = [2, 4, 3, 3, 2] + in_val = tf.constant( + 2 * np.random.random_sample(in_shape) - 1, + dtype=tf.float32) + filter_shape = [3, 3, 3, 2, 3] + strides = [1, stride, stride, stride, 1] + # Make a convolution op with the current settings, just to easily get + # the shape of the output. + conv_out = tf.nn.conv3d(in_val, tf.zeros(filter_shape), strides, + padding) + out_backprop_shape = conv_out.get_shape().as_list() + out_backprop_val = tf.constant( + 2 * np.random.random_sample(out_backprop_shape) - 1, + dtype=tf.float32) + output = tf.nn.conv3d_backprop_filter_v2(in_val, filter_shape, + out_backprop_val, + strides, padding) + err = tf.test.compute_gradient_error([in_val, out_backprop_val], + [in_shape, out_backprop_shape], + output, filter_shape) + print("conv3d_backprop_filter gradient err = %g " % err) + err_tolerance = 1e-3 + self.assertLess(err, err_tolerance) + + class Conv1DTest(tf.test.TestCase): def testBasic(self): diff --git a/tensorflow/python/ops/nn_grad.py b/tensorflow/python/ops/nn_grad.py index 3f4bb0e0688..a396f06ebad 100644 --- a/tensorflow/python/ops/nn_grad.py +++ b/tensorflow/python/ops/nn_grad.py @@ -82,6 +82,34 @@ def _Conv3DGrad(op, grad): padding=op.get_attr("padding"))] +@ops.RegisterGradient("Conv3DBackpropInputV2") +def _Conv3DBackpropInputGrad(op, grad): + return [None, + nn_ops.conv3d_backprop_filter_v2(grad, + array_ops.shape(op.inputs[1]), + op.inputs[2], + strides=op.get_attr("strides"), + padding=op.get_attr("padding")), + nn_ops.conv3d(grad, + op.inputs[1], + strides=op.get_attr("strides"), + padding=op.get_attr("padding"))] + + +@ops.RegisterGradient("Conv3DBackpropFilterV2") +def _Conv3DBackpropFilterGrad(op, grad): + return [nn_ops.conv3d_backprop_input_v2(array_ops.shape(op.inputs[0]), + grad, + op.inputs[2], + strides=op.get_attr("strides"), + padding=op.get_attr("padding")), + None, + nn_ops.conv3d(op.inputs[0], + grad, + strides=op.get_attr("strides"), + padding=op.get_attr("padding"))] + + @ops.RegisterGradient("AvgPool3D") def _AvgPool3DGrad(op, grad): return nn_ops.avg_pool3d_grad( diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py index d17536daebd..486ef6efb97 100644 --- a/tensorflow/python/ops/nn_ops.py +++ b/tensorflow/python/ops/nn_ops.py @@ -297,6 +297,73 @@ def conv2d_transpose(value, name=name) +def conv3d_transpose(value, + filter, + output_shape, + strides, + padding="SAME", + name=None): + """The transpose of `conv3d`. + + This operation is sometimes called "deconvolution" after [Deconvolutional + Networks](http://www.matthewzeiler.com/pubs/cvpr2010/cvpr2010.pdf), but is + actually the transpose (gradient) of `conv3d` rather than an actual + deconvolution. + + Args: + value: A 5-D `Tensor` of type `float` and shape + `[batch, depth, height, width, in_channels]`. + filter: A 5-D `Tensor` with the same type as `value` and shape + `[depth, height, width, output_channels, in_channels]`. `filter`'s + `in_channels` dimension must match that of `value`. + output_shape: A 1-D `Tensor` representing the output shape of the + deconvolution op. + strides: A list of ints. The stride of the sliding window for each + dimension of the input tensor. + padding: A string, either `'VALID'` or `'SAME'`. The padding algorithm. + See the [comment here](https://www.tensorflow.org/api_docs/python/nn.html#convolution) + name: Optional name for the returned tensor. + + Returns: + A `Tensor` with the same type as `value`. + + Raises: + ValueError: If input/output depth does not match `filter`'s shape, or if + padding is other than `'VALID'` or `'SAME'`. + """ + with ops.op_scope([value, filter, output_shape], name, + "conv3d_transpose") as name: + value = ops.convert_to_tensor(value, name="value") + filter = ops.convert_to_tensor(filter, name="filter") + if not value.get_shape()[4].is_compatible_with(filter.get_shape()[4]): + raise ValueError("input channels does not match filter's input channels, " + "{} != {}".format(value.get_shape()[4], filter.get_shape( + )[4])) + + output_shape_ = ops.convert_to_tensor(output_shape, name="output_shape") + if not output_shape_.get_shape().is_compatible_with(tensor_shape.vector(5)): + raise ValueError("output_shape must have shape (5,), got {}" + .format(output_shape_.get_shape())) + + if isinstance(output_shape, (list, np.ndarray)): + # output_shape's shape should be == [5] if reached this point. + if not filter.get_shape()[3].is_compatible_with(output_shape[4]): + raise ValueError( + "output_shape does not match filter's output channels, " + "{} != {}".format(output_shape[4], filter.get_shape()[3])) + + if padding != "VALID" and padding != "SAME": + raise ValueError("padding must be either VALID or SAME:" + " {}".format(padding)) + + return gen_nn_ops.conv3d_backprop_input_v2(input_sizes=output_shape_, + filter=filter, + out_backprop=value, + strides=strides, + padding=padding, + name=name) + + # pylint: disable=protected-access def bias_add(value, bias, data_format=None, name=None): """Adds `bias` to `value`. diff --git a/tensorflow/python/ops/rnn.py b/tensorflow/python/ops/rnn.py index 0ad22b2e4dc..8917a38a56d 100644 --- a/tensorflow/python/ops/rnn.py +++ b/tensorflow/python/ops/rnn.py @@ -535,6 +535,125 @@ def bidirectional_rnn(cell_fw, cell_bw, inputs, return (outputs, output_state_fw, output_state_bw) +def bidirectional_dynamic_rnn(cell_fw, cell_bw, inputs, sequence_length=None, + initial_state_fw=None, initial_state_bw=None, + dtype=None, parallel_iterations=None, + swap_memory=False, time_major=False, scope=None): + """Creates a dynamic version of bidirectional recurrent neural network. + + Similar to the unidirectional case above (rnn) but takes input and builds + independent forward and backward RNNs. The input_size of forward and + backward cell must match. The initial state for both directions is zero by + default (but can be set optionally) and no intermediate states are ever + returned -- the network is fully unrolled for the given (passed in) + length(s) of the sequence(s) or completely unrolled if length(s) is not + given. + + Args: + cell_fw: An instance of RNNCell, to be used for forward direction. + cell_bw: An instance of RNNCell, to be used for backward direction. + inputs: The RNN inputs. + If time_major == False (default), this must be a tensor of shape: + `[batch_size, max_time, input_size]`. + If time_major == True, this must be a tensor of shape: + `[max_time, batch_size, input_size]`. + [batch_size, input_size]. + sequence_length: An int32/int64 vector, size `[batch_size]`, + containing the actual lengths for each of the sequences. + initial_state_fw: (optional) An initial state for the forward RNN. + This must be a tensor of appropriate type and shape + `[batch_size x cell_fw.state_size]`. + If `cell_fw.state_size` is a tuple, this should be a tuple of + tensors having shapes `[batch_size, s] for s in cell_fw.state_size`. + initial_state_bw: (optional) Same as for `initial_state_fw`, but using + the corresponding properties of `cell_bw`. + parallel_iterations: (Default: 32). The number of iterations to run in + parallel. Those operations which do not have any temporal dependency + and can be run in parallel, will be. This parameter trades off + time for space. Values >> 1 use more memory but take less time, + while smaller values use less memory but computations take longer. + swap_memory: Transparently swap the tensors produced in forward inference + but needed for back prop from GPU to CPU. This allows training RNNs + which would typically not fit on a single GPU, with very minimal (or no) + performance penalty. + time_major: The shape format of the `inputs` and `outputs` Tensors. + If true, these `Tensors` must be shaped `[max_time, batch_size, depth]`. + If false, these `Tensors` must be shaped `[batch_size, max_time, depth]`. + Using `time_major = True` is a bit more efficient because it avoids + transposes at the beginning and end of the RNN calculation. However, + most TensorFlow data is batch-major, so by default this function + accepts input and emits output in batch-major form. + dtype: (optional) The data type for the initial state. Required if + initial_state is not provided. + sequence_length: An int32/int64 vector, size `[batch_size]`, + containing the actual lengths for each of the sequences. + either of the initial states are not provided. + scope: VariableScope for the created subgraph; defaults to "BiRNN" + + Returns: + A tuple (outputs, output_states) where: + outputs: A tuple (output_fw, output_bw) containing the forward and + the backward rnn output `Tensor`. + If time_major == False (default), + output_fw will be a `Tensor` shaped: + `[batch_size, max_time, cell_fw.output_size]` + and output_bw will be a `Tensor` shaped: + `[batch_size, max_time, cell_bw.output_size]`. + If time_major == True, + output_fw will be a `Tensor` shaped: + `[max_time, batch_size, cell_fw.output_size]` + and output_bw will be a `Tensor` shaped: + `[max_time, batch_size, cell_bw.output_size]`. + It returns a tuple instead of a single concatenated `Tensor`, unlike + in the `bidirectional_rnn`. If the concatenated one is preferred, + the forward and backward outputs can be concatenated as + `tf.concat(2, outputs)`. + output_states: A tuple (output_state_fw, output_state_bw) containing + the forward and the backward final states of bidirectional rnn. + + Raises: + TypeError: If `cell_fw` or `cell_bw` is not an instance of `RNNCell`. + """ + + if not isinstance(cell_fw, rnn_cell.RNNCell): + raise TypeError("cell_fw must be an instance of RNNCell") + if not isinstance(cell_bw, rnn_cell.RNNCell): + raise TypeError("cell_bw must be an instance of RNNCell") + + name = scope or "BiRNN" + # Forward direction + with vs.variable_scope(name + "_FW") as fw_scope: + output_fw, output_state_fw = dynamic_rnn( + cell=cell_fw, inputs=inputs, sequence_length=sequence_length, + initial_state=initial_state_fw, dtype=dtype, + parallel_iterations=parallel_iterations, swap_memory=swap_memory, + time_major=time_major, scope=fw_scope) + # Backward direction + if not time_major: + time_dim = 1 + batch_dim = 0 + else: + time_dim = 0 + batch_dim = 1 + with vs.variable_scope(name + "_BW") as bw_scope: + inputs_reverse = array_ops.reverse_sequence( + input=inputs, seq_lengths=sequence_length, + seq_dim=time_dim, batch_dim=batch_dim) + tmp, output_state_bw = dynamic_rnn( + cell=cell_bw, inputs=inputs_reverse, sequence_length=sequence_length, + initial_state=initial_state_bw, dtype=dtype, + parallel_iterations=parallel_iterations, swap_memory=swap_memory, + time_major=time_major, scope=bw_scope) + output_bw = array_ops.reverse_sequence( + input=tmp, seq_lengths=sequence_length, + seq_dim = time_dim, batch_dim=batch_dim) + + outputs = (output_fw, output_bw) + output_states = (output_state_fw, output_state_bw) + + return (outputs, output_states) + + def dynamic_rnn(cell, inputs, sequence_length=None, initial_state=None, dtype=None, parallel_iterations=None, swap_memory=False, time_major=False, scope=None): diff --git a/tensorflow/python/training/saver_test.py b/tensorflow/python/training/saver_test.py index 3aeceaac125..5a42472a298 100644 --- a/tensorflow/python/training/saver_test.py +++ b/tensorflow/python/training/saver_test.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# ============================================================================== +# ============================================================================= """Tests for tensorflow.python.training.saver.py.""" from __future__ import absolute_import diff --git a/tensorflow/tensorboard/README.md b/tensorflow/tensorboard/README.md index b95a4177bc5..c3c8ade6c6c 100644 --- a/tensorflow/tensorboard/README.md +++ b/tensorflow/tensorboard/README.md @@ -54,18 +54,18 @@ work, but there may be bugs or performance issues. The first step in using TensorBoard is acquiring data from your TensorFlow run. For this, you need [summary -ops](https://www.tensorflow.org/versions/r0.8/api_docs/python/train.html#summary-operations). +ops](https://www.tensorflow.org/versions/r0.9/api_docs/python/train.html#summary-operations). Summary ops are ops, like -[`tf.matmul`](https://www.tensorflow.org/versions/r0.8/api_docs/python/math_ops.html#matmul) +[`tf.matmul`](https://www.tensorflow.org/versions/r0.9/api_docs/python/math_ops.html#matmul) or -[`tf.nn.relu`](https://www.tensorflow.org/versions/r0.8/api_docs/python/nn.html#relu), +[`tf.nn.relu`](https://www.tensorflow.org/versions/r0.9/api_docs/python/nn.html#relu), which means they take in tensors, produce tensors, and are evaluated from within a TensorFlow graph. However, summary ops have a twist: the Tensors they produce contain serialized protobufs, which are written to disk and sent to TensorBoard. To visualize the summary data in TensorBoard, you should evaluate the summary op, retrieve the result, and then write that result to disk using a SummaryWriter. A full explanation, with examples, is in [the -tutorial](https://www.tensorflow.org/versions/r0.8/how_tos/summaries_and_tensorboard/index.html). +tutorial](https://www.tensorflow.org/versions/r0.9/how_tos/summaries_and_tensorboard/index.html). ### Tags: Giving names to data @@ -178,7 +178,7 @@ TensorFlow model. To get best use of the graph visualizer, you should use name scopes to hierarchically group the ops in your graph - otherwise, the graph may be difficult to decipher. For more information, including examples, see [the graph visualizer -tutorial](https://www.tensorflow.org/versions/r0.8/how_tos/graph_viz/index.html#tensorboard-graph-visualization). +tutorial](https://www.tensorflow.org/versions/r0.9/how_tos/graph_viz/index.html#tensorboard-graph-visualization). # Frequently Asked Questions diff --git a/tensorflow/tools/ci_build/builds/test_installation.sh b/tensorflow/tools/ci_build/builds/test_installation.sh index 173e3b254ec..6fb40c7b650 100755 --- a/tensorflow/tools/ci_build/builds/test_installation.sh +++ b/tensorflow/tools/ci_build/builds/test_installation.sh @@ -89,6 +89,7 @@ PY_TEST_BLACKLIST="${PY_TEST_BLACKLIST}:"\ "tensorflow/contrib/quantization/python/dequantize_op_test.py:"\ "tensorflow/contrib/quantization/python/quantized_conv_ops_test.py:"\ "tensorflow/contrib/quantization/tools/quantize_graph_test.py:"\ +"tensorflow/contrib/session_bundle/exporter_test.py:"\ "tensorflow/python/platform/default/_resource_loader_test.py:"\ "tensorflow/python/platform/default/flags_test.py:"\ "tensorflow/python/platform/default/logging_test.py:"\ diff --git a/tensorflow/tools/dist_test/Dockerfile b/tensorflow/tools/dist_test/Dockerfile index 5705767c7da..66787ca7f8b 100644 --- a/tensorflow/tools/dist_test/Dockerfile +++ b/tensorflow/tools/dist_test/Dockerfile @@ -20,7 +20,7 @@ RUN /var/gcloud/google-cloud-sdk/bin/gcloud components install kubectl # Install nightly TensorFlow pip # TODO(cais): Should we build it locally instead? RUN pip install \ - http://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_CONTAINER_TYPE=CPU,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.8.0-cp27-none-linux_x86_64.whl + http://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_CONTAINER_TYPE=CPU,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.9.0-cp27-none-linux_x86_64.whl # Copy test files COPY scripts /var/tf-dist-test/scripts diff --git a/tensorflow/tools/dist_test/server/Dockerfile b/tensorflow/tools/dist_test/server/Dockerfile index 4cbc2f0645b..c3bf751735e 100644 --- a/tensorflow/tools/dist_test/server/Dockerfile +++ b/tensorflow/tools/dist_test/server/Dockerfile @@ -36,7 +36,7 @@ RUN curl -O https://bootstrap.pypa.io/get-pip.py && \ # Install TensorFlow CPU version from nightly build RUN pip --no-cache-dir install \ - http://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_CONTAINER_TYPE=CPU,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.8.0-cp27-none-linux_x86_64.whl + http://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_CONTAINER_TYPE=CPU,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.9.0-cp27-none-linux_x86_64.whl # Copy files, including the GRPC server binary at # server/grpc_tensorflow_server.py diff --git a/tensorflow/tools/dist_test/server/Dockerfile.test b/tensorflow/tools/dist_test/server/Dockerfile.test index af795e8066f..de4411a05cd 100644 --- a/tensorflow/tools/dist_test/server/Dockerfile.test +++ b/tensorflow/tools/dist_test/server/Dockerfile.test @@ -42,7 +42,7 @@ RUN pip install --upgrade pandas==0.18.1 # Install TensorFlow CPU version. RUN pip --no-cache-dir install \ - http://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_CONTAINER_TYPE=CPU,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.8.0-cp27-none-linux_x86_64.whl + http://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_CONTAINER_TYPE=CPU,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.9.0-cp27-none-linux_x86_64.whl # Copy files, including the GRPC server binary at # server/grpc_tensorflow_server.py diff --git a/tensorflow/tools/docker/Dockerfile b/tensorflow/tools/docker/Dockerfile index a5927e79a50..31c3cd4d30a 100644 --- a/tensorflow/tools/docker/Dockerfile +++ b/tensorflow/tools/docker/Dockerfile @@ -32,7 +32,7 @@ RUN pip --no-cache-dir install \ && \ python -m ipykernel.kernelspec -ENV TENSORFLOW_VERSION 0.8.0 +ENV TENSORFLOW_VERSION 0.9.0 # --- DO NOT EDIT OR DELETE BETWEEN THE LINES --- # # These lines will be edited automatically by parameterized_docker_build.sh. # diff --git a/tensorflow/tools/docker/Dockerfile.gpu b/tensorflow/tools/docker/Dockerfile.gpu index 4dd97a6f20b..db91720cd9e 100644 --- a/tensorflow/tools/docker/Dockerfile.gpu +++ b/tensorflow/tools/docker/Dockerfile.gpu @@ -32,7 +32,7 @@ RUN pip --no-cache-dir install \ && \ python -m ipykernel.kernelspec -ENV TENSORFLOW_VERSION 0.8.0 +ENV TENSORFLOW_VERSION 0.9.0 # --- DO NOT EDIT OR DELETE BETWEEN THE LINES --- # # These lines will be edited automatically by parameterized_docker_build.sh. # diff --git a/tensorflow/tools/docker/README.md b/tensorflow/tools/docker/README.md index 35bd03c62df..921d4e53536 100644 --- a/tensorflow/tools/docker/README.md +++ b/tensorflow/tools/docker/README.md @@ -31,7 +31,7 @@ Run non-GPU container using $ docker run -it -p 8888:8888 gcr.io/tensorflow/tensorflow -For GPU support install Nvidia drivers (ideally latest) and +For GPU support install NVidia drivers (ideally latest) and [nvidia-docker](https://github.com/NVIDIA/nvidia-docker). Run using $ nvidia-docker run -it -p 8888:8888 gcr.io/tensorflow/tensorflow:latest-gpu @@ -46,6 +46,12 @@ it there please and try using the nvidia-docker as described above. $ docker run -it -p 8888:8888 $CUDA_SO $DEVICES gcr.io/tensorflow/tensorflow:latest-gpu +## More containers + +See all available [tags](https://hub.docker.com/r/tensorflow/tensorflow/tags/) +for additional containers like release candidates or nighlty builds. + + ## Rebuilding the containers Just pick the dockerfile corresponding to the container you want to build, and run diff --git a/tensorflow/tools/docker/docker_run_gpu.sh b/tensorflow/tools/docker/docker_run_gpu.sh deleted file mode 100755 index 08f391ddf9b..00000000000 --- a/tensorflow/tools/docker/docker_run_gpu.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -set -e - -export CUDA_SO=$(\ls /usr/lib/x86_64-linux-gnu/libcuda.* | \ - xargs -I{} echo '-v {}:{}') -export DEVICES=$(\ls /dev/nvidia* | \ - xargs -I{} echo '--device {}:{}') - -if [[ "${DEVICES}" = "" ]]; then - echo "Failed to locate NVidia device(s). Did you want the non-GPU container?" - exit 1 -fi - -docker run -it $CUDA_SO $DEVICES "$@" diff --git a/tensorflow/tools/docker/parameterized_docker_build.sh b/tensorflow/tools/docker/parameterized_docker_build.sh old mode 100644 new mode 100755 index dd4680bc0d2..56770bed643 --- a/tensorflow/tools/docker/parameterized_docker_build.sh +++ b/tensorflow/tools/docker/parameterized_docker_build.sh @@ -179,6 +179,9 @@ if [[ "${DO_PIP_BUILD}" == "1" ]]; then export TF_BUILD_IS_OPT="OPT" export TF_BUILD_IS_PIP="PIP" + export TF_BUILD_APPEND_CI_DOCKER_EXTRA_PARAMS=\ +"-e TF_CUDA_COMPUTE_CAPABILITIES=3.0,3.5,5.2" + pushd "${SCRIPT_DIR}/../../../" rm -rf pip_test/whl && tensorflow/tools/ci_build/ci_parameterized_build.sh diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD index cf83a767e7f..ae631812586 100644 --- a/tensorflow/tools/pip_package/BUILD +++ b/tensorflow/tools/pip_package/BUILD @@ -30,9 +30,9 @@ sh_binary( ":other_headers", ":simple_console", "//tensorflow:tensorflow_py", - "//tensorflow/contrib/session_bundle:all_files", - "//tensorflow/contrib/session_bundle:manifest_proto_py", - "//tensorflow/contrib/session_bundle/example:half_plus_two", +# "//tensorflow/contrib/session_bundle:all_files", +# "//tensorflow/contrib/session_bundle:manifest_proto_py", +# "//tensorflow/contrib/session_bundle/example:half_plus_two", "//tensorflow/contrib/slim:all_files", "//tensorflow/contrib/slim/python/slim/data:all_files", "//tensorflow/core:framework_headers", diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py index 07b10de08cf..59cb647e450 100644 --- a/tensorflow/tools/pip_package/setup.py +++ b/tensorflow/tools/pip_package/setup.py @@ -27,7 +27,7 @@ from setuptools import find_packages, setup, Command, Extension from setuptools.command.install import install as InstallCommandBase from setuptools.dist import Distribution -_VERSION = '0.8.0' +_VERSION = '0.9.0' numpy_version = "1.8.2" if platform.system() == "Darwin": diff --git a/tensorflow/tools/proto_text/BUILD b/tensorflow/tools/proto_text/BUILD index 19f14748c54..980a6d651e9 100644 --- a/tensorflow/tools/proto_text/BUILD +++ b/tensorflow/tools/proto_text/BUILD @@ -42,6 +42,13 @@ cc_library( name = "gen_proto_text_functions_lib", srcs = ["gen_proto_text_functions_lib.cc"], hdrs = ["gen_proto_text_functions_lib.h"], + linkopts = [ + "-lm", + "-lpthread", + ] + select({ + "//tensorflow:darwin": [], + "//conditions:default": ["-lrt"], + }), deps = [ "//tensorflow/core:lib", ], diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index 49f37ed1aba..74026bbcd7f 100644 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -143,13 +143,13 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""): native.new_git_repository( name = "boringssl_git", commit = "e72df93461c6d9d2b5698f10e16d3ab82f5adde3", - remote = "https://boringssl.googlesource.com/boringssl", + remote = "https://github.com/google/boringssl.git", build_file = path_prefix + "boringssl.BUILD", ) native.bind( name = "boringssl_err_data_c", - actual = "@//" + path_prefix + "third_party/boringssl:err_data_c", + actual = tf_repo_name + "//third_party/boringssl:err_data_c", ) native.new_git_repository( diff --git a/third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_is_not_gcc b/third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_is_not_gcc index 7e7633192c4..071997ca44b 100755 --- a/third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_is_not_gcc +++ b/third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_is_not_gcc @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # Copyright 2015 The TensorFlow Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -34,6 +34,8 @@ NOTES: //third_party/gpus/crosstool/v*/*/clang/bin/crosstool_wrapper_is_not_gcc """ +from __future__ import print_function + __author__ = 'keveman@google.com (Manjunath Kudlur)' from argparse import ArgumentParser @@ -54,7 +56,7 @@ LLVM_HOST_COMPILER_PATH = ('/usr/bin/gcc') PREFIX_DIR = os.path.dirname(GCC_HOST_COMPILER_PATH) def Log(s): - print 'gpus/crosstool: {0}'.format(s) + print('gpus/crosstool: {0}'.format(s)) def GetOptionValue(argv, option): From bd8ac369f9253e4327ce82e2e2f6fca088dd1dd4 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 30 Jun 2016 12:24:28 -0800 Subject: [PATCH 06/36] Fixed bug in fully_connected_feed.py in which checkpoints were being saved to the directory below the desired output directory. Change: 126336688 --- tensorflow/examples/tutorials/mnist/fully_connected_feed.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tensorflow/examples/tutorials/mnist/fully_connected_feed.py b/tensorflow/examples/tutorials/mnist/fully_connected_feed.py index 773d4678276..5ab6024c2b8 100644 --- a/tensorflow/examples/tutorials/mnist/fully_connected_feed.py +++ b/tensorflow/examples/tutorials/mnist/fully_connected_feed.py @@ -19,6 +19,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import os.path import time from six.moves import xrange # pylint: disable=redefined-builtin @@ -198,7 +199,8 @@ def run_training(): # Save a checkpoint and evaluate the model periodically. if (step + 1) % 1000 == 0 or (step + 1) == FLAGS.max_steps: - saver.save(sess, FLAGS.train_dir, global_step=step) + checkpoint_file = os.path.join(FLAGS.train_dir, 'checkpoint') + saver.save(sess, checkpoint_file, global_step=step) # Evaluate against the training set. print('Training Data Eval:') do_eval(sess, From 78cf08951e95cc3afd5d2e6677db6bc85b06d43e Mon Sep 17 00:00:00 2001 From: Xiaoqiang Zheng Date: Thu, 30 Jun 2016 12:39:44 -0800 Subject: [PATCH 07/36] Clip the padding for negative values. Change: 126338283 --- tensorflow/core/kernels/conv_grad_ops.cc | 6 ++++++ tensorflow/core/kernels/conv_grad_ops_3d.cc | 22 +++++++++++++-------- tensorflow/core/kernels/conv_ops.cc | 9 +++++++-- tensorflow/core/kernels/conv_ops_3d.cc | 9 +++++++-- 4 files changed, 34 insertions(+), 12 deletions(-) diff --git a/tensorflow/core/kernels/conv_grad_ops.cc b/tensorflow/core/kernels/conv_grad_ops.cc index 508ffc04029..487daa7c2da 100644 --- a/tensorflow/core/kernels/conv_grad_ops.cc +++ b/tensorflow/core/kernels/conv_grad_ops.cc @@ -1024,6 +1024,9 @@ class Conv2DSlowBackpropInputOp : public OpKernel { compatible_input_shape = input_shape; } + CHECK(padding_rows >= 0 && padding_cols >= 0) + << "Negative row or col paddings: (" << padding_rows << ", " + << padding_cols << ")"; perftools::gputools::dnn::BatchDescriptor input_desc; input_desc.set_count(dims.batch_size) .set_height(GetTensorDim(compatible_input_shape, data_format_, 'H')) @@ -1382,6 +1385,9 @@ class Conv2DSlowBackpropFilterOp : public OpKernel { compatible_input = input; } + CHECK(padding_rows >= 0 && padding_cols >= 0) + << "Negative row or col paddings: (" << padding_rows << ", " + << padding_cols << ")"; perftools::gputools::dnn::BatchDescriptor input_desc; input_desc.set_count(dims.batch_size) .set_height(GetTensorDim(compatible_input, data_format_, 'H')) diff --git a/tensorflow/core/kernels/conv_grad_ops_3d.cc b/tensorflow/core/kernels/conv_grad_ops_3d.cc index d0c6865951e..62e60d018b5 100644 --- a/tensorflow/core/kernels/conv_grad_ops_3d.cc +++ b/tensorflow/core/kernels/conv_grad_ops_3d.cc @@ -438,10 +438,10 @@ class Conv3DBackpropInputOp : public OpKernel { if (padding_ == Padding::SAME) { padding_planes = (output_planes - 1) * strides[0] + filter_size[0] - input_size[0]; - padding_cols = - (output_cols - 1) * strides[2] + filter_size[2] - input_size[2]; - padding_rows = - (output_rows - 1) * strides[1] + filter_size[1] - input_size[1]; + padding_cols = std::max( + 0, (output_cols - 1) * strides[2] + filter_size[2] - input_size[2]); + padding_rows = std::max( + 0, (output_rows - 1) * strides[1] + filter_size[1] - input_size[1]); } const bool rows_odd = (padding_rows % 2 != 0); const bool cols_odd = (padding_cols % 2 != 0); @@ -462,6 +462,9 @@ class Conv3DBackpropInputOp : public OpKernel { input_size[2]}; } + CHECK(padding_rows >= 0 && padding_cols >= 0) + << "Negative row or col paddings: (" << padding_rows << ", " + << padding_cols << ")"; perftools::gputools::dnn::BatchDescriptor input_desc(3); input_desc.set_count(batch) .set_spatial_dim(DimIndex::X, compatible_input_shape.dim_size(4)) @@ -659,10 +662,10 @@ class Conv3DBackpropFilterOp : public OpKernel { if (padding_ == Padding::SAME) { padding_planes = (output_planes - 1) * strides[0] + filter_size[0] - input_size[0]; - padding_cols = - (output_cols - 1) * strides[2] + filter_size[2] - input_size[2]; - padding_rows = - (output_rows - 1) * strides[1] + filter_size[1] - input_size[1]; + padding_cols = std::max( + 0, (output_cols - 1) * strides[2] + filter_size[2] - input_size[2]); + padding_rows = std::max( + 0, (output_rows - 1) * strides[1] + filter_size[1] - input_size[1]); } bool rows_odd = (padding_rows % 2 != 0); bool cols_odd = (padding_cols % 2 != 0); @@ -686,6 +689,9 @@ class Conv3DBackpropFilterOp : public OpKernel { compatible_input = input; } + CHECK(padding_rows >= 0 && padding_cols >= 0) + << "Negative row or col paddings: (" << padding_rows << ", " + << padding_cols << ")"; perftools::gputools::dnn::BatchDescriptor input_desc(3); input_desc.set_count(batch) .set_spatial_dim(DimIndex::X, compatible_input.dim_size(3)) diff --git a/tensorflow/core/kernels/conv_ops.cc b/tensorflow/core/kernels/conv_ops.cc index ede9a77ed0f..e0aff98854d 100644 --- a/tensorflow/core/kernels/conv_ops.cc +++ b/tensorflow/core/kernels/conv_ops.cc @@ -334,8 +334,10 @@ class LaunchConvOp { // We pad Pr/2 on the left and Pr - Pr/2 on the right, Pc/2 on the top // and Pc - Pc/2 on the bottom. When Pr or Pc is odd, this means // we pad more on the right and bottom than on the top and left. - padding_rows = (out_rows - 1) * row_stride + patch_rows - in_rows; - padding_cols = (out_cols - 1) * col_stride + patch_cols - in_cols; + padding_rows = + std::max(0, (out_rows - 1) * row_stride + patch_rows - in_rows); + padding_cols = + std::max(0, (out_cols - 1) * col_stride + patch_cols - in_cols); const bool rows_odd = (padding_rows % 2 != 0); const bool cols_odd = (padding_cols % 2 != 0); if (rows_odd || cols_odd) { @@ -375,6 +377,9 @@ class LaunchConvOp { input = transformed_input; } + CHECK(padding_rows >= 0 && padding_cols >= 0) + << "Negative row or col paddings: (" << padding_rows << ", " + << padding_cols << ")"; perftools::gputools::dnn::BatchDescriptor input_desc; input_desc.set_count(in_batch) .set_feature_map_count(in_depths) diff --git a/tensorflow/core/kernels/conv_ops_3d.cc b/tensorflow/core/kernels/conv_ops_3d.cc index 697b3f62679..e236edfc0d3 100644 --- a/tensorflow/core/kernels/conv_ops_3d.cc +++ b/tensorflow/core/kernels/conv_ops_3d.cc @@ -160,8 +160,10 @@ struct LaunchConvOp { if (padding == Padding::SAME) { pad_planes = (out_planes - 1) * strides[0] + filter_planes - in_planes; - pad_rows = (out_rows - 1) * strides[1] + filter_rows - in_rows; - pad_cols = (out_cols - 1) * strides[2] + filter_cols - in_cols; + pad_rows = std::max( + 0, (out_rows - 1) * strides[1] + filter_rows - in_rows); + pad_cols = std::max( + 0, (out_cols - 1) * strides[2] + filter_cols - in_cols); } // NOTE: This only works in NHWC. @@ -239,6 +241,9 @@ struct LaunchConvOp { transformed_input.tensor()); input = transformed_input; + CHECK(pad_rows >= 0 && pad_cols >= 0) << "Negative row or col paddings: (" + << pad_rows << ", " << pad_cols + << ")"; perftools::gputools::dnn::BatchDescriptor input_desc(3); input_desc.set_count(in_batch) .set_feature_map_count(in_depth) From 8975dca9beaab8c89c3f888116270a6c277151ee Mon Sep 17 00:00:00 2001 From: Robin Nabel Date: Thu, 30 Jun 2016 13:00:27 -0800 Subject: [PATCH 08/36] Bug fix: Correct erroneous parent fading when tracing inputs, correctly fading series nodes when loading run metadata. Change: 126340706 --- .../components/tf-graph-common/lib/scene/node.ts | 4 ++-- .../components/tf-graph/tf-graph-scene.html | 14 +++++++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/tensorflow/tensorboard/components/tf-graph-common/lib/scene/node.ts b/tensorflow/tensorboard/components/tf-graph-common/lib/scene/node.ts index dbc0213ac7b..64f245c7311 100644 --- a/tensorflow/tensorboard/components/tf-graph-common/lib/scene/node.ts +++ b/tensorflow/tensorboard/components/tf-graph-common/lib/scene/node.ts @@ -827,7 +827,7 @@ export function traceAllInputsOfOpNode( // Get visible parent. let currentVisibleParent = getVisibleParent(renderGraphInfo, startNode); // Mark as input node. - d3.selectAll(`[data-name="${currentVisibleParent.name}"]`) + d3.select(`.node[data-name="${currentVisibleParent.name}"]`) .classed('input-highlight', true); // Find the visible parent of each input. @@ -1018,7 +1018,7 @@ function _markParentsOfNodes(visibleNodes: {[nodeName: string]: Node}) { let currentNode = nodeInstance; while (currentNode.name !== tf.graph.ROOT_NAME) { - let renderedElement = d3.select(`[data-name="${currentNode.name}"]`); + let renderedElement = d3.select(`.node[data-name="${currentNode.name}"]`); // Only mark the element as a parent node to an input if it is not // marked as input node itself. if (renderedElement[0][0] && diff --git a/tensorflow/tensorboard/components/tf-graph/tf-graph-scene.html b/tensorflow/tensorboard/components/tf-graph/tf-graph-scene.html index d7bc9b8c04d..04c34bf935c 100644 --- a/tensorflow/tensorboard/components/tf-graph/tf-graph-scene.html +++ b/tensorflow/tensorboard/components/tf-graph/tf-graph-scene.html @@ -72,6 +72,7 @@ ::content .faded rect, ::content .faded ellipse, ::content .faded path, +::content .faded use, ::content #rectHatch line, ::content #ellipseHatch line { color: #e0d4b3 !important; @@ -88,7 +89,8 @@ fill: url(#rectHatch) !important; } -::content .faded ellipse { +::content .faded ellipse, +::content .faded use { fill: url(#ellipseHatch) !important; } @@ -111,8 +113,12 @@ ::content .non-input > * > use, /* For Const nodes. */ ::content .non-input > * > .constant:not([class*="input-highlight"]) > -.annotation-node > ellipse { + .annotation-node > ellipse, +/* For styling of annotation nodes of non-input nodes. */ +::content .non-input > g > .annotation > .annotation-node > rect { stroke: #e0d4b3 !important; + stroke-width: inherit; + stroke-dasharray: inherit; } @@ -121,7 +127,9 @@ } ::content .non-input > .nodeshape > rect, -::content .non-input > .annotation-node > rect +::content .non-input > .annotation-node > rect, +/* For styling of annotation nodes of non-input nodes. */ +::content .non-input > g > .annotation > .annotation-node > rect { fill: url(#rectHatch) !important; } From 1445e8054c65b776f63d43833da30dc3debdbc31 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 30 Jun 2016 13:16:29 -0800 Subject: [PATCH 09/36] Fix stack-use-after-scope detected by asan in gtl test inserted_count is used by RefCounted which is used by RefCountedVec. So inserted_count must outlive RefCountedVec. Change: 126342639 --- tensorflow/core/lib/gtl/inlined_vector_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/lib/gtl/inlined_vector_test.cc b/tensorflow/core/lib/gtl/inlined_vector_test.cc index 88ec1069c5d..7ce1a1d395f 100644 --- a/tensorflow/core/lib/gtl/inlined_vector_test.cc +++ b/tensorflow/core/lib/gtl/inlined_vector_test.cc @@ -285,6 +285,7 @@ TEST(RefCountedVec, InsertConstructorDestructor) { for (int pos = 0; pos <= len; pos++) { SCOPED_TRACE(pos); std::vector counts(len, 0); + int inserted_count = 0; RefCountedVec v; for (int i = 0; i < len; ++i) { SCOPED_TRACE(i); @@ -295,7 +296,6 @@ TEST(RefCountedVec, InsertConstructorDestructor) { EXPECT_EQ(1, elem); } - int inserted_count = 0; RefCounted insert_element(9999, &inserted_count); EXPECT_EQ(1, inserted_count); v.insert(v.begin() + pos, insert_element); From 5a8faefbad514c88ca5161ebc901bb9bd74d932d Mon Sep 17 00:00:00 2001 From: Benoit Steiner Date: Thu, 30 Jun 2016 13:17:39 -0800 Subject: [PATCH 10/36] Improved the gradients for tanh and sigmoid. This improves the speed of the ptb word model from 6800 to 7800 words per second. Change: 126342788 --- tensorflow/core/kernels/BUILD | 1 + .../core/kernels/cwise_op_gpu_sigmoid.cu.cc | 2 + .../core/kernels/cwise_op_gpu_tanh.cu.cc | 2 + tensorflow/core/kernels/cwise_op_sigmoid.cc | 9 ++ tensorflow/core/kernels/cwise_op_tanh.cc | 8 ++ tensorflow/core/kernels/cwise_ops_common.h | 30 +++++ .../core/kernels/cwise_ops_gpu_gradients.cu.h | 71 ++++++++++++ tensorflow/core/kernels/cwise_ops_gradients.h | 107 ++++++++++++++++++ tensorflow/core/ops/math_ops.cc | 21 ++++ tensorflow/python/BUILD | 2 + tensorflow/python/ops/math_grad.py | 5 +- tensorflow/python/ops/math_ops.py | 2 + 12 files changed, 258 insertions(+), 2 deletions(-) create mode 100644 tensorflow/core/kernels/cwise_ops_gpu_gradients.cu.h create mode 100644 tensorflow/core/kernels/cwise_ops_gradients.h diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index 5cf48bfab5c..142f63c6b47 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -1753,6 +1753,7 @@ filegroup( "cwise_ops.h", "cwise_ops_common.cc", "cwise_ops_common.h", + "cwise_ops_gradients.h", "dense_update_ops.cc", "dense_update_ops.h", "example_parsing_ops.cc", diff --git a/tensorflow/core/kernels/cwise_op_gpu_sigmoid.cu.cc b/tensorflow/core/kernels/cwise_op_gpu_sigmoid.cu.cc index a7ac9baca08..b59d22310e0 100644 --- a/tensorflow/core/kernels/cwise_op_gpu_sigmoid.cu.cc +++ b/tensorflow/core/kernels/cwise_op_gpu_sigmoid.cu.cc @@ -16,10 +16,12 @@ limitations under the License. #if GOOGLE_CUDA #include "tensorflow/core/kernels/cwise_ops_gpu_common.cu.h" +#include "tensorflow/core/kernels/cwise_ops_gpu_gradients.cu.h" namespace tensorflow { namespace functor { DEFINE_UNARY3(sigmoid, Eigen::half, float, double); +DEFINE_SIMPLE_BINARY3(sigmoid_grad, Eigen::half, float, double); } // namespace functor } // namespace tensorflow diff --git a/tensorflow/core/kernels/cwise_op_gpu_tanh.cu.cc b/tensorflow/core/kernels/cwise_op_gpu_tanh.cu.cc index 1678086c35e..66ee3c193e0 100644 --- a/tensorflow/core/kernels/cwise_op_gpu_tanh.cu.cc +++ b/tensorflow/core/kernels/cwise_op_gpu_tanh.cu.cc @@ -16,10 +16,12 @@ limitations under the License. #if GOOGLE_CUDA #include "tensorflow/core/kernels/cwise_ops_gpu_common.cu.h" +#include "tensorflow/core/kernels/cwise_ops_gpu_gradients.cu.h" namespace tensorflow { namespace functor { DEFINE_UNARY3(tanh, Eigen::half, float, double); +DEFINE_SIMPLE_BINARY3(tanh_grad, Eigen::half, float, double); } // namespace functor } // namespace tensorflow diff --git a/tensorflow/core/kernels/cwise_op_sigmoid.cc b/tensorflow/core/kernels/cwise_op_sigmoid.cc index 9d8a849bd33..cc1f9b8f03e 100644 --- a/tensorflow/core/kernels/cwise_op_sigmoid.cc +++ b/tensorflow/core/kernels/cwise_op_sigmoid.cc @@ -14,6 +14,7 @@ limitations under the License. ==============================================================================*/ #include "tensorflow/core/kernels/cwise_ops_common.h" +#include "tensorflow/core/kernels/cwise_ops_gradients.h" namespace tensorflow { REGISTER5(UnaryOp, CPU, "Sigmoid", functor::sigmoid, float, Eigen::half, double, @@ -22,4 +23,12 @@ REGISTER5(UnaryOp, CPU, "Sigmoid", functor::sigmoid, float, Eigen::half, double, REGISTER3(UnaryOp, GPU, "Sigmoid", functor::sigmoid, float, Eigen::half, double); #endif + +REGISTER5(SimpleBinaryOp, CPU, "SigmoidGrad", functor::sigmoid_grad, float, + Eigen::half, double, complex64, complex128); +#if GOOGLE_CUDA +REGISTER3(SimpleBinaryOp, GPU, "SigmoidGrad", functor::sigmoid_grad, float, + Eigen::half, double); +#endif + } // namespace tensorflow diff --git a/tensorflow/core/kernels/cwise_op_tanh.cc b/tensorflow/core/kernels/cwise_op_tanh.cc index 6604d71d14c..a4c4aad053f 100644 --- a/tensorflow/core/kernels/cwise_op_tanh.cc +++ b/tensorflow/core/kernels/cwise_op_tanh.cc @@ -14,6 +14,7 @@ limitations under the License. ==============================================================================*/ #include "tensorflow/core/kernels/cwise_ops_common.h" +#include "tensorflow/core/kernels/cwise_ops_gradients.h" namespace tensorflow { REGISTER5(UnaryOp, CPU, "Tanh", functor::tanh, float, Eigen::half, double, @@ -21,4 +22,11 @@ REGISTER5(UnaryOp, CPU, "Tanh", functor::tanh, float, Eigen::half, double, #if GOOGLE_CUDA REGISTER3(UnaryOp, GPU, "Tanh", functor::tanh, float, Eigen::half, double); #endif + +REGISTER5(SimpleBinaryOp, CPU, "TanhGrad", functor::tanh_grad, float, + Eigen::half, double, complex64, complex128); +#if GOOGLE_CUDA +REGISTER3(SimpleBinaryOp, GPU, "TanhGrad", functor::tanh_grad, float, + Eigen::half, double); +#endif } // namespace tensorflow diff --git a/tensorflow/core/kernels/cwise_ops_common.h b/tensorflow/core/kernels/cwise_ops_common.h index 02a82c00bf0..6ccbe46c7fa 100644 --- a/tensorflow/core/kernels/cwise_ops_common.h +++ b/tensorflow/core/kernels/cwise_ops_common.h @@ -21,6 +21,7 @@ limitations under the License. #define EIGEN_USE_THREADS #include "tensorflow/core/kernels/cwise_ops.h" +#include "tensorflow/core/kernels/cwise_ops_gradients.h" #include "tensorflow/core/framework/op.h" #include "tensorflow/core/framework/op_kernel.h" @@ -130,6 +131,35 @@ class BinaryOp : public BinaryOpShared { } }; +// Basic coefficient-wise binary operations that are known to not require +// any broadcasting. This is the case for example of the gradients of +// unary operations. +// Device: E.g., CPUDevice, GPUDevice. +// Functor: defined above. E.g., functor::tanh_grad. +template +class SimpleBinaryOp : public OpKernel { + public: + typedef typename Functor::in_type Tin; // Input scalar data type. + typedef typename Functor::out_type Tout; // Output scalar data type. + + explicit SimpleBinaryOp(OpKernelConstruction* ctx) : OpKernel(ctx) {} + + void Compute(OpKernelContext* ctx) override { + const Tensor& in0 = ctx->input(0); + const Tensor& in1 = ctx->input(1); + + Tensor* out; + OP_REQUIRES_OK(ctx, ctx->allocate_output(0, in0.shape(), &out)); + auto out_flat = out->flat(); + auto in0_flat = in0.flat(); + auto in1_flat = in1.flat(); + const Device& eigen_device = ctx->eigen_device(); + + functor::SimpleBinaryFunctor()(eigen_device, out_flat, + in0_flat, in1_flat); + } +}; + // Coefficient-wise unary operations: // Device: E.g., CPUDevice, GPUDevice. // Functor: defined in cwise_functors.h. E.g., functor::sqrt. diff --git a/tensorflow/core/kernels/cwise_ops_gpu_gradients.cu.h b/tensorflow/core/kernels/cwise_ops_gpu_gradients.cu.h new file mode 100644 index 00000000000..43947707089 --- /dev/null +++ b/tensorflow/core/kernels/cwise_ops_gpu_gradients.cu.h @@ -0,0 +1,71 @@ +/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#if !GOOGLE_CUDA +#error This file must only be included when building with Cuda support +#endif + +#ifndef TENSORFLOW_KERNELS_CWISE_OPS_GPU_GRADIENTS_CU_H_ +#define TENSORFLOW_KERNELS_CWISE_OPS_GPU_GRADIENTS_CU_H_ + +#define EIGEN_USE_GPU + +#include + +#include "tensorflow/core/framework/tensor_types.h" +#include "tensorflow/core/kernels/cwise_ops.h" +#include "tensorflow/core/kernels/cwise_ops_gradients.h" +#include "tensorflow/core/platform/types.h" + +#include "tensorflow/core/platform/logging.h" +namespace tensorflow { +namespace functor { + +typedef Eigen::GpuDevice GPUDevice; +typedef std::complex complex64; +typedef std::complex complex128; + +// Partial specialization of SimpleBinaryFunctor. +template +struct SimpleBinaryFunctor { + void operator()(const GPUDevice& d, typename Functor::tout_type out, + typename Functor::tin_type in1, + typename Functor::tin_type in2) { + To32Bit(out).device(d) = + To32Bit(in1).binaryExpr(in2, typename Functor::func()); + } +}; + +// Macros to explicitly instantiate kernels on GPU for multiple types +// (T0, T1, etc.) for SimpleBiaryFunctor (e.g., functor::tanh_grad). +#define DEFINE_SIMPLE_BINARY1(F, T) \ + template struct SimpleBinaryFunctor > +#define DEFINE_SIMPLE_BINARY2(F, T0, T1) \ + DEFINE_SIMPLE_BINARY1(F, T0); \ + DEFINE_SIMPLE_BINARY1(F, T1) +#define DEFINE_SIMPLE_BINARY3(F, T0, T1, T2) \ + DEFINE_SIMPLE_BINARY2(F, T0, T1); \ + DEFINE_SIMPLE_BINARY1(F, T2) +#define DEFINE_SIMPLE_BINARY4(F, T0, T1, T2, T3) \ + DEFINE_SIMPLE_BINARY2(F, T0, T1); \ + DEFINE_SIMPLE_BINARY2(F, T2, T3) +#define DEFINE_SIMPLE_BINARY5(F, T0, T1, T2, T3, T4) \ + DEFINE_SIMPLE_BINARY2(F, T0, T1); \ + DEFINE_SIMPLE_BINARY3(F, T2, T3, T4) + +} // end namespace functor +} // end namespace tensorflow + +#endif // TENSORFLOW_KERNELS_CWISE_OPS_GPU_GRADIENTS_CU_H_ diff --git a/tensorflow/core/kernels/cwise_ops_gradients.h b/tensorflow/core/kernels/cwise_ops_gradients.h new file mode 100644 index 00000000000..a59f1572810 --- /dev/null +++ b/tensorflow/core/kernels/cwise_ops_gradients.h @@ -0,0 +1,107 @@ +/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_KERNELS_CWISE_OPS_GRADIENTS_H_ +#define TENSORFLOW_KERNELS_CWISE_OPS_GRADIENTS_H_ + +#define EIGEN_USE_THREADS +#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" + +namespace Eigen { +namespace internal { + +// Gradient for the tanh function +template +struct scalar_tanh_gradient_op { + EIGEN_EMPTY_STRUCT_CTOR(scalar_tanh_gradient_op) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T + operator()(const T& output, const T& output_gradient) const { + return output_gradient * (T(1) - output * output); + } + template + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet + packetOp(const Packet& output, const Packet& output_gradient) const { + return pmul(output_gradient, + psub(pset1(T(1)), pmul(output, output))); + } +}; +template +struct functor_traits> { + enum { + Cost = NumTraits::AddCost + 2 * NumTraits::MulCost, + PacketAccess = packet_traits::HasSub && packet_traits::HasMul, + }; +}; + +// Gradient for the sigmoid function +template +struct scalar_sigmoid_gradient_op { + EIGEN_EMPTY_STRUCT_CTOR(scalar_sigmoid_gradient_op) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T + operator()(const T& output, const T& output_gradient) const { + return output_gradient * output * (T(1) - output); + } + template + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet + packetOp(const Packet& output, const Packet& output_gradient) const { + return pmul(output_gradient, + pmul(output, psub(pset1(T(1)), output))); + } +}; +template +struct functor_traits> { + enum { + Cost = NumTraits::AddCost + 2 * NumTraits::MulCost, + PacketAccess = packet_traits::HasSub && packet_traits::HasMul, + }; +}; + +} // end namespace internal +} // end namespace Eigen + +namespace tensorflow { + +namespace functor { + +template +struct SimpleBinaryFunctor { + void operator()(const Device& d, typename Functor::tout_type out, + typename Functor::tin_type in0, + typename Functor::tin_type in1); +}; + +// Partial specialization of BinaryFunctor for CPU devices +typedef Eigen::ThreadPoolDevice CPUDevice; + +template +struct SimpleBinaryFunctor { + void operator()(const CPUDevice& d, typename Functor::tout_type out, + typename Functor::tin_type in0, + typename Functor::tin_type in1) { + out.device(d) = in0.binaryExpr(in1, typename Functor::func()); + } +}; + +template +struct tanh_grad : base> {}; + +template +struct sigmoid_grad : base> { +}; + +} // end namespace functor + +} // end namespace tensorflow +#endif // TENSORFLOW_KERNELS_CWISE_OPS_GRADIENTS_H_ diff --git a/tensorflow/core/ops/math_ops.cc b/tensorflow/core/ops/math_ops.cc index 0f9ee4942aa..b220a2d2d62 100644 --- a/tensorflow/core/ops/math_ops.cc +++ b/tensorflow/core/ops/math_ops.cc @@ -238,6 +238,13 @@ tf.complex_abs(x) ==> [5.25594902, 6.60492229] .Attr("T: {half, float, double, complex64, complex128}") \ .SetShapeFn(OpShapeInferenceFn(shape_inference::UnchangedShape)) +#define UNARY_GRADIENT_COMPLEX() \ + Input("x: T") \ + .Input("y: T") \ + .Output("z: T") \ + .Attr("T: {half, float, double, complex64, complex128}") \ + .SetShapeFn(OpShapeInferenceFn(shape_inference::UnchangedShape)) + REGISTER_OP("Neg") .UNARY() .Doc(R"doc( @@ -292,6 +299,13 @@ REGISTER_OP("Tanh") Computes hyperbolic tangent of `x` element-wise. )doc"); +REGISTER_OP("TanhGrad").UNARY_GRADIENT_COMPLEX().Doc(R"doc( +Computes the gradient for the tanh of `x` wrt its input. + +Specifically, `grad = dy * (1 - y*y)`, where `y = tanh(x)`, and `dy` +is the corresponding input gradient. +)doc"); + REGISTER_OP("Lgamma") .UNARY_REAL() .Doc(R"doc( @@ -325,6 +339,13 @@ Computes sigmoid of `x` element-wise. Specifically, `y = 1 / (1 + exp(-x))`. )doc"); +REGISTER_OP("SigmoidGrad").UNARY_GRADIENT_COMPLEX().Doc(R"doc( +Computes the gradient of the sigmoid of `x` wrt its input. + +Specifically, `grad = dy * y * (1 - y)`, where `y = sigmoid(x)`, and +`dy` is the corresponding input gradient. +)doc"); + REGISTER_OP("Sin") .UNARY_COMPLEX() .Doc(R"doc( diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index c2e5b0cc1c8..c5418cf076f 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -670,6 +670,8 @@ tf_gen_op_wrapper_py( "MatMul", "Sigmoid", "Tanh", + "SigmoidGrad", + "TanhGrad", ], require_shape_functions = True, ) diff --git a/tensorflow/python/ops/math_grad.py b/tensorflow/python/ops/math_grad.py index 8bfd9ce8bf8..348ab9fd121 100644 --- a/tensorflow/python/ops/math_grad.py +++ b/tensorflow/python/ops/math_grad.py @@ -26,6 +26,7 @@ from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_array_ops +from tensorflow.python.ops import gen_math_ops from tensorflow.python.ops import math_ops @@ -272,7 +273,7 @@ def _TanhGrad(op, grad): with ops.control_dependencies([grad.op]): if y.dtype.is_complex: y = math_ops.conj(y) - return grad * (1 - math_ops.square(y)) + return gen_math_ops._tanh_grad(y, grad) @ops.RegisterGradient("Erf") @@ -374,7 +375,7 @@ def _SigmoidGrad(op, grad): with ops.control_dependencies([grad.op]): if y.dtype.is_complex: y = math_ops.conj(y) - return grad * (y * (1 - y)) + return gen_math_ops._sigmoid_grad(y, grad) @ops.RegisterGradient("Sign") diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py index 0a76450c5b1..0bcf45db76f 100644 --- a/tensorflow/python/ops/math_ops.py +++ b/tensorflow/python/ops/math_ops.py @@ -1609,6 +1609,8 @@ ops.RegisterShape("BatchFFT2D")(common_shapes.unchanged_shape) ops.RegisterShape("BatchIFFT2D")(common_shapes.unchanged_shape) ops.RegisterShape("BatchFFT3D")(common_shapes.unchanged_shape) ops.RegisterShape("BatchIFFT3D")(common_shapes.unchanged_shape) +ops.RegisterShape("TanhGrad")(common_shapes.unchanged_shape) +ops.RegisterShape("SigmoidGrad")(common_shapes.unchanged_shape) @ops.RegisterShape("Add") From 8ebd94e71bf61c9b86bf20980775772e8a08f513 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 30 Jun 2016 13:32:57 -0800 Subject: [PATCH 11/36] Update ops-related pbtxt files. Change: 126344587 --- .../core/ops/compat/ops_history.v0.pbtxt | 56 +++++++++++++++++ tensorflow/core/ops/ops.pbtxt | 60 +++++++++++++++++++ 2 files changed, 116 insertions(+) diff --git a/tensorflow/core/ops/compat/ops_history.v0.pbtxt b/tensorflow/core/ops/compat/ops_history.v0.pbtxt index adaa47ab8c5..2dba61efe78 100644 --- a/tensorflow/core/ops/compat/ops_history.v0.pbtxt +++ b/tensorflow/core/ops/compat/ops_history.v0.pbtxt @@ -20208,6 +20208,34 @@ op { } } } +op { + name: "SigmoidGrad" + input_arg { + name: "x" + type_attr: "T" + } + input_arg { + name: "y" + type_attr: "T" + } + output_arg { + name: "z" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_HALF + type: DT_FLOAT + type: DT_DOUBLE + type: DT_COMPLEX64 + type: DT_COMPLEX128 + } + } + } +} op { name: "Sign" input_arg { @@ -24557,6 +24585,34 @@ op { } } } +op { + name: "TanhGrad" + input_arg { + name: "x" + type_attr: "T" + } + input_arg { + name: "y" + type_attr: "T" + } + output_arg { + name: "z" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_HALF + type: DT_FLOAT + type: DT_DOUBLE + type: DT_COMPLEX64 + type: DT_COMPLEX128 + } + } + } +} op { name: "TemporaryVariable" output_arg { diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt index 474516bf4c6..afd6507b0d8 100644 --- a/tensorflow/core/ops/ops.pbtxt +++ b/tensorflow/core/ops/ops.pbtxt @@ -11796,6 +11796,36 @@ op { summary: "Computes sigmoid of `x` element-wise." description: "Specifically, `y = 1 / (1 + exp(-x))`." } +op { + name: "SigmoidGrad" + input_arg { + name: "x" + type_attr: "T" + } + input_arg { + name: "y" + type_attr: "T" + } + output_arg { + name: "z" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_HALF + type: DT_FLOAT + type: DT_DOUBLE + type: DT_COMPLEX64 + type: DT_COMPLEX128 + } + } + } + summary: "Computes the gradient of the sigmoid of `x` wrt its input." + description: "Specifically, `grad = dy * y * (1 - y)`, where `y = sigmoid(x)`, and\n`dy` is the corresponding input gradient." +} op { name: "Sign" input_arg { @@ -14643,6 +14673,36 @@ op { } summary: "Computes hyperbolic tangent of `x` element-wise." } +op { + name: "TanhGrad" + input_arg { + name: "x" + type_attr: "T" + } + input_arg { + name: "y" + type_attr: "T" + } + output_arg { + name: "z" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_HALF + type: DT_FLOAT + type: DT_DOUBLE + type: DT_COMPLEX64 + type: DT_COMPLEX128 + } + } + } + summary: "Computes the gradient for the tanh of `x` wrt its input." + description: "Specifically, `grad = dy * (1 - y*y)`, where `y = tanh(x)`, and `dy`\nis the corresponding input gradient." +} op { name: "TemporaryVariable" output_arg { From 4f63c813dd60536d71ee105efdbdd9afb2d12e08 Mon Sep 17 00:00:00 2001 From: Derek Murray Date: Thu, 30 Jun 2016 13:33:36 -0800 Subject: [PATCH 12/36] Raise an Unimplemented error when using MasterSession with place_pruned_graphs. This change has the unfortunate side-effect of preventing the use of tf.InteractiveSession with a gRPC session. This is intended as a temporary measure, while we fix the implementation of SimpleGraphExecutionState. Change: 126344674 --- .../core/distributed_runtime/master_session.cc | 6 ++++++ tensorflow/python/training/server_lib_test.py | 15 +++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/tensorflow/core/distributed_runtime/master_session.cc b/tensorflow/core/distributed_runtime/master_session.cc index 870970b7cab..46dd7913d32 100644 --- a/tensorflow/core/distributed_runtime/master_session.cc +++ b/tensorflow/core/distributed_runtime/master_session.cc @@ -746,6 +746,12 @@ void MasterSession::UpdateLastAccessTime() { } Status MasterSession::Create(GraphDef* graph_def) { + if (session_opts_.config.graph_options().place_pruned_graph()) { + // TODO(b/29900832): Fix this or remove the option. + return errors::Unimplemented( + "MasterSession does not support the place_pruned_graph option."); + } + // Keeps a copy of graph_def->library() and flib_def_ serves the // OpRegistryInterface used by the SimpleGraphExecutionState to construct the // pre-partitioned graphs during DoRunWithLocalExecution(). diff --git a/tensorflow/python/training/server_lib_test.py b/tensorflow/python/training/server_lib_test.py index 40d399ccedf..f72e96f5af4 100644 --- a/tensorflow/python/training/server_lib_test.py +++ b/tensorflow/python/training/server_lib_test.py @@ -280,6 +280,21 @@ class GrpcServerTest(tf.test.TestCase): job_name="local", task_index=0) + def testInteractiveSession(self): + server = tf.train.Server.create_local_server() + # TODO(b/29900832): Remove this assertion when the bug is fixed. + a = tf.constant(1.0) + with self.assertRaisesRegexp(tf.errors.UnimplementedError, "pruned"): + sess = tf.InteractiveSession(target=server.target) + sess.run(a) + + # TODO(b/29900832): The following code fails (without the unimplemented + # check in `tensorflow::MasterSession`): + # a = tf.constant(1.0) + # b = tf.constant(2.0) + # self.assertEqual(1.0, sess.run(a)) + # self.assertEqual(2.0, sess.run(b)) + class ServerDefTest(tf.test.TestCase): From 76fcc17bb1252862f1a640e741142b20239387d6 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 30 Jun 2016 13:47:22 -0800 Subject: [PATCH 13/36] Prepare move of safe_embedding_lookup_sparse from framework/ to layers/ Change: 126346230 --- .../framework/python/ops/embedding_ops.py | 99 ++----------- .../layers/python/layers/embedding_ops.py | 131 +++++++++++++++++- 2 files changed, 139 insertions(+), 91 deletions(-) diff --git a/tensorflow/contrib/framework/python/ops/embedding_ops.py b/tensorflow/contrib/framework/python/ops/embedding_ops.py index af51042944d..76f4143c09a 100644 --- a/tensorflow/contrib/framework/python/ops/embedding_ops.py +++ b/tensorflow/contrib/framework/python/ops/embedding_ops.py @@ -17,18 +17,14 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from tensorflow.contrib.framework.python.framework import tensor_util as contrib_tensor_util -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import embedding_ops as tf_embedding_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import sparse_ops +from tensorflow.contrib.framework.python.framework.deprecation import deprecated +from tensorflow.contrib.layers import embedding_ops as embedding_ops __all__ = ["safe_embedding_lookup_sparse",] +@deprecated("2016-09-01", + "Please use tf.contrib.layers.safe_embedding_lookup_sparse.") def safe_embedding_lookup_sparse(embedding_weights, sparse_ids, sparse_weights=None, @@ -74,82 +70,11 @@ def safe_embedding_lookup_sparse(embedding_weights, Raises: ValueError: if `embedding_weights` is empty. """ - if embedding_weights is None or len(embedding_weights) < 1: - raise ValueError("Missing embedding_weights %s." % embedding_weights) - - dtype = sparse_weights.dtype if sparse_weights is not None else None - embedding_weights = [ - ops.convert_to_tensor(w, dtype=dtype) for w in embedding_weights - ] - - contrib_tensor_util.assert_same_float_dtype(embedding_weights + - [sparse_weights]) - - with ops.op_scope(embedding_weights + [sparse_ids, sparse_weights], name, - "embedding_lookup") as scope: - # Reshape higher-rank sparse ids and weights to linear segment ids. - original_shape = sparse_ids.shape - original_rank_dim = sparse_ids.shape.get_shape()[0] - original_rank = ( - array_ops.size(original_shape) - if original_rank_dim.value is None - else original_rank_dim.value) - sparse_ids = sparse_ops.sparse_reshape(sparse_ids, [ - math_ops.reduce_prod( - array_ops.slice(original_shape, [0], [original_rank - 1])), - array_ops.gather(original_shape, original_rank - 1)]) - if sparse_weights is not None: - sparse_weights = ops.SparseTensor(sparse_ids.indices, - sparse_weights.values, sparse_ids.shape) - - # Prune invalid ids and weights. - sparse_ids, sparse_weights = _prune_invalid_ids(sparse_ids, sparse_weights) - - # Fill in dummy values for empty features, if necessary. - sparse_ids, is_row_empty = sparse_ops.sparse_fill_empty_rows(sparse_ids, - default_id or - 0) - if sparse_weights is not None: - sparse_weights, _ = sparse_ops.sparse_fill_empty_rows(sparse_weights, 1.0) - - result = tf_embedding_ops.embedding_lookup_sparse( - embedding_weights, - sparse_ids, - sparse_weights, - combiner=combiner, - partition_strategy=partition_strategy, - name=None if default_id is None else scope) - - if default_id is None: - # Broadcast is_row_empty to the same shape as embedding_lookup_result, - # for use in Select. - is_row_empty = array_ops.tile( - array_ops.reshape(is_row_empty, [-1, 1]), - array_ops.pack([1, array_ops.shape(result)[1]])) - - result = math_ops.select(is_row_empty, - array_ops.zeros_like(result), - result, - name=scope) - - # Reshape back from linear ids back into higher-dimensional dense result. - final_result = array_ops.reshape(result, array_ops.concat(0, [ - array_ops.slice( - math_ops.cast(original_shape, dtypes.int32), - [0], [original_rank - 1]), - array_ops.slice(array_ops.shape(result), [1], [-1])])) - final_result.set_shape(tensor_shape.unknown_shape( - (original_rank_dim - 1).value).concatenate(result.get_shape()[1:])) - return final_result - - -def _prune_invalid_ids(sparse_ids, sparse_weights): - """Prune invalid IDs (< 0) from the input ids and weights.""" - is_id_valid = math_ops.greater_equal(sparse_ids.values, 0) - if sparse_weights is not None: - is_id_valid = math_ops.logical_and( - is_id_valid, math_ops.greater(sparse_weights.values, 0)) - sparse_ids = sparse_ops.sparse_retain(sparse_ids, is_id_valid) - if sparse_weights is not None: - sparse_weights = sparse_ops.sparse_retain(sparse_weights, is_id_valid) - return sparse_ids, sparse_weights + return embedding_ops.safe_embedding_lookup_sparse( + embedding_weights, + sparse_ids, + sparse_weights=sparse_weights, + combiner=combiner, + default_id=default_id, + name=name, + partition_strategy=partition_strategy) diff --git a/tensorflow/contrib/layers/python/layers/embedding_ops.py b/tensorflow/contrib/layers/python/layers/embedding_ops.py index 4904c16a9cd..b40b622b8f1 100644 --- a/tensorflow/contrib/layers/python/layers/embedding_ops.py +++ b/tensorflow/contrib/layers/python/layers/embedding_ops.py @@ -18,11 +18,12 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from tensorflow.contrib.framework.python.ops import embedding_ops as contrib_embedding_ops +from tensorflow.contrib.framework.python.framework import tensor_util as contrib_tensor_util from tensorflow.contrib.layers.python.ops import sparse_feature_cross_op -from tensorflow.python.framework import dtypes +from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import tensor_shape from tensorflow.python.ops import array_ops from tensorflow.python.ops import embedding_ops from tensorflow.python.ops import math_ops @@ -32,8 +33,130 @@ __all__ = ["safe_embedding_lookup_sparse", "hashed_embedding_lookup", "hashed_embedding_lookup_sparse"] -# TODO(chapelle): move the safe_embedding_lookup_sparse code here (b/29826543) -safe_embedding_lookup_sparse = contrib_embedding_ops.safe_embedding_lookup_sparse # pylint: disable=line-too-long +def safe_embedding_lookup_sparse(embedding_weights, + sparse_ids, + sparse_weights=None, + combiner="mean", + default_id=None, + name=None, + partition_strategy="div"): + """Lookup embedding results, accounting for invalid IDs and empty features. + + The partitioned embedding in `embedding_weights` must all be the same shape + except for the first dimension. The first dimension is allowed to vary as the + vocabulary size is not necessarily a multiple of `P`. + + Invalid IDs (< 0) are pruned from input IDs and weights, as well as any IDs + with non-positive weight. For an entry with no features, the embedding vector + for `default_id` is returned, or the 0-vector if `default_id` is not supplied. + + The ids and weights may be multi-dimensional. Embeddings are always aggregated + along the last dimension. + + Args: + embedding_weights: A list of `P` float tensors or values representing + partitioned embedding tensors. The total unpartitioned shape should be + `[e_0, e_1, ..., e_m]`, where `e_0` represents the vocab size and + `e_1, ..., e_m` are the embedding dimensions. + sparse_ids: `SparseTensor` of shape `[d_0, d_1, ..., d_n]` containing the + ids. `d_0` is typically batch size. + sparse_weights: `SparseTensor` of same shape as `sparse_ids`, containing + float weights corresponding to `sparse_ids`, or `None` if all weights + are be assumed to be 1.0. + combiner: A string specifying how to combine embedding results for each + entry. Currently "mean", "sqrtn" and "sum" are supported, with "mean" + the default. + default_id: The id to use for an entry with no features. + name: A name for this operation (optional). + partition_strategy: A string specifying the partitioning strategy. + Currently `"div"` and `"mod"` are supported. Default is `"div"`. + + + Returns: + Dense tensor of shape `[d_0, d_1, ..., d_{n-1}, e_1, ..., e_m]`. + + Raises: + ValueError: if `embedding_weights` is empty. + """ + if embedding_weights is None or len(embedding_weights) < 1: + raise ValueError("Missing embedding_weights %s." % embedding_weights) + + dtype = sparse_weights.dtype if sparse_weights is not None else None + embedding_weights = [ + ops.convert_to_tensor(w, dtype=dtype) for w in embedding_weights + ] + + contrib_tensor_util.assert_same_float_dtype(embedding_weights + + [sparse_weights]) + + with ops.op_scope(embedding_weights + [sparse_ids, sparse_weights], name, + "embedding_lookup") as scope: + # Reshape higher-rank sparse ids and weights to linear segment ids. + original_shape = sparse_ids.shape + original_rank_dim = sparse_ids.shape.get_shape()[0] + original_rank = ( + array_ops.size(original_shape) + if original_rank_dim.value is None + else original_rank_dim.value) + sparse_ids = sparse_ops.sparse_reshape(sparse_ids, [ + math_ops.reduce_prod( + array_ops.slice(original_shape, [0], [original_rank - 1])), + array_ops.gather(original_shape, original_rank - 1)]) + if sparse_weights is not None: + sparse_weights = ops.SparseTensor(sparse_ids.indices, + sparse_weights.values, sparse_ids.shape) + + # Prune invalid ids and weights. + sparse_ids, sparse_weights = _prune_invalid_ids(sparse_ids, sparse_weights) + + # Fill in dummy values for empty features, if necessary. + sparse_ids, is_row_empty = sparse_ops.sparse_fill_empty_rows(sparse_ids, + default_id or + 0) + if sparse_weights is not None: + sparse_weights, _ = sparse_ops.sparse_fill_empty_rows(sparse_weights, 1.0) + + result = embedding_ops.embedding_lookup_sparse( + embedding_weights, + sparse_ids, + sparse_weights, + combiner=combiner, + partition_strategy=partition_strategy, + name=None if default_id is None else scope) + + if default_id is None: + # Broadcast is_row_empty to the same shape as embedding_lookup_result, + # for use in Select. + is_row_empty = array_ops.tile( + array_ops.reshape(is_row_empty, [-1, 1]), + array_ops.pack([1, array_ops.shape(result)[1]])) + + result = math_ops.select(is_row_empty, + array_ops.zeros_like(result), + result, + name=scope) + + # Reshape back from linear ids back into higher-dimensional dense result. + final_result = array_ops.reshape(result, array_ops.concat(0, [ + array_ops.slice( + math_ops.cast(original_shape, dtypes.int32), + [0], [original_rank - 1]), + array_ops.slice(array_ops.shape(result), [1], [-1])])) + final_result.set_shape(tensor_shape.unknown_shape( + (original_rank_dim - 1).value).concatenate(result.get_shape()[1:])) + return final_result + + +def _prune_invalid_ids(sparse_ids, sparse_weights): + """Prune invalid IDs (< 0) from the input ids and weights.""" + is_id_valid = math_ops.greater_equal(sparse_ids.values, 0) + if sparse_weights is not None: + is_id_valid = math_ops.logical_and( + is_id_valid, math_ops.greater(sparse_weights.values, 0)) + sparse_ids = sparse_ops.sparse_retain(sparse_ids, is_id_valid) + if sparse_weights is not None: + sparse_weights = sparse_ops.sparse_retain(sparse_weights, is_id_valid) + return sparse_ids, sparse_weights def hashed_embedding_lookup(params, values, dimension, name=None): From 379df09118ddfdbad19375d6d853254312ccf1ae Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 30 Jun 2016 14:00:39 -0800 Subject: [PATCH 14/36] Fix initialization issues with Variables whose shape contains a zero. Fixes #2099. Tries to give Variables the same behavior as non-Variable tensors in this respect. Useful for not having to special case e.g. coefficients of a feature vector which may sometimes not have any features. Change: 126347791 --- tensorflow/core/client/tensor_c_api.cc | 2 +- tensorflow/core/framework/op_kernel.cc | 4 +-- tensorflow/core/framework/op_kernel.h | 4 ++- tensorflow/core/framework/tensor.cc | 13 +++++---- tensorflow/core/framework/tensor.h | 5 +++- .../framework/unique_tensor_references.cc | 2 +- tensorflow/core/kernels/barrier_ops.cc | 5 ++-- tensorflow/core/kernels/tensor_array.h | 4 +-- .../python/kernel_tests/variables_test.py | 28 +++++++++++++++++++ 9 files changed, 51 insertions(+), 16 deletions(-) diff --git a/tensorflow/core/client/tensor_c_api.cc b/tensorflow/core/client/tensor_c_api.cc index a7f1a4fa78b..9959280029e 100644 --- a/tensorflow/core/client/tensor_c_api.cc +++ b/tensorflow/core/client/tensor_c_api.cc @@ -475,7 +475,7 @@ void TF_Run_Helper(TF_Session* s, const char* handle, // Store results in c_outputs[] for (int i = 0; i < noutputs; i++) { const Tensor& src = outputs[i]; - if (!src.IsInitialized()) { + if (!src.IsInitialized() || src.NumElements() == 0) { c_outputs[i] = tensorflow::EmptyTensor( static_cast(src.dtype()), src.shape()); continue; diff --git a/tensorflow/core/framework/op_kernel.cc b/tensorflow/core/framework/op_kernel.cc index 6bfc55df41f..12df379e8f0 100644 --- a/tensorflow/core/framework/op_kernel.cc +++ b/tensorflow/core/framework/op_kernel.cc @@ -159,7 +159,7 @@ Status OpKernelConstruction::allocate_temp(DataType type, attr.allocation_will_be_logged = true; Tensor new_temp(allocator_, type, shape, attr); - if (!new_temp.IsInitialized() && shape.num_elements() > 0) { + if (!new_temp.IsInitialized()) { return errors::ResourceExhausted( "OOM when allocating temporary tensor with shape", shape.DebugString()); } @@ -447,7 +447,7 @@ Status OpKernelContext::allocate_tensor( logged_attr.allocation_will_be_logged = true; Tensor new_tensor(a, type, shape, logged_attr); - if (!new_tensor.IsInitialized() && shape.num_elements() > 0) { + if (!new_tensor.IsInitialized()) { return errors::ResourceExhausted("OOM when allocating tensor with shape", shape.DebugString()); } diff --git a/tensorflow/core/framework/op_kernel.h b/tensorflow/core/framework/op_kernel.h index 0092c6286f8..a6cc323ceae 100644 --- a/tensorflow/core/framework/op_kernel.h +++ b/tensorflow/core/framework/op_kernel.h @@ -199,7 +199,9 @@ class PersistentTensor { // The check for initialization does not need to access the // underlying tensor buffer. - bool IsInitialized() { return tensor_.IsInitialized(); } + bool IsInitialized() const { return tensor_.IsInitialized(); } + + int64 NumElements() const { return tensor_.NumElements(); } private: Tensor tensor_; diff --git a/tensorflow/core/framework/tensor.cc b/tensorflow/core/framework/tensor.cc index 7b85ff9c364..de15d82269c 100644 --- a/tensorflow/core/framework/tensor.cc +++ b/tensorflow/core/framework/tensor.cc @@ -416,7 +416,8 @@ Tensor::Tensor(DataType type, const TensorShape& shape, TensorBuffer* buf) } bool Tensor::IsInitialized() const { - return buf_ != nullptr && buf_->data() != nullptr; + return (buf_ != nullptr && buf_->data() != nullptr) || + shape_.num_elements() == 0; } void Tensor::CheckType(DataType expected_dtype) const { @@ -507,7 +508,7 @@ Tensor::Tensor(Allocator* a, DataType type, const TensorShape& shape) if (shape_.num_elements() > 0 || a->ShouldAllocateEmptyTensors()) { CASES(type, buf_ = new Buffer(a, shape.num_elements())); } - if (IsInitialized() && LogMemory::IsEnabled()) { + if (buf_ != nullptr && buf_->data() != nullptr && LogMemory::IsEnabled()) { LogMemory::RecordTensorAllocation("Unknown", LogMemory::UNKNOWN_STEP_ID, *this); } @@ -521,8 +522,8 @@ Tensor::Tensor(Allocator* a, DataType type, const TensorShape& shape, if (shape_.num_elements() > 0 || a->ShouldAllocateEmptyTensors()) { CASES(type, buf_ = new Buffer(a, shape.num_elements(), allocation_attr)); } - if (!allocation_attr.allocation_will_be_logged && IsInitialized() && - LogMemory::IsEnabled()) { + if (!allocation_attr.allocation_will_be_logged && buf_ != nullptr && + buf_->data() != nullptr && LogMemory::IsEnabled()) { LogMemory::RecordTensorAllocation("Unknown (with attributes)", LogMemory::UNKNOWN_STEP_ID, *this); } @@ -617,7 +618,7 @@ bool Tensor::FromProto(Allocator* a, const TensorProto& proto) { buf_ = p; // TODO(misard) add tracking of which kernels and steps are calling // FromProto. - if (IsInitialized() && LogMemory::IsEnabled()) { + if (buf_ != nullptr && buf_->data() != nullptr && LogMemory::IsEnabled()) { LogMemory::RecordTensorAllocation("Unknown (from Proto)", LogMemory::UNKNOWN_STEP_ID, *this); } @@ -765,7 +766,7 @@ string Tensor::DebugString() const { void Tensor::FillDescription(TensorDescription* description) const { description->set_dtype(dtype()); shape().AsProto(description->mutable_shape()); - if (IsInitialized()) { + if (buf_ != nullptr && buf_->data() != nullptr) { buf_->FillAllocationDescription( description->mutable_allocation_description()); } diff --git a/tensorflow/core/framework/tensor.h b/tensorflow/core/framework/tensor.h index dd2d9a4c863..48fbd38e0c4 100644 --- a/tensorflow/core/framework/tensor.h +++ b/tensorflow/core/framework/tensor.h @@ -120,7 +120,10 @@ class Tensor { // underlying refcounted storage size_t BufferHash() const; - /// Has this Tensor been initialized? + /// \brief If necessary, has this Tensor been initialized? + /// + /// Zero-element Tensors are always considered initialized, even if they + /// have never been assigned to and do not have any memory allocated. bool IsInitialized() const; /// Returns the estimated memory usage of this tensor. diff --git a/tensorflow/core/framework/unique_tensor_references.cc b/tensorflow/core/framework/unique_tensor_references.cc index 2ac6431c54b..ab33d9ede6c 100644 --- a/tensorflow/core/framework/unique_tensor_references.cc +++ b/tensorflow/core/framework/unique_tensor_references.cc @@ -33,7 +33,7 @@ UniqueTensorReferences::~UniqueTensorReferences() { void UniqueTensorReferences::Add(const Tensor& tensor) { DCHECK(!frozen_); // Do nothing if the tensor has a null buffer. - if (tensor.IsInitialized()) { + if (tensor.IsInitialized() && tensor.NumElements() > 0) { if (referenced_tensors_set_ != nullptr) { // There are enough tensors that we are using a hash set to // de-duplicate. diff --git a/tensorflow/core/kernels/barrier_ops.cc b/tensorflow/core/kernels/barrier_ops.cc index 533b03db0e2..e66b9d41689 100644 --- a/tensorflow/core/kernels/barrier_ops.cc +++ b/tensorflow/core/kernels/barrier_ops.cc @@ -354,7 +354,8 @@ class Barrier : public ResourceBase { element.push_back(PersistentTensor(uninitialized)); } } - if (element[1 + component_index].IsInitialized()) { + const PersistentTensor& component = element[1 + component_index]; + if (component.IsInitialized() && component.NumElements() > 0) { return errors::InvalidArgument("Key ", keys_vec(i), " already has a value for component ", component_index, " in barrier ", name()); @@ -374,7 +375,7 @@ class Barrier : public ResourceBase { // ready queue. bool is_complete = true; for (int j = 0; is_complete && j < element.size(); ++j) { - is_complete = element[j].IsInitialized(); + is_complete = element[j].IsInitialized() && element[j].NumElements() > 0; } if (is_complete) { // Add tuple to the ready queue. A queue tuple has the index diff --git a/tensorflow/core/kernels/tensor_array.h b/tensorflow/core/kernels/tensor_array.h index 03b2c3b68b9..1456ec28447 100644 --- a/tensorflow/core/kernels/tensor_array.h +++ b/tensorflow/core/kernels/tensor_array.h @@ -441,7 +441,7 @@ Status TensorArray::LockedWriteOrAggregate(OpKernelContext* ctx, " but the new input shape is ", value_t->shape().DebugString(), "."); } - if (!t.tensor.IsInitialized()) { + if (!t.tensor.IsInitialized() || t.tensor.NumElements() == 0) { // If existing_t == nullptr but written == true, then what was stored // was just a shape, which just means zeros. So all we must do in this // case is copy the reference over and return early. @@ -502,7 +502,7 @@ Status TensorArray::LockedRead(OpKernelContext* ctx, const int32 index, "clear_after_read = false?)."); } - if (!t.tensor.IsInitialized()) { + if (!t.tensor.IsInitialized() || t.tensor.NumElements() == 0) { // We stored just a shape, but no value. This means create and // return zeros of the appropriate shape. Tensor* tensor_t; diff --git a/tensorflow/python/kernel_tests/variables_test.py b/tensorflow/python/kernel_tests/variables_test.py index 82f010cf5b2..23a7f3717cc 100644 --- a/tensorflow/python/kernel_tests/variables_test.py +++ b/tensorflow/python/kernel_tests/variables_test.py @@ -197,6 +197,17 @@ class VariablesTestCase(tf.test.TestCase): self.assertAllClose(3.0, var_y.eval()) self.assertAllClose(5.0, tf.add(var_x, var_y).eval()) + def testZeroSizeVarSameAsConst(self): + with self.test_session(): + zero_size_var = tf.Variable(tf.zeros([0, 2])) + zero_size_const = tf.ones([2, 0]) + variable_mul = tf.matmul(zero_size_const, zero_size_var) + const_mul = tf.matmul(zero_size_const, zero_size_const, transpose_b=True) + tf.initialize_all_variables().run() + variable_output = variable_mul.eval() + self.assertAllClose(const_mul.eval(), variable_output) + self.assertAllClose([[0., 0.], [0., 0.]], variable_output) + def testCachingDevice(self): with self.test_session(): var = tf.Variable(2.0) @@ -387,6 +398,23 @@ class IsInitializedTest(tf.test.TestCase): v.initializer.run() self.assertEqual(0, sess.run(uninited).size) + def testZeroSizeVarInitialized(self): + with tf.Graph().as_default(), self.test_session() as sess: + v = tf.Variable(tf.zeros([0, 2]), name="v") + uninited = tf.report_uninitialized_variables() + v.initializer.run() # not strictly necessary + self.assertEqual(0, sess.run(uninited).size) + + def testTrainingWIthZeroSizeVar(self): + with tf.Graph().as_default(), self.test_session() as sess: + a = tf.Variable(tf.zeros([0, 2])) + b = tf.Variable(tf.ones([2, 2])) + objective = tf.reduce_sum(b + tf.matmul(a, a, transpose_a=True)) + tf.initialize_all_variables().run() + do_opt = tf.train.GradientDescentOptimizer(0.1).minimize(objective) + sess.run([do_opt]) + self.assertAllClose([[0.9, 0.9], [0.9, 0.9]], b.eval()) + class ObsoleteIsInitializedTest(tf.test.TestCase): From d04c05def547a84bdc78f04e44e69c6f634b6df0 Mon Sep 17 00:00:00 2001 From: Zongheng Yang Date: Thu, 30 Jun 2016 14:04:12 -0800 Subject: [PATCH 15/36] Support Sparse-Sparse cwise ops; use for tf.sparse_{minimum,maximum}(). This change adds the CPU kernel and Python ifaces. For now, assumes both operands have the same shapes. Change: 126348349 --- tensorflow/core/kernels/BUILD | 1 + tensorflow/core/kernels/sparse_add_op.cc | 35 +-- .../kernels/sparse_sparse_binary_op_shared.cc | 230 ++++++++++++++++++ tensorflow/core/ops/sparse_ops.cc | 54 ++++ .../python/kernel_tests/sparse_ops_test.py | 63 +++++ tensorflow/python/ops/sparse_grad.py | 12 + tensorflow/python/ops/sparse_ops.py | 83 +++++++ 7 files changed, 461 insertions(+), 17 deletions(-) create mode 100644 tensorflow/core/kernels/sparse_sparse_binary_op_shared.cc diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index 142f63c6b47..06c3c86c673 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -1562,6 +1562,7 @@ tf_kernel_libraries( "sparse_concat_op", "sparse_reduce_sum_op", "sparse_dense_binary_op_shared", + "sparse_sparse_binary_op_shared", "sparse_reorder_op", "sparse_reshape_op", "sparse_softmax", diff --git a/tensorflow/core/kernels/sparse_add_op.cc b/tensorflow/core/kernels/sparse_add_op.cc index 0cb77d785ad..bd91dfdce64 100644 --- a/tensorflow/core/kernels/sparse_add_op.cc +++ b/tensorflow/core/kernels/sparse_add_op.cc @@ -54,31 +54,32 @@ class SparseAddOp : public OpKernel { b_values_t->shape().DebugString())); auto a_values = ctx->input(1).vec(); auto b_values = ctx->input(4).vec(); - - OP_REQUIRES_OK(ctx, ctx->input("a_shape", &a_shape)); - OP_REQUIRES_OK(ctx, ctx->input("b_shape", &b_shape)); - OP_REQUIRES(ctx, TensorShapeUtils::IsVector(a_shape->shape()) && - TensorShapeUtils::IsVector(b_shape->shape()), - errors::InvalidArgument( - "Input shape should be a vector but received shapes ", - a_shape->shape().DebugString(), " and ", - b_shape->shape().DebugString())); - OP_REQUIRES( ctx, a_values.size() == a_nnz && b_values.size() == b_nnz, errors::InvalidArgument("Expected ", a_nnz, " and ", b_nnz, " non-empty input values, got ", a_values.size(), " and ", b_values.size())); - OP_REQUIRES(ctx, a_shape->dims() == b_shape->dims(), + OP_REQUIRES_OK(ctx, ctx->input("a_shape", &a_shape)); + OP_REQUIRES_OK(ctx, ctx->input("b_shape", &b_shape)); + OP_REQUIRES(ctx, TensorShapeUtils::IsVector(a_shape->shape()) && + TensorShapeUtils::IsVector(b_shape->shape()), errors::InvalidArgument( - "Ranks of input tensors must match, but saw ranks: ", - a_shape->dims(), " and ", b_shape->dims())); - for (int i = 0; i < a_shape->dims(); ++i) { - OP_REQUIRES(ctx, a_shape->dim_size(i) == b_shape->dim_size(i), + "Input shapes should be a vector but received shapes ", + a_shape->shape().DebugString(), " and ", + b_shape->shape().DebugString())); + OP_REQUIRES( + ctx, a_shape->IsSameSize(*b_shape), + errors::InvalidArgument( + "Operands do not have the same ranks; got shapes: ", + a_shape->SummarizeValue(10), " and ", b_shape->SummarizeValue(10))); + const auto a_shape_flat = a_shape->flat(); + const auto b_shape_flat = b_shape->flat(); + for (int i = 0; i < a_shape->NumElements(); ++i) { + OP_REQUIRES(ctx, a_shape_flat(i) == b_shape_flat(i), errors::InvalidArgument( - "Input shapes must match: got ", a_shape->dim_size(i), - " and ", b_shape->dim_size(i), " for dimension ", i)); + "Operands' shapes do not match: got ", a_shape_flat(i), + " and ", b_shape_flat(i), " for dimension ", i)); } OP_REQUIRES_OK(ctx, ctx->input("thresh", &thresh_t)); diff --git a/tensorflow/core/kernels/sparse_sparse_binary_op_shared.cc b/tensorflow/core/kernels/sparse_sparse_binary_op_shared.cc new file mode 100644 index 00000000000..a8a116ebf58 --- /dev/null +++ b/tensorflow/core/kernels/sparse_sparse_binary_op_shared.cc @@ -0,0 +1,230 @@ +/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// SparseSparseBinaryOpShared is the shared code for binary coefficient-wise +// (cwise) operations of the following form: +// +// sparse_t sparse_t -> new sparse_t +// +// The output SparseTensor may store up to "a_nnz + b_nnz" elements. + +// IMPLEMENTATION DETAILS (not part of the interface specification). +// +// This kernel implements the "union" semantics on the non-zeros: namely, any +// non-zero from either side participate in the calculations, and any resultant +// zeros will NOT be excluded from the output storage. +// +// (In the future, we could always add a pruning op the prunes away the zeros, +// if desirable.) + +// See docs of all registered ops in ../ops/sparse_ops.cc. + +#define EIGEN_USE_THREADS + +#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor_util.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/kernels/cwise_ops.h" +#include "tensorflow/core/kernels/cwise_ops_common.h" +#include "tensorflow/core/util/sparse/sparse_tensor.h" + +namespace tensorflow { + +typedef Eigen::ThreadPoolDevice CPUDevice; + +namespace { +// Unions the sparse indices and outputs corresponding values: namely, if a +// non-zero appear in one side, it will participate in the calculation, where +// the counterpart on the other side is either a value or an implicit zero. +// +// On exit, outputs the augmented values in "{a,b}_augmented_values", and fills +// "entries_to_copy" with "(from_a?, index)" pairs. All three vectors have the +// same size. +// +// The input and output sparse tensors are assumed ordered in the canonical +// row-major order. +template +void UnionSparseIndicesAndValues( + typename TTypes::ConstMatrix a_indices_mat, + typename TTypes::ConstFlat a_values, int64 a_nnz, + typename TTypes::ConstMatrix b_indices_mat, + typename TTypes::ConstFlat b_values, int64 b_nnz, int num_dims, + std::vector *a_augmented_values, std::vector *b_augmented_values, + std::vector> *entries_to_copy) { + entries_to_copy->reserve(a_nnz + b_nnz); + a_augmented_values->reserve(a_nnz); + b_augmented_values->reserve(b_nnz); + + int64 i = 0, j = 0; + const T kZero = T(0); + while (i < a_nnz && j < b_nnz) { + switch (sparse::DimComparator::cmp(a_indices_mat, b_indices_mat, i, j, + num_dims)) { + case -1: + entries_to_copy->emplace_back(true, i); + a_augmented_values->push_back(a_values(i)); + b_augmented_values->push_back(kZero); + ++i; + break; + case 0: + entries_to_copy->emplace_back(true, i); + a_augmented_values->push_back(a_values(i)); + b_augmented_values->push_back(b_values(j)); + ++i; + ++j; + break; + case 1: + entries_to_copy->emplace_back(false, j); + a_augmented_values->push_back(kZero); + b_augmented_values->push_back(b_values(j)); + ++j; + break; + } + } + // Handles leftovers; at most one loop runs. + while (i < a_nnz) { + entries_to_copy->emplace_back(/* is_a */ true, i); + a_augmented_values->push_back(a_values(i++)); + b_augmented_values->push_back(kZero); + } + while (j < b_nnz) { + entries_to_copy->emplace_back(/* is_a */ false, j); + a_augmented_values->push_back(kZero); + b_augmented_values->push_back(b_values(j++)); + } +} +} // anonymous namespace + +// Device: CPUDevice. GPU kernel is not supported currently. +// T: dtype of the SparseTensor's. +// Functor: binary cwise operation to perform on the corresponding operand +// values. See cwise_ops.h for a list of possible functors to register with. +template +class SparseSparseBinaryOpShared : public OpKernel { + public: + explicit SparseSparseBinaryOpShared(OpKernelConstruction *ctx) + : OpKernel(ctx) {} + + void Compute(OpKernelContext *ctx) override { + const Tensor *a_indices_t, *a_values_t, *a_shape_t, *b_indices_t, + *b_values_t, *b_shape_t; + OP_REQUIRES_OK(ctx, ctx->input("a_indices", &a_indices_t)); + OP_REQUIRES_OK(ctx, ctx->input("a_values", &a_values_t)); + OP_REQUIRES_OK(ctx, ctx->input("a_shape", &a_shape_t)); + OP_REQUIRES_OK(ctx, ctx->input("b_indices", &b_indices_t)); + OP_REQUIRES_OK(ctx, ctx->input("b_values", &b_values_t)); + OP_REQUIRES_OK(ctx, ctx->input("b_shape", &b_shape_t)); + + // Validations. + OP_REQUIRES( + ctx, TensorShapeUtils::IsMatrix(a_indices_t->shape()) && + TensorShapeUtils::IsMatrix(b_indices_t->shape()), + errors::InvalidArgument("Inputs a_indices and b_indices should be " + "matrices but received shapes: ", + a_indices_t->shape().DebugString(), ", ", + b_indices_t->shape().DebugString())); + OP_REQUIRES(ctx, TensorShapeUtils::IsVector(a_values_t->shape()) && + TensorShapeUtils::IsVector(b_values_t->shape()), + errors::InvalidArgument( + "Inputs a_values and b_values should be vectors " + "but received shapes: ", + a_values_t->shape().DebugString(), " and ", + b_values_t->shape().DebugString())); + + const int64 a_nnz = a_indices_t->dim_size(0); + const int64 b_nnz = b_indices_t->dim_size(0); + const auto a_values = a_values_t->vec(); + const auto b_values = b_values_t->vec(); + + OP_REQUIRES( + ctx, a_values.size() == a_nnz && b_values.size() == b_nnz, + errors::InvalidArgument("Expected ", a_nnz, " and ", b_nnz, + " non-empty input values, got ", + a_values.size(), " and ", b_values.size())); + + OP_REQUIRES(ctx, TensorShapeUtils::IsVector(a_shape_t->shape()) && + TensorShapeUtils::IsVector(b_shape_t->shape()), + errors::InvalidArgument( + "Input shapes should be a vector but received shapes ", + a_shape_t->shape().DebugString(), " and ", + b_shape_t->shape().DebugString())); + OP_REQUIRES(ctx, a_shape_t->IsSameSize(*b_shape_t), + errors::InvalidArgument( + "Operands do not have the same ranks; got shapes: ", + a_shape_t->SummarizeValue(10), " and ", + b_shape_t->SummarizeValue(10))); + const auto a_shape = a_shape_t->flat(); + const auto b_shape = b_shape_t->flat(); + for (int i = 0; i < a_shape_t->NumElements(); ++i) { + OP_REQUIRES(ctx, a_shape(i) == b_shape(i), + errors::InvalidArgument("Operands' shapes do not match: got ", + a_shape(i), " and ", b_shape(i), + " for dimension ", i)); + } + + const int num_dims = a_indices_t->dim_size(1); + const auto a_indices_mat = a_indices_t->matrix(); + const auto b_indices_mat = b_indices_t->matrix(); + std::vector a_augmented_values, b_augmented_values; + std::vector> entries_to_copy; // from_a?, idx + UnionSparseIndicesAndValues(a_indices_mat, a_values, a_nnz, b_indices_mat, + b_values, b_nnz, num_dims, &a_augmented_values, + &b_augmented_values, &entries_to_copy); + + // Allocates and fills output tensors. + const int64 sum_nnz = a_augmented_values.size(); + Tensor *output_indices_t, *output_values_t; + OP_REQUIRES_OK(ctx, + ctx->allocate_output(0, TensorShape({sum_nnz, num_dims}), + &output_indices_t)); + OP_REQUIRES_OK( + ctx, ctx->allocate_output(1, TensorShape({sum_nnz}), &output_values_t)); + auto output_indices_mat = output_indices_t->matrix(); + + for (int64 i = 0; i < sum_nnz; ++i) { + const bool from_a = entries_to_copy[i].first; + const int64 idx = entries_to_copy[i].second; + output_indices_mat.chip<0>(i) = + from_a ? a_indices_mat.chip<0>(idx) : b_indices_mat.chip<0>(idx); + } + + // Performs the functor operation using Eigen. + using TensorMap = + Eigen::TensorMap, + Eigen::Aligned>; + auto a_augmented_values_t = TensorMap(a_augmented_values.data(), sum_nnz); + auto b_augmented_values_t = TensorMap(b_augmented_values.data(), sum_nnz); + output_values_t->flat().device(ctx->eigen_device()) = + a_augmented_values_t.binaryExpr(b_augmented_values_t, + typename Functor::func()); + } +}; + +#define REGISTER_KERNELS(T) \ + REGISTER_KERNEL_BUILDER( \ + Name("SparseSparseMinimum").Device(DEVICE_CPU).TypeConstraint("T"), \ + SparseSparseBinaryOpShared>) \ + \ + REGISTER_KERNEL_BUILDER( \ + Name("SparseSparseMaximum").Device(DEVICE_CPU).TypeConstraint("T"), \ + SparseSparseBinaryOpShared>) + +TF_CALL_REAL_NUMBER_TYPES(REGISTER_KERNELS); +#undef REGISTER_KERNELS + +} // namespace tensorflow diff --git a/tensorflow/core/ops/sparse_ops.cc b/tensorflow/core/ops/sparse_ops.cc index d80a5b1f9bc..a39f2f70cba 100644 --- a/tensorflow/core/ops/sparse_ops.cc +++ b/tensorflow/core/ops/sparse_ops.cc @@ -570,4 +570,58 @@ sp_shape: 1-D. Shape of the input SparseTensor. output: 1-D. The `NNZ` values for the result `SparseTensor`. )doc"); +REGISTER_OP("SparseSparseMaximum") + .Input("a_indices: int64") + .Input("a_values: T") + .Input("a_shape: int64") + .Input("b_indices: int64") + .Input("b_values: T") + .Input("b_shape: int64") + .Output("output_indices: int64") + .Output("output_values: T") + .Attr("T: realnumbertype") + .Doc(R"doc( +Returns the element-wise max of two SparseTensors. + +Assumes the two SparseTensors have the same shape, i.e., no broadcasting. + +a_indices: 2-D. `N x R` matrix with the indices of non-empty values in a + SparseTensor, in the canonical lexicographic ordering. +a_values: 1-D. `N` non-empty values corresponding to `a_indices`. +a_shape: 1-D. Shape of the input SparseTensor. +b_indices: counterpart to `a_indices` for the other operand. +b_values: counterpart to `a_values` for the other operand; must be of the same dtype. +b_shape: counterpart to `a_shape` for the other operand; the two shapes must be equal. + +output_indices: 2-D. The indices of the output SparseTensor. +output_values: 1-D. The values of the output SparseTensor. +)doc"); + +REGISTER_OP("SparseSparseMinimum") + .Input("a_indices: int64") + .Input("a_values: T") + .Input("a_shape: int64") + .Input("b_indices: int64") + .Input("b_values: T") + .Input("b_shape: int64") + .Output("output_indices: int64") + .Output("output_values: T") + .Attr("T: numbertype") + .Doc(R"doc( +Returns the element-wise min of two SparseTensors. + +Assumes the two SparseTensors have the same shape, i.e., no broadcasting. + +a_indices: 2-D. `N x R` matrix with the indices of non-empty values in a + SparseTensor, in the canonical lexicographic ordering. +a_values: 1-D. `N` non-empty values corresponding to `a_indices`. +a_shape: 1-D. Shape of the input SparseTensor. +b_indices: counterpart to `a_indices` for the other operand. +b_values: counterpart to `a_values` for the other operand; must be of the same dtype. +b_shape: counterpart to `a_shape` for the other operand; the two shapes must be equal. + +output_indices: 2-D. The indices of the output SparseTensor. +output_values: 1-D. The values of the output SparseTensor. +)doc"); + } // namespace tensorflow diff --git a/tensorflow/python/kernel_tests/sparse_ops_test.py b/tensorflow/python/kernel_tests/sparse_ops_test.py index 867bfc5b369..a0394851d11 100644 --- a/tensorflow/python/kernel_tests/sparse_ops_test.py +++ b/tensorflow/python/kernel_tests/sparse_ops_test.py @@ -649,5 +649,68 @@ class SparseSoftmaxTest(test_util.TensorFlowTestCase): self.assertLess(err, 1e-4) +class SparseMinimumMaximumTest(test_util.TensorFlowTestCase): + + def _assertSparseTensorValueEqual(self, a, b): + self.assertAllEqual(a.indices, b.indices) + self.assertAllEqual(a.values, b.values) + self.assertAllEqual(a.shape, b.shape) + + def testBasic(self): + with self.test_session(use_gpu=False): + # 1-D, values at index 0. + sp_zero = ops.SparseTensor([[0]], [0], [7]) + sp_one = ops.SparseTensor([[0]], [1], [7]) + max_tf = tf.sparse_maximum(sp_zero, sp_one).eval() + min_tf = tf.sparse_minimum(sp_zero, sp_one).eval() + self._assertSparseTensorValueEqual(sp_one.eval(), max_tf) + self._assertSparseTensorValueEqual(sp_zero.eval(), min_tf) + + # Values at different indices. + sp_zero = ops.SparseTensor([[0]], [0], [7]) + sp_zero_2 = ops.SparseTensor([[1]], [0], [7]) + expected = ops.SparseTensor([[0], [1]], [0, 0], [7]) + max_tf = tf.sparse_maximum(sp_zero, sp_zero_2).eval() + min_tf = tf.sparse_minimum(sp_zero, sp_zero_2).eval() + self._assertSparseTensorValueEqual(expected.eval(), max_tf) + self._assertSparseTensorValueEqual(expected.eval(), min_tf) + + def testRandom(self): + np.random.seed(1618) + shapes = [(13,), (6, 8), (1, 7, 1)] + for shape in shapes: + for dtype in [np.int32, np.int64, np.float16, np.float32, np.float64]: + a_np = np.random.randn(*shape).astype(dtype) + b_np = np.random.randn(*shape).astype(dtype) + sp_a, unused_a_nnz = _sparsify(a_np, thresh=-.5) + sp_b, unused_b_nnz = _sparsify(b_np, thresh=-.5) + + with self.test_session(use_gpu=False): + maximum_tf = tf.sparse_maximum(sp_a, sp_b) + maximum_tf_densified = tf.sparse_tensor_to_dense(maximum_tf).eval() + minimum_tf = tf.sparse_minimum(sp_a, sp_b) + minimum_tf_densified = tf.sparse_tensor_to_dense(minimum_tf).eval() + + a_densified = tf.sparse_tensor_to_dense(sp_a).eval() + b_densified = tf.sparse_tensor_to_dense(sp_b).eval() + + self.assertAllEqual(np.maximum(a_densified, b_densified), + maximum_tf_densified) + self.assertAllEqual(np.minimum(a_densified, b_densified), + minimum_tf_densified) + + def testMismatchedShapes(self): + with self.test_session(use_gpu=False): + sp_zero = ops.SparseTensor([[0, 0]], [0], [1, 1]) + sp_one = ops.SparseTensor([[0]], [1], [2]) + with self.assertRaisesOpError("Operands do not have the same ranks"): + tf.sparse_maximum(sp_zero, sp_one).eval() + + sp_zero = ops.SparseTensor([[0]], [0], [1]) + sp_one = ops.SparseTensor([[0]], [1], [2]) + with self.assertRaisesOpError("Operands' shapes do not match"): + tf.sparse_maximum(sp_zero, sp_one).eval() + + if __name__ == "__main__": googletest.main() diff --git a/tensorflow/python/ops/sparse_grad.py b/tensorflow/python/ops/sparse_grad.py index 93c026f2471..57350253ab8 100644 --- a/tensorflow/python/ops/sparse_grad.py +++ b/tensorflow/python/ops/sparse_grad.py @@ -256,3 +256,15 @@ def _SparseSoftmaxGrad(op, grad): grad_x = sp_sum.values * sp_output.values return [None, grad_x, None] + + +@ops.RegisterGradient("SparseSparseMaximum") +def _SparseSparseMaximumGrad(unused_op, unused_grad): + raise NotImplementedError("Gradient for SparseSparseMaximum is currently not" + " implemented yet.") + + +@ops.RegisterGradient("SparseSparseMinimum") +def _SparseSparseMinimumGrad(unused_op, unused_grad): + raise NotImplementedError("Gradient for SparseSparseMinimum is currently not" + " implemented yet.") diff --git a/tensorflow/python/ops/sparse_ops.py b/tensorflow/python/ops/sparse_ops.py index e0a922739bc..b5877d27423 100644 --- a/tensorflow/python/ops/sparse_ops.py +++ b/tensorflow/python/ops/sparse_ops.py @@ -48,6 +48,8 @@ dimension, and dense along all other dimensions. @@sparse_add @@sparse_softmax @@sparse_tensor_dense_matmul +@@sparse_maximum +@@sparse_minimum """ from __future__ import absolute_import from __future__ import division @@ -1487,3 +1489,84 @@ def _SparseSoftmaxShape(op): # pylint: disable=invalid-name unused_shape_shape = op.inputs[2].get_shape().with_rank(1) nnz = values_shape[0] return [tensor_shape.vector(nnz)] + + +def sparse_maximum(sp_a, sp_b, name=None): + """Returns the element-wise max of two SparseTensors. + + Assumes the two SparseTensors have the same shape, i.e., no broadcasting. + Example: + + ```python + sp_zero = ops.SparseTensor([[0]], [0], [7]) + sp_one = ops.SparseTensor([[1]], [1], [7]) + res = tf.sparse_maximum(sp_zero, sp_one).eval() + # "res" should be equal to SparseTensor([[0], [1]], [0, 1], [7]). + ``` + + Args: + sp_a: a `SparseTensor` operand whose dtype is real, and indices + lexicographically ordered. + sp_b: the other `SparseTensor` operand with the same requirements (and the + same shape). + name: optional name of the operation. + Returns: + output: the output SparseTensor. + """ + with ops.op_scope([sp_a.indices, sp_a.values, sp_b.indices, sp_b.values], + name, "SparseSparseMaximum") as name: + out_indices, out_values = gen_sparse_ops.sparse_sparse_maximum(sp_a.indices, + sp_a.values, + sp_a.shape, + sp_b.indices, + sp_b.values, + sp_b.shape, + name=name) + return ops.SparseTensor(out_indices, out_values, sp_a.shape) + + +def sparse_minimum(sp_a, sp_b, name=None): + """Returns the element-wise min of two SparseTensors. + + Assumes the two SparseTensors have the same shape, i.e., no broadcasting. + Example: + + ```python + sp_zero = ops.SparseTensor([[0]], [0], [7]) + sp_one = ops.SparseTensor([[1]], [1], [7]) + res = tf.sparse_minimum(sp_zero, sp_one).eval() + # "res" should be equal to SparseTensor([[0], [1]], [0, 0], [7]). + ``` + + Args: + sp_a: a `SparseTensor` operand whose dtype is real, and indices + lexicographically ordered. + sp_b: the other `SparseTensor` operand with the same requirements (and the + same shape). + name: optional name of the operation. + Returns: + output: the output SparseTensor. + """ + with ops.op_scope([sp_a.indices, sp_a.values, sp_b.indices, sp_b.values], + name, "SparseSparseMinimum") as name: + out_indices, out_values = gen_sparse_ops.sparse_sparse_minimum(sp_a.indices, + sp_a.values, + sp_a.shape, + sp_b.indices, + sp_b.values, + sp_b.shape, + name=name) + return ops.SparseTensor(out_indices, out_values, sp_a.shape) + + +@ops.RegisterShape("SparseSparseMaximum") +@ops.RegisterShape("SparseSparseMinimum") +def _SparseSparseMaximumMinimumShape(op): # pylint: disable=invalid-name + """Shape function for SparseSparseMaximum and SparseSparseMinimum.""" + op.inputs[0].get_shape().assert_has_rank(2) # a_indices + op.inputs[1].get_shape().assert_has_rank(1) # a_values + op.inputs[2].get_shape().assert_has_rank(1) # a_shape + op.inputs[3].get_shape().assert_has_rank(2) # b_indices + op.inputs[4].get_shape().assert_has_rank(1) # b_values + op.inputs[5].get_shape().assert_has_rank(1) # b_shape + return [tensor_shape.unknown_shape(2), tensor_shape.unknown_shape(1)] From b5c493301acf018a7d3323025c8ad637c104e7a9 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 30 Jun 2016 14:18:14 -0800 Subject: [PATCH 16/36] Update ops-related pbtxt files. Change: 126349886 --- .../core/ops/compat/ops_history.v0.pbtxt | 109 +++++++++++++++ tensorflow/core/ops/ops.pbtxt | 129 ++++++++++++++++++ 2 files changed, 238 insertions(+) diff --git a/tensorflow/core/ops/compat/ops_history.v0.pbtxt b/tensorflow/core/ops/compat/ops_history.v0.pbtxt index 2dba61efe78..2dc59179b82 100644 --- a/tensorflow/core/ops/compat/ops_history.v0.pbtxt +++ b/tensorflow/core/ops/compat/ops_history.v0.pbtxt @@ -23173,6 +23173,115 @@ op { } } } +op { + name: "SparseSparseMaximum" + input_arg { + name: "a_indices" + type: DT_INT64 + } + input_arg { + name: "a_values" + type_attr: "T" + } + input_arg { + name: "a_shape" + type: DT_INT64 + } + input_arg { + name: "b_indices" + type: DT_INT64 + } + input_arg { + name: "b_values" + type_attr: "T" + } + input_arg { + name: "b_shape" + type: DT_INT64 + } + output_arg { + name: "output_indices" + type: DT_INT64 + } + output_arg { + name: "output_values" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_FLOAT + type: DT_DOUBLE + type: DT_INT32 + type: DT_INT64 + type: DT_UINT8 + type: DT_INT16 + type: DT_INT8 + type: DT_UINT16 + type: DT_HALF + } + } + } +} +op { + name: "SparseSparseMinimum" + input_arg { + name: "a_indices" + type: DT_INT64 + } + input_arg { + name: "a_values" + type_attr: "T" + } + input_arg { + name: "a_shape" + type: DT_INT64 + } + input_arg { + name: "b_indices" + type: DT_INT64 + } + input_arg { + name: "b_values" + type_attr: "T" + } + input_arg { + name: "b_shape" + type: DT_INT64 + } + output_arg { + name: "output_indices" + type: DT_INT64 + } + output_arg { + name: "output_values" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_FLOAT + type: DT_DOUBLE + type: DT_INT64 + type: DT_INT32 + type: DT_UINT8 + type: DT_UINT16 + type: DT_INT16 + type: DT_INT8 + type: DT_COMPLEX64 + type: DT_COMPLEX128 + type: DT_QINT8 + type: DT_QUINT8 + type: DT_QINT32 + type: DT_HALF + } + } + } +} op { name: "SparseSplit" input_arg { diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt index afd6507b0d8..3668ecf5aae 100644 --- a/tensorflow/core/ops/ops.pbtxt +++ b/tensorflow/core/ops/ops.pbtxt @@ -13696,6 +13696,135 @@ op { summary: "Computes softmax cross entropy cost and gradients to backpropagate." description: "Unlike `SoftmaxCrossEntropyWithLogits`, this operation does not accept\na matrix of label probabilities, but rather a single label per row\nof features. This label is considered to have probability 1.0 for the\ngiven row.\n\nInputs are the logits, not probabilities." } +op { + name: "SparseSparseMaximum" + input_arg { + name: "a_indices" + description: "2-D. `N x R` matrix with the indices of non-empty values in a\nSparseTensor, in the canonical lexicographic ordering." + type: DT_INT64 + } + input_arg { + name: "a_values" + description: "1-D. `N` non-empty values corresponding to `a_indices`." + type_attr: "T" + } + input_arg { + name: "a_shape" + description: "1-D. Shape of the input SparseTensor." + type: DT_INT64 + } + input_arg { + name: "b_indices" + description: "counterpart to `a_indices` for the other operand." + type: DT_INT64 + } + input_arg { + name: "b_values" + description: "counterpart to `a_values` for the other operand; must be of the same dtype." + type_attr: "T" + } + input_arg { + name: "b_shape" + description: "counterpart to `a_shape` for the other operand; the two shapes must be equal." + type: DT_INT64 + } + output_arg { + name: "output_indices" + description: "2-D. The indices of the output SparseTensor." + type: DT_INT64 + } + output_arg { + name: "output_values" + description: "1-D. The values of the output SparseTensor." + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_FLOAT + type: DT_DOUBLE + type: DT_INT32 + type: DT_INT64 + type: DT_UINT8 + type: DT_INT16 + type: DT_INT8 + type: DT_UINT16 + type: DT_HALF + } + } + } + summary: "Returns the element-wise max of two SparseTensors." + description: "Assumes the two SparseTensors have the same shape, i.e., no broadcasting." +} +op { + name: "SparseSparseMinimum" + input_arg { + name: "a_indices" + description: "2-D. `N x R` matrix with the indices of non-empty values in a\nSparseTensor, in the canonical lexicographic ordering." + type: DT_INT64 + } + input_arg { + name: "a_values" + description: "1-D. `N` non-empty values corresponding to `a_indices`." + type_attr: "T" + } + input_arg { + name: "a_shape" + description: "1-D. Shape of the input SparseTensor." + type: DT_INT64 + } + input_arg { + name: "b_indices" + description: "counterpart to `a_indices` for the other operand." + type: DT_INT64 + } + input_arg { + name: "b_values" + description: "counterpart to `a_values` for the other operand; must be of the same dtype." + type_attr: "T" + } + input_arg { + name: "b_shape" + description: "counterpart to `a_shape` for the other operand; the two shapes must be equal." + type: DT_INT64 + } + output_arg { + name: "output_indices" + description: "2-D. The indices of the output SparseTensor." + type: DT_INT64 + } + output_arg { + name: "output_values" + description: "1-D. The values of the output SparseTensor." + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_FLOAT + type: DT_DOUBLE + type: DT_INT64 + type: DT_INT32 + type: DT_UINT8 + type: DT_UINT16 + type: DT_INT16 + type: DT_INT8 + type: DT_COMPLEX64 + type: DT_COMPLEX128 + type: DT_QINT8 + type: DT_QUINT8 + type: DT_QINT32 + type: DT_HALF + } + } + } + summary: "Returns the element-wise min of two SparseTensors." + description: "Assumes the two SparseTensors have the same shape, i.e., no broadcasting." +} op { name: "SparseSplit" input_arg { From 605aa53d2bb65a8a38dc72725e28ebe75a949d5a Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 30 Jun 2016 14:21:08 -0800 Subject: [PATCH 17/36] Add C++ shape inference for Pack, Unpack, and Const. Add GetAttr to shape_inference::InferenceContext. Allow setting NodeDef in shape_inference_testutil INFER calls (with new INFER*_WITH_DEF macro). Fix a bug that caused a crash when an INFER..ERROR macro called a shape inference function that did not return an error. Change: 126350221 --- tensorflow/core/BUILD | 7 +- tensorflow/core/framework/shape_inference.cc | 6 +- tensorflow/core/framework/shape_inference.h | 23 ++- .../core/framework/shape_inference_test.cc | 61 ++++++-- .../framework/shape_inference_testutil.cc | 9 +- .../core/framework/shape_inference_testutil.h | 11 +- tensorflow/core/ops/array_ops.cc | 85 +++++++++++ tensorflow/core/ops/array_ops_test.cc | 137 ++++++++++++++++++ 8 files changed, 315 insertions(+), 24 deletions(-) create mode 100644 tensorflow/core/ops/array_ops_test.cc diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index b684522eb6d..b2a928867f4 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -1812,10 +1812,13 @@ tf_cc_test( ], ) -tf_cc_test( - name = "ops/math_ops_test", +tf_cc_tests( size = "small", linkstatic = tf_kernel_tests_linkstatic(), + tests = [ + "ops/array_ops_test.cc", + "ops/math_ops_test.cc", + ], deps = [ ":core", ":core_cpu", diff --git a/tensorflow/core/framework/shape_inference.cc b/tensorflow/core/framework/shape_inference.cc index 2df57d6cab3..bd8e6ea3094 100644 --- a/tensorflow/core/framework/shape_inference.cc +++ b/tensorflow/core/framework/shape_inference.cc @@ -25,9 +25,9 @@ constexpr int32 InferenceContext::kUnknownRank; constexpr int64 InferenceContext::kUnknownDim; InferenceContext::InferenceContext( - const std::vector& input_shapes, int num_outputs, - const std::vector& input_tensors) - : input_tensors_(input_tensors) { + const NodeDef* node_def, const std::vector& input_shapes, + int num_outputs, const std::vector& input_tensors) + : input_tensors_(input_tensors), node_def_(*CHECK_NOTNULL(node_def)) { for (const string& spec : input_shapes) { if (spec == "?") { inputs_.push_back(CreateUnknownShape()); diff --git a/tensorflow/core/framework/shape_inference.h b/tensorflow/core/framework/shape_inference.h index bb6a66dc533..6385177bc19 100644 --- a/tensorflow/core/framework/shape_inference.h +++ b/tensorflow/core/framework/shape_inference.h @@ -17,6 +17,8 @@ limitations under the License. #include +#include "tensorflow/core/framework/graph.pb.h" +#include "tensorflow/core/framework/node_def_util.h" #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/gtl/inlined_vector.h" @@ -80,7 +82,10 @@ class InferenceContext { // the same Dimension*. // // is NULL-padded to be the same size as . - InferenceContext(const std::vector& input_shapes, int num_outputs, + // + // REQUIRES: is not NULL, and must outlive the InferenceContext. + InferenceContext(const NodeDef* node_def, + const std::vector& input_shapes, int num_outputs, const std::vector& input_tensors = {}); ~InferenceContext(); @@ -162,6 +167,12 @@ class InferenceContext { const Dimension* CreateDim(int64 value); const Dimension* CreateUnknownDim(); + // Look up the attr for the NodeDef being evaluated with name attr_name and + // set *value to its value. If no attr with attr_name is found in def(), or + // the attr does not have a matching type, a non-ok status will be returned. + template + Status GetAttr(StringPiece attr_name, T* value) const; + private: Status ReturnUnknownShape(const Shape** out) { *out = CreateUnknownShape(); @@ -181,9 +192,14 @@ class InferenceContext { std::vector input_tensors_; std::vector outputs_; + const NodeDef& node_def_; + TF_DISALLOW_COPY_AND_ASSIGN(InferenceContext); }; +// ----------------------------------------------------------------------------- +// Template and inline method implementations, please ignore + inline Dimension::Dimension() : value_(InferenceContext::kUnknownDim) {} inline Dimension::Dimension(int64 value) : value_(value) {} @@ -191,6 +207,11 @@ inline Shape::Shape() : rank_(InferenceContext::kUnknownRank) {} inline Shape::Shape(const std::vector dims) : rank_(dims.size()), dims_(dims) {} +template +Status InferenceContext::GetAttr(StringPiece attr_name, T* value) const { + return GetNodeAttr(node_def_, attr_name, value); +} + } // namespace shape_inference } // namespace tensorflow diff --git a/tensorflow/core/framework/shape_inference_test.cc b/tensorflow/core/framework/shape_inference_test.cc index e4ca7645b2e..e52d1c5a2d6 100644 --- a/tensorflow/core/framework/shape_inference_test.cc +++ b/tensorflow/core/framework/shape_inference_test.cc @@ -14,6 +14,8 @@ limitations under the License. ==============================================================================*/ #include "tensorflow/core/framework/shape_inference.h" +#include "tensorflow/core/framework/node_def_builder.h" +#include "tensorflow/core/framework/op_def_builder.h" #include "tensorflow/core/framework/tensor_testutil.h" #include "tensorflow/core/platform/test.h" @@ -21,7 +23,8 @@ namespace tensorflow { namespace shape_inference { TEST(ShapeInferenceTest, RankAndDimInspection) { - InferenceContext c({"?", "[1,?,3]", "[]"}, 2 /* num_outputs */); + NodeDef def; + InferenceContext c(&def, {"?", "[1,?,3]", "[]"}, 2 /* num_outputs */); EXPECT_EQ(3, c.num_inputs()); EXPECT_EQ(2, c.num_outputs()); @@ -54,7 +57,8 @@ TEST(ShapeInferenceTest, RankAndDimInspection) { } TEST(ShapeInferenceTest, WithRank) { - InferenceContext c({"?", "[1,?,3]"}, 2 /* num_outputs */); + NodeDef def; + InferenceContext c(&def, {"?", "[1,?,3]"}, 2 /* num_outputs */); auto in0 = c.input(0); auto in1 = c.input(1); @@ -91,7 +95,8 @@ TEST(ShapeInferenceTest, WithRank) { } TEST(ShapeInferenceTest, WithRankAtLeast) { - InferenceContext c({"?", "[1,?,3]"}, 2 /* num_outputs */); + NodeDef def; + InferenceContext c(&def, {"?", "[1,?,3]"}, 2 /* num_outputs */); auto in0 = c.input(0); auto in1 = c.input(1); @@ -125,7 +130,8 @@ TEST(ShapeInferenceTest, WithRankAtLeast) { } TEST(ShapeInferenceTest, WithValue) { - InferenceContext c({"[1,?]"}, 2 /* num_outputs */); + NodeDef def; + InferenceContext c(&def, {"[1,?]"}, 2 /* num_outputs */); auto d0 = c.Dim(c.input(0), 0); auto d1 = c.Dim(c.input(0), 1); @@ -163,7 +169,8 @@ TEST(ShapeInferenceTest, WithValue) { } TEST(ShapeInferenceTest, MergeDim) { - InferenceContext c({"[2,?,2,1,?]"}, 2 /* num_outputs */); + NodeDef def; + InferenceContext c(&def, {"[2,?,2,1,?]"}, 2 /* num_outputs */); auto d2 = c.Dim(c.input(0), 0); auto d_unknown = c.Dim(c.input(0), 1); @@ -202,7 +209,9 @@ TEST(ShapeInferenceTest, MergeDim) { } TEST(ShapeInferenceTest, MergeShape) { - InferenceContext c({"?", "[1,2]", "[?,2]", "[1,?]", "[1,3]", "?", "[1]"}, + NodeDef def; + InferenceContext c(&def, + {"?", "[1,2]", "[?,2]", "[1,?]", "[1,3]", "?", "[1]"}, 2 /* num_outputs */); auto s_unknown = c.input(0); @@ -260,7 +269,8 @@ TEST(ShapeInferenceTest, MergeShape) { } TEST(ShapeInferenceTest, Subshape) { - InferenceContext c({"[1,2,3,?,5]", "?"}, 2 /* num_outputs */); + NodeDef def; + InferenceContext c(&def, {"[1,2,3,?,5]", "?"}, 2 /* num_outputs */); const Shape* unknown = c.input(1); const Shape* out; @@ -297,7 +307,8 @@ TEST(ShapeInferenceTest, Subshape) { } TEST(ShapeInferenceTest, Concatenate) { - InferenceContext c({"[1,?,3]", "[4,5]", "?"}, 2 /* num_outputs */); + NodeDef def; + InferenceContext c(&def, {"[1,?,3]", "[4,5]", "?"}, 2 /* num_outputs */); auto in0 = c.input(0); auto in1 = c.input(1); @@ -322,7 +333,8 @@ TEST(ShapeInferenceTest, Concatenate) { } TEST(ShapeInferenceTest, CreateShape) { - InferenceContext c({"[1,2,3,?,5]"}, 2 /* num_outputs */); + NodeDef def; + InferenceContext c(&def, {"[1,2,3,?,5]"}, 2 /* num_outputs */); std::vector dims; auto in0 = c.input(0); @@ -341,7 +353,8 @@ TEST(ShapeInferenceTest, CreateShape) { } TEST(ShapeInferenceTest, CreateUnknownShape) { - InferenceContext c({}, 2 /* num_outputs */); + NodeDef def; + InferenceContext c(&def, {}, 2 /* num_outputs */); auto u0 = c.CreateUnknownShape(); auto u1 = c.CreateUnknownShape(); @@ -352,7 +365,8 @@ TEST(ShapeInferenceTest, CreateUnknownShape) { TEST(ShapeInferenceTest, CreateShapeFromShapeTensor) { auto create = [](Tensor* t) { - InferenceContext c({"?"}, 0 /* num_outputs */, {t}); + NodeDef def; + InferenceContext c(&def, {"?"}, 0 /* num_outputs */, {t}); const Shape* out; Status s = c.CreateShapeFromShapeTensor(0, &out); if (s.ok()) { @@ -386,7 +400,8 @@ TEST(ShapeInferenceTest, CreateShapeFromShapeTensor) { } TEST(ShapeInferenceTest, CreateDim) { - InferenceContext c({}, 2 /* num_outputs */); + NodeDef def; + InferenceContext c(&def, {}, 2 /* num_outputs */); auto* d0 = c.CreateDim(1); auto* d1 = c.CreateDim(1); @@ -398,7 +413,8 @@ TEST(ShapeInferenceTest, CreateDim) { } TEST(ShapeInferenceTest, CreateUnknownDim) { - InferenceContext c({}, 2 /* num_outputs */); + NodeDef def; + InferenceContext c(&def, {}, 2 /* num_outputs */); auto* d0 = c.CreateUnknownDim(); auto* d1 = c.CreateUnknownDim(); @@ -410,12 +426,29 @@ TEST(ShapeInferenceTest, CreateUnknownDim) { TEST(ShapeInferenceTest, InputTensors) { const Tensor t1 = tensorflow::test::AsTensor({10}); const Tensor t2 = tensorflow::test::AsTensor({20, 30}); - InferenceContext c({"[1]", "[2]", "[3]"}, 2 /* num_outputs */, {&t1, &t2}); + NodeDef def; + InferenceContext c(&def, {"[1]", "[2]", "[3]"}, 2 /* num_outputs */, + {&t1, &t2}); EXPECT_TRUE(c.input_tensor(0) == &t1); EXPECT_TRUE(c.input_tensor(1) == &t2); EXPECT_TRUE(c.input_tensor(2) == nullptr); } +TEST(ShapeInferenceTest, GetAttr) { + OpRegistrationData op_reg_data; + CHECK(OpDefBuilder("dummy").Attr("foo:string").Finalize(&op_reg_data).ok()); + NodeDef def; + CHECK(NodeDefBuilder("dummy", &op_reg_data.op_def) + .Attr("foo", "bar") + .Finalize(&def) + .ok()); + + InferenceContext c(&def, {}, 2 /* num_outputs */); + string value; + EXPECT_TRUE(c.GetAttr("foo", &value).ok()); + EXPECT_EQ("bar", value); +} + } // namespace shape_inference } // namespace tensorflow diff --git a/tensorflow/core/framework/shape_inference_testutil.cc b/tensorflow/core/framework/shape_inference_testutil.cc index 9b56014edbe..f771e477644 100644 --- a/tensorflow/core/framework/shape_inference_testutil.cc +++ b/tensorflow/core/framework/shape_inference_testutil.cc @@ -29,13 +29,18 @@ using shape_inference::Shape; using errors::Unknown; Status InferShapes(const string& op_name, const string& ins, - const string& expected_outs) { + const string& expected_outs, const NodeDef* node_def) { const OpRegistrationData* op_reg_data; TF_RETURN_IF_ERROR(OpRegistry::Global()->LookUp(op_name, &op_reg_data)); const int num_outputs = op_reg_data->op_def.output_arg_size(); std::vector ins_v = str_util::Split(ins, ';'); - shape_inference::InferenceContext c(ins_v, num_outputs); + std::unique_ptr new_node_def; + if (node_def == nullptr) { + new_node_def.reset(new NodeDef); + node_def = new_node_def.get(); + } + shape_inference::InferenceContext c(node_def, ins_v, num_outputs); TF_RETURN_IF_ERROR(op_reg_data->shape_inference_fn(&c)); std::unordered_map> diff --git a/tensorflow/core/framework/shape_inference_testutil.h b/tensorflow/core/framework/shape_inference_testutil.h index f2581247d9e..221ec875fb0 100644 --- a/tensorflow/core/framework/shape_inference_testutil.h +++ b/tensorflow/core/framework/shape_inference_testutil.h @@ -23,6 +23,8 @@ limitations under the License. namespace tensorflow { +class NodeDef; + // Run shape inference for , given inputs specified by // and returns an error if the inferred shape does not match expected_outs. // @@ -45,11 +47,16 @@ namespace tensorflow { // can be "e"; this is used to indicate that shape inference // should have failed. Status InferShapes(const string& op_name, const string& ins, - const string& expected_outs); + const string& expected_outs, + const NodeDef* node_def = nullptr); #define INFER_OK(op, i, o) EXPECT_EQ("", InferShapes(op, i, o).error_message()) #define INFER_ERROR(s, op, i) \ - EXPECT_EQ(s, InferShapes(op, i, "x").error_message()) + EXPECT_EQ(s, InferShapes(op, i, "e").error_message()) +#define INFER_OK_WITH_DEF(op, nd, i, o) \ + EXPECT_EQ("", InferShapes(op, i, o, nd).error_message()) +#define INFER_ERROR_WITH_DEF(s, op, nd, i) \ + EXPECT_EQ(s, InferShapes(op, i, "e", nd).error_message()) } // namespace tensorflow diff --git a/tensorflow/core/ops/array_ops.cc b/tensorflow/core/ops/array_ops.cc index dc96588f73a..4ef3a48221a 100644 --- a/tensorflow/core/ops/array_ops.cc +++ b/tensorflow/core/ops/array_ops.cc @@ -14,17 +14,67 @@ limitations under the License. ==============================================================================*/ #include "tensorflow/core/framework/op.h" +#include "tensorflow/core/framework/shape_inference.h" #include "tensorflow/core/util/mirror_pad_mode.h" #include "tensorflow/core/util/padding.h" namespace tensorflow { +typedef shape_inference::Dimension Dimension; +typedef shape_inference::InferenceContext InferenceContext; +typedef shape_inference::Shape Shape; + +namespace { + +Status GetAxisForPackAndUnpack(InferenceContext* c, int32 rank_after_pack, + int32* axis) { + TF_RETURN_IF_ERROR(c->GetAttr("axis", axis)); + if (*axis < -1 * rank_after_pack || *axis >= rank_after_pack) { + return errors::InvalidArgument("Invalid axis: ", *axis, "; must be in [", + -1 * rank_after_pack, ",", rank_after_pack, + ")"); + } + if (*axis < 0) *axis = (rank_after_pack + *axis); + return Status::OK(); +} + +} // namespace + REGISTER_OP("Pack") .Input("values: N * T") .Output("output: T") .Attr("N: int >= 1") .Attr("T: type") .Attr("axis: int = 0") + .SetShapeFn(OpShapeInferenceFn([](InferenceContext* c) { + // Validate shapes of all inputs are compatible + const Shape* cur = c->input(c->num_inputs() - 1); + for (int i = c->num_inputs() - 2; i >= 0; --i) { + TF_RETURN_WITH_CONTEXT_IF_ERROR(c->Merge(c->input(i), cur, &cur), + "From merging shape ", i, + " with other shapes."); + } + if (!c->RankKnown(cur)) { + c->set_output(0, c->CreateUnknownShape()); + return Status::OK(); + } + // Determine the axis that will be added, converting from negative + // axes to a positive point per negative indexing rules. + int32 rank = c->Rank(cur); + int32 axis; + TF_RETURN_IF_ERROR(GetAxisForPackAndUnpack(c, rank + 1, &axis)); + + // Copy all dimensions over, inserting a dimension of value #inputs + // at . + std::vector dims; + int index = 0; + while (index < axis) dims.push_back(c->Dim(cur, index++)); + dims.push_back(c->CreateDim(c->num_inputs())); + while (index < rank) dims.push_back(c->Dim(cur, index++)); + + c->set_output(0, c->CreateShape(dims)); + return Status::OK(); + })) .Doc(R"doc( Packs a list of `N` rank-`R` tensors into one rank-`(R+1)` tensor. @@ -61,6 +111,29 @@ REGISTER_OP("Unpack") .Attr("num: int >= 0") .Attr("T: type") .Attr("axis: int = 0") + .SetShapeFn(OpShapeInferenceFn([](InferenceContext* c) { + const Shape* s = c->input(0); + const Shape* out; + if (c->RankKnown(s)) { + // Determine the axis that will be removed, converting from negative + // axes to a positive point per negative indexing rules. + int32 rank = c->Rank(s); + int32 axis; + TF_RETURN_IF_ERROR(GetAxisForPackAndUnpack(c, rank, &axis)); + + // Copy all dimensions, removing the dimension. + std::vector dims; + for (int i = 0; i < rank; ++i) { + if (i != axis) dims.push_back(c->Dim(s, i)); + } + out = c->CreateShape(dims); + } else { + // All outputs are the same shape, but it's not known. + out = c->CreateUnknownShape(); + } + for (int i = 0; i < c->num_outputs(); ++i) c->set_output(i, out); + return Status::OK(); + })) .Doc(R"doc( Unpacks a given dimension of a rank-`R` tensor into `num` rank-`(R-1)` tensors. @@ -154,6 +227,18 @@ REGISTER_OP("Const") .Output("output: dtype") .Attr("value: tensor") .Attr("dtype: type") + .SetShapeFn(OpShapeInferenceFn([](InferenceContext* c) { + const TensorProto* proto = nullptr; + TF_RETURN_IF_ERROR(c->GetAttr("value", &proto)); + TF_RETURN_IF_ERROR(TensorShape::IsValidShape(proto->tensor_shape())); + TensorShape shape(proto->tensor_shape()); + std::vector dims; + for (int i = 0; i < shape.dims(); ++i) { + dims.push_back(c->CreateDim(shape.dim_size(i))); + } + c->set_output(0, c->CreateShape(dims)); + return Status::OK(); + })) .Doc(R"doc( Returns a constant tensor. diff --git a/tensorflow/core/ops/array_ops_test.cc b/tensorflow/core/ops/array_ops_test.cc new file mode 100644 index 00000000000..19dfa293584 --- /dev/null +++ b/tensorflow/core/ops/array_ops_test.cc @@ -0,0 +1,137 @@ +/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); + +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/framework/graph.pb.h" +#include "tensorflow/core/framework/node_def_builder.h" +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/framework/shape_inference_testutil.h" +#include "tensorflow/core/platform/test.h" + +namespace tensorflow { + +TEST(ArrayOpsTest, Pack_ShapeFn) { + std::unique_ptr def_storage(new NodeDef); + NodeDef* def = def_storage.get(); + auto set_axis = [def](int axis) { + TF_CHECK_OK(NodeDefBuilder("test", "Pack") + .Input({{"a", 0, DT_FLOAT}}) + .Attr("axis", axis) + .Finalize(def)); + }; + const char op[] = "Pack"; + + set_axis(0); + INFER_OK_WITH_DEF(op, def, "?;?;?", "?"); + + for (int axis : {0, -3}) { + set_axis(axis); + INFER_OK_WITH_DEF(op, def, "?;?", "?"); + INFER_OK_WITH_DEF(op, def, "[1,3];[1,3];?", "[3,d0_0|d1_0,d0_1|d1_1]"); + INFER_OK_WITH_DEF(op, def, "[?,3];[1,3];?", "[3,d1_0,d0_1|d1_1]"); + INFER_OK_WITH_DEF(op, def, "[?,?];[1,3];?", "[3,d1_0,d1_1]"); + } + for (int axis : {1, -2}) { + set_axis(axis); + INFER_OK_WITH_DEF(op, def, "?;?", "?"); + INFER_OK_WITH_DEF(op, def, "[1,3];[1,3];?", "[d0_0|d1_0,3,d0_1|d1_1]"); + INFER_OK_WITH_DEF(op, def, "[?,3];[1,3];?", "[d1_0,3,d0_1|d1_1]"); + INFER_OK_WITH_DEF(op, def, "[?,?];[1,3];?", "[d1_0,3,d1_1]"); + } + for (int axis : {2, -1}) { + set_axis(axis); + INFER_OK_WITH_DEF(op, def, "?;?", "?"); + INFER_OK_WITH_DEF(op, def, "[1,3];[1,3];?", "[d0_0|d1_0,d0_1|d1_1,3]"); + INFER_OK_WITH_DEF(op, def, "[?,3];[1,3];?", "[d1_0,d0_1|d1_1,3]"); + INFER_OK_WITH_DEF(op, def, "[?,?];[1,3];?", "[d1_0,d1_1,3]"); + } + + set_axis(-4); + INFER_ERROR_WITH_DEF("Invalid axis: -4; must be in [-3,3)", op, def, + "[1,3];[1,3];?"); + set_axis(3); + INFER_ERROR_WITH_DEF("Invalid axis: 3; must be in [-3,3)", op, def, + "[1,3];[1,3];?"); + + set_axis(0); + INFER_ERROR_WITH_DEF(("Shapes must be equal rank, but are 3 and 2" + "\n\tFrom merging shape 0 with other shapes."), + op, def, "[1,2,3];?;[1,4]"); +} + +TEST(ArrayOpsTest, UnPack_ShapeFn) { + std::unique_ptr def_storage(new NodeDef); + NodeDef* def = def_storage.get(); + auto set_axis = [def](int axis) { + TF_CHECK_OK(NodeDefBuilder("test", "Unpack") + .Input("a", 0, DT_FLOAT) + .Attr("axis", axis) + .Finalize(def)); + }; + const char op[] = "Unpack"; + + set_axis(0); + INFER_OK_WITH_DEF(op, def, "?;?;?", "?"); + + for (int axis : {0, -3}) { + set_axis(axis); + INFER_OK_WITH_DEF(op, def, "?", "?"); + INFER_OK_WITH_DEF(op, def, "[1,2,3]", "[d0_1,d0_2]"); + INFER_OK_WITH_DEF(op, def, "[?,?,?]", "[d0_1,d0_2]"); + } + for (int axis : {1, -2}) { + set_axis(axis); + INFER_OK_WITH_DEF(op, def, "[1,2,3]", "[d0_0,d0_2]"); + INFER_OK_WITH_DEF(op, def, "[?,?,?]", "[d0_0,d0_2]"); + } + for (int axis : {2, -1}) { + set_axis(axis); + INFER_OK_WITH_DEF(op, def, "[1,2,3]", "[d0_0,d0_1]"); + INFER_OK_WITH_DEF(op, def, "[?,?,?]", "[d0_0,d0_1]"); + } + + set_axis(-4); + INFER_ERROR_WITH_DEF("Invalid axis: -4; must be in [-3,3)", op, def, + "[1,2,3]"); + set_axis(3); + INFER_ERROR_WITH_DEF("Invalid axis: 3; must be in [-3,3)", op, def, + "[1,2,3]"); +} + +TEST(ArrayOpsTest, Const_ShapeFn) { + std::unique_ptr def_storage(new NodeDef); + NodeDef* def = def_storage.get(); + TensorProto tensor_proto; + auto* shape_proto = tensor_proto.mutable_tensor_shape(); + auto rebuild_node_def = [def, &tensor_proto]() { + TF_CHECK_OK(NodeDefBuilder("test", "Const") + .Attr("value", tensor_proto) + .Finalize(def)); + }; + const char op[] = "Const"; + + TensorShape{}.AsProto(shape_proto); + rebuild_node_def(); + INFER_OK_WITH_DEF(op, def, "", "[]"); + TensorShape{1, 2, 3, 4}.AsProto(shape_proto); + rebuild_node_def(); + INFER_OK_WITH_DEF(op, def, "", "[1,2,3,4]"); + + shape_proto->add_dim()->set_size(-1); + rebuild_node_def(); + INFER_ERROR_WITH_DEF("Shape [1,2,3,4,-1] has negative dimensions", op, def, + ""); +} + +} // end namespace tensorflow From d3067c338425bdf97fa782d834399b89bce18309 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 30 Jun 2016 14:25:47 -0800 Subject: [PATCH 18/36] Update generated Python Op docs. Change: 126350719 --- .../api_docs/python/contrib.framework.md | 71 +++++++++---------- ....framework.safe_embedding_lookup_sparse.md | 71 +++++++++---------- .../shard1/tf.sparse_minimum.md | 28 ++++++++ .../shard5/tf.sparse_maximum.md | 28 ++++++++ tensorflow/g3doc/api_docs/python/index.md | 2 + .../g3doc/api_docs/python/sparse_ops.md | 62 ++++++++++++++++ 6 files changed, 190 insertions(+), 72 deletions(-) create mode 100644 tensorflow/g3doc/api_docs/python/functions_and_classes/shard1/tf.sparse_minimum.md create mode 100644 tensorflow/g3doc/api_docs/python/functions_and_classes/shard5/tf.sparse_maximum.md diff --git a/tensorflow/g3doc/api_docs/python/contrib.framework.md b/tensorflow/g3doc/api_docs/python/contrib.framework.md index 4c234555975..df4df30d199 100644 --- a/tensorflow/g3doc/api_docs/python/contrib.framework.md +++ b/tensorflow/g3doc/api_docs/python/contrib.framework.md @@ -227,50 +227,49 @@ adds them via `tf.add_n`. - - - -### `tf.contrib.framework.safe_embedding_lookup_sparse(embedding_weights, sparse_ids, sparse_weights=None, combiner='mean', default_id=None, name=None, partition_strategy='div')` {#safe_embedding_lookup_sparse} +### `tf.contrib.framework.safe_embedding_lookup_sparse(*args, **kwargs)` {#safe_embedding_lookup_sparse} -Lookup embedding results, accounting for invalid IDs and empty features. +Lookup embedding results, accounting for invalid IDs and empty features. (deprecated) -The partitioned embedding in `embedding_weights` must all be the same shape -except for the first dimension. The first dimension is allowed to vary as the -vocabulary size is not necessarily a multiple of `P`. +THIS FUNCTION IS DEPRECATED. It will be removed after 2016-09-01. +Instructions for updating: +Please use tf.contrib.layers.safe_embedding_lookup_sparse. -Invalid IDs (< 0) are pruned from input IDs and weights, as well as any IDs -with non-positive weight. For an entry with no features, the embedding vector -for `default_id` is returned, or the 0-vector if `default_id` is not supplied. + The partitioned embedding in `embedding_weights` must all be the same shape + except for the first dimension. The first dimension is allowed to vary as the + vocabulary size is not necessarily a multiple of `P`. -The ids and weights may be multi-dimensional. Embeddings are always aggregated -along the last dimension. + Invalid IDs (< 0) are pruned from input IDs and weights, as well as any IDs + with non-positive weight. For an entry with no features, the embedding vector + for `default_id` is returned, or the 0-vector if `default_id` is not supplied. -##### Args: + The ids and weights may be multi-dimensional. Embeddings are always aggregated + along the last dimension. + + Args: + embedding_weights: A list of `P` float tensors or values representing + partitioned embedding tensors. The total unpartitioned shape should be + `[e_0, e_1, ..., e_m]`, where `e_0` represents the vocab size and + `e_1, ..., e_m` are the embedding dimensions. + sparse_ids: `SparseTensor` of shape `[d_0, d_1, ..., d_n]` containing the + ids. `d_0` is typically batch size. + sparse_weights: `SparseTensor` of same shape as `sparse_ids`, containing + float weights corresponding to `sparse_ids`, or `None` if all weights + are be assumed to be 1.0. + combiner: A string specifying how to combine embedding results for each + entry. Currently "mean", "sqrtn" and "sum" are supported, with "mean" + the default. + default_id: The id to use for an entry with no features. + name: A name for this operation (optional). + partition_strategy: A string specifying the partitioning strategy. + Currently `"div"` and `"mod"` are supported. Default is `"div"`. -* `embedding_weights`: A list of `P` float tensors or values representing - partitioned embedding tensors. The total unpartitioned shape should be - `[e_0, e_1, ..., e_m]`, where `e_0` represents the vocab size and - `e_1, ..., e_m` are the embedding dimensions. -* `sparse_ids`: `SparseTensor` of shape `[d_0, d_1, ..., d_n]` containing the - ids. `d_0` is typically batch size. -* `sparse_weights`: `SparseTensor` of same shape as `sparse_ids`, containing - float weights corresponding to `sparse_ids`, or `None` if all weights - are be assumed to be 1.0. -* `combiner`: A string specifying how to combine embedding results for each - entry. Currently "mean", "sqrtn" and "sum" are supported, with "mean" - the default. -* `default_id`: The id to use for an entry with no features. -* `name`: A name for this operation (optional). -* `partition_strategy`: A string specifying the partitioning strategy. - Currently `"div"` and `"mod"` are supported. Default is `"div"`. + Returns: + Dense tensor of shape `[d_0, d_1, ..., d_{n-1}, e_1, ..., e_m]`. - -##### Returns: - - Dense tensor of shape `[d_0, d_1, ..., d_{n-1}, e_1, ..., e_m]`. - -##### Raises: - - -* `ValueError`: if `embedding_weights` is empty. + Raises: + ValueError: if `embedding_weights` is empty. - - - diff --git a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard0/tf.contrib.framework.safe_embedding_lookup_sparse.md b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard0/tf.contrib.framework.safe_embedding_lookup_sparse.md index f56043cc0dc..3d5491c98aa 100644 --- a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard0/tf.contrib.framework.safe_embedding_lookup_sparse.md +++ b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard0/tf.contrib.framework.safe_embedding_lookup_sparse.md @@ -1,45 +1,44 @@ -### `tf.contrib.framework.safe_embedding_lookup_sparse(embedding_weights, sparse_ids, sparse_weights=None, combiner='mean', default_id=None, name=None, partition_strategy='div')` {#safe_embedding_lookup_sparse} +### `tf.contrib.framework.safe_embedding_lookup_sparse(*args, **kwargs)` {#safe_embedding_lookup_sparse} -Lookup embedding results, accounting for invalid IDs and empty features. +Lookup embedding results, accounting for invalid IDs and empty features. (deprecated) -The partitioned embedding in `embedding_weights` must all be the same shape -except for the first dimension. The first dimension is allowed to vary as the -vocabulary size is not necessarily a multiple of `P`. +THIS FUNCTION IS DEPRECATED. It will be removed after 2016-09-01. +Instructions for updating: +Please use tf.contrib.layers.safe_embedding_lookup_sparse. -Invalid IDs (< 0) are pruned from input IDs and weights, as well as any IDs -with non-positive weight. For an entry with no features, the embedding vector -for `default_id` is returned, or the 0-vector if `default_id` is not supplied. + The partitioned embedding in `embedding_weights` must all be the same shape + except for the first dimension. The first dimension is allowed to vary as the + vocabulary size is not necessarily a multiple of `P`. -The ids and weights may be multi-dimensional. Embeddings are always aggregated -along the last dimension. + Invalid IDs (< 0) are pruned from input IDs and weights, as well as any IDs + with non-positive weight. For an entry with no features, the embedding vector + for `default_id` is returned, or the 0-vector if `default_id` is not supplied. -##### Args: + The ids and weights may be multi-dimensional. Embeddings are always aggregated + along the last dimension. + + Args: + embedding_weights: A list of `P` float tensors or values representing + partitioned embedding tensors. The total unpartitioned shape should be + `[e_0, e_1, ..., e_m]`, where `e_0` represents the vocab size and + `e_1, ..., e_m` are the embedding dimensions. + sparse_ids: `SparseTensor` of shape `[d_0, d_1, ..., d_n]` containing the + ids. `d_0` is typically batch size. + sparse_weights: `SparseTensor` of same shape as `sparse_ids`, containing + float weights corresponding to `sparse_ids`, or `None` if all weights + are be assumed to be 1.0. + combiner: A string specifying how to combine embedding results for each + entry. Currently "mean", "sqrtn" and "sum" are supported, with "mean" + the default. + default_id: The id to use for an entry with no features. + name: A name for this operation (optional). + partition_strategy: A string specifying the partitioning strategy. + Currently `"div"` and `"mod"` are supported. Default is `"div"`. -* `embedding_weights`: A list of `P` float tensors or values representing - partitioned embedding tensors. The total unpartitioned shape should be - `[e_0, e_1, ..., e_m]`, where `e_0` represents the vocab size and - `e_1, ..., e_m` are the embedding dimensions. -* `sparse_ids`: `SparseTensor` of shape `[d_0, d_1, ..., d_n]` containing the - ids. `d_0` is typically batch size. -* `sparse_weights`: `SparseTensor` of same shape as `sparse_ids`, containing - float weights corresponding to `sparse_ids`, or `None` if all weights - are be assumed to be 1.0. -* `combiner`: A string specifying how to combine embedding results for each - entry. Currently "mean", "sqrtn" and "sum" are supported, with "mean" - the default. -* `default_id`: The id to use for an entry with no features. -* `name`: A name for this operation (optional). -* `partition_strategy`: A string specifying the partitioning strategy. - Currently `"div"` and `"mod"` are supported. Default is `"div"`. + Returns: + Dense tensor of shape `[d_0, d_1, ..., d_{n-1}, e_1, ..., e_m]`. - -##### Returns: - - Dense tensor of shape `[d_0, d_1, ..., d_{n-1}, e_1, ..., e_m]`. - -##### Raises: - - -* `ValueError`: if `embedding_weights` is empty. + Raises: + ValueError: if `embedding_weights` is empty. diff --git a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard1/tf.sparse_minimum.md b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard1/tf.sparse_minimum.md new file mode 100644 index 00000000000..4419f736b94 --- /dev/null +++ b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard1/tf.sparse_minimum.md @@ -0,0 +1,28 @@ +### `tf.sparse_minimum(sp_a, sp_b, name=None)` {#sparse_minimum} + +Returns the element-wise min of two SparseTensors. + +Assumes the two SparseTensors have the same shape, i.e., no broadcasting. +Example: + +```python +sp_zero = ops.SparseTensor([[0]], [0], [7]) +sp_one = ops.SparseTensor([[1]], [1], [7]) +res = tf.sparse_minimum(sp_zero, sp_one).eval() +# "res" should be equal to SparseTensor([[0], [1]], [0, 0], [7]). +``` + +##### Args: + + +* `sp_a`: a `SparseTensor` operand whose dtype is real, and indices + lexicographically ordered. +* `sp_b`: the other `SparseTensor` operand with the same requirements (and the + same shape). +* `name`: optional name of the operation. + +##### Returns: + + +* `output`: the output SparseTensor. + diff --git a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard5/tf.sparse_maximum.md b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard5/tf.sparse_maximum.md new file mode 100644 index 00000000000..b934c3b1cdb --- /dev/null +++ b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard5/tf.sparse_maximum.md @@ -0,0 +1,28 @@ +### `tf.sparse_maximum(sp_a, sp_b, name=None)` {#sparse_maximum} + +Returns the element-wise max of two SparseTensors. + +Assumes the two SparseTensors have the same shape, i.e., no broadcasting. +Example: + +```python +sp_zero = ops.SparseTensor([[0]], [0], [7]) +sp_one = ops.SparseTensor([[1]], [1], [7]) +res = tf.sparse_maximum(sp_zero, sp_one).eval() +# "res" should be equal to SparseTensor([[0], [1]], [0, 1], [7]). +``` + +##### Args: + + +* `sp_a`: a `SparseTensor` operand whose dtype is real, and indices + lexicographically ordered. +* `sp_b`: the other `SparseTensor` operand with the same requirements (and the + same shape). +* `name`: optional name of the operation. + +##### Returns: + + +* `output`: the output SparseTensor. + diff --git a/tensorflow/g3doc/api_docs/python/index.md b/tensorflow/g3doc/api_docs/python/index.md index 5c4f7107d19..e9af0e65751 100644 --- a/tensorflow/g3doc/api_docs/python/index.md +++ b/tensorflow/g3doc/api_docs/python/index.md @@ -367,7 +367,9 @@ * [`sparse_add`](../../api_docs/python/sparse_ops.md#sparse_add) * [`sparse_concat`](../../api_docs/python/sparse_ops.md#sparse_concat) * [`sparse_fill_empty_rows`](../../api_docs/python/sparse_ops.md#sparse_fill_empty_rows) + * [`sparse_maximum`](../../api_docs/python/sparse_ops.md#sparse_maximum) * [`sparse_merge`](../../api_docs/python/sparse_ops.md#sparse_merge) + * [`sparse_minimum`](../../api_docs/python/sparse_ops.md#sparse_minimum) * [`sparse_reduce_sum`](../../api_docs/python/sparse_ops.md#sparse_reduce_sum) * [`sparse_reorder`](../../api_docs/python/sparse_ops.md#sparse_reorder) * [`sparse_reset_shape`](../../api_docs/python/sparse_ops.md#sparse_reset_shape) diff --git a/tensorflow/g3doc/api_docs/python/sparse_ops.md b/tensorflow/g3doc/api_docs/python/sparse_ops.md index 6781eadef41..1665420b5bb 100644 --- a/tensorflow/g3doc/api_docs/python/sparse_ops.md +++ b/tensorflow/g3doc/api_docs/python/sparse_ops.md @@ -1155,3 +1155,65 @@ B dense [k, n] return A*B +- - - + +### `tf.sparse_maximum(sp_a, sp_b, name=None)` {#sparse_maximum} + +Returns the element-wise max of two SparseTensors. + +Assumes the two SparseTensors have the same shape, i.e., no broadcasting. +Example: + +```python +sp_zero = ops.SparseTensor([[0]], [0], [7]) +sp_one = ops.SparseTensor([[1]], [1], [7]) +res = tf.sparse_maximum(sp_zero, sp_one).eval() +# "res" should be equal to SparseTensor([[0], [1]], [0, 1], [7]). +``` + +##### Args: + + +* `sp_a`: a `SparseTensor` operand whose dtype is real, and indices + lexicographically ordered. +* `sp_b`: the other `SparseTensor` operand with the same requirements (and the + same shape). +* `name`: optional name of the operation. + +##### Returns: + + +* `output`: the output SparseTensor. + + +- - - + +### `tf.sparse_minimum(sp_a, sp_b, name=None)` {#sparse_minimum} + +Returns the element-wise min of two SparseTensors. + +Assumes the two SparseTensors have the same shape, i.e., no broadcasting. +Example: + +```python +sp_zero = ops.SparseTensor([[0]], [0], [7]) +sp_one = ops.SparseTensor([[1]], [1], [7]) +res = tf.sparse_minimum(sp_zero, sp_one).eval() +# "res" should be equal to SparseTensor([[0], [1]], [0, 0], [7]). +``` + +##### Args: + + +* `sp_a`: a `SparseTensor` operand whose dtype is real, and indices + lexicographically ordered. +* `sp_b`: the other `SparseTensor` operand with the same requirements (and the + same shape). +* `name`: optional name of the operation. + +##### Returns: + + +* `output`: the output SparseTensor. + + From b70103502b41df370906e8988b6593e55caf69cf Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 30 Jun 2016 14:40:02 -0800 Subject: [PATCH 19/36] Improvements to tensor_forest, including support for sparse and categorical inputs. Add tf.learn.Estimator for random forests. Change: 126352221 --- tensorflow/contrib/learn/BUILD | 13 + .../learn/python/learn/estimators/__init__.py | 1 + .../python/learn/estimators/random_forest.py | 191 +++++++ .../learn/estimators/random_forest_test.py | 55 ++ tensorflow/contrib/tensor_forest/BUILD | 51 ++ tensorflow/contrib/tensor_forest/__init__.py | 2 + .../contrib/tensor_forest/client/__init__.py | 21 + .../tensor_forest/client/eval_metrics.py | 69 +++ .../ops/count_extremely_random_stats_op.cc | 251 ++++++--- .../core/ops/finished_nodes_op.cc | 132 ++++- .../core/ops/sample_inputs_op.cc | 114 +++- .../core/ops/tree_predictions_op.cc | 94 +++- .../tensor_forest/core/ops/tree_utils.cc | 272 ++++++++-- .../tensor_forest/core/ops/tree_utils.h | 93 +++- .../core/ops/update_fertile_slots_op.cc | 101 ++-- .../contrib/tensor_forest/data/__init__.py | 21 + .../contrib/tensor_forest/data/data_ops.py | 109 ++++ .../tensor_forest/data/string_to_float_op.cc | 111 ++++ .../contrib/tensor_forest/python/__init__.py | 1 + .../contrib/tensor_forest/python/constants.py | 26 + .../kernel_tests/best_splits_op_test.py | 18 +- .../count_extremely_random_stats_op_test.py | 101 +++- .../kernel_tests/finished_nodes_op_test.py | 56 +- .../kernel_tests/sample_inputs_op_test.py | 34 +- .../kernel_tests/tree_predictions_op_test.py | 70 ++- .../update_fertile_slots_op_test.py | 29 +- .../tensor_forest/python/ops/inference_ops.py | 24 +- .../tensor_forest/python/ops/training_ops.py | 28 +- .../tensor_forest/python/tensor_forest.py | 506 +++++++++++------- .../python/tensor_forest_test.py | 41 ++ tensorflow/tools/pip_package/BUILD | 1 + 31 files changed, 2115 insertions(+), 521 deletions(-) create mode 100644 tensorflow/contrib/learn/python/learn/estimators/random_forest.py create mode 100644 tensorflow/contrib/learn/python/learn/estimators/random_forest_test.py create mode 100644 tensorflow/contrib/tensor_forest/client/__init__.py create mode 100644 tensorflow/contrib/tensor_forest/client/eval_metrics.py create mode 100644 tensorflow/contrib/tensor_forest/data/__init__.py create mode 100644 tensorflow/contrib/tensor_forest/data/data_ops.py create mode 100644 tensorflow/contrib/tensor_forest/data/string_to_float_op.cc create mode 100644 tensorflow/contrib/tensor_forest/python/constants.py diff --git a/tensorflow/contrib/learn/BUILD b/tensorflow/contrib/learn/BUILD index ff834723f44..5db69a8b5d0 100644 --- a/tensorflow/contrib/learn/BUILD +++ b/tensorflow/contrib/learn/BUILD @@ -19,6 +19,7 @@ py_library( deps = [ "//tensorflow/contrib/learn/python/learn/datasets", "//tensorflow/contrib/session_bundle:exporter", + "//tensorflow/contrib/tensor_forest:client_lib", "//tensorflow/python:framework", ], ) @@ -390,6 +391,18 @@ py_test( ], ) +py_test( + name = "random_forest_test", + size = "medium", + srcs = ["python/learn/estimators/random_forest_test.py"], + srcs_version = "PY2AND3", + deps = [ + ":learn", + "//tensorflow:tensorflow_py", + "//tensorflow/python:framework_test_lib", + ], +) + py_test( name = "rnn_test", size = "medium", diff --git a/tensorflow/contrib/learn/python/learn/estimators/__init__.py b/tensorflow/contrib/learn/python/learn/estimators/__init__.py index 12a8b7cfe51..b6e6e57ebe4 100644 --- a/tensorflow/contrib/learn/python/learn/estimators/__init__.py +++ b/tensorflow/contrib/learn/python/learn/estimators/__init__.py @@ -40,6 +40,7 @@ from tensorflow.contrib.learn.python.learn.estimators.linear import TensorFlowLi from tensorflow.contrib.learn.python.learn.estimators.linear import TensorFlowLinearRegressor from tensorflow.contrib.learn.python.learn.estimators.linear import TensorFlowRegressor from tensorflow.contrib.learn.python.learn.estimators.logistic_regressor import LogisticRegressor +from tensorflow.contrib.learn.python.learn.estimators.random_forest import TensorForestEstimator from tensorflow.contrib.learn.python.learn.estimators.rnn import TensorFlowRNNClassifier from tensorflow.contrib.learn.python.learn.estimators.rnn import TensorFlowRNNRegressor from tensorflow.contrib.learn.python.learn.estimators.run_config import RunConfig diff --git a/tensorflow/contrib/learn/python/learn/estimators/random_forest.py b/tensorflow/contrib/learn/python/learn/estimators/random_forest.py new file mode 100644 index 00000000000..b9f50701ee3 --- /dev/null +++ b/tensorflow/contrib/learn/python/learn/estimators/random_forest.py @@ -0,0 +1,191 @@ +# pylint: disable=g-bad-file-header +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""A tf.learn implementation of tensor_forest (extremely random forests).""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import time + +import numpy as np +import six + +from tensorflow.contrib import framework as contrib_framework +from tensorflow.contrib.learn.python.learn import monitors as mon + +from tensorflow.contrib.learn.python.learn.estimators import estimator +from tensorflow.contrib.learn.python.learn.estimators import run_config + +from tensorflow.contrib.tensor_forest.client import eval_metrics +from tensorflow.contrib.tensor_forest.data import data_ops +from tensorflow.contrib.tensor_forest.python import tensor_forest + +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import state_ops + + +class LossMonitor(mon.EveryN): + """Terminates training when training loss stops decreasing.""" + + def __init__(self, + early_stopping_rounds, + every_n_steps): + super(LossMonitor, self).__init__(every_n_steps=every_n_steps) + self.early_stopping_rounds = early_stopping_rounds + self.min_loss = None + self.min_loss_step = 0 + + def set_estimator(self, est): + """This function gets called in the same graph as _get_train_ops.""" + super(LossMonitor, self).set_estimator(est) + self._loss_op_name = est.training_loss.name + + def every_n_step_end(self, step, outputs): + super(LossMonitor, self).every_n_step_end(step, outputs) + current_loss = outputs[self._loss_op_name] + if self.min_loss is None or current_loss < self.min_loss: + self.min_loss = current_loss + self.min_loss_step = step + return step - self.min_loss_step >= self.early_stopping_rounds + + +class TensorForestEstimator(estimator.BaseEstimator): + """An estimator that can train and evaluate a random forest.""" + + def __init__(self, params, device_assigner=None, model_dir=None, + graph_builder_class=tensor_forest.RandomForestGraphs, + master='', accuracy_metric=None, + tf_random_seed=None, continue_training=False, verbose=1, + max_to_keep=5, save_checkpoint_secs=300): + self.params = params.fill() + self.accuracy_metric = (accuracy_metric or + ('r2' if self.params.regression else 'accuracy')) + self.data_feeder = None + self.device_assigner = ( + device_assigner or tensor_forest.RandomForestDeviceAssigner()) + self.graph_builder_class = graph_builder_class + self.training_args = {} + self.construction_args = {} + + config = run_config.RunConfig( + master=master, + tf_random_seed=(tf_random_seed or int((time.time() * 1000) % 1000)), + save_checkpoints_secs=save_checkpoint_secs, + keep_checkpoint_max=max_to_keep) + + super(TensorForestEstimator, self).__init__(model_dir=model_dir, + config=config) + + def predict_proba(self, x=None, input_fn=None, batch_size=None): + """Returns prediction probabilities for given features (classification). + + Args: + x: features. + input_fn: Input function. If set, x and y must be None. + batch_size: Override default batch size. + + Returns: + Numpy array of predicted probabilities. + + Raises: + ValueError: If both or neither of x and input_fn were given. + """ + return super(TensorForestEstimator, self).predict( + x=x, input_fn=input_fn, batch_size=batch_size) + + def predict(self, x=None, input_fn=None, axis=None, batch_size=None): + """Returns predictions for given features. + + Args: + x: features. + input_fn: Input function. If set, x must be None. + axis: Axis on which to argmax (for classification). + Last axis is used by default. + batch_size: Override default batch size. + + Returns: + Numpy array of predicted classes or regression values. + """ + probabilities = self.predict_proba(x, input_fn, batch_size) + if self.params.regression: + return probabilities + else: + return np.argmax(probabilities, axis=1) + + def _get_train_ops(self, features, targets): + """Method that builds model graph and returns trainer ops. + + Args: + features: `Tensor` or `dict` of `Tensor` objects. + targets: `Tensor` or `dict` of `Tensor` objects. + + Returns: + Tuple of train `Operation` and loss `Tensor`. + """ + features, spec = data_ops.ParseDataTensorOrDict(features) + labels = data_ops.ParseLabelTensorOrDict(targets) + + graph_builder = self.graph_builder_class( + self.params, device_assigner=self.device_assigner, + **self.construction_args) + + epoch = None + if self.data_feeder: + epoch = self.data_feeder.make_epoch_variable() + + train = control_flow_ops.group( + graph_builder.training_graph( + features, labels, data_spec=spec, epoch=epoch, + **self.training_args), + state_ops.assign_add(contrib_framework.get_global_step(), 1)) + + self.training_loss = graph_builder.training_loss() + + return train, self.training_loss + + def _get_predict_ops(self, features): + graph_builder = self.graph_builder_class( + self.params, device_assigner=self.device_assigner, training=False, + **self.construction_args) + features, spec = data_ops.ParseDataTensorOrDict(features) + return graph_builder.inference_graph(features, data_spec=spec) + + def _get_eval_ops(self, features, targets, metrics): + features, spec = data_ops.ParseDataTensorOrDict(features) + labels = data_ops.ParseLabelTensorOrDict(targets) + + graph_builder = self.graph_builder_class( + self.params, device_assigner=self.device_assigner, training=False, + **self.construction_args) + + probabilities = graph_builder.inference_graph(features, data_spec=spec) + + # One-hot the labels. + if not self.params.regression: + labels = math_ops.to_int64(array_ops.one_hot(math_ops.to_int64( + array_ops.squeeze(labels)), self.params.num_classes, 1, 0)) + + if metrics is None: + metrics = {self.accuracy_metric: + eval_metrics.get_metric(self.accuracy_metric)} + + result = {} + for name, metric in six.iteritems(metrics): + result[name] = metric(probabilities, labels) + + return result diff --git a/tensorflow/contrib/learn/python/learn/estimators/random_forest_test.py b/tensorflow/contrib/learn/python/learn/estimators/random_forest_test.py new file mode 100644 index 00000000000..15db20906d1 --- /dev/null +++ b/tensorflow/contrib/learn/python/learn/estimators/random_forest_test.py @@ -0,0 +1,55 @@ +# pylint: disable=g-bad-file-header +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Tests for TensorForestTrainer.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf + + +class TensorForestTrainerTests(tf.test.TestCase): + + def testClassification(self): + """Tests multi-class classification using matrix data as input.""" + hparams = tf.contrib.tensor_forest.python.tensor_forest.ForestHParams( + num_trees=3, max_nodes=1000, num_classes=3, num_features=4) + classifier = tf.contrib.learn.TensorForestEstimator(hparams) + + iris = tf.contrib.learn.datasets.load_iris() + + classifier.fit(x=iris.data, y=iris.target, steps=100) + classifier.evaluate(x=iris.data, y=iris.target, steps=10) + + def testRegression(self): + """Tests multi-class classification using matrix data as input.""" + + hparams = tf.contrib.tensor_forest.python.tensor_forest.ForestHParams( + num_trees=3, max_nodes=1000, num_classes=1, num_features=13, + regression=True) + + regressor = tf.contrib.learn.TensorForestEstimator(hparams) + + boston = tf.contrib.learn.datasets.load_boston() + + regressor.fit(x=boston.data, y=boston.target, steps=100) + regressor.evaluate(x=boston.data, y=boston.target, steps=10) + + +if __name__ == '__main__': + tf.test.main() diff --git a/tensorflow/contrib/tensor_forest/BUILD b/tensorflow/contrib/tensor_forest/BUILD index 792243790ce..8c9dc742228 100644 --- a/tensorflow/contrib/tensor_forest/BUILD +++ b/tensorflow/contrib/tensor_forest/BUILD @@ -18,6 +18,54 @@ filegroup( ), ) +py_library( + name = "constants", + srcs = [ + "python/constants.py", + ], + srcs_version = "PY2AND3", +) + +tf_custom_op_library( + name = "data/_data_ops.so", + srcs = [ + "data/string_to_float_op.cc", + ], + deps = [ + ":tree_utils", + ], +) + +py_library( + name = "data_ops_lib", + srcs = [ + "data/data_ops.py", + ], + data = [ + "data/_data_ops.so", + ], + srcs_version = "PY2AND3", + deps = [ + ":constants", + ], +) + +py_library( + name = "eval_metrics", + srcs = ["client/eval_metrics.py"], + srcs_version = "PY2AND3", +) + +py_library( + name = "client_lib", + srcs_version = "PY2AND3", + deps = [ + ":data_ops_lib", + ":eval_metrics", + ":tensor_forest_py", + ], +) + cc_library( name = "tree_utils", srcs = ["core/ops/tree_utils.cc"], @@ -86,6 +134,7 @@ py_test( srcs = ["python/kernel_tests/count_extremely_random_stats_op_test.py"], srcs_version = "PY2AND3", deps = [ + ":constants", ":ops_lib", "//tensorflow:tensorflow_py", "//tensorflow/python:framework_test_lib", @@ -151,6 +200,7 @@ py_test( srcs = ["python/kernel_tests/tree_predictions_op_test.py"], srcs_version = "PY2AND3", deps = [ + ":constants", ":ops_lib", "//tensorflow:tensorflow_py", "//tensorflow/python:framework_test_lib", @@ -176,6 +226,7 @@ py_library( srcs = ["python/tensor_forest.py"], srcs_version = "PY2AND3", deps = [ + ":constants", ":ops_lib", ], ) diff --git a/tensorflow/contrib/tensor_forest/__init__.py b/tensorflow/contrib/tensor_forest/__init__.py index 7cf05299c4b..7d97e01df08 100644 --- a/tensorflow/contrib/tensor_forest/__init__.py +++ b/tensorflow/contrib/tensor_forest/__init__.py @@ -18,4 +18,6 @@ from __future__ import division from __future__ import print_function # pylint: disable=unused-import,wildcard-import +from tensorflow.contrib.tensor_forest.client import * +from tensorflow.contrib.tensor_forest.data import * from tensorflow.contrib.tensor_forest.python import * diff --git a/tensorflow/contrib/tensor_forest/client/__init__.py b/tensorflow/contrib/tensor_forest/client/__init__.py new file mode 100644 index 00000000000..753f406cbc7 --- /dev/null +++ b/tensorflow/contrib/tensor_forest/client/__init__.py @@ -0,0 +1,21 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Random forest implementation in tensorflow.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +# pylint: disable=unused-import,wildcard-import +from tensorflow.contrib.tensor_forest.client import eval_metrics diff --git a/tensorflow/contrib/tensor_forest/client/eval_metrics.py b/tensorflow/contrib/tensor_forest/client/eval_metrics.py new file mode 100644 index 00000000000..f41794a886e --- /dev/null +++ b/tensorflow/contrib/tensor_forest/client/eval_metrics.py @@ -0,0 +1,69 @@ +# pylint: disable=g-bad-file-header +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""A collection of functions to be used as evaluation metrics.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.contrib import losses +from tensorflow.contrib.metrics.python.ops import metric_ops + +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import math_ops + + +def _accuracy(probabilities, targets): + predictions = math_ops.argmax(probabilities, 1) + # undo one-hot + labels = math_ops.argmax(targets, 1) + return metric_ops.streaming_accuracy(predictions, labels) + + +def _r2(probabilities, targets): + if targets.get_shape().ndims == 1: + targets = array_ops.expand_dims(targets, -1) + y_mean = math_ops.reduce_mean(targets, 0) + squares_total = math_ops.reduce_sum(math_ops.square(targets - y_mean), 0) + squares_residuals = math_ops.reduce_sum(math_ops.square( + targets - probabilities), 0) + score = 1 - math_ops.reduce_sum(squares_residuals / squares_total) + return metric_ops.streaming_mean(score) + + +def _sigmoid_entropy(probabilities, targets): + return metric_ops.streaming_mean(losses.sigmoid_cross_entropy( + probabilities, targets)) + + +def _softmax_entropy(probabilities, targets): + return metric_ops.streaming_mean(losses.softmax_cross_entropy( + probabilities, targets)) + + +def _predictions(probabilities, unused_targets): + return math_ops.argmax(probabilities, 1) + + +_EVAL_METRICS = {'sigmoid_entropy': _sigmoid_entropy, + 'softmax_entropy': _softmax_entropy, + 'accuracy': _accuracy, + 'r2': _r2, + 'predictions': _predictions} + + +def get_metric(metric_name): + """Given a metric name, return the corresponding metric function.""" + return _EVAL_METRICS[metric_name] diff --git a/tensorflow/contrib/tensor_forest/core/ops/count_extremely_random_stats_op.cc b/tensorflow/contrib/tensor_forest/core/ops/count_extremely_random_stats_op.cc index 0ccf75bcc6f..0413f1e20a1 100644 --- a/tensorflow/contrib/tensor_forest/core/ops/count_extremely_random_stats_op.cc +++ b/tensorflow/contrib/tensor_forest/core/ops/count_extremely_random_stats_op.cc @@ -43,7 +43,7 @@ using tensorforest::LEAF_NODE; using tensorforest::FREE_NODE; using tensorforest::CheckTensorBounds; -using tensorforest::DecideNode; +using tensorforest::DataColumnTypes; using tensorforest::Initialize; using tensorforest::IsAllInitialized; @@ -61,61 +61,77 @@ struct InputDataResult { bool splits_initialized; }; -void Evaluate(const Tensor& input_data, const Tensor& input_labels, - const Tensor& tree_tensor, const Tensor& tree_thresholds, - const Tensor& node_to_accumulator, - const Tensor& candidate_split_features, - const Tensor& candidate_split_thresholds, - InputDataResult* results, int32 start, int32 end) { - const auto tree = tree_tensor.tensor(); - const auto thresholds = tree_thresholds.unaligned_flat(); - const auto node_map = node_to_accumulator.unaligned_flat(); - const auto split_features = candidate_split_features.tensor(); - const auto split_thresholds = candidate_split_thresholds.tensor(); + +struct EvaluateParams { + std::function decide_function; + Tensor input_spec; + Tensor input_labels; + Tensor tree_tensor; + Tensor tree_thresholds; + Tensor node_to_accumulator; + Tensor candidate_split_features; + Tensor candidate_split_thresholds; + InputDataResult* results; +}; + +void Evaluate(const EvaluateParams& params, int32 start, int32 end) { + const auto tree = params.tree_tensor.tensor(); + const auto thresholds = params.tree_thresholds.unaligned_flat(); + const auto node_map = params.node_to_accumulator.unaligned_flat(); + const auto split_features = + params.candidate_split_features.tensor(); + const auto split_thresholds = + params.candidate_split_thresholds.tensor(); + const auto spec = params.input_spec.unaligned_flat(); const int32 num_splits = static_cast( - candidate_split_features.shape().dim_size(1)); - const int32 num_nodes = static_cast(tree_tensor.shape().dim_size(0)); + params.candidate_split_features.shape().dim_size(1)); + const int32 num_nodes = static_cast( + params.tree_tensor.shape().dim_size(0)); const int32 num_accumulators = static_cast( - candidate_split_features.shape().dim_size(0)); + params.candidate_split_features.shape().dim_size(0)); for (int32 i = start; i < end; ++i) { - const Tensor point = input_data.Slice(i, i + 1); int node_index = 0; - results[i].splits_initialized = false; + params.results[i].splits_initialized = false; while (true) { - results[i].node_indices.push_back(node_index); + params.results[i].node_indices.push_back(node_index); CHECK_LT(node_index, num_nodes); int32 left_child = internal::SubtleMustCopy( tree(node_index, CHILDREN_INDEX)); if (left_child == LEAF_NODE) { const int32 accumulator = internal::SubtleMustCopy( node_map(node_index)); - results[i].leaf_accumulator = accumulator; + params.results[i].leaf_accumulator = accumulator; // If the leaf is not fertile or is not yet initialized, we don't // count it in the candidate/total split per-class-weights because // it won't have any candidate splits yet. if (accumulator >= 0 && - IsAllInitialized(candidate_split_features.Slice( + IsAllInitialized(params.candidate_split_features.Slice( accumulator, accumulator + 1))) { CHECK_LT(accumulator, num_accumulators); - results[i].splits_initialized = true; + params.results[i].splits_initialized = true; for (int split = 0; split < num_splits; split++) { - if (!DecideNode(point, split_features(accumulator, split), - split_thresholds(accumulator, split))) { - results[i].split_adds.push_back(split); + const int32 feature = split_features(accumulator, split); + if (!params.decide_function( + i, feature, split_thresholds(accumulator, split), + static_cast(spec(feature)))) { + params.results[i].split_adds.push_back(split); } } } break; } else if (left_child == FREE_NODE) { LOG(ERROR) << "Reached a free node, not good."; - results[i].node_indices.push_back(FREE_NODE); + params.results[i].node_indices.push_back(FREE_NODE); break; } + const int32 feature = tree(node_index, FEATURE_INDEX); node_index = - left_child + DecideNode(point, tree(node_index, FEATURE_INDEX), - thresholds(node_index)); + left_child + params.decide_function( + i, feature, thresholds(node_index), + static_cast(spec(feature))); } } } @@ -124,16 +140,18 @@ REGISTER_OP("CountExtremelyRandomStats") .Attr("num_classes: int") .Attr("regression: bool = false") .Input("input_data: float") + .Input("sparse_input_indices: int64") + .Input("sparse_input_values: float") + .Input("sparse_input_shape: int64") + .Input("input_spec: int32") .Input("input_labels: float") - .Input("tree: int32") .Input("tree_thresholds: float") - .Input("node_to_accumulator: int32") - .Input("candidate_split_features: int32") .Input("candidate_split_thresholds: float") - + .Input("birth_epochs: int32") + .Input("current_epoch: int32") .Output("pcw_node_sums_delta: float") .Output("pcw_node_squares_delta: float") .Output("pcw_splits_indices: int32") @@ -142,7 +160,6 @@ REGISTER_OP("CountExtremelyRandomStats") .Output("pcw_totals_indices: int32") .Output("pcw_totals_sums_delta: float") .Output("pcw_totals_squares_delta: float") - .Output("leaves: int32") .Doc(R"doc( Calculates incremental statistics for a batch of training data. @@ -156,7 +173,7 @@ For `regression` = false (classification), `pcw_node_sums_delta[i]` is incremented for every node i that it passes through, and the leaf it ends up in is recorded in `leaves[i]`. Then, if the leaf is fertile and initialized, the statistics for its corresponding accumulator slot -are updated in `pcw_candidate_splits_delta` and `pcw_total_splits_delta`. +are updated in `pcw_candidate_sums_delta` and `pcw_totals_sums_delta`. For `regression` = true, outputs contain the sum of the input_labels for the appropriate nodes. In adddition, the *_squares outputs are filled @@ -171,6 +188,11 @@ The attr `num_classes` is needed to appropriately size the outputs. input_data: The training batch's features as a 2-d tensor; `input_data[i][j]` gives the j-th feature of the i-th input. +sparse_input_indices: The indices tensor from the SparseTensor input. +sparse_input_values: The values tensor from the SparseTensor input. +sparse_input_shape: The shape tensor from the SparseTensor input. +input_spec: A 1-D tensor containing the type of each column in input_data, + (e.g. continuous float, categorical). input_labels: The training batch's labels; `input_labels[i]` is the class of the i-th input. tree:= A 2-d int32 tensor. `tree[i][0]` gives the index of the left child @@ -185,6 +207,10 @@ candidate_split_features: `candidate_split_features[a][s]` is the index of the feature being considered by split s of accumulator slot a. candidate_split_thresholds: `candidate_split_thresholds[a][s]` is the threshold value being considered by split s of accumulator slot a. +birth_epochs: `birth_epoch[i]` is the epoch node i was born in. Only + nodes satisfying `current_epoch - birth_epoch <= 1` accumulate statistics. +current_epoch:= A 1-d int32 tensor with shape (1). current_epoch[0] contains + the current epoch. pcw_node_sums_delta: `pcw_node_sums_delta[i][c]` is the number of training examples in this training batch with class c that passed through node i for classification. For regression, it is the sum of the input_labels that @@ -236,17 +262,57 @@ class CountExtremelyRandomStats : public OpKernel { void Compute(OpKernelContext* context) override { const Tensor& input_data = context->input(0); - const Tensor& input_labels = context->input(1); - const Tensor& tree_tensor = context->input(2); - const Tensor& tree_thresholds = context->input(3); - const Tensor& node_to_accumulator = context->input(4); - const Tensor& candidate_split_features = context->input(5); - const Tensor& candidate_split_thresholds = context->input(6); + const Tensor& sparse_input_indices = context->input(1); + const Tensor& sparse_input_values = context->input(2); + const Tensor& sparse_input_shape = context->input(3); + const Tensor& input_spec = context->input(4); + const Tensor& input_labels = context->input(5); + const Tensor& tree_tensor = context->input(6); + const Tensor& tree_thresholds = context->input(7); + const Tensor& node_to_accumulator = context->input(8); + const Tensor& candidate_split_features = context->input(9); + const Tensor& candidate_split_thresholds = context->input(10); + const Tensor& birth_epochs = context->input(11); + const Tensor& current_epoch = context->input(12); + + bool sparse_input = (sparse_input_indices.shape().dims() == 2); // Check inputs. - OP_REQUIRES(context, input_data.shape().dims() == 2, + if (sparse_input) { + OP_REQUIRES(context, sparse_input_shape.shape().dims() == 1, + errors::InvalidArgument( + "sparse_input_shape should be one-dimensional")); + OP_REQUIRES(context, + sparse_input_shape.shape().dim_size(0) == 2, + errors::InvalidArgument( + "The sparse input data should be two-dimensional")); + OP_REQUIRES(context, sparse_input_values.shape().dims() == 1, + errors::InvalidArgument( + "sparse_input_values should be one-dimensional")); + OP_REQUIRES(context, sparse_input_indices.shape().dims() == 2, + errors::InvalidArgument( + "The sparse input data should be two-dimensional")); + OP_REQUIRES(context, + sparse_input_indices.shape().dim_size(0) == + sparse_input_values.shape().dim_size(0), + errors::InvalidArgument( + "sparse_input_indices and sparse_input_values should " + "agree on the number of non-zero values")); + } else { + OP_REQUIRES(context, input_data.shape().dims() == 2, + errors::InvalidArgument( + "input_data should be two-dimensional")); + OP_REQUIRES( + context, + input_data.shape().dim_size(0) == input_labels.shape().dim_size(0), + errors::InvalidArgument( + "Number of inputs should be the same in " + "input_data and input_labels.")); + } + + OP_REQUIRES(context, input_labels.shape().dims() >= 1, errors::InvalidArgument( - "input_data should be two-dimensional")); + "input_labels should be at least one-dimensional")); OP_REQUIRES(context, tree_tensor.shape().dims() == 2, errors::InvalidArgument( "tree should be two-dimensional")); @@ -262,58 +328,93 @@ class CountExtremelyRandomStats : public OpKernel { OP_REQUIRES(context, candidate_split_thresholds.shape().dims() == 2, errors::InvalidArgument( "candidate_split_thresholds should be two-dimensional")); - - OP_REQUIRES( - context, - input_data.shape().dim_size(0) == input_labels.shape().dim_size(0), - errors::InvalidArgument( - "Number of inputs should be the same in " - "input_data and input_labels.")); + OP_REQUIRES(context, birth_epochs.shape().dims() == 1, + errors::InvalidArgument( + "birth_epochs should be one-dimensional")); + OP_REQUIRES(context, current_epoch.shape().dims() == 1, + errors::InvalidArgument( + "current_epoch should be one-dimensional")); OP_REQUIRES( context, tree_tensor.shape().dim_size(0) == tree_thresholds.shape().dim_size(0) && tree_tensor.shape().dim_size(0) == - node_to_accumulator.shape().dim_size(0), + node_to_accumulator.shape().dim_size(0) && + tree_tensor.shape().dim_size(0) == + birth_epochs.shape().dim_size(0), errors::InvalidArgument( "Number of nodes should be the same in " - "tree, tree_thresholds, and node_to_accumulator")); + "tree, tree_thresholds, node_to_accumulator, and birth_epoch.")); OP_REQUIRES( context, candidate_split_features.shape() == candidate_split_thresholds.shape(), errors::InvalidArgument( "candidate_split_features and candidate_split_thresholds should be " "the same shape.")); + OP_REQUIRES( + context, + current_epoch.shape().dim_size(0) == 1, + errors::InvalidArgument( + "The current_epoch should be a tensor of shape (1).")); // Check tensor bounds. if (!CheckTensorBounds(context, input_data)) return; + if (!CheckTensorBounds(context, sparse_input_indices)) return; + if (!CheckTensorBounds(context, sparse_input_values)) return; + if (!CheckTensorBounds(context, sparse_input_shape)) return; if (!CheckTensorBounds(context, input_labels)) return; if (!CheckTensorBounds(context, tree_tensor)) return; if (!CheckTensorBounds(context, tree_thresholds)) return; if (!CheckTensorBounds(context, node_to_accumulator)) return; if (!CheckTensorBounds(context, candidate_split_features)) return; if (!CheckTensorBounds(context, candidate_split_thresholds)) return; + if (!CheckTensorBounds(context, birth_epochs)) return; + if (!CheckTensorBounds(context, current_epoch)) return; // Evaluate input data in parallel. - const int32 num_data = static_cast(input_data.shape().dim_size(0)); + const int32 epoch = current_epoch.unaligned_flat()(0); + int32 num_data; + std::function decide_function; + if (sparse_input) { + num_data = sparse_input_shape.unaligned_flat()(0); + decide_function = [&sparse_input_indices, &sparse_input_values]( + int32 i, int32 feature, float bias, DataColumnTypes type) { + const auto sparse_indices = sparse_input_indices.matrix(); + const auto sparse_values = sparse_input_values.vec(); + return tensorforest::DecideSparseNode( + sparse_indices, sparse_values, i, feature, bias, type); + }; + } else { + num_data = static_cast(input_data.shape().dim_size(0)); + decide_function = [&input_data]( + int32 i, int32 feature, float bias, DataColumnTypes type) { + const auto input_matrix = input_data.matrix(); + return tensorforest::DecideDenseNode( + input_matrix, i, feature, bias, type); + }; + } std::unique_ptr results(new InputDataResult[num_data]); auto worker_threads = context->device()->tensorflow_cpu_worker_threads(); int num_threads = worker_threads->num_threads; + EvaluateParams params; + params.decide_function = decide_function; + params.input_spec = input_spec; + params.input_labels = input_labels; + params.tree_tensor = tree_tensor; + params.tree_thresholds = tree_thresholds; + params.node_to_accumulator = node_to_accumulator; + params.candidate_split_features = candidate_split_features; + params.candidate_split_thresholds = candidate_split_thresholds; + params.results = results.get(); if (num_threads <= 1) { - Evaluate(input_data, input_labels, tree_tensor, tree_thresholds, - node_to_accumulator, candidate_split_features, - candidate_split_thresholds, results.get(), 0, num_data); + Evaluate(params, 0, num_data); } else { - auto work = [&input_data, &input_labels, &tree_tensor, &tree_thresholds, - &node_to_accumulator, &candidate_split_features, - &candidate_split_thresholds, &num_data, - &results](int64 start, int64 end) { + auto work = [¶ms, num_data](int64 start, int64 end) { CHECK(start <= end); CHECK(end <= num_data); - Evaluate(input_data, input_labels, tree_tensor, tree_thresholds, - node_to_accumulator, candidate_split_features, - candidate_split_thresholds, results.get(), + Evaluate(params, static_cast(start), static_cast(end)); }; Shard(num_threads, worker_threads->workers, num_data, 100, work); @@ -321,11 +422,13 @@ class CountExtremelyRandomStats : public OpKernel { const int32 num_nodes = static_cast(tree_tensor.shape().dim_size(0)); if (regression_) { - ProcessResultsRegression(context, input_labels, std::move(results), - num_nodes); + ProcessResultsRegression( + context, input_labels, birth_epochs, epoch, std::move(results), + num_nodes); } else { - ProcessResultsClassification(context, input_labels, std::move(results), - num_nodes); + ProcessResultsClassification( + context, input_labels, birth_epochs, epoch, std::move(results), + num_nodes); } } @@ -333,10 +436,13 @@ class CountExtremelyRandomStats : public OpKernel { void ProcessResultsClassification( OpKernelContext* context, const Tensor &input_labels, + const Tensor &birth_epochs, + int32 epoch, std::unique_ptr results, int32 num_nodes) { const int32 num_data = static_cast(input_labels.shape().dim_size(0)); const auto labels = input_labels.unaligned_flat(); + const auto start_epochs = birth_epochs.unaligned_flat(); // Unused outputs for classification. Still have to specify them or // tensorflow complains. @@ -381,10 +487,16 @@ class CountExtremelyRandomStats : public OpKernel { CHECK_LT(column, num_classes_); const int32 accumulator = results[i].leaf_accumulator; for (const int32 node : results[i].node_indices) { + if (epoch > start_epochs(node) + 1) { + continue; + } ++out_node_sums(node, column); ++out_node_sums(node, 0); } out_leaves(i) = results[i].node_indices.back(); + if (epoch > start_epochs(out_leaves(i)) + 1) { + continue; + } if (accumulator >= 0 && results[i].splits_initialized) { ++total_delta[make_pair(accumulator, column)]; ++total_delta[make_pair(accumulator, 0)]; @@ -457,6 +569,8 @@ class CountExtremelyRandomStats : public OpKernel { void ProcessResultsRegression( OpKernelContext* context, const Tensor &input_labels, + const Tensor &birth_epochs, + const int32 epoch, std::unique_ptr results, int32 num_nodes) { const int32 num_data = static_cast(input_labels.shape().dim_size(0)); @@ -465,6 +579,7 @@ class CountExtremelyRandomStats : public OpKernel { num_outputs = static_cast(input_labels.shape().dim_size(1)); } const auto labels = input_labels.unaligned_flat(); + const auto start_epochs = birth_epochs.unaligned_flat(); // node pcw delta Tensor* output_node_pcw_sums_delta = nullptr; @@ -503,6 +618,9 @@ class CountExtremelyRandomStats : public OpKernel { for (int32 i = 0; i < num_data; ++i) { const int32 accumulator = results[i].leaf_accumulator; for (const int32 node : results[i].node_indices) { + if (epoch > start_epochs(node) + 1) { + continue; + } for (int32 j = 0; j < num_outputs; ++j) { const float output = labels(i * num_outputs + j); out_node_sums(node, j + 1) += output; @@ -512,6 +630,9 @@ class CountExtremelyRandomStats : public OpKernel { } } out_leaves(i) = results[i].node_indices.back(); + if (epoch > start_epochs(out_leaves(i)) + 1) { + continue; + } if (accumulator >= 0 && results[i].splits_initialized) { total_delta[accumulator].insert(i); for (const int32 split : results[i].split_adds) { diff --git a/tensorflow/contrib/tensor_forest/core/ops/finished_nodes_op.cc b/tensorflow/contrib/tensor_forest/core/ops/finished_nodes_op.cc index e1369f9d8cb..d179f5b84e9 100644 --- a/tensorflow/contrib/tensor_forest/core/ops/finished_nodes_op.cc +++ b/tensorflow/contrib/tensor_forest/core/ops/finished_nodes_op.cc @@ -24,42 +24,84 @@ namespace tensorflow { using tensorforest::CheckTensorBounds; using tensorforest::Sum; +using tensorforest::BestSplitDominatesClassification; +using tensorforest::BestSplitDominatesRegression; REGISTER_OP("FinishedNodes") + .Attr("regression: bool = false") .Attr("num_split_after_samples: int") + .Attr("min_split_samples: int") + .Attr("dominate_fraction: float = 0.95") .Input("leaves: int32") .Input("node_to_accumulator: int32") + .Input("split_sums: float") + .Input("split_squares: float") .Input("accumulator_sums: float") - + .Input("accumulator_squares: float") + .Input("birth_epochs: int32") + .Input("current_epoch: int32") .Output("finished: int32") + .Output("stale: int32") .Doc(R"doc( Determines which of the given leaf nodes are done accumulating. leaves:= A 1-d int32 tensor. Lists the nodes that are currently leaves. node_to_accumulator: If the i-th node is fertile, `node_to_accumulator[i]` is it's accumulator slot. Otherwise, `node_to_accumulator[i]` is -1. -accumulator_sums: For classification, `accumulator_sums[a][c]` records how many - training examples have class c and have ended up in the fertile node +split_sums:= a 3-d tensor where `split_sums[a][s]` summarizes the + training labels for examples that fall into the fertile node associated with + accumulator slot s and have then taken the *left* branch of candidate split + s. For a classification problem, `split_sums[a][s][c]` is the count of such + examples with class c and for regression problems, `split_sums[a][s]` is the + sum of the regression labels for such examples. +split_squares: Same as split_sums, but it contains the sum of the + squares of the regression labels. Only used for regression. For + classification problems, pass a dummy tensor into this. +accumulator_sums: For classification, `accumulator_sums[a][c]` records how + many training examples have class c and have ended up in the fertile node associated with accumulator slot a. It has the total sum in entry 0 for convenience. For regression, it is the same except it contains the sum of the input labels that have been seen, and entry 0 contains the number of training examples that have been seen. -finished:= A 1-d int32 tensor. Contains the nodes that have total split - counts greater or equal to the num_split_after_samples attribute. +accumulator_squares: Same as accumulator_sums, but it contains the sum of the + squares of the regression labels. Only used for regression. For + classification problems, pass a dummy tensor into this. +birth_epochs:= A 1-d int32 tensor. `birth_epochs[i]` contains the epoch + the i-th node was created in. +current_epoch:= A 1-d int32 tensor with shape (1). `current_epoch[0]` + stores the current epoch number. +finished:= A 1-d int32 tensor containing the indices of the finished nodes. + Nodes are finished if they have received at least num_split_after_samples + samples, or if they have received min_split_samples and the best scoring + split is sufficiently greater than the next best split. +stale:= A 1-d int32 tensor containing the fertile nodes that were created two + or more epochs ago. + )doc"); class FinishedNodes : public OpKernel { public: explicit FinishedNodes(OpKernelConstruction* context) : OpKernel(context) { + OP_REQUIRES_OK(context, context->GetAttr( + "regression", ®ression_)); OP_REQUIRES_OK(context, context->GetAttr( "num_split_after_samples", &num_split_after_samples_)); + OP_REQUIRES_OK(context, context->GetAttr( + "min_split_samples", &min_split_samples_)); + OP_REQUIRES_OK(context, context->GetAttr( + "dominate_fraction", &dominate_fraction_)); } void Compute(OpKernelContext* context) override { const Tensor& leaf_tensor = context->input(0); const Tensor& node_to_accumulator = context->input(1); - const Tensor& accumulator_sums = context->input(2); + const Tensor& split_sums = context->input(2); + const Tensor& split_squares = context->input(3); + const Tensor& accumulator_sums = context->input(4); + const Tensor& accumulator_squares = context->input(5); + const Tensor& birth_epochs = context->input(6); + const Tensor& current_epoch = context->input(7); OP_REQUIRES(context, leaf_tensor.shape().dims() == 1, errors::InvalidArgument( @@ -67,25 +109,45 @@ class FinishedNodes : public OpKernel { OP_REQUIRES(context, node_to_accumulator.shape().dims() == 1, errors::InvalidArgument( "node_to_accumulator should be one-dimensional")); + OP_REQUIRES(context, split_sums.shape().dims() == 3, + errors::InvalidArgument( + "split_sums should be three-dimensional")); OP_REQUIRES(context, accumulator_sums.shape().dims() == 2, errors::InvalidArgument( "accumulator_sums should be two-dimensional")); + OP_REQUIRES(context, birth_epochs.shape().dims() == 1, + errors::InvalidArgument( + "birth_epochs should be one-dimensional")); + OP_REQUIRES( + context, + birth_epochs.shape().dim_size(0) == + node_to_accumulator.shape().dim_size(0), + errors::InvalidArgument( + "birth_epochs and node_to_accumulator should be the same size.")); // Check tensor bounds. if (!CheckTensorBounds(context, leaf_tensor)) return; if (!CheckTensorBounds(context, node_to_accumulator)) return; + if (!CheckTensorBounds(context, split_sums)) return; + if (!CheckTensorBounds(context, split_squares)) return; if (!CheckTensorBounds(context, accumulator_sums)) return; + if (!CheckTensorBounds(context, accumulator_squares)) return; + if (!CheckTensorBounds(context, birth_epochs)) return; + if (!CheckTensorBounds(context, current_epoch)) return; const auto leaves = leaf_tensor.unaligned_flat(); const auto node_map = node_to_accumulator.unaligned_flat(); const auto sums = accumulator_sums.tensor(); + const auto start_epochs = birth_epochs.unaligned_flat(); + const int32 epoch = current_epoch.unaligned_flat()(0); const int32 num_leaves = static_cast( leaf_tensor.shape().dim_size(0)); const int32 num_accumulators = static_cast( accumulator_sums.shape().dim_size(0)); - std::vector finished; + std::vector finished_leaves; + std::vector stale; for (int32 i = 0; i < num_leaves; i++) { const int32 leaf = internal::SubtleMustCopy(leaves(i)); OP_REQUIRES(context, FastBoundsCheck(leaf, node_map.size()), @@ -97,30 +159,74 @@ class FinishedNodes : public OpKernel { OP_REQUIRES(context, FastBoundsCheck(accumulator, num_accumulators), errors::InvalidArgument("accumulator not in valid range.")) - // The first column holds the number of samples seen. // For classification, this should be the sum of the other columns. - if (sums(accumulator, 0) >= num_split_after_samples_) { - finished.push_back(leaf); + int32 count = sums(accumulator, 0); + + if (epoch > start_epochs(leaf) + 1) { + if (count >= min_split_samples_) { + finished_leaves.push_back(leaf); + } else { + stale.push_back(leaf); + } + continue; + } + + if (count >= num_split_after_samples_) { + finished_leaves.push_back(leaf); + continue; + } + + if (count < min_split_samples_) { + continue; + } + + bool finished = false; + if (regression_) { + finished = BestSplitDominatesRegression( + accumulator_sums, accumulator_squares, + split_sums, split_squares, accumulator); + } else { + finished = BestSplitDominatesClassification( + accumulator_sums, split_sums, accumulator, dominate_fraction_); + } + + if (finished) { + finished_leaves.push_back(leaf); } } // Copy to output. Tensor* output_finished = nullptr; TensorShape finished_shape; - finished_shape.AddDim(finished.size()); + finished_shape.AddDim(finished_leaves.size()); OP_REQUIRES_OK(context, context->allocate_output(0, finished_shape, &output_finished)); auto out_finished = output_finished->unaligned_flat(); - for (int32 i = 0; i < finished.size(); i++) { - out_finished(i) = finished[i]; + for (int32 i = 0; i < finished_leaves.size(); i++) { + out_finished(i) = finished_leaves[i]; + } + + Tensor* output_stale = nullptr; + TensorShape stale_shape; + stale_shape.AddDim(stale.size()); + OP_REQUIRES_OK(context, + context->allocate_output(1, stale_shape, + &output_stale)); + auto out_stale = output_stale->unaligned_flat(); + + for (int32 i = 0; i < stale.size(); i++) { + out_stale(i) = stale[i]; } } private: + bool regression_; int32 num_split_after_samples_; + int32 min_split_samples_; + float dominate_fraction_; }; REGISTER_KERNEL_BUILDER(Name("FinishedNodes").Device(DEVICE_CPU), diff --git a/tensorflow/contrib/tensor_forest/core/ops/sample_inputs_op.cc b/tensorflow/contrib/tensor_forest/core/ops/sample_inputs_op.cc index 182b1257b6e..8b15f8a0b5f 100644 --- a/tensorflow/contrib/tensor_forest/core/ops/sample_inputs_op.cc +++ b/tensorflow/contrib/tensor_forest/core/ops/sample_inputs_op.cc @@ -35,6 +35,9 @@ REGISTER_OP("SampleInputs") .Attr("split_initializations_per_input: int") .Attr("split_sampling_random_seed: int") .Input("input_data: float") + .Input("sparse_input_indices: int64") + .Input("sparse_input_values: float") + .Input("sparse_input_shape: int64") .Input("node_to_accumulator: int32") .Input("leaves: int32") .Input("candidate_split_features: int32") @@ -60,6 +63,9 @@ a single training example can initialize, and the attribute input_data: The features for the current batch of training data. `input_data[i][j]` is the j-th feature of the i-th input. +sparse_input_indices: The indices tensor from the SparseTensor input. +sparse_input_values: The values tensor from the SparseTensor input. +sparse_input_shape: The shape tensor from the SparseTensor input. node_to_accumulator: For a fertile node i, node_to_accumulator[i] is the associated accumulator slot. For non-fertile nodes, it is -1. leaves: `leaves[i]` is the leaf that the i-th input landed in, as @@ -82,6 +88,7 @@ new_split_threshold_rows: The new values for the candidate_split_thresholds `tf.scatter_update(candidate_split_thresholds, accumulators_to_update, new_split_feature_thresholds)` + )doc"); class SampleInputs : public OpKernel { @@ -106,16 +113,74 @@ class SampleInputs : public OpKernel { new random::SimplePhilox(single_rand_.get())); } + template + void GetRandomFeatureDense(const T& inputs, int32 num_features, + int32 input_index, int32* index, float* val) { + *index = rng_->Uniform(num_features); + *val = inputs(input_index, *index); + } + + template + void GetRandomFeatureSparse(const T1& sparse_indices, const T2& sparse_values, + int32 input_index, int32* index, float* val) { + int32 low = 0; + int32 high = sparse_values.dimension(0); + while (low < high) { + int32 vi = low + rng_->Uniform(high - low); + int64 i = internal::SubtleMustCopy(sparse_indices(vi, 0)); + if (i == input_index) { + *index = internal::SubtleMustCopy(sparse_indices(vi, 1)); + *val = sparse_values(vi); + return; + } + if (i < input_index) { + low = vi + 1; + } else { + high = vi; + } + } + LOG(FATAL) << "Could not find any values for input " << input_index + << " inside sparse_input_indices"; + } + void Compute(OpKernelContext* context) override { const Tensor& input_data = context->input(0); - const Tensor& node_to_accumulator = context->input(1); - const Tensor& leaves = context->input(2); - const Tensor& split_features = context->input(3); - const Tensor& split_thresholds = context->input(4); + const Tensor& sparse_input_indices = context->input(1); + const Tensor& sparse_input_values = context->input(2); + const Tensor& sparse_input_shape = context->input(3); + const Tensor& node_to_accumulator = context->input(4); + const Tensor& leaves = context->input(5); + const Tensor& split_features = context->input(6); + const Tensor& split_thresholds = context->input(7); + + bool sparse_input = (sparse_input_indices.shape().dims() == 2); + + if (sparse_input) { + OP_REQUIRES(context, sparse_input_shape.shape().dims() == 1, + errors::InvalidArgument( + "sparse_input_shape should be one-dimensional")); + OP_REQUIRES(context, + sparse_input_shape.shape().dim_size(0) == 2, + errors::InvalidArgument( + "The sparse input data should be two-dimensional")); + OP_REQUIRES(context, sparse_input_values.shape().dims() == 1, + errors::InvalidArgument( + "sparse_input_values should be one-dimensional")); + OP_REQUIRES(context, sparse_input_indices.shape().dims() == 2, + errors::InvalidArgument( + "The sparse input data should be two-dimensional")); + OP_REQUIRES(context, + sparse_input_indices.shape().dim_size(0) == + sparse_input_values.shape().dim_size(0), + errors::InvalidArgument( + "sparse_input_indices and sparse_input_values should " + "agree on the number of non-zero values")); + } else { + OP_REQUIRES(context, input_data.shape().dims() == 2, + errors::InvalidArgument( + "input_data should be two-dimensional")); + } - OP_REQUIRES(context, input_data.shape().dims() == 2, - errors::InvalidArgument( - "input_data should be two-dimensional")); OP_REQUIRES(context, node_to_accumulator.shape().dims() == 1, errors::InvalidArgument( "node_to_accumulator should be one-dimensional")); @@ -137,12 +202,36 @@ class SampleInputs : public OpKernel { // Check tensor bounds. if (!CheckTensorBounds(context, input_data)) return; + if (!CheckTensorBounds(context, sparse_input_indices)) return; + if (!CheckTensorBounds(context, sparse_input_values)) return; + if (!CheckTensorBounds(context, sparse_input_shape)) return; if (!CheckTensorBounds(context, node_to_accumulator)) return; if (!CheckTensorBounds(context, leaves)) return; if (!CheckTensorBounds(context, split_features)) return; if (!CheckTensorBounds(context, split_thresholds)) return; - const auto inputs = input_data.tensor(); + int32 num_features; + std::function get_random_feature; + // TODO(thomaswc): Figure out a way to avoid calling .vec, etc. over and + // over again + if (sparse_input) { + num_features = sparse_input_shape.unaligned_flat()(1); + get_random_feature = [&sparse_input_indices, &sparse_input_values, this]( + int32 input_index, int32* index, float* val) { + const auto sparse_indices = sparse_input_indices.matrix(); + const auto sparse_values = sparse_input_values.vec(); + GetRandomFeatureSparse(sparse_indices, sparse_values, input_index, + index, val); + }; + } else { + num_features = static_cast(input_data.shape().dim_size(1)); + get_random_feature = [&input_data, num_features, this]( + int32 input_index, int32* index, float* val) { + const auto inputs = input_data.tensor(); + GetRandomFeatureDense(inputs, num_features, input_index, index, val); + }; + } + const auto leaves_vec = leaves.unaligned_flat(); const auto node_map = node_to_accumulator.unaligned_flat(); const auto features = split_features.tensor(); @@ -151,8 +240,6 @@ class SampleInputs : public OpKernel { const int32 num_data = static_cast(leaves.shape().dim_size(0)); const int32 num_splits = static_cast( split_features.shape().dim_size(1)); - const int32 num_features = static_cast( - input_data.shape().dim_size(1)); const int32 num_accumulators = static_cast( split_features.shape().dim_size(0)); @@ -234,10 +321,11 @@ class SampleInputs : public OpKernel { for (int split = 0; split < num_splits && num_inits > 0; split++) { if (new_split_feature_rows_flat(output_slot, split) < 0) { VLOG(1) << "Over-writing @ " << output_slot << "," << split; - const int32 index = rng_->Uniform(num_features); + int32 index; + float val; + get_random_feature(i, &index, &val); new_split_feature_rows_flat(output_slot, split) = index; - new_split_threshold_rows_flat(output_slot, split) = - inputs(i, index); + new_split_threshold_rows_flat(output_slot, split) = val; --num_inits; } } diff --git a/tensorflow/contrib/tensor_forest/core/ops/tree_predictions_op.cc b/tensorflow/contrib/tensor_forest/core/ops/tree_predictions_op.cc index 7db52ec3cae..1f77212d20e 100644 --- a/tensorflow/contrib/tensor_forest/core/ops/tree_predictions_op.cc +++ b/tensorflow/contrib/tensor_forest/core/ops/tree_predictions_op.cc @@ -30,12 +30,17 @@ using tensorforest::LEAF_NODE; using tensorforest::FREE_NODE; using tensorforest::CheckTensorBounds; -using tensorforest::DecideNode; +using tensorforest::DataColumnTypes; using tensorforest::Sum; REGISTER_OP("TreePredictions") .Attr("valid_leaf_threshold: float") .Input("input_data: float") + .Input("sparse_input_indices: int64") + .Input("sparse_input_values: float") + .Input("sparse_input_shape: int64") + .Input("input_spec: int32") + .Input("tree: int32") .Input("tree_thresholds: float") .Input("node_per_class_weights: float") @@ -46,6 +51,11 @@ REGISTER_OP("TreePredictions") input_data: The training batch's features as a 2-d tensor; `input_data[i][j]` gives the j-th feature of the i-th input. + sparse_input_indices: The indices tensor from the SparseTensor input. + sparse_input_values: The values tensor from the SparseTensor input. + sparse_input_shape: The shape tensor from the SparseTensor input. + input_spec: A 1-D tensor containing the type of each column in input_data, + (e.g. continuous float, categorical). tree:= A 2-d int32 tensor. `tree[i][0]` gives the index of the left child of the i-th node, `tree[i][0] + 1` gives the index of the right child of the i-th node, and `tree[i][1]` gives the index of the feature used to @@ -70,10 +80,42 @@ class TreePredictions : public OpKernel { void Compute(OpKernelContext* context) override { const Tensor& input_data = context->input(0); + const Tensor& sparse_input_indices = context->input(1); + const Tensor& sparse_input_values = context->input(2); + const Tensor& sparse_input_shape = context->input(3); + const Tensor& input_spec = context->input(4); + const Tensor& tree_tensor = context->input(5); + const Tensor& tree_thresholds = context->input(6); + const Tensor& node_per_class_weights = context->input(7); - const Tensor& tree_tensor = context->input(1); - const Tensor& tree_thresholds = context->input(2); - const Tensor& node_per_class_weights = context->input(3); + bool sparse_input = (sparse_input_indices.shape().dims() == 2); + + if (sparse_input) { + OP_REQUIRES(context, sparse_input_values.shape().dims() == 1, + errors::InvalidArgument( + "sparse_input_values should be one-dimensional")); + OP_REQUIRES(context, sparse_input_shape.shape().dims() == 1, + errors::InvalidArgument( + "sparse_input_shape should be one-dimensional")); + OP_REQUIRES(context, + sparse_input_indices.shape().dim_size(0) == + sparse_input_values.shape().dim_size(0), + errors::InvalidArgument( + "sparse_input_indices and sparse_input_values should " + "agree on the number of non-zero values")); + OP_REQUIRES(context, + sparse_input_indices.shape().dim_size(1) == + sparse_input_shape.shape().dim_size(0), + errors::InvalidArgument( + "sparse_input_indices and sparse_input_shape should " + "agree on the dimensionality of data points")); + } else { + if (input_data.shape().dim_size(0) > 0) { + OP_REQUIRES(context, input_data.shape().dims() == 2, + errors::InvalidArgument( + "input_data should be two-dimensional")); + } + } OP_REQUIRES(context, tree_tensor.shape().dims() == 2, errors::InvalidArgument( @@ -85,11 +127,6 @@ class TreePredictions : public OpKernel { errors::InvalidArgument( "node_pcw should be two-dimensional")); - if (input_data.shape().dim_size(0) > 0) { - OP_REQUIRES(context, input_data.shape().dims() == 2, - errors::InvalidArgument( - "input_data should be two-dimensional")); - } OP_REQUIRES( context, tree_tensor.shape().dim_size(0) == @@ -102,16 +139,43 @@ class TreePredictions : public OpKernel { // Check tensor bounds. if (!CheckTensorBounds(context, input_data)) return; + if (!CheckTensorBounds(context, sparse_input_indices)) return; + if (!CheckTensorBounds(context, sparse_input_values)) return; + if (!CheckTensorBounds(context, sparse_input_shape)) return; if (!CheckTensorBounds(context, tree_tensor)) return; if (!CheckTensorBounds(context, tree_thresholds)) return; if (!CheckTensorBounds(context, node_per_class_weights)) return; const int32 num_classes = static_cast( node_per_class_weights.shape().dim_size(1)); - const int32 num_data = static_cast( - input_data.shape().dim_size(0)); const int32 num_nodes = static_cast( tree_tensor.shape().dim_size(0)); + int32 num_data; + std::function decide_function; + + if (sparse_input) { + num_data = sparse_input_shape.unaligned_flat()(0); + decide_function = [&sparse_input_indices, &sparse_input_values]( + int32 i, int32 feature, float bias, DataColumnTypes type) { + const auto sparse_indices = sparse_input_indices.matrix(); + const auto sparse_values = sparse_input_values.vec(); + return tensorforest::DecideSparseNode( + sparse_indices, sparse_values, i, feature, bias, type); + }; + } else { + num_data = static_cast(input_data.shape().dim_size(0)); + int32 num_features = 0; + if (num_data > 0) { + num_features = input_data.NumElements() / num_data; + } + decide_function = [&input_data]( + int32 i, int32 feature, float bias, DataColumnTypes type) { + const auto input_matrix = input_data.matrix(); + return tensorforest::DecideDenseNode( + input_matrix, i, feature, bias, type); + }; + } Tensor* output_predictions = nullptr; TensorShape output_shape; @@ -124,10 +188,10 @@ class TreePredictions : public OpKernel { const auto node_pcw = node_per_class_weights.tensor(); const auto tree = tree_tensor.tensor(); + const auto spec = input_spec.unaligned_flat(); const auto thresholds = tree_thresholds.unaligned_flat(); for (int i = 0; i < num_data; i++) { - const Tensor point = input_data.Slice(i, i+1); int node_index = 0; int parent = -1; while (true) { @@ -162,9 +226,11 @@ class TreePredictions : public OpKernel { return; } parent = node_index; + const int32 feature = tree(node_index, FEATURE_INDEX); node_index = left_child + - DecideNode(point, tree(node_index, FEATURE_INDEX), - thresholds(node_index)); + decide_function( + i, feature, thresholds(node_index), + static_cast(spec(feature))); } } diff --git a/tensorflow/contrib/tensor_forest/core/ops/tree_utils.cc b/tensorflow/contrib/tensor_forest/core/ops/tree_utils.cc index f3fc4160554..398990780cd 100644 --- a/tensorflow/contrib/tensor_forest/core/ops/tree_utils.cc +++ b/tensorflow/contrib/tensor_forest/core/ops/tree_utils.cc @@ -13,56 +13,127 @@ // limitations under the License. // ============================================================================= #include "tensorflow/contrib/tensor_forest/core/ops/tree_utils.h" +#include +#include "tensorflow/core/platform/logging.h" namespace tensorflow { namespace tensorforest { using tensorflow::Tensor; -int32 BestFeatureClassification( +void GetTwoBest(int max, std::function score_fn, + float *best_score, int *best_index, + float *second_best_score) { + *best_index = -1; + *best_score = FLT_MAX; + *second_best_score = FLT_MAX; + for (int i = 0; i < max; i++) { + float score = score_fn(i); + if (score < *best_score) { + *second_best_score = *best_score; + *best_score = score; + *best_index = i; + } else if (score < *second_best_score) { + *second_best_score = score; + } + } +} + +float ClassificationSplitScore( + const Eigen::Tensor& splits, + const Eigen::Tensor& rights, + int32 num_classes, int i) { + Eigen::array offsets; + offsets[0] = i * num_classes + 1; + Eigen::array extents; + extents[0] = num_classes - 1; + return WeightedGiniImpurity(splits.slice(offsets, extents)) + + WeightedGiniImpurity(rights.slice(offsets, extents)); +} + +void GetTwoBestClassification( const Tensor& total_counts, const Tensor& split_counts, - int32 accumulator) { - int32 best_feature_index = -1; - // We choose the split with the lowest score. - float best_score = kint64max; + int32 accumulator, + float *best_score, int *best_index, + float *second_best_score) { const int32 num_splits = static_cast(split_counts.shape().dim_size(1)); const int32 num_classes = static_cast( split_counts.shape().dim_size(2)); + // Ideally, Eigen::Tensor::chip would be best to use here but it results // in seg faults, so we have to go with flat views of these tensors. However, // it is still pretty efficient because we put off evaluation until the // score is actually returned. const auto tc = total_counts.Slice( accumulator, accumulator + 1).unaligned_flat(); - const auto splits = split_counts.Slice( + + // TODO(gilberth): See if we can delay evaluation here by templating the + // arguments to ClassificationSplitScore. + const Eigen::Tensor splits = split_counts.Slice( accumulator, accumulator + 1).unaligned_flat(); Eigen::array bcast; bcast[0] = num_splits; - const auto rights = tc.broadcast(bcast) - splits; + const Eigen::Tensor rights = + tc.broadcast(bcast) - splits; - for (int i = 0; i < num_splits; i++) { - Eigen::array offsets; - offsets[0] = i * num_classes; - Eigen::array extents; - extents[0] = num_classes; - float score = WeightedGiniImpurity(splits.slice(offsets, extents)) + - WeightedGiniImpurity(rights.slice(offsets, extents)); + std::function score_fn = std::bind( + ClassificationSplitScore, splits, rights, num_classes, + std::placeholders::_1); - if (score < best_score) { - best_score = score; - best_feature_index = i; - } - } + GetTwoBest( + num_splits, score_fn, + best_score, best_index, second_best_score); +} + +int32 BestFeatureClassification( + const Tensor& total_counts, const Tensor& split_counts, + int32 accumulator) { + float best_score; + float second_best_score; + int best_feature_index; + GetTwoBestClassification( + total_counts, split_counts, accumulator, + &best_score, &best_feature_index, &second_best_score); return best_feature_index; } -int32 BestFeatureRegression( +float RegressionSplitScore( + const Eigen::Tensor& splits_count_accessor, + const Eigen::Tensor& totals_count_accessor, + const Eigen::Tensor& splits_sum, + const Eigen::Tensor& splits_square, + const Eigen::Tensor& right_sums, + const Eigen::Tensor& right_squares, + int32 accumulator, + int32 num_regression_dims, int i) { + Eigen::array offsets = {i * num_regression_dims + 1}; + Eigen::array extents = {num_regression_dims - 1}; + float left_count = splits_count_accessor(accumulator, i, 0); + float right_count = totals_count_accessor(accumulator, 0) - left_count; + + float score = 0; + + // Guard against divide-by-zero. + if (left_count > 0) { + score += WeightedVariance( + splits_sum.slice(offsets, extents), + splits_square.slice(offsets, extents), left_count); + } + + if (right_count > 0) { + score += WeightedVariance(right_sums.slice(offsets, extents), + right_squares.slice(offsets, extents), + right_count); + } + return score; +} + +void GetTwoBestRegression( const Tensor& total_sums, const Tensor& total_squares, const Tensor& split_sums, const Tensor& split_squares, - int32 accumulator) { - int32 best_feature_index = -1; - // We choose the split with the lowest score. - float best_score = kint64max; + int32 accumulator, + float *best_score, int *best_index, + float *second_best_score) { const int32 num_splits = static_cast(split_sums.shape().dim_size(1)); const int32 num_regression_dims = static_cast( split_sums.shape().dim_size(2)); @@ -90,43 +161,138 @@ int32 BestFeatureRegression( const auto right_sums = tc_sum.broadcast(bcast) - splits_sum; const auto right_squares = tc_square.broadcast(bcast) - splits_square; - for (int i = 0; i < num_splits; i++) { - Eigen::array offsets; - offsets[0] = i * num_regression_dims; - Eigen::array extents; - extents[0] = num_regression_dims; - float left_count = splits_count_accessor(accumulator, i, 0); - float right_count = totals_count_accessor(accumulator, 0) - left_count; + GetTwoBest( + num_splits, + std::bind(RegressionSplitScore, + splits_count_accessor, totals_count_accessor, + splits_sum, splits_square, right_sums, right_squares, + accumulator, num_regression_dims, std::placeholders::_1), + best_score, best_index, second_best_score); +} - float score = 0; - - // Guard against divide-by-zero. - if (left_count > 0) { - score += WeightedVariance( - splits_sum.slice(offsets, extents), - splits_square.slice(offsets, extents), left_count); - } - - if (right_count > 0) { - score += WeightedVariance(right_sums.slice(offsets, extents), - right_squares.slice(offsets, extents), - right_count); - } - - if (score < best_score) { - best_score = score; - best_feature_index = i; - } - } +int32 BestFeatureRegression( + const Tensor& total_sums, const Tensor& total_squares, + const Tensor& split_sums, const Tensor& split_squares, + int32 accumulator) { + float best_score; + float second_best_score; + int best_feature_index; + GetTwoBestRegression( + total_sums, total_squares, split_sums, split_squares, accumulator, + &best_score, &best_feature_index, &second_best_score); return best_feature_index; } -bool DecideNode(const Tensor& point, int32 feature, float bias) { + +bool BestSplitDominatesRegression( + const Tensor& total_sums, const Tensor& total_squares, + const Tensor& split_sums, const Tensor& split_squares, + int32 accumulator) { + // TODO(thomaswc): Implement this, probably as part of v3. + return false; +} + +bool BestSplitDominatesClassification( + const Tensor& total_counts, + const Tensor& split_counts, int32 accumulator, + float dominate_fraction) { + float best_score; + float second_best_score; + int best_feature_index; + GetTwoBestClassification( + total_counts, split_counts, accumulator, + &best_score, &best_feature_index, &second_best_score); + + // Total counts are stored in the first column. + const int32 num_classes = split_counts.shape().dim_size(2) - 1; + + // total_class_counts(c) is the # of class c examples seen by this + // accumulator. + auto total_class_counts = total_counts.Slice( + accumulator, accumulator + 1).unaligned_flat(); + + const Eigen::Tensor splits = split_counts.Slice( + accumulator, accumulator + 1).unaligned_flat(); + + // For some reason, Eigen is fine with offsets being an array in + // ClassificationSplitScore, but it demands an array here. + const Eigen::array offsets = + {num_classes * best_feature_index}; + const Eigen::array extents = {num_classes}; + + const Eigen::Tensor left_counts = + splits.slice(offsets, extents); + // I can find no other way using Eigen to copy a const Tensor into a + // non-const Tensor. + Eigen::Tensor left_counts_copy(num_classes+1); + for (int i = 0; i <= num_classes; i++) { + left_counts_copy(i) = left_counts(i); + } + + Eigen::Tensor right_counts_copy = + total_class_counts - left_counts_copy; + + // "Reverse-jackknife" estimate of how often the chosen best split is + // truly better than the second best split. We use the reverse jackknife + // (in which counts are incremented) rather than the normal jackknife + // (in which counts are decremented) because the later badly underestimates + // the score variance of perfect splits. + float better_count = 0.0; + float worse_count = 0.0; + for (int i = 1; i <= num_classes; i++) { + left_counts_copy(i) += 1.0; + float weight = left_counts_copy(i); + float v = WeightedGiniImpurity(left_counts_copy) + + WeightedGiniImpurity(right_counts_copy); + left_counts_copy(i) -= 1.0; + if (v < second_best_score) { + better_count += weight; + } else { + worse_count += weight; + } + + right_counts_copy(i) += 1.0; + weight = right_counts_copy(i); + v = WeightedGiniImpurity(left_counts) + + WeightedGiniImpurity(right_counts_copy); + right_counts_copy(i) -= 1.0; + if (v < second_best_score) { + better_count += weight; + } else { + worse_count += weight; + } + } + + VLOG(1) << "Better count = " << better_count; + VLOG(1) << "Worse count = " << worse_count; + return better_count > dominate_fraction * (better_count + worse_count); +} + + +bool DecideNode(const Tensor& point, int32 feature, float bias, + DataColumnTypes type) { const auto p = point.unaligned_flat(); CHECK_LT(feature, p.size()); - return p(feature) > bias; + return Decide(p(feature), bias, type); } + +bool Decide(float value, float bias, DataColumnTypes type) { + switch (type) { + case kDataFloat: + return value > bias; + + case kDataCategorical: + // We arbitrarily define categorical equality as going left. + return value != bias; + + default: + LOG(ERROR) << "Got unknown column type: " << type; + return false; + } +} + + bool IsAllInitialized(const Tensor& features) { const auto feature_vec = features.unaligned_flat(); return feature_vec(feature_vec.size() - 1) >= 0; diff --git a/tensorflow/contrib/tensor_forest/core/ops/tree_utils.h b/tensorflow/contrib/tensor_forest/core/ops/tree_utils.h index 19b02e379e7..067f0768d3c 100644 --- a/tensorflow/contrib/tensor_forest/core/ops/tree_utils.h +++ b/tensorflow/contrib/tensor_forest/core/ops/tree_utils.h @@ -19,6 +19,7 @@ #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/kernels/bounds_check.h" #include "tensorflow/core/lib/strings/strcat.h" #include "tensorflow/core/platform/macros.h" #include "tensorflow/core/platform/types.h" @@ -26,6 +27,7 @@ namespace tensorflow { namespace tensorforest { +// TODO(gilberth): Put these in protos so they can be shared by C++ and python. // Indexes in the tree representation's 2nd dimension for children and features. const int32 CHILDREN_INDEX = 0; const int32 FEATURE_INDEX = 1; @@ -34,6 +36,14 @@ const int32 FEATURE_INDEX = 1; const int32 LEAF_NODE = -1; const int32 FREE_NODE = -2; +// Used to indicate column types, e.g. categorical vs. float +enum DataColumnTypes { + kDataFloat = 0, + kDataCategorical = 1 +}; + + + // Calculates the sum of a tensor. template T Sum(Tensor counts) { @@ -80,6 +90,20 @@ int32 BestFeatureRegression(const Tensor& total_sums, const Tensor& split_sums, const Tensor& split_squares, int32 accumulator); +// Returns true if the best split's variance is sufficiently smaller than +// that of the next best split. +bool BestSplitDominatesRegression( + const Tensor& total_sums, const Tensor& total_squares, + const Tensor& split_sums, const Tensor& split_squares, + int32 accumulator); + +// Returns true if the best split's Gini impurity is sufficiently smaller than +// that of the next best split. +bool BestSplitDominatesClassification( + const Tensor& total_counts, + const Tensor& split_counts, int32 accumulator, + float dominate_fraction); + // Initializes everything in the given tensor to the given value. template void Initialize(Tensor counts, T val = 0) { @@ -90,7 +114,74 @@ void Initialize(Tensor counts, T val = 0) { // Returns true if the point falls to the right (i.e., the selected feature // of the input point is greater than the bias threshold), and false if it // falls to the left. -bool DecideNode(const Tensor& point, int32 feature, float bias); +// Even though our input data is forced into float Tensors, it could have +// originally been something else (e.g. categorical string data) which +// we treat differently. +bool DecideNode(const Tensor& point, int32 feature, float bias, + DataColumnTypes type = kDataFloat); + +// Returns input_data(i, feature) > bias. +template +bool DecideDenseNode(const T& input_data, + int32 i, int32 feature, float bias, + DataColumnTypes type = kDataFloat) { + CHECK_LT(i, input_data.dimensions()[0]); + CHECK_LT(feature, input_data.dimensions()[1]); + return Decide(input_data(i, feature), bias, type); +} + +// If T is a sparse float matrix represented by sparse_input_indices and +// sparse_input_values, FindSparseValue returns T(i,j), or 0.0 if (i,j) +// isn't present in sparse_input_indices. sparse_input_indices is assumed +// to be sorted. +template +float FindSparseValue( + const T1& sparse_input_indices, + const T2& sparse_input_values, + int32 i, int32 j) { + int32 low = 0; + int32 high = sparse_input_values.dimension(0); + while (low < high) { + int32 mid = (low + high) / 2; + int64 midi = internal::SubtleMustCopy(sparse_input_indices(mid, 0)); + int64 midj = internal::SubtleMustCopy(sparse_input_indices(mid, 1)); + if (midi == i) { + if (midj == j) { + return sparse_input_values(mid); + } + if (midj < j) { + low = mid + 1; + } else { + high = mid; + } + continue; + } + if (midi < i) { + low = mid + 1; + } else { + high = mid; + } + } + return 0.0; +} + +// Returns t(i, feature) > bias, where t is the sparse tensor represented by +// sparse_input_indices and sparse_input_values. +template +bool DecideSparseNode( + const T1& sparse_input_indices, + const T2& sparse_input_values, + int32 i, int32 feature, float bias, + DataColumnTypes type = kDataFloat) { + return Decide( + FindSparseValue(sparse_input_indices, sparse_input_values, i, feature), + bias, type); +} + +// Returns left/right decision between the input value and the threshold bias. +// For floating point types, the decision is value > bias, but for +// categorical data, it is value != bias. +bool Decide(float value, float bias, DataColumnTypes type = kDataFloat); // Returns true if all the splits are initialized. Since they get initialized // in order, we can simply infer this from the last split. diff --git a/tensorflow/contrib/tensor_forest/core/ops/update_fertile_slots_op.cc b/tensorflow/contrib/tensor_forest/core/ops/update_fertile_slots_op.cc index 026262e47ff..33638ca7e67 100644 --- a/tensorflow/contrib/tensor_forest/core/ops/update_fertile_slots_op.cc +++ b/tensorflow/contrib/tensor_forest/core/ops/update_fertile_slots_op.cc @@ -36,7 +36,7 @@ using tensorforest::Initialize; using tensorforest::WeightedGiniImpurity; REGISTER_OP("UpdateFertileSlots") - .Attr("max_depth: int") + .Attr("max_depth: int") .Attr("regression: bool = False") .Input("finished: int32") .Input("non_fertile_leaves: int32") @@ -45,11 +45,10 @@ REGISTER_OP("UpdateFertileSlots") .Input("tree_depths: int32") .Input("accumulator_sums: float") .Input("node_to_accumulator: int32") + .Input("stale_leaves: int32") .Output("node_map_updates: int32") .Output("accumulators_cleared: int32") .Output("accumulators_allocated: int32") - .Output("new_nonfertile_leaves: int32") - .Output("new_nonfertile_leaves_scores: float") .Doc(R"doc( Updates accumulator slots to reflect finished or newly fertile nodes. @@ -77,6 +76,8 @@ accumulator_sums: For classification, `accumulator_sums[a][c]` records how of training examples that have been seen. node_to_accumulator: `node_to_accumulator[i]` is the accumulator slot used by fertile node i, or -1 if node i isn't fertile. +stale_leaves:= A 1-d int32 tensor containing the indices of all leaves that + have stopped accumulating statistics because they are too old. node_map_updates:= A 2-d int32 tensor describing the changes that need to be applied to the node_to_accumulator map. Intended to be used with `tf.scatter_update(node_to_accumulator, @@ -86,10 +87,7 @@ accumulators_cleared:= A 1-d int32 tensor containing the indices of all the accumulator slots that need to be cleared. accumulators_allocated:= A 1-d int32 tensor containing the indices of all the accumulator slots that need to be allocated. -new_nonfertile_leaves:= A 1-d int32 tensor containing the indices of all the - leaves that are now non-fertile. -new_nonfertile_leaves_scores: `new_nonfertile_leaves_scores[i]` contains the - splitting score for the non-fertile leaf `new_nonfertile_leaves[i]`. + )doc"); class UpdateFertileSlots : public OpKernel { @@ -112,6 +110,7 @@ class UpdateFertileSlots : public OpKernel { const Tensor& accumulator_sums = context->input(5); const Tensor& node_to_accumulator = context->input(6); + const Tensor& stale_leaves = context->input(7); OP_REQUIRES(context, finished.shape().dims() == 1, errors::InvalidArgument( @@ -134,6 +133,9 @@ class UpdateFertileSlots : public OpKernel { OP_REQUIRES(context, node_to_accumulator.shape().dims() == 1, errors::InvalidArgument( "node_to_accumulator should be one-dimensional")); + OP_REQUIRES(context, stale_leaves.shape().dims() == 1, + errors::InvalidArgument( + "stale_leaves should be one-dimensional")); OP_REQUIRES( context, @@ -151,6 +153,7 @@ class UpdateFertileSlots : public OpKernel { if (!CheckTensorBounds(context, tree_depths)) return; if (!CheckTensorBounds(context, accumulator_sums)) return; if (!CheckTensorBounds(context, node_to_accumulator)) return; + if (!CheckTensorBounds(context, stale_leaves)) return; // Read finished accumulators into a set for quick lookup. const auto node_map = node_to_accumulator.unaligned_flat(); @@ -164,6 +167,16 @@ class UpdateFertileSlots : public OpKernel { errors::InvalidArgument("finished node is outside the valid range")); finished_accumulators.insert(node_map(node)); } + // Stale accumulators are also finished for the purposes of clearing + // and re-allocating. + const auto stale_vec = stale_leaves.unaligned_flat(); + for (int32 i = 0; i < stale_vec.size(); ++i) { + const int32 node = internal::SubtleMustCopy(stale_vec(i)); + OP_REQUIRES( + context, FastBoundsCheck(node, node_map.size()), + errors::InvalidArgument("stale node is outside the valid range")); + finished_accumulators.insert(node_map(node)); + } // Construct leaf heap to sort leaves to allocate accumulators to. const int32 num_nodes = static_cast(tree_depths.shape().dim_size(0)); @@ -210,11 +223,10 @@ class UpdateFertileSlots : public OpKernel { } // Construct and fill outputs. - SetNodeMapUpdates(accumulators_to_node, finished, context); + SetNodeMapUpdates(accumulators_to_node, finished, stale_leaves, context); SetAccumulatorsCleared(finished_accumulators, accumulators_to_node, context); SetAccumulatorsAllocated(accumulators_to_node, context); - SetNewNonFertileLeaves(values.get(), i, context); } private: @@ -228,18 +240,20 @@ class UpdateFertileSlots : public OpKernel { typedef TopN, OrderBySecondGreater> LeafHeapType; typedef std::vector> HeapValuesType; - // Creates an update tensor for node to accumulator map. Sets finished nodes - // to -1 (no accumulator assigned) and newly allocated nodes to their - // accumulator. + // Creates an update tensor for node to accumulator map. Sets finished and + // stale nodes to -1 (no accumulator assigned) and newly allocated nodes to + // their accumulator. void SetNodeMapUpdates( const std::unordered_map& accumulators_to_node, - const Tensor& finished, OpKernelContext* context) { + const Tensor& finished, const Tensor& stale, OpKernelContext* context) { // Node map updates. Tensor* output_node_map = nullptr; TensorShape node_map_shape; node_map_shape.AddDim(2); - node_map_shape.AddDim(accumulators_to_node.size() + - static_cast(finished.shape().dim_size(0))); + node_map_shape.AddDim( + accumulators_to_node.size() + + static_cast(stale.shape().dim_size(0) + + finished.shape().dim_size(0))); OP_REQUIRES_OK(context, context->allocate_output(0, node_map_shape, &output_node_map)); @@ -254,6 +268,13 @@ class UpdateFertileSlots : public OpKernel { out_node(1, output_slot) = -1; ++output_slot; } + // Set stale nodes to -1. + const auto stale_vec = stale.unaligned_flat(); + for (int32 i = 0; i < stale_vec.size(); ++i) { + out_node(0, output_slot) = stale_vec(i); + out_node(1, output_slot) = -1; + ++output_slot; + } // Set newly allocated nodes to their allocator. for (const auto& node_alloc_pair : accumulators_to_node) { @@ -315,56 +336,6 @@ class UpdateFertileSlots : public OpKernel { } } - // Creates output tensors for non-fertile leaves and non-fertile leaf scores. - // Start indicates the index in values where the leaves that weren't - // allocated this round begin, and should thus be placed in the new - // nonfertile_leaves tensors. - void SetNewNonFertileLeaves(HeapValuesType* values, int32 start, - OpKernelContext* context) { - // Node map updates. - int32 num_values = static_cast(values->size()) - start; - - // Unfortunately, a zero-sized Variable results in an uninitialized - // error, probably because they check for zero size instead of - // a real inititalization condition. - bool fill_with_garbage = false; - if (num_values == 0) { - num_values = 1; - fill_with_garbage = true; - } - Tensor* output_nonfertile_leaves = nullptr; - TensorShape nonfertile_leaves_shape; - nonfertile_leaves_shape.AddDim(num_values); - OP_REQUIRES_OK(context, - context->allocate_output(3, nonfertile_leaves_shape, - &output_nonfertile_leaves)); - - auto out_nonfertile_leaves = - output_nonfertile_leaves->unaligned_flat(); - - Tensor* output_nonfertile_leaves_scores = nullptr; - TensorShape nonfertile_leaves_scores_shape; - nonfertile_leaves_scores_shape.AddDim(num_values); - OP_REQUIRES_OK(context, - context->allocate_output(4, nonfertile_leaves_scores_shape, - &output_nonfertile_leaves_scores)); - - auto out_nonfertile_leaves_scores = - output_nonfertile_leaves_scores->unaligned_flat(); - - if (fill_with_garbage) { - out_nonfertile_leaves(0) = -1; - out_nonfertile_leaves_scores(0) = 0.0; - return; - } - - for (int32 i = start; i < values->size(); ++i) { - const std::pair& node = (*values)[i]; - out_nonfertile_leaves(i -start) = node.first; - out_nonfertile_leaves_scores(i - start) = node.second; - } - } - void ConstructLeafHeap(const Tensor& non_fertile_leaves, const Tensor& non_fertile_leaf_scores, const Tensor& tree_depths, int32 end_of_tree, diff --git a/tensorflow/contrib/tensor_forest/data/__init__.py b/tensorflow/contrib/tensor_forest/data/__init__.py new file mode 100644 index 00000000000..3d04705878d --- /dev/null +++ b/tensorflow/contrib/tensor_forest/data/__init__.py @@ -0,0 +1,21 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Random forest implementation in tensorflow.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +# pylint: disable=unused-import,wildcard-import +from tensorflow.contrib.tensor_forest.data import data_ops diff --git a/tensorflow/contrib/tensor_forest/data/data_ops.py b/tensorflow/contrib/tensor_forest/data/data_ops.py new file mode 100644 index 00000000000..ca229f4ce93 --- /dev/null +++ b/tensorflow/contrib/tensor_forest/data/data_ops.py @@ -0,0 +1,109 @@ +# pylint: disable=g-bad-file-header +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Ops for preprocessing data.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import threading + +from tensorflow.contrib.tensor_forest.python import constants + +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import load_library +from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.platform import resource_loader +from tensorflow.python.platform import tf_logging as logging + +DATA_OPS_FILE = '_data_ops.so' + +_data_ops = None +_ops_lock = threading.Lock() + + +ops.NoGradient('StringToFloat') + + +@ops.RegisterShape('StringToFloat') +def StringToFloatShape(op): + """Shape function for StringToFloat Op.""" + return [op.inputs[0].get_shape()] + + +# Workaround for the fact that importing tensorflow imports contrib +# (even if a user isn't using this or any other contrib op), but +# there's not yet any guarantee that the shared object exists. +# In which case, "import tensorflow" will always crash, even for users that +# never use contrib. +def Load(): + """Load the data ops library and return the loaded module.""" + with _ops_lock: + global _data_ops + if not _data_ops: + ops_path = resource_loader.get_path_to_datafile(DATA_OPS_FILE) + logging.info('data path: %s', ops_path) + _data_ops = load_library.load_op_library(ops_path) + + assert _data_ops, 'Could not load _data_ops.so' + return _data_ops + + +def ParseDataTensorOrDict(data): + """Return a tensor to use for input data. + + The incoming features can be a dict where keys are the string names of the + columns, which we turn into a single 2-D tensor. + + Args: + data: `Tensor` or `dict` of `Tensor` objects. + + Returns: + A 2-D tensor for input to tensor_forest and a 1-D tensor of the + type of each column (e.g. continuous float, categorical). + """ + convert_ops = Load() + if isinstance(data, dict): + data_spec = [constants.DATA_CATEGORICAL if data[k].dtype == dtypes.string + else constants.DATA_FLOAT + for k in sorted(data.keys())] + return array_ops.concat(1, [ + convert_ops.string_to_float(data[k]) + if data[k].dtype == dtypes.string else data[k] + for k in sorted(data.keys())]), data_spec + else: + return data, [constants.DATA_FLOAT] * data.get_shape().as_list()[1] + + +def ParseLabelTensorOrDict(labels): + """Return a tensor to use for input labels to tensor_forest. + + The incoming targets can be a dict where keys are the string names of the + columns, which we turn into a single 1-D tensor for classification or + 2-D tensor for regression. + + Args: + labels: `Tensor` or `dict` of `Tensor` objects. + + Returns: + A 2-D tensor for labels/outputs. + """ + if isinstance(labels, dict): + return math_ops.to_float(array_ops.concat( + 1, [labels[k] for k in sorted(labels.keys())])) + else: + return math_ops.to_float(labels) diff --git a/tensorflow/contrib/tensor_forest/data/string_to_float_op.cc b/tensorflow/contrib/tensor_forest/data/string_to_float_op.cc new file mode 100644 index 00000000000..3908855063d --- /dev/null +++ b/tensorflow/contrib/tensor_forest/data/string_to_float_op.cc @@ -0,0 +1,111 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================= +// Converts strings of arbitrary length to float values by +// hashing and cramming bits. +#include + +#include "tensorflow/contrib/tensor_forest/core/ops/tree_utils.h" + +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/lib/strings/numbers.h" +#include "tensorflow/core/lib/strings/strcat.h" + +#include "tensorflow/core/util/work_sharder.h" + +namespace tensorflow { + +using tensorforest::CheckTensorBounds; + + +float Convert(const string& in) { + const std::size_t intval = std::hash()(in); + return static_cast(intval); +} + + +void Evaluate(const Tensor& input_data, Tensor output_data, + int32 start, int32 end) { + auto out_data = output_data.tensor(); + const auto in_data = input_data.tensor(); + + for (int32 i = start; i < end; ++i) { + for (int32 j = 0; j < output_data.dim_size(1); ++j) { + out_data(i, j) = Convert(in_data(i, j)); + } + } +} + + +REGISTER_OP("StringToFloat") + .Input("input_data: string") + .Output("output_data: float") + + .Doc(R"doc( + Converts byte arrays represented by strings to 32-bit + floating point numbers. The output numbers themselves are meaningless, and + should only be used in == comparisons. + + input_data: A batch of string features as a 2-d tensor; `input_data[i][j]` + gives the j-th feature of the i-th input. + output_data: A tensor of the same shape as input_data but the values are + float32. + +)doc"); + +class StringToFloat : public OpKernel { + public: + explicit StringToFloat(OpKernelConstruction* context) + : OpKernel(context) {} + + void Compute(OpKernelContext* context) override { + const Tensor& input_data = context->input(0); + + // Check inputs. + OP_REQUIRES(context, input_data.shape().dims() == 2, + errors::InvalidArgument( + "input_data should be two-dimensional")); + + // Check tensor bounds. + if (!CheckTensorBounds(context, input_data)) return; + + Tensor* output_data = nullptr; + OP_REQUIRES_OK(context, + context->allocate_output(0, input_data.shape(), + &output_data)); + + // Evaluate input data in parallel. + const int32 num_data = static_cast(input_data.shape().dim_size(0)); + auto worker_threads = context->device()->tensorflow_cpu_worker_threads(); + int num_threads = worker_threads->num_threads; + if (num_threads <= 1) { + Evaluate(input_data, *output_data, 0, num_data); + } else { + auto work = [&input_data, output_data, num_data](int64 start, int64 end) { + CHECK(start <= end); + CHECK(end <= num_data); + Evaluate(input_data, *output_data, + static_cast(start), static_cast(end)); + }; + Shard(num_threads, worker_threads->workers, num_data, 100, work); + } + } +}; + + +REGISTER_KERNEL_BUILDER(Name("StringToFloat").Device(DEVICE_CPU), + StringToFloat); + +} // namespace tensorflow diff --git a/tensorflow/contrib/tensor_forest/python/__init__.py b/tensorflow/contrib/tensor_forest/python/__init__.py index 0f692bbe972..a9dd599c970 100644 --- a/tensorflow/contrib/tensor_forest/python/__init__.py +++ b/tensorflow/contrib/tensor_forest/python/__init__.py @@ -18,6 +18,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.contrib.tensor_forest.python import constants from tensorflow.contrib.tensor_forest.python import tensor_forest from tensorflow.contrib.tensor_forest.python.ops import inference_ops from tensorflow.contrib.tensor_forest.python.ops import training_ops diff --git a/tensorflow/contrib/tensor_forest/python/constants.py b/tensorflow/contrib/tensor_forest/python/constants.py new file mode 100644 index 00000000000..029c7824615 --- /dev/null +++ b/tensorflow/contrib/tensor_forest/python/constants.py @@ -0,0 +1,26 @@ +# pylint: disable=g-bad-file-header +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Constants used by tensorforest. Some of these map to values in C++ ops.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +# If tree[i][0] equals this value, then i is a leaf node. +LEAF_NODE = -1 + +# Data column types for indicating categorical or other non-float values. +DATA_FLOAT = 0 +DATA_CATEGORICAL = 1 diff --git a/tensorflow/contrib/tensor_forest/python/kernel_tests/best_splits_op_test.py b/tensorflow/contrib/tensor_forest/python/kernel_tests/best_splits_op_test.py index c5b5981adba..3641ab0ee06 100644 --- a/tensorflow/contrib/tensor_forest/python/kernel_tests/best_splits_op_test.py +++ b/tensorflow/contrib/tensor_forest/python/kernel_tests/best_splits_op_test.py @@ -30,14 +30,16 @@ class BestSplitsClassificationTests(test_util.TensorFlowTestCase): def setUp(self): self.finished = [3, 5] self.node_map = [-1, -1, -1, 0, -1, 3, -1, -1, -1] - self.candidate_counts = [[[50., 60., 40., 3.], [70., 30., 70., 30.]], - [[0., 0., 0., 0.], [0., 0., 0., 0.]], - [[0., 0., 0., 0.], [0., 0., 0., 0.]], - [[10., 10., 10., 10.], [10., 5., 5., 10.]]] - self.total_counts = [[100., 100., 100., 100.], - [0., 0., 0., 0.], - [0., 0., 0., 0.], - [100., 100., 100., 100.]] + self.candidate_counts = [[[153., 50., 60., 40., 3.], + [200., 70., 30., 70., 30.]], + [[0., 0., 0., 0., 0.], [0., 0., 0., 0., 0.]], + [[0., 0., 0., 0., 0.], [0., 0., 0., 0., 0.]], + [[40., 10., 10., 10., 10.], + [30., 10., 5., 5., 10.]]] + self.total_counts = [[400., 100., 100., 100., 100.], + [0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0.], + [400., 100., 100., 100., 100.]] self.squares = [] self.ops = training_ops.Load() diff --git a/tensorflow/contrib/tensor_forest/python/kernel_tests/count_extremely_random_stats_op_test.py b/tensorflow/contrib/tensor_forest/python/kernel_tests/count_extremely_random_stats_op_test.py index eb61573f24f..a50eb22795c 100644 --- a/tensorflow/contrib/tensor_forest/python/kernel_tests/count_extremely_random_stats_op_test.py +++ b/tensorflow/contrib/tensor_forest/python/kernel_tests/count_extremely_random_stats_op_test.py @@ -19,6 +19,7 @@ from __future__ import print_function import tensorflow as tf +from tensorflow.contrib.tensor_forest.python import constants from tensorflow.contrib.tensor_forest.python.ops import training_ops from tensorflow.python.framework import test_util @@ -37,16 +38,20 @@ class CountExtremelyRandomStatsClassificationTest(test_util.TensorFlowTestCase): self.split_features = [[1], [-1]] self.split_thresholds = [[1.], [0.]] self.ops = training_ops.Load() + self.epochs = [0, 1, 1] + self.current_epoch = [1] + self.data_spec = [constants.DATA_FLOAT] * 2 def testSimple(self): with self.test_session(): (pcw_node_sums, _, pcw_splits_indices, pcw_splits_sums, _, pcw_totals_indices, pcw_totals_sums, _, leaves) = ( self.ops.count_extremely_random_stats( - self.input_data, self.input_labels, self.tree, - self.tree_thresholds, self.node_map, - self.split_features, self.split_thresholds, num_classes=5, - regression=False)) + self.input_data, [], [], [], self.data_spec, self.input_labels, + self.tree, self.tree_thresholds, self.node_map, + self.split_features, self.split_thresholds, self.epochs, + self.current_epoch, + num_classes=5, regression=False)) self.assertAllEqual( [[4., 1., 1., 1., 1.], [2., 1., 1., 0., 0.], [2., 0., 0., 1., 1.]], @@ -57,15 +62,68 @@ class CountExtremelyRandomStatsClassificationTest(test_util.TensorFlowTestCase): self.assertAllEqual([1., 2., 1.], pcw_totals_sums.eval()) self.assertAllEqual([1, 1, 2, 2], leaves.eval()) + def testSparseInput(self): + sparse_shape = [4, 10] + sparse_indices = [[0, 0], [0, 4], [0, 9], + [1, 0], [1, 7], + [2, 0], + [3, 1], [3, 4]] + sparse_values = [3.0, -1.0, 0.5, + 1.5, 6.0, + -2.0, + -0.5, 2.0] + with self.test_session(): + (pcw_node_sums, _, pcw_splits_indices, pcw_splits_sums, _, + pcw_totals_indices, pcw_totals_sums, _, leaves) = ( + self.ops.count_extremely_random_stats( + [], sparse_indices, sparse_values, sparse_shape, self.data_spec, + self.input_labels, self.tree, + self.tree_thresholds, self.node_map, + self.split_features, self.split_thresholds, self.epochs, + self.current_epoch, + num_classes=5, regression=False)) + + self.assertAllEqual( + [[4., 1., 1., 1., 1.], + [2., 0., 0., 1., 1.], + [2., 1., 1., 0., 0.]], + pcw_node_sums.eval()) + self.assertAllEqual([[0, 0, 4], [0, 0, 0], [0, 0, 3]], + pcw_splits_indices.eval()) + self.assertAllEqual([1., 2., 1.], pcw_splits_sums.eval()) + self.assertAllEqual([[0, 4], [0, 0], [0, 3]], pcw_totals_indices.eval()) + self.assertAllEqual([1., 2., 1.], pcw_totals_sums.eval()) + self.assertAllEqual([2, 2, 1, 1], leaves.eval()) + + def testFutureEpoch(self): + current_epoch = [3] + with self.test_session(): + (pcw_node_sums, _, _, pcw_splits_sums, _, + _, pcw_totals_sums, _, leaves) = ( + self.ops.count_extremely_random_stats( + self.input_data, [], [], [], self.data_spec, self.input_labels, + self.tree, self.tree_thresholds, self.node_map, + self.split_features, self.split_thresholds, self.epochs, + current_epoch, num_classes=5, regression=False)) + + self.assertAllEqual( + [[0., 0., 0., 0., 0.], [0., 0., 0., 0., 0.], [0., 0., 0., 0., 0.]], + pcw_node_sums.eval()) + self.assertAllEqual([], pcw_splits_sums.eval()) + self.assertAllEqual([], pcw_totals_sums.eval()) + self.assertAllEqual([1, 1, 2, 2], leaves.eval()) + def testThreaded(self): with self.test_session( config=tf.ConfigProto(intra_op_parallelism_threads=2)): (pcw_node_sums, _, pcw_splits_indices, pcw_splits_sums, _, pcw_totals_indices, pcw_totals_sums, _, leaves) = ( self.ops.count_extremely_random_stats( - self.input_data, self.input_labels, self.tree, - self.tree_thresholds, self.node_map, self.split_features, - self.split_thresholds, num_classes=5, regression=False)) + self.input_data, [], [], [], self.data_spec, self.input_labels, + self.tree, self.tree_thresholds, self.node_map, + self.split_features, + self.split_thresholds, self.epochs, self.current_epoch, + num_classes=5, regression=False)) self.assertAllEqual([[4., 1., 1., 1., 1.], [2., 1., 1., 0., 0.], [2., 0., 0., 1., 1.]], @@ -81,10 +139,10 @@ class CountExtremelyRandomStatsClassificationTest(test_util.TensorFlowTestCase): (pcw_node_sums, _, pcw_splits_indices, pcw_splits_sums, _, pcw_totals_indices, pcw_totals_sums, _, leaves) = ( self.ops.count_extremely_random_stats( - self.input_data, self.input_labels, self.tree, - self.tree_thresholds, [-1] * 3, - self.split_features, self.split_thresholds, num_classes=5, - regression=False)) + self.input_data, [], [], [], self.data_spec, self.input_labels, + self.tree, self.tree_thresholds, [-1] * 3, + self.split_features, self.split_thresholds, self.epochs, + self.current_epoch, num_classes=5, regression=False)) self.assertAllEqual([[4., 1., 1., 1., 1.], [2., 1., 1., 0., 0.], [2., 0., 0., 1., 1.]], @@ -101,13 +159,13 @@ class CountExtremelyRandomStatsClassificationTest(test_util.TensorFlowTestCase): with self.test_session(): with self.assertRaisesOpError( 'Number of nodes should be the same in ' - 'tree, tree_thresholds, and node_to_accumulator'): + 'tree, tree_thresholds, node_to_accumulator, and birth_epoch.'): pcw_node, _, _, _, _, _, _, _, _ = ( self.ops.count_extremely_random_stats( - self.input_data, self.input_labels, self.tree, - self.tree_thresholds, self.node_map, - self.split_features, self.split_thresholds, num_classes=5, - regression=False)) + self.input_data, [], [], [], self.data_spec, self.input_labels, + self.tree, self.tree_thresholds, self.node_map, + self.split_features, self.split_thresholds, self.epochs, + self.current_epoch, num_classes=5, regression=False)) self.assertAllEqual([], pcw_node.eval()) @@ -124,6 +182,9 @@ class CountExtremelyRandomStatsRegressionTest(test_util.TensorFlowTestCase): self.split_features = [[1], [-1]] self.split_thresholds = [[1.], [0.]] self.ops = training_ops.Load() + self.epochs = [0, 1, 1] + self.current_epoch = [1] + self.data_spec = [constants.DATA_FLOAT] * 2 def testSimple(self): with self.test_session(): @@ -131,10 +192,10 @@ class CountExtremelyRandomStatsRegressionTest(test_util.TensorFlowTestCase): pcw_splits_squares, pcw_totals_indices, pcw_totals_sums, pcw_totals_squares, leaves) = ( self.ops.count_extremely_random_stats( - self.input_data, self.input_labels, self.tree, - self.tree_thresholds, self.node_map, - self.split_features, self.split_thresholds, num_classes=2, - regression=True)) + self.input_data, [], [], [], self.data_spec, self.input_labels, + self.tree, self.tree_thresholds, self.node_map, + self.split_features, self.split_thresholds, self.epochs, + self.current_epoch, num_classes=2, regression=True)) self.assertAllEqual( [[4., 14.], [2., 9.], [2., 5.]], pcw_node_sums.eval()) diff --git a/tensorflow/contrib/tensor_forest/python/kernel_tests/finished_nodes_op_test.py b/tensorflow/contrib/tensor_forest/python/kernel_tests/finished_nodes_op_test.py index 24fbe2c11d6..222ef2b2eb7 100644 --- a/tensorflow/contrib/tensor_forest/python/kernel_tests/finished_nodes_op_test.py +++ b/tensorflow/contrib/tensor_forest/python/kernel_tests/finished_nodes_op_test.py @@ -30,35 +30,71 @@ class FinishedNodesTest(test_util.TensorFlowTestCase): def setUp(self): self.leaves = [1, 3, 4] self.node_map = [-1, -1, -1, 0, 1, -1] - self.pcw_total_splits = [[6, 3, 3], [11, 4, 7], [0, 0, 0], [0, 0, 0], + self.split_sums = [ + # Accumulator 1 + [[3, 0, 3], [2, 1, 1], [3, 1, 2]], + # Accumulator 2 + [[6, 3, 3], [6, 2, 4], [5, 0, 5]], + # Accumulator 3 + [[0, 0, 0], [0, 0, 0], [0, 0, 0]], + # Accumulator 4 + [[0, 0, 0], [0, 0, 0], [0, 0, 0]], + # Accumulator 5 + [[0, 0, 0], [0, 0, 0], [0, 0, 0]] + ] + self.split_squares = [] + self.accumulator_sums = [[6, 3, 3], [11, 4, 7], [0, 0, 0], [0, 0, 0], [0, 0, 0]] + self.accumulator_squares = [] self.ops = training_ops.Load() + self.birth_epochs = [0, 0, 0, 1, 1, 1] + self.current_epoch = [1] def testSimple(self): with self.test_session(): - finished = self.ops.finished_nodes(self.leaves, self.node_map, - self.pcw_total_splits, - num_split_after_samples=10) + finished, stale = self.ops.finished_nodes( + self.leaves, self.node_map, self.split_sums, + self.split_squares, self.accumulator_sums, self.accumulator_squares, + self.birth_epochs, self.current_epoch, + regression=False, num_split_after_samples=10, min_split_samples=10) self.assertAllEqual([4], finished.eval()) + self.assertAllEqual([], stale.eval()) def testNoAccumulators(self): with self.test_session(): - finished = self.ops.finished_nodes(self.leaves, [-1] * 6, - self.pcw_total_splits, - num_split_after_samples=10) + finished, stale = self.ops.finished_nodes( + self.leaves, [-1] * 6, self.split_sums, + self.split_squares, self.accumulator_sums, self.accumulator_squares, + self.birth_epochs, self.current_epoch, + regression=False, num_split_after_samples=10, min_split_samples=10) self.assertAllEqual([], finished.eval()) + self.assertAllEqual([], stale.eval()) def testBadInput(self): with self.test_session(): with self.assertRaisesOpError( 'leaf_tensor should be one-dimensional'): - finished = self.ops.finished_nodes([self.leaves], self.node_map, - self.pcw_total_splits, - num_split_after_samples=10) + finished, stale = self.ops.finished_nodes( + [self.leaves], self.node_map, self.split_sums, + self.split_squares, self.accumulator_sums, self.accumulator_squares, + self.birth_epochs, self.current_epoch, + regression=False, num_split_after_samples=10, min_split_samples=10) self.assertAllEqual([], finished.eval()) + self.assertAllEqual([], stale.eval()) + + def testEarlyDominates(self): + with self.test_session(): + finished, stale = self.ops.finished_nodes( + self.leaves, self.node_map, self.split_sums, + self.split_squares, self.accumulator_sums, self.accumulator_squares, + self.birth_epochs, self.current_epoch, + regression=False, num_split_after_samples=10, min_split_samples=5) + + self.assertAllEqual([4], finished.eval()) + self.assertAllEqual([], stale.eval()) if __name__ == '__main__': googletest.main() diff --git a/tensorflow/contrib/tensor_forest/python/kernel_tests/sample_inputs_op_test.py b/tensorflow/contrib/tensor_forest/python/kernel_tests/sample_inputs_op_test.py index 0bbd94a2a4a..9830651a5d0 100644 --- a/tensorflow/contrib/tensor_forest/python/kernel_tests/sample_inputs_op_test.py +++ b/tensorflow/contrib/tensor_forest/python/kernel_tests/sample_inputs_op_test.py @@ -41,7 +41,8 @@ class SampleInputsTest(test_util.TensorFlowTestCase): tf.initialize_all_variables().run() indices, feature_updates, threshold_updates = ( self.ops.sample_inputs( - self.input_data, self.node_map, self.leaves, self.split_features, + self.input_data, [], [], [], + self.node_map, self.leaves, self.split_features, self.split_thresholds, split_initializations_per_input=1, split_sampling_random_seed=3)) self.assertAllEqual([1, 0], indices.eval()) @@ -50,12 +51,38 @@ class SampleInputsTest(test_util.TensorFlowTestCase): self.assertAllEqual([[5., -2., 50.], [-1., -10., 0.]], threshold_updates.eval()) + def testSparse(self): + sparse_shape = [4, 10] + sparse_indices = [[0, 0], [0, 4], [0, 9], + [1, 0], [1, 7], + [2, 0], + [3, 1], [3, 4]] + sparse_values = [3.0, -1.0, 0.5, + 1.5, 6.0, + -2.0, + -0.5, 2.0] + + with self.test_session(): + tf.initialize_all_variables().run() + indices, feature_updates, threshold_updates = ( + self.ops.sample_inputs( + [], sparse_indices, sparse_values, sparse_shape, + self.node_map, self.leaves, self.split_features, + self.split_thresholds, split_initializations_per_input=1, + split_sampling_random_seed=3)) + self.assertAllEqual([1, 0], indices.eval()) + self.assertAllEqual([[1, 0, 0], [4, 7, -1]], + feature_updates.eval()) + self.assertAllEqual([[5., -2., -2.], [-1., 6., 0.]], + threshold_updates.eval()) + def testNoAccumulators(self): with self.test_session(): tf.initialize_all_variables().run() indices, feature_updates, threshold_updates = ( self.ops.sample_inputs( - self.input_data, [-1] * 3, self.leaves, self.split_features, + self.input_data, [], [], [], + [-1] * 3, self.leaves, self.split_features, self.split_thresholds, split_initializations_per_input=1, split_sampling_random_seed=3)) self.assertAllEqual([], indices.eval()) @@ -69,7 +96,8 @@ class SampleInputsTest(test_util.TensorFlowTestCase): with self.assertRaisesOpError( 'split_features and split_thresholds should be the same shape.'): indices, _, _ = self.ops.sample_inputs( - self.input_data, self.node_map, self.leaves, self.split_features, + self.input_data, [], [], [], + self.node_map, self.leaves, self.split_features, self.split_thresholds, split_initializations_per_input=1, split_sampling_random_seed=3) self.assertAllEqual([], indices.eval()) diff --git a/tensorflow/contrib/tensor_forest/python/kernel_tests/tree_predictions_op_test.py b/tensorflow/contrib/tensor_forest/python/kernel_tests/tree_predictions_op_test.py index e61085657a1..aaead5610f5 100644 --- a/tensorflow/contrib/tensor_forest/python/kernel_tests/tree_predictions_op_test.py +++ b/tensorflow/contrib/tensor_forest/python/kernel_tests/tree_predictions_op_test.py @@ -19,6 +19,7 @@ from __future__ import print_function import tensorflow # pylint: disable=unused-import +from tensorflow.contrib.tensor_forest.python import constants from tensorflow.contrib.tensor_forest.python.ops import inference_ops from tensorflow.python.framework import test_util @@ -29,6 +30,7 @@ class TreePredictionsTest(test_util.TensorFlowTestCase): def setUp(self): self.ops = inference_ops.Load() + self.data_spec = [constants.DATA_FLOAT] * 2 def testSimple(self): input_data = [[-1., 0.], [-1., 2.], # node 1 @@ -41,13 +43,65 @@ class TreePredictionsTest(test_util.TensorFlowTestCase): with self.test_session(): predictions = self.ops.tree_predictions( - input_data, tree, tree_thresholds, node_pcw, - valid_leaf_threshold=1) + input_data, [], [], [], self.data_spec, tree, tree_thresholds, + node_pcw, valid_leaf_threshold=1) self.assertAllClose([[0.1, 0.1, 0.8], [0.1, 0.1, 0.8], [0.5, 0.25, 0.25], [0.5, 0.25, 0.25]], predictions.eval()) + def testSparseInput(self): + sparse_shape = [3, 10] + sparse_indices = [[0, 0], [0, 4], [0, 9], + [1, 0], [1, 7], + [2, 0]] + sparse_values = [3.0, -1.0, 0.5, + 1.5, 6.0, + -2.0] + sparse_data_spec = [constants.DATA_FLOAT] * 10 + + tree = [[1, 0], [-1, 0], [-1, 0]] + tree_thresholds = [0., 0., 0.] + node_pcw = [[1.0, 0.3, 0.4, 0.3], [1.0, 0.1, 0.1, 0.8], + [1.0, 0.5, 0.25, 0.25]] + + with self.test_session(): + predictions = self.ops.tree_predictions( + [], sparse_indices, sparse_values, sparse_shape, sparse_data_spec, + tree, tree_thresholds, node_pcw, + valid_leaf_threshold=1) + + self.assertAllClose([[0.5, 0.25, 0.25], + [0.5, 0.25, 0.25], + [0.1, 0.1, 0.8]], + predictions.eval()) + + def testSparseInputDefaultIsZero(self): + sparse_shape = [3, 10] + sparse_indices = [[0, 0], [0, 4], [0, 9], + [1, 0], [1, 7], + [2, 0]] + sparse_values = [3.0, -1.0, 0.5, + 1.5, 6.0, + -2.0] + sparse_data_spec = [constants.DATA_FLOAT] * 10 + + tree = [[1, 7], [-1, 0], [-1, 0]] + tree_thresholds = [3.0, 0., 0.] + node_pcw = [[1.0, 0.3, 0.4, 0.3], [1.0, 0.1, 0.1, 0.8], + [1.0, 0.5, 0.25, 0.25]] + + with self.test_session(): + predictions = self.ops.tree_predictions( + [], sparse_indices, sparse_values, sparse_shape, sparse_data_spec, + tree, tree_thresholds, node_pcw, + valid_leaf_threshold=1) + + self.assertAllClose([[0.1, 0.1, 0.8], + [0.5, 0.25, 0.25], + [0.1, 0.1, 0.8]], + predictions.eval()) + def testBackoffToParent(self): input_data = [[-1., 0.], [-1., 2.], # node 1 [1., 0.], [1., -2.]] # node 2 @@ -59,8 +113,8 @@ class TreePredictionsTest(test_util.TensorFlowTestCase): with self.test_session(): predictions = self.ops.tree_predictions( - input_data, tree, tree_thresholds, node_pcw, - valid_leaf_threshold=10) + input_data, [], [], [], self.data_spec, tree, tree_thresholds, + node_pcw, valid_leaf_threshold=10) # Node 2 has enough data, but Node 1 needs to combine with the parent # counts. @@ -78,8 +132,8 @@ class TreePredictionsTest(test_util.TensorFlowTestCase): with self.test_session(): predictions = self.ops.tree_predictions( - input_data, tree, tree_thresholds, node_pcw, - valid_leaf_threshold=10) + input_data, [], [], [], self.data_spec, tree, tree_thresholds, + node_pcw, valid_leaf_threshold=10) self.assertEquals((0, 3), predictions.eval().shape) @@ -97,8 +151,8 @@ class TreePredictionsTest(test_util.TensorFlowTestCase): 'Number of nodes should be the same in tree, tree_thresholds ' 'and node_pcw.'): predictions = self.ops.tree_predictions( - input_data, tree, tree_thresholds, node_pcw, - valid_leaf_threshold=10) + input_data, [], [], [], self.data_spec, tree, tree_thresholds, + node_pcw, valid_leaf_threshold=10) self.assertEquals((0, 3), predictions.eval().shape) diff --git a/tensorflow/contrib/tensor_forest/python/kernel_tests/update_fertile_slots_op_test.py b/tensorflow/contrib/tensor_forest/python/kernel_tests/update_fertile_slots_op_test.py index f370903b3c6..c9af01c50b7 100644 --- a/tensorflow/contrib/tensor_forest/python/kernel_tests/update_fertile_slots_op_test.py +++ b/tensorflow/contrib/tensor_forest/python/kernel_tests/update_fertile_slots_op_test.py @@ -40,48 +40,43 @@ class UpdateFertileSlotsTest(test_util.TensorFlowTestCase): self.node_map = [-1, -1, 0, -1, -1, -1, -1] self.total_counts = [[80., 40., 40.]] self.ops = training_ops.Load() + self.stale_leaves = [] def testSimple(self): with self.test_session(): - (node_map_updates, accumulators_cleared, accumulators_allocated, - new_nfl, new_nfl_scores) = self.ops.update_fertile_slots( + (node_map_updates, accumulators_cleared, + accumulators_allocated) = self.ops.update_fertile_slots( self.finished, self.non_fertile_leaves, self.non_fertile_leaf_scores, self.end_of_tree, self.depths, - self.total_counts, self.node_map, max_depth=4) + self.total_counts, self.node_map, self.stale_leaves, max_depth=4) self.assertAllEqual([[2, 4], [-1, 0]], node_map_updates.eval()) self.assertAllEqual([], accumulators_cleared.eval()) self.assertAllEqual([0], accumulators_allocated.eval()) - self.assertAllEqual([3, 5, 6], new_nfl.eval()) - self.assertAllEqual([10., 1., 1.], new_nfl_scores.eval()) def testReachedMaxDepth(self): with self.test_session(): - (node_map_updates, accumulators_cleared, accumulators_allocated, - new_nfl, new_nfl_scores) = self.ops.update_fertile_slots( + (node_map_updates, accumulators_cleared, + accumulators_allocated) = self.ops.update_fertile_slots( self.finished, self.non_fertile_leaves, self.non_fertile_leaf_scores, self.end_of_tree, self.depths, - self.total_counts, self.node_map, max_depth=3) + self.total_counts, self.node_map, self.stale_leaves, max_depth=3) self.assertAllEqual([[2], [-1]], node_map_updates.eval()) self.assertAllEqual([0], accumulators_cleared.eval()) self.assertAllEqual([], accumulators_allocated.eval()) - self.assertAllEqual([-1], new_nfl.eval()) - self.assertAllEqual([0.0], new_nfl_scores.eval()) def testNoFinished(self): with self.test_session(): - (node_map_updates, accumulators_cleared, accumulators_allocated, - new_nfl, new_nfl_scores) = self.ops.update_fertile_slots( + (node_map_updates, accumulators_cleared, + accumulators_allocated) = self.ops.update_fertile_slots( [], self.non_fertile_leaves, self.non_fertile_leaf_scores, self.end_of_tree, self.depths, - self.total_counts, self.node_map, max_depth=4) + self.total_counts, self.node_map, self.stale_leaves, max_depth=4) self.assertAllEqual((2, 0), node_map_updates.eval().shape) self.assertAllEqual([], accumulators_cleared.eval()) self.assertAllEqual([], accumulators_allocated.eval()) - self.assertAllEqual([4, 3], new_nfl.eval()) - self.assertAllEqual([15., 10.], new_nfl_scores.eval()) def testBadInput(self): del self.non_fertile_leaf_scores[-1] @@ -89,10 +84,10 @@ class UpdateFertileSlotsTest(test_util.TensorFlowTestCase): with self.assertRaisesOpError( 'Number of non fertile leaves should be the same in ' 'non_fertile_leaves and non_fertile_leaf_scores.'): - (node_map_updates, _, _, _, _) = self.ops.update_fertile_slots( + (node_map_updates, _, _) = self.ops.update_fertile_slots( self.finished, self.non_fertile_leaves, self.non_fertile_leaf_scores, self.end_of_tree, self.depths, - self.total_counts, self.node_map, max_depth=4) + self.total_counts, self.node_map, self.stale_leaves, max_depth=4) self.assertAllEqual((2, 0), node_map_updates.eval().shape) diff --git a/tensorflow/contrib/tensor_forest/python/ops/inference_ops.py b/tensorflow/contrib/tensor_forest/python/ops/inference_ops.py index 6f4e6fff401..88f8112ed4c 100644 --- a/tensorflow/contrib/tensor_forest/python/ops/inference_ops.py +++ b/tensorflow/contrib/tensor_forest/python/ops/inference_ops.py @@ -1,3 +1,4 @@ +# pylint: disable=g-bad-file-header # Copyright 2016 The TensorFlow Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,13 +18,14 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import os import threading -import tensorflow as tf - +from tensorflow.python.framework import load_library from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape +from tensorflow.python.platform import resource_loader +from tensorflow.python.platform import tf_logging as logging + INFERENCE_OPS_FILE = '_inference_ops.so' @@ -38,7 +40,11 @@ ops.NoGradient('TreePredictions') def TreePredictions(op): """Shape function for TreePredictions Op.""" num_points = op.inputs[0].get_shape()[0].value - num_classes = op.inputs[3].get_shape()[1].value + sparse_shape = op.inputs[3].get_shape() + if sparse_shape.ndims == 2: + num_points = sparse_shape[0].value + num_classes = op.inputs[7].get_shape()[1].value + # The output of TreePredictions is # [node_pcw(evaluate_tree(x), c) for c in classes for x in input_data]. return [tensor_shape.TensorShape([num_points, num_classes - 1])] @@ -49,16 +55,14 @@ def TreePredictions(op): # there's not yet any guarantee that the shared object exists. # In which case, "import tensorflow" will always crash, even for users that # never use contrib. -def Load(library_base_dir=''): +def Load(): """Load the inference ops library and return the loaded module.""" with _ops_lock: global _inference_ops if not _inference_ops: - data_files_path = os.path.join(library_base_dir, - tf.resource_loader.get_data_files_path()) - tf.logging.info('data path: %s', data_files_path) - _inference_ops = tf.load_op_library(os.path.join( - data_files_path, INFERENCE_OPS_FILE)) + ops_path = resource_loader.get_path_to_datafile(INFERENCE_OPS_FILE) + logging.info('data path: %s', ops_path) + _inference_ops = load_library.load_op_library(ops_path) assert _inference_ops, 'Could not load inference_ops.so' return _inference_ops diff --git a/tensorflow/contrib/tensor_forest/python/ops/training_ops.py b/tensorflow/contrib/tensor_forest/python/ops/training_ops.py index 7a108baf426..d25d5ce50bf 100644 --- a/tensorflow/contrib/tensor_forest/python/ops/training_ops.py +++ b/tensorflow/contrib/tensor_forest/python/ops/training_ops.py @@ -1,3 +1,4 @@ +# pylint: disable=g-bad-file-header # Copyright 2016 The TensorFlow Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,13 +18,13 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import os import threading -import tensorflow as tf - +from tensorflow.python.framework import load_library from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape +from tensorflow.python.platform import resource_loader +from tensorflow.python.platform import tf_logging as logging TRAINING_OPS_FILE = '_training_ops.so' @@ -45,7 +46,10 @@ def _CountExtremelyRandomStatsShape(op): """Shape function for CountExtremelyRandomStats Op.""" regression = op.get_attr('regression') num_points = op.inputs[0].get_shape()[0].value - num_nodes = op.inputs[2].get_shape()[0].value + sparse_shape = op.inputs[3].get_shape() + if sparse_shape.ndims == 2: + num_points = sparse_shape[0].value + num_nodes = op.inputs[6].get_shape()[0].value num_classes = op.get_attr('num_classes') # The output of TraverseTree is [leaf_node_index(x) for x in input_data]. return [tensor_shape.TensorShape([num_nodes, num_classes]), # node sums @@ -66,7 +70,7 @@ def _CountExtremelyRandomStatsShape(op): @ops.RegisterShape('SampleInputs') def _SampleInputsShape(op): """Shape function for SampleInputs Op.""" - num_splits = op.inputs[3].get_shape()[1].value + num_splits = op.inputs[6].get_shape()[1].value return [[None], [None, num_splits], [None, num_splits]] @@ -85,7 +89,7 @@ def _GrowTreeShape(unused_op): @ops.RegisterShape('FinishedNodes') def _FinishedNodesShape(unused_op): """Shape function for FinishedNodes Op.""" - return [[None]] + return [[None], [None]] @ops.RegisterShape('ScatterAddNdim') @@ -97,7 +101,7 @@ def _ScatterAddNdimShape(unused_op): @ops.RegisterShape('UpdateFertileSlots') def _UpdateFertileSlotsShape(unused_op): """Shape function for UpdateFertileSlots Op.""" - return [[None, 2], [None], [None], [None], [None]] + return [[None, 2], [None], [None]] # Workaround for the fact that importing tensorflow imports contrib @@ -105,16 +109,14 @@ def _UpdateFertileSlotsShape(unused_op): # there's not yet any guarantee that the shared object exists. # In which case, "import tensorflow" will always crash, even for users that # never use contrib. -def Load(library_base_dir=''): +def Load(): """Load training ops library and return the loaded module.""" with _ops_lock: global _training_ops if not _training_ops: - data_files_path = os.path.join(library_base_dir, - tf.resource_loader.get_data_files_path()) - tf.logging.info('data path: %s', data_files_path) - _training_ops = tf.load_op_library(os.path.join( - data_files_path, TRAINING_OPS_FILE)) + ops_path = resource_loader.get_path_to_datafile(TRAINING_OPS_FILE) + logging.info('data path: %s', ops_path) + _training_ops = load_library.load_op_library(ops_path) assert _training_ops, 'Could not load _training_ops.so' return _training_ops diff --git a/tensorflow/contrib/tensor_forest/python/tensor_forest.py b/tensorflow/contrib/tensor_forest/python/tensor_forest.py index f48efaa5db1..791954c51f4 100644 --- a/tensorflow/contrib/tensor_forest/python/tensor_forest.py +++ b/tensorflow/contrib/tensor_forest/python/tensor_forest.py @@ -1,3 +1,4 @@ +# pylint: disable=g-bad-file-header # Copyright 2016 The TensorFlow Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,14 +21,22 @@ from __future__ import print_function import math import random -import tensorflow as tf - +from tensorflow.contrib.tensor_forest.python import constants from tensorflow.contrib.tensor_forest.python.ops import inference_ops from tensorflow.contrib.tensor_forest.python.ops import training_ops - -# If tree[i][0] equals this value, then i is a leaf node. -LEAF_NODE = -1 +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import init_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import random_ops +from tensorflow.python.ops import state_ops +from tensorflow.python.ops import variable_scope +from tensorflow.python.ops import variables as tf_variables +from tensorflow.python.platform import tf_logging as logging # A convenience class for holding random forest hyperparameters. @@ -49,6 +58,7 @@ class ForestHParams(object): max_depth=0, num_splits_to_consider=0, feature_bagging_fraction=1.0, max_fertile_nodes=0, split_after_samples=250, + min_split_samples=5, valid_leaf_threshold=1, **kwargs): self.num_trees = num_trees self.max_nodes = max_nodes @@ -58,6 +68,7 @@ class ForestHParams(object): self.num_splits_to_consider = num_splits_to_consider self.max_fertile_nodes = max_fertile_nodes self.split_after_samples = split_after_samples + self.min_split_samples = min_split_samples self.valid_leaf_threshold = valid_leaf_threshold for name, value in kwargs.items(): @@ -72,11 +83,6 @@ class ForestHParams(object): _ = getattr(self, 'num_classes') _ = getattr(self, 'num_features') - self.training_library_base_dir = getattr( - self, 'training_library_base_dir', '') - self.inference_library_base_dir = getattr( - self, 'inference_library_base_dir', '') - self.bagged_num_features = int(self.feature_bagging_fraction * self.num_features) @@ -147,92 +153,86 @@ class TreeTrainingVariables(object): """ def __init__(self, params, tree_num, training): - self.tree = tf.get_variable( - name=self.get_tree_name('tree', tree_num), dtype=tf.int32, - initializer=tf.constant( - [[-1, -1]] + [[-2, -1]] * (params.max_nodes - 1))) - self.tree_thresholds = tf.get_variable( + self.tree = variable_scope.get_variable( + name=self.get_tree_name('tree', tree_num), dtype=dtypes.int32, + shape=[params.max_nodes, 2], + initializer=init_ops.constant_initializer(-2)) + self.tree_thresholds = variable_scope.get_variable( name=self.get_tree_name('tree_thresholds', tree_num), shape=[params.max_nodes], - initializer=tf.constant_initializer(-1.0)) - self.tree_depths = tf.get_variable( + initializer=init_ops.constant_initializer(-1.0)) + self.tree_depths = variable_scope.get_variable( name=self.get_tree_name('tree_depths', tree_num), shape=[params.max_nodes], - dtype=tf.int32, - initializer=tf.constant_initializer(1)) - self.end_of_tree = tf.get_variable( + dtype=dtypes.int32, + initializer=init_ops.constant_initializer(1)) + self.end_of_tree = variable_scope.get_variable( name=self.get_tree_name('end_of_tree', tree_num), - dtype=tf.int32, - initializer=tf.constant([1])) + dtype=dtypes.int32, + initializer=constant_op.constant([1])) + self.start_epoch = tf_variables.Variable( + [0] * (params.max_nodes), name='start_epoch') if training: - self.non_fertile_leaves = tf.get_variable( - name=self.get_tree_name('non_fertile_leaves', tree_num), - dtype=tf.int32, - initializer=tf.constant([0])) - self.non_fertile_leaf_scores = tf.get_variable( - name=self.get_tree_name('non_fertile_leaf_scores', tree_num), - initializer=tf.constant([1.0])) - - self.node_to_accumulator_map = tf.get_variable( + self.node_to_accumulator_map = variable_scope.get_variable( name=self.get_tree_name('node_to_accumulator_map', tree_num), shape=[params.max_nodes], - dtype=tf.int32, - initializer=tf.constant_initializer(-1)) + dtype=dtypes.int32, + initializer=init_ops.constant_initializer(-1)) - self.candidate_split_features = tf.get_variable( + self.candidate_split_features = variable_scope.get_variable( name=self.get_tree_name('candidate_split_features', tree_num), shape=[params.max_fertile_nodes, params.num_splits_to_consider], - dtype=tf.int32, - initializer=tf.constant_initializer(-1)) - self.candidate_split_thresholds = tf.get_variable( + dtype=dtypes.int32, + initializer=init_ops.constant_initializer(-1)) + self.candidate_split_thresholds = variable_scope.get_variable( name=self.get_tree_name('candidate_split_thresholds', tree_num), shape=[params.max_fertile_nodes, params.num_splits_to_consider], - initializer=tf.constant_initializer(0.0)) + initializer=init_ops.constant_initializer(0.0)) # Statistics shared by classification and regression. - self.node_sums = tf.get_variable( + self.node_sums = variable_scope.get_variable( name=self.get_tree_name('node_sums', tree_num), shape=[params.max_nodes, params.num_output_columns], - initializer=tf.constant_initializer(0.0)) + initializer=init_ops.constant_initializer(0.0)) if training: - self.candidate_split_sums = tf.get_variable( + self.candidate_split_sums = variable_scope.get_variable( name=self.get_tree_name('candidate_split_sums', tree_num), shape=[params.max_fertile_nodes, params.num_splits_to_consider, params.num_output_columns], - initializer=tf.constant_initializer(0.0)) - self.accumulator_sums = tf.get_variable( + initializer=init_ops.constant_initializer(0.0)) + self.accumulator_sums = variable_scope.get_variable( name=self.get_tree_name('accumulator_sums', tree_num), shape=[params.max_fertile_nodes, params.num_output_columns], - initializer=tf.constant_initializer(-1.0)) + initializer=init_ops.constant_initializer(-1.0)) # Regression also tracks second order stats. if params.regression: - self.node_squares = tf.get_variable( + self.node_squares = variable_scope.get_variable( name=self.get_tree_name('node_squares', tree_num), shape=[params.max_nodes, params.num_output_columns], - initializer=tf.constant_initializer(0.0)) + initializer=init_ops.constant_initializer(0.0)) - self.candidate_split_squares = tf.get_variable( + self.candidate_split_squares = variable_scope.get_variable( name=self.get_tree_name('candidate_split_squares', tree_num), shape=[params.max_fertile_nodes, params.num_splits_to_consider, params.num_output_columns], - initializer=tf.constant_initializer(0.0)) + initializer=init_ops.constant_initializer(0.0)) - self.accumulator_squares = tf.get_variable( + self.accumulator_squares = variable_scope.get_variable( name=self.get_tree_name('accumulator_squares', tree_num), shape=[params.max_fertile_nodes, params.num_output_columns], - initializer=tf.constant_initializer(-1.0)) + initializer=init_ops.constant_initializer(-1.0)) else: - self.node_squares = tf.constant( + self.node_squares = constant_op.constant( 0.0, name=self.get_tree_name('node_squares', tree_num)) - self.candidate_split_squares = tf.constant( + self.candidate_split_squares = constant_op.constant( 0.0, name=self.get_tree_name('candidate_split_squares', tree_num)) - self.accumulator_squares = tf.constant( + self.accumulator_squares = constant_op.constant( 0.0, name=self.get_tree_name('accumulator_squares', tree_num)) def get_tree_name(self, name, num): @@ -273,11 +273,11 @@ class ForestTrainingVariables(object): """ def __init__(self, params, device_assigner, training=True, - tree_variable_class=TreeTrainingVariables): + tree_variables_class=TreeTrainingVariables): self.variables = [] for i in range(params.num_trees): - with tf.device(device_assigner.get_device(i)): - self.variables.append(tree_variable_class(params, i, training)) + with ops.device(device_assigner.get_device(i)): + self.variables.append(tree_variables_class(params, i, training)) def __setitem__(self, t, val): self.variables[t] = val @@ -299,7 +299,7 @@ class RandomForestDeviceAssigner(object): def get_device(self, unused_tree_num): if not self.cached: - dummy = tf.constant(0) + dummy = constant_op.constant(0) self.cached = dummy.device return self.cached @@ -308,43 +308,51 @@ class RandomForestDeviceAssigner(object): class RandomForestGraphs(object): """Builds TF graphs for random forest training and inference.""" - def __init__(self, params, device_assigner=None, variables=None, - tree_graphs=None, + def __init__(self, params, device_assigner=None, + variables=None, tree_variables_class=TreeTrainingVariables, + tree_graphs=None, training=True, t_ops=training_ops, i_ops=inference_ops): self.params = params self.device_assigner = device_assigner or RandomForestDeviceAssigner() - tf.logging.info('Constructing forest with params = ') - tf.logging.info(self.params.__dict__) + logging.info('Constructing forest with params = ') + logging.info(self.params.__dict__) self.variables = variables or ForestTrainingVariables( - self.params, device_assigner=self.device_assigner) + self.params, device_assigner=self.device_assigner, training=training, + tree_variables_class=tree_variables_class) tree_graph_class = tree_graphs or RandomTreeGraphs self.trees = [ tree_graph_class( self.variables[i], self.params, - t_ops.Load(self.params.training_library_base_dir), - i_ops.Load(self.params.inference_library_base_dir), i) + t_ops.Load(), i_ops.Load(), i) for i in range(self.params.num_trees)] def _bag_features(self, tree_num, input_data): - split_data = tf.split(1, self.params.num_features, input_data) - return tf.concat(1, [split_data[ind] - for ind in self.params.bagged_features[tree_num]]) + split_data = array_ops.split(1, self.params.num_features, input_data) + return array_ops.concat( + 1, [split_data[ind] for ind in self.params.bagged_features[tree_num]]) - def training_graph(self, input_data, input_labels): + def training_graph(self, input_data, input_labels, data_spec=None, + epoch=None, **tree_kwargs): """Constructs a TF graph for training a random forest. Args: - input_data: A tensor or placeholder for input data. + input_data: A tensor or SparseTensor or placeholder for input data. input_labels: A tensor or placeholder for labels associated with input_data. + data_spec: A list of tf.dtype values specifying the original types of + each column. + epoch: A tensor or placeholder for the epoch the training data comes from. + **tree_kwargs: Keyword arguments passed to each tree's training_graph. Returns: The last op in the random forest training graph. """ + data_spec = ([constants.DATA_FLOAT] * self.params.num_features + if data_spec is None else data_spec) tree_graphs = [] for i in range(self.params.num_trees): - with tf.device(self.device_assigner.get_device(i)): + with ops.device(self.device_assigner.get_device(i)): seed = self.params.base_random_seed if seed != 0: seed += i @@ -354,40 +362,54 @@ class RandomForestGraphs(object): if self.params.bagging_fraction < 1.0: # TODO(thomaswc): This does sampling without replacment. Consider # also allowing sampling with replacement as an option. - batch_size = tf.slice(tf.shape(input_data), [0], [1]) - r = tf.random_uniform(batch_size, seed=seed) - mask = tf.less(r, tf.ones_like(r) * self.params.bagging_fraction) - gather_indices = tf.squeeze(tf.where(mask), squeeze_dims=[1]) + batch_size = array_ops.slice(array_ops.shape(input_data), [0], [1]) + r = random_ops.random_uniform(batch_size, seed=seed) + mask = math_ops.less( + r, array_ops.ones_like(r) * self.params.bagging_fraction) + gather_indices = array_ops.squeeze( + array_ops.where(mask), squeeze_dims=[1]) # TODO(thomaswc): Calculate out-of-bag data and labels, and store # them for use in calculating statistics later. - tree_data = tf.gather(input_data, gather_indices) - tree_labels = tf.gather(input_labels, gather_indices) + tree_data = array_ops.gather(input_data, gather_indices) + tree_labels = array_ops.gather(input_labels, gather_indices) if self.params.bagged_features: tree_data = self._bag_features(i, tree_data) - tree_graphs.append( - self.trees[i].training_graph(tree_data, tree_labels, seed)) - return tf.group(*tree_graphs) + initialization = self.trees[i].tree_initialization() - def inference_graph(self, input_data): + with ops.control_dependencies([initialization]): + tree_graphs.append( + self.trees[i].training_graph( + tree_data, tree_labels, seed, data_spec=data_spec, + epoch=([0] if epoch is None else epoch), + **tree_kwargs)) + + return control_flow_ops.group(*tree_graphs) + + def inference_graph(self, input_data, data_spec=None): """Constructs a TF graph for evaluating a random forest. Args: - input_data: A tensor or placeholder for input data. + input_data: A tensor or SparseTensor or placeholder for input data. + data_spec: A list of tf.dtype values specifying the original types of + each column. Returns: The last op in the random forest inference graph. """ + data_spec = ([constants.DATA_FLOAT] * self.params.num_features + if data_spec is None else data_spec) probabilities = [] for i in range(self.params.num_trees): - with tf.device(self.device_assigner.get_device(i)): + with ops.device(self.device_assigner.get_device(i)): tree_data = input_data if self.params.bagged_features: tree_data = self._bag_features(i, input_data) - probabilities.append(self.trees[i].inference_graph(tree_data)) - with tf.device(self.device_assigner.get_device(0)): - all_predict = tf.pack(probabilities) - return tf.reduce_sum(all_predict, 0) / self.params.num_trees + probabilities.append(self.trees[i].inference_graph(tree_data, + data_spec)) + with ops.device(self.device_assigner.get_device(0)): + all_predict = array_ops.pack(probabilities) + return math_ops.reduce_sum(all_predict, 0) / self.params.num_trees def average_size(self): """Constructs a TF graph for evaluating the average size of a forest. @@ -397,9 +419,16 @@ class RandomForestGraphs(object): """ sizes = [] for i in range(self.params.num_trees): - with tf.device(self.device_assigner.get_device(i)): + with ops.device(self.device_assigner.get_device(i)): sizes.append(self.trees[i].size()) - return tf.reduce_mean(tf.pack(sizes)) + return math_ops.reduce_mean(array_ops.pack(sizes)) + + def training_loss(self): + return math_ops.neg(self.average_size()) + + # pylint: disable=unused-argument + def validation_loss(self, features, labels): + return math_ops.neg(self.average_size()) def average_impurity(self): """Constructs a TF graph for evaluating the leaf impurity of a forest. @@ -409,14 +438,14 @@ class RandomForestGraphs(object): """ impurities = [] for i in range(self.params.num_trees): - with tf.device(self.device_assigner.get_device(i)): + with ops.device(self.device_assigner.get_device(i)): impurities.append(self.trees[i].average_impurity()) - return tf.reduce_mean(tf.pack(impurities)) + return math_ops.reduce_mean(array_ops.pack(impurities)) def get_stats(self, session): tree_stats = [] for i in range(self.params.num_trees): - with tf.device(self.device_assigner.get_device(i)): + with ops.device(self.device_assigner.get_device(i)): tree_stats.append(self.trees[i].get_stats(session)) return ForestStats(tree_stats, self.params) @@ -431,6 +460,18 @@ class RandomTreeGraphs(object): self.params = params self.tree_num = tree_num + def tree_initialization(self): + def _init_tree(): + return state_ops.scatter_update(self.variables.tree, [0], [[-1, -1]]).op + + def _nothing(): + return control_flow_ops.no_op() + + return control_flow_ops.cond( + math_ops.equal(array_ops.squeeze(array_ops.slice( + self.variables.tree, [0, 0], [1, 1])), -2), + _init_tree, _nothing) + def _gini(self, class_counts): """Calculate the Gini impurity. @@ -444,9 +485,9 @@ class RandomTreeGraphs(object): Returns: A 1-D tensor of the Gini impurities for each row in the input. """ - smoothed = 1.0 + tf.slice(class_counts, [0, 1], [-1, -1]) - sums = tf.reduce_sum(smoothed, 1) - sum_squares = tf.reduce_sum(tf.square(smoothed), 1) + smoothed = 1.0 + array_ops.slice(class_counts, [0, 1], [-1, -1]) + sums = math_ops.reduce_sum(smoothed, 1) + sum_squares = math_ops.reduce_sum(math_ops.square(smoothed), 1) return 1.0 - sum_squares / (sums * sums) @@ -463,9 +504,9 @@ class RandomTreeGraphs(object): Returns: A 1-D tensor of the Gini impurities for each row in the input. """ - smoothed = 1.0 + tf.slice(class_counts, [0, 1], [-1, -1]) - sums = tf.reduce_sum(smoothed, 1) - sum_squares = tf.reduce_sum(tf.square(smoothed), 1) + smoothed = 1.0 + array_ops.slice(class_counts, [0, 1], [-1, -1]) + sums = math_ops.reduce_sum(smoothed, 1) + sum_squares = math_ops.reduce_sum(math_ops.square(smoothed), 1) return sums - sum_squares / sums @@ -483,40 +524,58 @@ class RandomTreeGraphs(object): Returns: A 1-D tensor of the variances for each row in the input. """ - total_count = tf.slice(sums, [0, 0], [-1, 1]) + total_count = array_ops.slice(sums, [0, 0], [-1, 1]) e_x = sums / total_count e_x2 = squares / total_count - return tf.reduce_sum(e_x2 - tf.square(e_x), 1) + return math_ops.reduce_sum(e_x2 - math_ops.square(e_x), 1) + + def training_graph(self, input_data, input_labels, random_seed, + data_spec, epoch=None): - def training_graph(self, input_data, input_labels, random_seed): """Constructs a TF graph for training a random tree. Args: - input_data: A tensor or placeholder for input data. + input_data: A tensor or SparseTensor or placeholder for input data. input_labels: A tensor or placeholder for labels associated with input_data. random_seed: The random number generator seed to use for this tree. 0 means use the current time as the seed. + data_spec: A list of tf.dtype values specifying the original types of + each column. + epoch: A tensor or placeholder for the epoch the training data comes from. Returns: The last op in the random tree training graph. """ + epoch = [0] if epoch is None else epoch + + sparse_indices = [] + sparse_values = [] + sparse_shape = [] + if isinstance(input_data, ops.SparseTensor): + sparse_indices = input_data.indices + sparse_values = input_data.values + sparse_shape = input_data.shape + input_data = [] + # Count extremely random stats. (node_sums, node_squares, splits_indices, splits_sums, splits_squares, totals_indices, totals_sums, totals_squares, input_leaves) = ( self.training_ops.count_extremely_random_stats( - input_data, input_labels, self.variables.tree, + input_data, sparse_indices, sparse_values, sparse_shape, + data_spec, input_labels, self.variables.tree, self.variables.tree_thresholds, self.variables.node_to_accumulator_map, self.variables.candidate_split_features, self.variables.candidate_split_thresholds, + self.variables.start_epoch, epoch, num_classes=self.params.num_output_columns, regression=self.params.regression)) node_update_ops = [] node_update_ops.append( - tf.assign_add(self.variables.node_sums, node_sums)) + state_ops.assign_add(self.variables.node_sums, node_sums)) splits_update_ops = [] splits_update_ops.append(self.training_ops.scatter_add_ndim( @@ -527,8 +586,8 @@ class RandomTreeGraphs(object): totals_sums)) if self.params.regression: - node_update_ops.append(tf.assign_add(self.variables.node_squares, - node_squares)) + node_update_ops.append(state_ops.assign_add(self.variables.node_squares, + node_squares)) splits_update_ops.append(self.training_ops.scatter_add_ndim( self.variables.candidate_split_squares, splits_indices, splits_squares)) @@ -539,63 +598,56 @@ class RandomTreeGraphs(object): # Sample inputs. update_indices, feature_updates, threshold_updates = ( self.training_ops.sample_inputs( - input_data, self.variables.node_to_accumulator_map, + input_data, sparse_indices, sparse_values, sparse_shape, + self.variables.node_to_accumulator_map, input_leaves, self.variables.candidate_split_features, self.variables.candidate_split_thresholds, split_initializations_per_input=( self.params.split_initializations_per_input), split_sampling_random_seed=random_seed)) - update_features_op = tf.scatter_update( + update_features_op = state_ops.scatter_update( self.variables.candidate_split_features, update_indices, feature_updates) - update_thresholds_op = tf.scatter_update( + update_thresholds_op = state_ops.scatter_update( self.variables.candidate_split_thresholds, update_indices, threshold_updates) # Calculate finished nodes. - with tf.control_dependencies(splits_update_ops): - children = tf.squeeze(tf.slice(self.variables.tree, [0, 0], [-1, 1]), - squeeze_dims=[1]) - is_leaf = tf.equal(LEAF_NODE, children) - leaves = tf.to_int32(tf.squeeze(tf.where(is_leaf), squeeze_dims=[1])) - finished = self.training_ops.finished_nodes( + with ops.control_dependencies(splits_update_ops): + children = array_ops.squeeze(array_ops.slice( + self.variables.tree, [0, 0], [-1, 1]), squeeze_dims=[1]) + is_leaf = math_ops.equal(constants.LEAF_NODE, children) + leaves = math_ops.to_int32(array_ops.squeeze(array_ops.where(is_leaf), + squeeze_dims=[1])) + finished, stale = self.training_ops.finished_nodes( leaves, self.variables.node_to_accumulator_map, + self.variables.candidate_split_sums, + self.variables.candidate_split_squares, self.variables.accumulator_sums, - num_split_after_samples=self.params.split_after_samples) + self.variables.accumulator_squares, + self.variables.start_epoch, epoch, + num_split_after_samples=self.params.split_after_samples, + min_split_samples=self.params.min_split_samples) # Update leaf scores. - # TODO(gilberth): Optimize this. It currently calculates counts for - # every non-fertile leaf. - with tf.control_dependencies(node_update_ops): - def dont_update_leaf_scores(): - return self.variables.non_fertile_leaf_scores + non_fertile_leaves = array_ops.boolean_mask( + leaves, math_ops.less(array_ops.gather( + self.variables.node_to_accumulator_map, leaves), 0)) - def update_leaf_scores_regression(): - sums = tf.gather(self.variables.node_sums, - self.variables.non_fertile_leaves) - squares = tf.gather(self.variables.node_squares, - self.variables.non_fertile_leaves) - new_scores = self._variance(sums, squares) - return tf.assign(self.variables.non_fertile_leaf_scores, new_scores) - - def update_leaf_scores_classification(): - counts = tf.gather(self.variables.node_sums, - self.variables.non_fertile_leaves) - new_scores = self._weighted_gini(counts) - return tf.assign(self.variables.non_fertile_leaf_scores, new_scores) - - # Because we can't have tf.self.variables of size 0, we have to put in a - # garbage value of -1 in there. Here we check for that so we don't - # try to index into node_per_class_weights in a tf.gather with a negative - # number. - update_nonfertile_leaves_scores_op = tf.cond( - tf.less(self.variables.non_fertile_leaves[0], 0), - dont_update_leaf_scores, - update_leaf_scores_regression if self.params.regression else - update_leaf_scores_classification) + # TODO(gilberth): It should be possible to limit the number of non + # fertile leaves we calculate scores for, especially since we can only take + # at most array_ops.shape(finished)[0] of them. + with ops.control_dependencies(node_update_ops): + sums = array_ops.gather(self.variables.node_sums, non_fertile_leaves) + if self.params.regression: + squares = array_ops.gather(self.variables.node_squares, + non_fertile_leaves) + non_fertile_leaf_scores = self._variance(sums, squares) + else: + non_fertile_leaf_scores = self._weighted_gini(sums) # Calculate best splits. - with tf.control_dependencies(splits_update_ops): + with ops.control_dependencies(splits_update_ops): split_indices = self.training_ops.best_splits( finished, self.variables.node_to_accumulator_map, self.variables.candidate_split_sums, @@ -605,7 +657,7 @@ class RandomTreeGraphs(object): regression=self.params.regression) # Grow tree. - with tf.control_dependencies([update_features_op, update_thresholds_op]): + with ops.control_dependencies([update_features_op, update_thresholds_op]): (tree_update_indices, tree_children_updates, tree_threshold_updates, tree_depth_updates, new_eot) = ( self.training_ops.grow_tree( @@ -613,110 +665,138 @@ class RandomTreeGraphs(object): self.variables.node_to_accumulator_map, finished, split_indices, self.variables.candidate_split_features, self.variables.candidate_split_thresholds)) - tree_update_op = tf.scatter_update( + tree_update_op = state_ops.scatter_update( self.variables.tree, tree_update_indices, tree_children_updates) - threhsolds_update_op = tf.scatter_update( + thresholds_update_op = state_ops.scatter_update( self.variables.tree_thresholds, tree_update_indices, tree_threshold_updates) - depth_update_op = tf.scatter_update( + depth_update_op = state_ops.scatter_update( self.variables.tree_depths, tree_update_indices, tree_depth_updates) + # TODO(thomaswc): Only update the epoch on the new leaves. + new_epoch_updates = epoch * array_ops.ones_like(tree_depth_updates) + epoch_update_op = state_ops.scatter_update( + self.variables.start_epoch, tree_update_indices, + new_epoch_updates) # Update fertile slots. - with tf.control_dependencies([update_nonfertile_leaves_scores_op, - depth_update_op]): - (node_map_updates, accumulators_cleared, accumulators_allocated, - new_nonfertile_leaves, new_nonfertile_leaves_scores) = ( - self.training_ops.update_fertile_slots( - finished, self.variables.non_fertile_leaves, - self.variables.non_fertile_leaf_scores, - self.variables.end_of_tree, self.variables.tree_depths, - self.variables.accumulator_sums, - self.variables.node_to_accumulator_map, - max_depth=self.params.max_depth, - regression=self.params.regression)) + with ops.control_dependencies([depth_update_op]): + (node_map_updates, accumulators_cleared, accumulators_allocated) = ( + self.training_ops.update_fertile_slots( + finished, non_fertile_leaves, + non_fertile_leaf_scores, + self.variables.end_of_tree, self.variables.tree_depths, + self.variables.accumulator_sums, + self.variables.node_to_accumulator_map, + stale, + max_depth=self.params.max_depth, + regression=self.params.regression)) # Ensure end_of_tree doesn't get updated until UpdateFertileSlots has # used it to calculate new leaves. - gated_new_eot, = tf.tuple([new_eot], control_inputs=[new_nonfertile_leaves]) - eot_update_op = tf.assign(self.variables.end_of_tree, gated_new_eot) + gated_new_eot, = control_flow_ops.tuple([new_eot], + control_inputs=[node_map_updates]) + eot_update_op = state_ops.assign(self.variables.end_of_tree, gated_new_eot) updates = [] updates.append(eot_update_op) updates.append(tree_update_op) - updates.append(threhsolds_update_op) - updates.append(tf.assign( - self.variables.non_fertile_leaves, new_nonfertile_leaves, - validate_shape=False)) - updates.append(tf.assign( - self.variables.non_fertile_leaf_scores, - new_nonfertile_leaves_scores, validate_shape=False)) + updates.append(thresholds_update_op) + updates.append(epoch_update_op) - updates.append(tf.scatter_update( + updates.append(state_ops.scatter_update( self.variables.node_to_accumulator_map, - tf.squeeze(tf.slice(node_map_updates, [0, 0], [1, -1]), - squeeze_dims=[0]), - tf.squeeze(tf.slice(node_map_updates, [1, 0], [1, -1]), - squeeze_dims=[0]))) + array_ops.squeeze(array_ops.slice(node_map_updates, [0, 0], [1, -1]), + squeeze_dims=[0]), + array_ops.squeeze(array_ops.slice(node_map_updates, [1, 0], [1, -1]), + squeeze_dims=[0]))) - cleared_and_allocated_accumulators = tf.concat( + cleared_and_allocated_accumulators = array_ops.concat( 0, [accumulators_cleared, accumulators_allocated]) # Calculate values to put into scatter update for candidate counts. # Candidate split counts are always reset back to 0 for both cleared # and allocated accumulators. This means some accumulators might be doubly # reset to 0 if the were released and not allocated, then later allocated. - split_values = tf.tile( - tf.expand_dims(tf.expand_dims( - tf.zeros_like(cleared_and_allocated_accumulators, dtype=tf.float32), - 1), 2), + split_values = array_ops.tile( + array_ops.expand_dims(array_ops.expand_dims( + array_ops.zeros_like(cleared_and_allocated_accumulators, + dtype=dtypes.float32), 1), 2), [1, self.params.num_splits_to_consider, self.params.num_output_columns]) - updates.append(tf.scatter_update( + updates.append(state_ops.scatter_update( self.variables.candidate_split_sums, cleared_and_allocated_accumulators, split_values)) if self.params.regression: - updates.append(tf.scatter_update( + updates.append(state_ops.scatter_update( self.variables.candidate_split_squares, cleared_and_allocated_accumulators, split_values)) # Calculate values to put into scatter update for total counts. - total_cleared = tf.tile( - tf.expand_dims( - tf.neg(tf.ones_like(accumulators_cleared, dtype=tf.float32)), 1), + total_cleared = array_ops.tile( + array_ops.expand_dims( + math_ops.neg(array_ops.ones_like(accumulators_cleared, + dtype=dtypes.float32)), 1), [1, self.params.num_output_columns]) - total_reset = tf.tile( - tf.expand_dims( - tf.zeros_like(accumulators_allocated, dtype=tf.float32), 1), + total_reset = array_ops.tile( + array_ops.expand_dims( + array_ops.zeros_like(accumulators_allocated, + dtype=dtypes.float32), 1), [1, self.params.num_output_columns]) - accumulator_updates = tf.concat(0, [total_cleared, total_reset]) - updates.append(tf.scatter_update( + accumulator_updates = array_ops.concat(0, [total_cleared, total_reset]) + updates.append(state_ops.scatter_update( self.variables.accumulator_sums, cleared_and_allocated_accumulators, accumulator_updates)) if self.params.regression: - updates.append(tf.scatter_update( + updates.append(state_ops.scatter_update( self.variables.accumulator_squares, cleared_and_allocated_accumulators, accumulator_updates)) # Calculate values to put into scatter update for candidate splits. - split_features_updates = tf.tile( - tf.expand_dims( - tf.neg(tf.ones_like(cleared_and_allocated_accumulators)), 1), + split_features_updates = array_ops.tile( + array_ops.expand_dims( + math_ops.neg(array_ops.ones_like( + cleared_and_allocated_accumulators)), 1), [1, self.params.num_splits_to_consider]) - updates.append(tf.scatter_update( + updates.append(state_ops.scatter_update( self.variables.candidate_split_features, cleared_and_allocated_accumulators, split_features_updates)) - return tf.group(*updates) + updates += self.finish_iteration() - def inference_graph(self, input_data): + return control_flow_ops.group(*updates) + + def finish_iteration(self): + """Perform any operations that should be done at the end of an iteration. + + This is mostly useful for subclasses that need to reset variables after + an iteration, such as ones that are used to finish nodes. + + Returns: + A list of operations. + """ + return [] + + def inference_graph(self, input_data, data_spec): """Constructs a TF graph for evaluating a random tree. Args: - input_data: A tensor or placeholder for input data. + input_data: A tensor or SparseTensor or placeholder for input data. + data_spec: A list of tf.dtype values specifying the original types of + each column. Returns: The last op in the random tree inference graph. """ + sparse_indices = [] + sparse_values = [] + sparse_shape = [] + if isinstance(input_data, ops.SparseTensor): + sparse_indices = input_data.indices + sparse_values = input_data.values + sparse_shape = input_data.shape + input_data = [] return self.inference_ops.tree_predictions( - input_data, self.variables.tree, self.variables.tree_thresholds, + input_data, sparse_indices, sparse_values, sparse_shape, data_spec, + self.variables.tree, + self.variables.tree_thresholds, self.variables.node_sums, valid_leaf_threshold=self.params.valid_leaf_threshold) @@ -729,13 +809,22 @@ class RandomTreeGraphs(object): Returns: The last op in the graph. """ - children = tf.squeeze(tf.slice(self.variables.tree, [0, 0], [-1, 1]), - squeeze_dims=[1]) - is_leaf = tf.equal(LEAF_NODE, children) - leaves = tf.to_int32(tf.squeeze(tf.where(is_leaf), squeeze_dims=[1])) - counts = tf.gather(self.variables.node_sums, leaves) - impurity = self._weighted_gini(counts) - return tf.reduce_sum(impurity) / tf.reduce_sum(counts + 1.0) + children = array_ops.squeeze(array_ops.slice( + self.variables.tree, [0, 0], [-1, 1]), squeeze_dims=[1]) + is_leaf = math_ops.equal(constants.LEAF_NODE, children) + leaves = math_ops.to_int32(array_ops.squeeze(array_ops.where(is_leaf), + squeeze_dims=[1])) + counts = array_ops.gather(self.variables.node_sums, leaves) + gini = self._weighted_gini(counts) + # Guard against step 1, when there often are no leaves yet. + def impurity(): + return gini + # Since average impurity can be used for loss, when there's no data just + # return a big number so that loss always decreases. + def big(): + return array_ops.ones_like(gini, dtype=dtypes.float32) * 10000000. + return control_flow_ops.cond(math_ops.greater( + array_ops.shape(leaves)[0], 0), impurity, big) def size(self): """Constructs a TF graph for evaluating the current number of nodes. @@ -747,7 +836,8 @@ class RandomTreeGraphs(object): def get_stats(self, session): num_nodes = self.variables.end_of_tree.eval(session=session) - 1 - num_leaves = tf.where( - tf.equal(tf.squeeze(tf.slice(self.variables.tree, [0, 0], [-1, 1])), - LEAF_NODE)).eval(session=session).shape[0] + num_leaves = array_ops.where( + math_ops.equal(array_ops.squeeze(array_ops.slice( + self.variables.tree, [0, 0], [-1, 1])), constants.LEAF_NODE) + ).eval(session=session).shape[0] return TreeStats(num_nodes, num_leaves) diff --git a/tensorflow/contrib/tensor_forest/python/tensor_forest_test.py b/tensorflow/contrib/tensor_forest/python/tensor_forest_test.py index c3e1c8520d3..4e4cfcd1e82 100644 --- a/tensorflow/contrib/tensor_forest/python/tensor_forest_test.py +++ b/tensorflow/contrib/tensor_forest/python/tensor_forest_test.py @@ -105,6 +105,47 @@ class TensorForestTest(test_util.TensorFlowTestCase): graph = graph_builder.average_impurity() self.assertTrue(isinstance(graph, tf.Tensor)) + def testTrainingConstructionClassificationSparse(self): + input_data = tf.SparseTensor( + indices=[[0, 0], [0, 3], + [1, 0], [1, 7], + [2, 1], + [3, 9]], + values=[-1.0, 0.0, + -1., 2., + 1., + -2.0], + shape=[4, 10]) + input_labels = [0, 1, 2, 3] + + params = tensor_forest.ForestHParams( + num_classes=4, num_features=10, num_trees=10, max_nodes=1000, + split_after_samples=25).fill() + + graph_builder = tensor_forest.RandomForestGraphs(params) + graph = graph_builder.training_graph(input_data, input_labels) + self.assertTrue(isinstance(graph, tf.Operation)) + + def testInferenceConstructionSparse(self): + input_data = tf.SparseTensor( + indices=[[0, 0], [0, 3], + [1, 0], [1, 7], + [2, 1], + [3, 9]], + values=[-1.0, 0.0, + -1., 2., + 1., + -2.0], + shape=[4, 10]) + + params = tensor_forest.ForestHParams( + num_classes=4, num_features=10, num_trees=10, max_nodes=1000, + split_after_samples=25).fill() + + graph_builder = tensor_forest.RandomForestGraphs(params) + graph = graph_builder.inference_graph(input_data) + self.assertTrue(isinstance(graph, tf.Tensor)) + if __name__ == '__main__': googletest.main() diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD index ae631812586..b4fc9e4df2e 100644 --- a/tensorflow/tools/pip_package/BUILD +++ b/tensorflow/tools/pip_package/BUILD @@ -35,6 +35,7 @@ sh_binary( # "//tensorflow/contrib/session_bundle/example:half_plus_two", "//tensorflow/contrib/slim:all_files", "//tensorflow/contrib/slim/python/slim/data:all_files", + "//tensorflow/contrib/tensor_forest:all_files", "//tensorflow/core:framework_headers", "//tensorflow/examples/tutorials/mnist:package", "//tensorflow/models/embedding:package", From 01fb3ef3652d1040778c45c40b5517c4c474771d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 30 Jun 2016 14:59:48 -0800 Subject: [PATCH 20/36] Cleanup SDCA op based on additional review comments. Change: 126354256 --- .../linear_optimizer/kernels/sdca_ops.cc | 26 ++++++++++++------- .../python/kernel_tests/sdca_ops_test.py | 9 ++++++- .../linear_optimizer/python/ops/sdca_ops.py | 3 +-- 3 files changed, 25 insertions(+), 13 deletions(-) diff --git a/tensorflow/contrib/linear_optimizer/kernels/sdca_ops.cc b/tensorflow/contrib/linear_optimizer/kernels/sdca_ops.cc index 921dad614c2..15deca87e67 100644 --- a/tensorflow/contrib/linear_optimizer/kernels/sdca_ops.cc +++ b/tensorflow/contrib/linear_optimizer/kernels/sdca_ops.cc @@ -600,13 +600,13 @@ Status RunTrainStepsForMiniBatch( const DeviceBase::CpuWorkerThreads& worker_threads, const Regularizations& regularizations, const DualLossUpdater& loss_updater, FeaturesAndWeights* const features_and_weights, - TTypes::Matrix example_state_data) { + TTypes::Matrix* const example_state_data) { // Process examples in parallel, in a partitioned fashion. mutex mu; Status train_step_status GUARDED_BY(mu); auto train_step = [&](const int64 begin, const int64 end) { for (int64 example_index = begin; example_index < end; ++example_index) { - const float dual = example_state_data(example_index, 0); + const float dual = (*example_state_data)(example_index, 0); const float example_weight = example_weights(example_index); float example_label = example_labels(example_index); const Status conversion_status = @@ -641,10 +641,10 @@ Status RunTrainStepsForMiniBatch( example_index, bounded_dual_delta, regularizations.symmetric_l2()); // Update example data. - example_state_data(example_index, 0) = new_dual; - example_state_data(example_index, 1) = primal_loss; - example_state_data(example_index, 2) = dual_loss; - example_state_data(example_index, 3) = example_weight; + (*example_state_data)(example_index, 0) = new_dual; + (*example_state_data)(example_index, 1) = primal_loss; + (*example_state_data)(example_index, 2) = dual_loss; + (*example_state_data)(example_index, 3) = example_weight; } }; // TODO(sibyl-Aix6ihai): Current multiplier 100000 works well empirically @@ -737,11 +737,11 @@ class SdcaSolver : public OpKernel { num_examples, example_labels, example_weights, *context->device()->tensorflow_cpu_worker_threads(), regularizations_, *loss_updater_, &features_and_weights, - example_state_data)); + &example_state_data)); } features_and_weights.AddDeltaWeights(); - context->set_output(0, mutable_example_state_data_t); + context->set_output("example_data_data_out", mutable_example_state_data_t); } private: @@ -775,6 +775,12 @@ class SdcaShrinkL1 : public OpKernel { }; REGISTER_KERNEL_BUILDER(Name("SdcaShrinkL1").Device(DEVICE_CPU), SdcaShrinkL1); +// Computes platform independent, compact and unique (with very high +// probability) representation of an example id. It shouldn't be put in +// persistent storage, as its implementation may change in the future. +// +// The current probability of at least one collision for 1B example_ids is +// approximately 10^-21 (ie 2^60 / 2^129). class SdcaFprint : public OpKernel { public: explicit SdcaFprint(OpKernelConstruction* context) : OpKernel(context) {} @@ -788,8 +794,8 @@ class SdcaFprint : public OpKernel { auto out_values = out->flat(); for (int64 i = 0; i < in_values.size(); ++i) { - const string& s = in_values(i); - Fprint128 fprint = Fingerprint128(s); + const Fprint128 fprint = Fingerprint128(in_values(i)); + // Hex encode the fprint as a string (33 characters). out_values(i) = strings::StrCat(strings::FpToString(fprint.high64), "-", strings::FpToString(fprint.low64)); } diff --git a/tensorflow/contrib/linear_optimizer/python/kernel_tests/sdca_ops_test.py b/tensorflow/contrib/linear_optimizer/python/kernel_tests/sdca_ops_test.py index 8ecf4bfe89a..2b9a95a1d65 100644 --- a/tensorflow/contrib/linear_optimizer/python/kernel_tests/sdca_ops_test.py +++ b/tensorflow/contrib/linear_optimizer/python/kernel_tests/sdca_ops_test.py @@ -781,7 +781,14 @@ class SdcaWithHingeLossTest(SdcaOptimizerTest): class SdcaFprintTest(TensorFlowTestCase): - """Tests for the SdcaFprint op.""" + """Tests for the SdcaFprint op. + + This is one way of enforcing the platform-agnostic nature of SdcaFprint. + Basically we are checking against exact values and this test could be running + across different platforms. Note that it is fine for expected values to change + in the future, if the implementation of SdcaFprint changes (ie this is *not* a + frozen test). + """ def testFprint(self): with self.test_session(): diff --git a/tensorflow/contrib/linear_optimizer/python/ops/sdca_ops.py b/tensorflow/contrib/linear_optimizer/python/ops/sdca_ops.py index f669407c176..b4e9e5b23d9 100644 --- a/tensorflow/contrib/linear_optimizer/python/ops/sdca_ops.py +++ b/tensorflow/contrib/linear_optimizer/python/ops/sdca_ops.py @@ -95,8 +95,7 @@ class SdcaModel(object): ``` In the training program you will just have to run the returned Op from - minimize(). You should also eventually cleanup the temporary state used by - the model, by resetting its (possibly shared) container. + minimize(). ```python # Execute opt_op and train for num_steps. From 69abcd7ec28c8f6a58576d95fd59a8c7c537ade5 Mon Sep 17 00:00:00 2001 From: Mustafa Ispir Date: Thu, 30 Jun 2016 15:03:20 -0800 Subject: [PATCH 21/36] Support optimizer function in Dnn/Linear/.. Estimators. Change: 126354629 --- .../learn/estimators/dnn_linear_combined.py | 33 ++++++++++--------- .../estimators/dnn_linear_combined_test.py | 30 +++++++++++++++++ 2 files changed, 48 insertions(+), 15 deletions(-) diff --git a/tensorflow/contrib/learn/python/learn/estimators/dnn_linear_combined.py b/tensorflow/contrib/learn/python/learn/estimators/dnn_linear_combined.py index dcd4719c5cf..a5b1dc722f8 100644 --- a/tensorflow/contrib/learn/python/learn/estimators/dnn_linear_combined.py +++ b/tensorflow/contrib/learn/python/learn/estimators/dnn_linear_combined.py @@ -130,6 +130,14 @@ class _ComposableModel(object): return [] def _get_optimizer(self): + if (self._optimizer is None or isinstance(self._optimizer, + six.string_types)): + self._optimizer = self._get_default_optimizer(self._optimizer) + elif callable(self._optimizer): + self._optimizer = self._optimizer() + return self._optimizer + + def _get_default_optimizer(self, optimizer_name=None): raise NotImplementedError @@ -173,14 +181,12 @@ class _LinearComposableModel(_ComposableModel): name="linear") return logits - def _get_optimizer(self): - if self._optimizer is None: - self._optimizer = "Ftrl" - if isinstance(self._optimizer, six.string_types): - default_learning_rate = 1. / math.sqrt(len(self._get_feature_columns())) - self._optimizer = layers.OPTIMIZER_CLS_NAMES[self._optimizer]( - learning_rate=default_learning_rate) - return self._optimizer + def _get_default_optimizer(self, optimizer_name=None): + if optimizer_name is None: + optimizer_name = "Ftrl" + default_learning_rate = 1. / math.sqrt(len(self._get_feature_columns())) + return layers.OPTIMIZER_CLS_NAMES[optimizer_name]( + learning_rate=default_learning_rate) class _DNNComposableModel(_ComposableModel): @@ -269,13 +275,10 @@ class _DNNComposableModel(_ComposableModel): self._add_hidden_layer_summary(logits, "dnn_logits") return logits - def _get_optimizer(self): - if self._optimizer is None: - self._optimizer = "Adagrad" - if isinstance(self._optimizer, six.string_types): - self._optimizer = layers.OPTIMIZER_CLS_NAMES[self._optimizer]( - learning_rate=0.05) - return self._optimizer + def _get_default_optimizer(self, optimizer_name=None): + if optimizer_name is None: + optimizer_name = "Adagrad" + return layers.OPTIMIZER_CLS_NAMES[optimizer_name](learning_rate=0.05) # TODO(ispir): Increase test coverage diff --git a/tensorflow/contrib/learn/python/learn/estimators/dnn_linear_combined_test.py b/tensorflow/contrib/learn/python/learn/estimators/dnn_linear_combined_test.py index 345065c4832..00679a30b47 100644 --- a/tensorflow/contrib/learn/python/learn/estimators/dnn_linear_combined_test.py +++ b/tensorflow/contrib/learn/python/learn/estimators/dnn_linear_combined_test.py @@ -262,6 +262,36 @@ class DNNLinearCombinedClassifierTest(tf.test.TestCase): scores = classifier.evaluate(input_fn=_iris_input_logistic_fn, steps=100) self.assertGreater(scores['accuracy'], 0.9) + def testCustomOptimizerByFunction(self): + """Tests binary classification using matrix data as input.""" + iris = _prepare_iris_data_for_logistic_regression() + cont_features = [ + tf.contrib.layers.real_valued_column('feature', dimension=4) + ] + bucketized_features = [ + tf.contrib.layers.bucketized_column( + cont_features[0], _get_quantile_based_buckets(iris.data, 10)) + ] + + def _optimizer_exp_decay(): + global_step = tf.contrib.framework.get_global_step() + learning_rate = tf.train.exponential_decay(learning_rate=0.1, + global_step=global_step, + decay_steps=100, + decay_rate=0.001) + return tf.train.AdagradOptimizer(learning_rate=learning_rate) + + classifier = tf.contrib.learn.DNNLinearCombinedClassifier( + linear_feature_columns=bucketized_features, + linear_optimizer=_optimizer_exp_decay, + dnn_feature_columns=cont_features, + dnn_hidden_units=[3, 3], + dnn_optimizer=_optimizer_exp_decay) + + classifier.fit(input_fn=_iris_input_logistic_fn, steps=100) + scores = classifier.evaluate(input_fn=_iris_input_logistic_fn, steps=100) + self.assertGreater(scores['accuracy'], 0.9) + def testPredict(self): """Tests weight column in evaluation.""" def _input_fn_train(): From aabfe1d03ec518269f1b28793f70e4a95eb52574 Mon Sep 17 00:00:00 2001 From: Geoffrey Irving Date: Thu, 30 Jun 2016 15:49:37 -0800 Subject: [PATCH 22/36] Fix sparse_softmax_cross_entropy_with_logits for empty tensor If the batch size is zero, we need to avoid calling into Eigen because Eigen will explode. Zero classes is an error. Change: 126359444 --- tensorflow/core/kernels/sparse_xent_op.cc | 50 ++++++++++--------- .../kernel_tests/sparse_xent_op_test.py | 10 ++++ 2 files changed, 37 insertions(+), 23 deletions(-) diff --git a/tensorflow/core/kernels/sparse_xent_op.cc b/tensorflow/core/kernels/sparse_xent_op.cc index 34411c9bbb6..48124d20af9 100644 --- a/tensorflow/core/kernels/sparse_xent_op.cc +++ b/tensorflow/core/kernels/sparse_xent_op.cc @@ -35,38 +35,42 @@ class SparseSoftmaxXentWithLogitsOp : public OpKernel { : OpKernel(context) {} void Compute(OpKernelContext* context) override { - const Tensor& logits_in = context->input(0); - const Tensor& labels_in = context->input(1); - OP_REQUIRES(context, logits_in.shape().dim_size(0) == labels_in.NumElements(), + const Tensor& logits = context->input(0); + const Tensor& labels = context->input(1); + OP_REQUIRES(context, TensorShapeUtils::IsMatrix(logits.shape()), + errors::InvalidArgument("logits must be 2-D, but got shape ", + logits.shape().DebugString())); + OP_REQUIRES(context, TensorShapeUtils::IsVector(labels.shape()), + errors::InvalidArgument("labels must be 1-D, but got shape ", + labels.shape().DebugString())); + OP_REQUIRES(context, logits.dim_size(0) == labels.dim_size(0), errors::InvalidArgument( - "logits first dimension must match labels size. logits shape=", - logits_in.shape().DebugString(), " labels shape=", - labels_in.shape().DebugString())); - OP_REQUIRES(context, TensorShapeUtils::IsMatrix(logits_in.shape()), - errors::InvalidArgument("logits must be 2-dimensional")); - // As we already tested that both inputs have the same shape no need to - // check that "labels" is a matrix too. - - // loss is 1-D (one per example), and size is batch_size. + "logits and labels must have the same first dimension, " + "got logits shape ", + logits.shape().DebugString(), " and labels shape ", + labels.shape().DebugString())); + OP_REQUIRES(context, logits.dim_size(1) > 0, + errors::InvalidArgument( + "Must have at least one class, but got logits shape ", + logits.shape().DebugString())); Tensor scratch; - OP_REQUIRES_OK( - context, context->allocate_temp(DataTypeToEnum::value, - TensorShape({logits_in.dim_size(0)}), - &scratch)); + OP_REQUIRES_OK(context, context->allocate_temp(DataTypeToEnum::value, + labels.shape(), &scratch)); Tensor* loss_out = nullptr; OP_REQUIRES_OK(context, - context->allocate_output( - 0, TensorShape({logits_in.dim_size(0)}), &loss_out)); + context->allocate_output(0, labels.shape(), &loss_out)); Tensor* back_out = nullptr; OP_REQUIRES_OK(context, - context->allocate_output(1, logits_in.shape(), &back_out)); + context->allocate_output(1, logits.shape(), &back_out)); - functor::SparseXentFunctor functor; - functor(context->eigen_device(), logits_in.matrix(), - labels_in.vec(), scratch.vec(), loss_out->vec(), - back_out->matrix()); + if (logits.dim_size(0) > 0) { + functor::SparseXentFunctor functor; + functor(context->eigen_device(), logits.matrix(), + labels.vec(), scratch.vec(), loss_out->vec(), + back_out->matrix()); + } } }; diff --git a/tensorflow/python/kernel_tests/sparse_xent_op_test.py b/tensorflow/python/kernel_tests/sparse_xent_op_test.py index eb6bdff8b5a..ea379fbac01 100644 --- a/tensorflow/python/kernel_tests/sparse_xent_op_test.py +++ b/tensorflow/python/kernel_tests/sparse_xent_op_test.py @@ -120,6 +120,13 @@ class SparseXentTest(tf.test.TestCase): tf.nn.sparse_softmax_cross_entropy_with_logits( tf.constant(1.0), tf.constant(0)) + def testLabelsPlaceholderScalar(self): + with self.test_session(): + labels = tf.placeholder(np.int32) + y = tf.nn.sparse_softmax_cross_entropy_with_logits([[7.]], labels) + with self.assertRaisesOpError("labels must be 1-D"): + y.eval(feed_dict={labels: 0}) + def testVector(self): with self.test_session(): loss = tf.nn.sparse_softmax_cross_entropy_with_logits( @@ -145,6 +152,9 @@ class SparseXentTest(tf.test.TestCase): np.array([[1., 1., 1., 1.], [1., 2., 3., 4.]]).astype(np.float16), np.array([3, 0]).astype(label_dtype)) + def testEmpty(self): + self._testXent(np.zeros((0, 3)), np.zeros((0,), dtype=np.int32)) + def testGradient(self): with self.test_session(): l = tf.constant([3, 0, 1], name="l") From e2587a04769af616cd8a50133792071685cedd5d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 30 Jun 2016 16:15:10 -0800 Subject: [PATCH 23/36] Transformed Distribution Change: 126362065 --- tensorflow/contrib/distributions/BUILD | 10 + tensorflow/contrib/distributions/__init__.py | 5 + .../transformed_distribution_test.py | 79 ++++++ .../python/ops/transformed_distribution.py | 252 ++++++++++++++++++ 4 files changed, 346 insertions(+) create mode 100644 tensorflow/contrib/distributions/python/kernel_tests/transformed_distribution_test.py create mode 100644 tensorflow/contrib/distributions/python/ops/transformed_distribution.py diff --git a/tensorflow/contrib/distributions/BUILD b/tensorflow/contrib/distributions/BUILD index 8f1e5b860a4..7f831e67c41 100644 --- a/tensorflow/contrib/distributions/BUILD +++ b/tensorflow/contrib/distributions/BUILD @@ -171,6 +171,16 @@ cuda_py_tests( ], ) +cuda_py_tests( + name = "transformed_distribution_test", + size = "small", + srcs = ["python/kernel_tests/transformed_distribution_test.py"], + additional_deps = [ + ":distributions_py", + "//tensorflow/python:platform_test", + ], +) + filegroup( name = "all_files", srcs = glob( diff --git a/tensorflow/contrib/distributions/__init__.py b/tensorflow/contrib/distributions/__init__.py index d04693ce983..0957c681016 100644 --- a/tensorflow/contrib/distributions/__init__.py +++ b/tensorflow/contrib/distributions/__init__.py @@ -47,6 +47,10 @@ initialized with parameters that define the distributions. @@DirichletMultinomial +### Transformed distributions + +@@ContinuousTransformedDistribution + ## Operators allowing for matrix-free methods ### Positive definite operators @@ -95,4 +99,5 @@ from tensorflow.contrib.distributions.python.ops.operator_pd import * from tensorflow.contrib.distributions.python.ops.operator_pd_cholesky import * from tensorflow.contrib.distributions.python.ops.operator_pd_full import * from tensorflow.contrib.distributions.python.ops.student_t import * +from tensorflow.contrib.distributions.python.ops.transformed_distribution import * from tensorflow.contrib.distributions.python.ops.uniform import * diff --git a/tensorflow/contrib/distributions/python/kernel_tests/transformed_distribution_test.py b/tensorflow/contrib/distributions/python/kernel_tests/transformed_distribution_test.py new file mode 100644 index 00000000000..d78f4a92161 --- /dev/null +++ b/tensorflow/contrib/distributions/python/kernel_tests/transformed_distribution_test.py @@ -0,0 +1,79 @@ +# Copyright 2015 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for ContinuousTransformedDistribution.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np +from scipy import stats +import tensorflow as tf + + +class ContinuousTransformedDistributionTest(tf.test.TestCase): + + def testContinuousTransformedDistribution(self): + with self.test_session(): + mu = 3.0 + sigma = 0.02 + log_normal = tf.contrib.distributions.ContinuousTransformedDistribution( + base_dist_cls=tf.contrib.distributions.Normal, + mu=mu, + sigma=sigma, + transform=lambda x: tf.exp(x), + inverse=lambda y: tf.log(y), + log_det_jacobian=(lambda x: tf.reduce_sum(x))) + + # sample + self.assertAllClose([stats.lognorm.mean(s=sigma, scale=np.exp(mu))], + [np.mean(log_normal.sample(100000, seed=235).eval())], + atol=1e-2) + + # pdf, log_pdf + test_vals = np.linspace(0.00001, 10.).astype(np.float32) + for test_val in test_vals: + expected = stats.lognorm.logpdf(test_val, s=sigma, scale=np.exp(mu)) + self.assertAllClose([expected], [log_normal.log_pdf(test_val).eval()]) + self.assertAllClose([np.exp(expected)], + [log_normal.pdf(test_val).eval()]) + + def testCachedSamplesWithoutInverse(self): + with self.test_session() as sess: + mu = 3.0 + sigma = 0.02 + log_normal = tf.contrib.distributions.ContinuousTransformedDistribution( + base_dist_cls=tf.contrib.distributions.Normal, + mu=mu, + sigma=sigma, + transform=lambda x: tf.exp(x), + inverse=None, + log_det_jacobian=(lambda x: tf.reduce_sum(x))) + + sample = log_normal.sample(1) + sample_val, log_pdf_val = sess.run([sample, log_normal.log_pdf(sample)]) + self.assertAllClose( + stats.lognorm.logpdf(sample_val, s=sigma, + scale=np.exp(mu)), + log_pdf_val, + atol=1e-2) + + with self.assertRaisesRegexp(ValueError, + "was not returned from `sample`"): + log_normal.log_pdf(tf.constant(3.0)) + + +if __name__ == "__main__": + tf.test.main() diff --git a/tensorflow/contrib/distributions/python/ops/transformed_distribution.py b/tensorflow/contrib/distributions/python/ops/transformed_distribution.py new file mode 100644 index 00000000000..8899d7a59f7 --- /dev/null +++ b/tensorflow/contrib/distributions/python/ops/transformed_distribution.py @@ -0,0 +1,252 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""A Transformed Distribution class.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.contrib.distributions.python.ops import distribution # pylint: disable=line-too-long +from tensorflow.python.framework import ops + + +class ContinuousTransformedDistribution(distribution.ContinuousDistribution): + """A Transformed Distribution. + + A Transformed Distribution models `p(y)` given a base distribution `p(x)`, + an invertible transform, `y = f(x)`, and the determinant of the Jacobian of + `f(x)`. + + Shapes, type, and reparameterization are taken from the base distribution. + + #### Mathematical details + + * `p(x)` - probability distribution for random variable X + * `p(y)` - probability distribution for random variable Y + * `f` - transform + * `g` - inverse transform, `f(g(x)) = x` + * `J(x)` - Jacobian of f(x) + + A Transformed Distribution exposes `sample` and `pdf`: + + * `sample`: `y = f(x)`, after drawing a sample of X. + * `pdf`: `p(y) = p(x) / det|J(x)| = p(g(y)) / det|J(g(y))|` + + A simple example constructing a Log-Normal distribution from a Normal + distribution: + + ``` + logit_normal = ContinuousTransformedDistribution( + base_dist=Normal(mu, sigma), + transform=lambda x: tf.sigmoid(x), + inverse=lambda y: tf.log(y) - tf.log(1. - y), + log_det_jacobian=(lambda x: + tf.reduce_sum(tf.log(tf.sigmoid(x)) + tf.log(1. - tf.sigmoid(x)), + reduction_indices=[-1]))) + name="LogitNormalTransformedDistribution" + ) + ``` + """ + + def __init__(self, + base_dist_cls, + transform, + inverse, + log_det_jacobian, + name="ContinuousTransformedDistribution", + **base_dist_args): + """Construct a Transformed Distribution. + + Args: + base_dist_cls: the base distribution class to transform. Must be a + subclass of `ContinuousDistribution`. + transform: a callable that takes a `Tensor` sample from `base_dist` and + returns a `Tensor` of the same shape and type. `x => y`. + inverse: a callable that computes the inverse of transform. `y => x`. If + None, users can only call `log_pdf` on values returned by `sample`. + log_det_jacobian: a callable that takes a `Tensor` sample from `base_dist` + and returns the log of the determinant of the Jacobian of `transform`. + name: The name for the distribution. + **base_dist_args: kwargs to pass on to dist_cls on construction. + + Raises: + TypeError: if `base_dist_cls` is not a subclass of + `ContinuousDistribution`. + """ + if not issubclass(base_dist_cls, distribution.ContinuousDistribution): + raise TypeError("base_dist_cls must be a subclass of" + "ContinuousDistribution.") + with ops.op_scope(base_dist_args.values(), name) as scope: + self._name = scope + self._base_dist = base_dist_cls(**base_dist_args) + self._transform = transform + self._inverse = inverse + self._log_det_jacobian = log_det_jacobian + self._inverse_cache = {} + + @property + def name(self): + return self._name + + @property + def dtype(self): + return self._base_dist.dtype + + def batch_shape(self, name="batch_shape"): + """Batch dimensions of this instance as a 1-D int32 `Tensor`. + + The product of the dimensions of the `batch_shape` is the number of + independent distributions of this kind the instance represents. + + Args: + name: name to give to the op. + + Returns: + `Tensor` `batch_shape` + """ + with ops.name_scope(self.name): + return self._base_dist.batch_shape(name) + + def get_batch_shape(self): + """`TensorShape` available at graph construction time. + + Same meaning as `batch_shape`. May be only partially defined. + + Returns: + batch shape + """ + return self._base_dist.get_batch_shape() + + def event_shape(self, name="event_shape"): + """Shape of a sample from a single distribution as a 1-D int32 `Tensor`. + + Args: + name: name to give to the op. + + Returns: + `Tensor` `event_shape` + """ + with ops.name_scope(self.name): + return self._base_dist.event_shape(name) + + def get_event_shape(self): + """`TensorShape` available at graph construction time. + + Same meaning as `event_shape`. May be only partially defined. + + Returns: + event shape + """ + return self._base_dist.get_event_shape() + + @property + def base_distribution(self): + """Base distribution, p(x).""" + return self._base_dist + + @property + def transform(self): + """Function transforming x => y.""" + return self._transform + + @property + def inverse(self): + """Inverse function of transform, y => x.""" + return self._inverse + + @property + def log_det_jacobian(self): + """Function computing the log determinant of the Jacobian of transform.""" + return self._log_det_jacobian + + def log_pdf(self, y, name="log_pdf"): + """Log pdf of observations in `y`. + + `log ( p(g(y)) / det|J(g(y))| )`, where `g` is the inverse of `transform`. + + Args: + y: tensor of dtype `dtype`. + name: The name to give this op. + + Returns: + log_pdf: tensor of dtype `dtype`, the log-PDFs of `y`. + + Raises: + ValueError: if `inverse` was not provided to the distribution and `y` was + not returned from `sample`. + """ + with ops.name_scope(self.name): + with ops.op_scope([y], name): + y = ops.convert_to_tensor(y) + if y.dtype != self.dtype: + raise TypeError("Input x dtype does not match dtype: %s vs. %s" % + (y.dtype, self.dtype)) + with ops.name_scope("inverse"): + if y in self._inverse_cache: + x = self._inverse_cache[y] + elif self._inverse: + x = self._inverse(y) + else: + raise ValueError("No inverse function exists and input `y` was not " + "returned from `sample`.") + with ops.name_scope("log_det_jacobian"): + log_det_jacobian = self._log_det_jacobian(x) + return self._base_dist.log_likelihood(x) - log_det_jacobian + + def pdf(self, y, name="pdf"): + """The PDF of observations in `y`. + + `p(g(y)) / det|J(g(y))|`, where `g` is the inverse of `transform`. + + Args: + y: `Tensor` of dtype `dtype`. + name: The name to give this op. + + Returns: + pdf: `Tensor` of dtype `dtype`, the pdf values of `y`. + """ + return super(ContinuousTransformedDistribution, self).pdf(y, name=name) + + def sample(self, n, seed=None, name="sample"): + """Sample `n` observations. + + Samples from the base distribution and then passes through the transform. + + Args: + n: scalar, type int32, the number of observations to sample. + seed: Python integer, the random seed. + name: The name to give this op. + + Returns: + samples: `[n, ...]`, a `Tensor` of `n` samples. + """ + with ops.name_scope(self.name): + with ops.name_scope(name): + samples = self._base_dist.sample(n=n, seed=seed) + with ops.name_scope("transform"): + transformed = self._transform(samples) + self._inverse_cache[transformed] = samples + return transformed + + @property + def is_reparameterized(self): + return self._base_dist.is_reparameterized + + @property + def strict_statistics(self): + return self._base_dist.strict_statistics + + @property + def strict(self): + return self._base_dist.strict From be5fceb6dc17f1d791a9ec8c7262ac07e6daa5fd Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 30 Jun 2016 16:39:58 -0800 Subject: [PATCH 24/36] Update generated Python Op docs. Change: 126364170 --- .../api_docs/python/contrib.distributions.md | 317 ++++++++++++++++++ ...tions.ContinuousTransformedDistribution.md | 309 +++++++++++++++++ tensorflow/g3doc/api_docs/python/index.md | 1 + 3 files changed, 627 insertions(+) create mode 100644 tensorflow/g3doc/api_docs/python/functions_and_classes/shard0/tf.contrib.distributions.ContinuousTransformedDistribution.md diff --git a/tensorflow/g3doc/api_docs/python/contrib.distributions.md b/tensorflow/g3doc/api_docs/python/contrib.distributions.md index c2e67db7cb6..7bea8d72dd2 100644 --- a/tensorflow/g3doc/api_docs/python/contrib.distributions.md +++ b/tensorflow/g3doc/api_docs/python/contrib.distributions.md @@ -4022,6 +4022,323 @@ Variance of the distribution. +### Transformed distributions + +- - - + +### `class tf.contrib.distributions.ContinuousTransformedDistribution` {#ContinuousTransformedDistribution} + +A Transformed Distribution. + +A Transformed Distribution models `p(y)` given a base distribution `p(x)`, +an invertible transform, `y = f(x)`, and the determinant of the Jacobian of +`f(x)`. + +Shapes, type, and reparameterization are taken from the base distribution. + +#### Mathematical details + +* `p(x)` - probability distribution for random variable X +* `p(y)` - probability distribution for random variable Y +* `f` - transform +* `g` - inverse transform, `f(g(x)) = x` +* `J(x)` - Jacobian of f(x) + +A Transformed Distribution exposes `sample` and `pdf`: + + * `sample`: `y = f(x)`, after drawing a sample of X. + * `pdf`: `p(y) = p(x) / det|J(x)| = p(g(y)) / det|J(g(y))|` + +A simple example constructing a Log-Normal distribution from a Normal +distribution: + +``` +logit_normal = ContinuousTransformedDistribution( + base_dist=Normal(mu, sigma), + transform=lambda x: tf.sigmoid(x), + inverse=lambda y: tf.log(y) - tf.log(1. - y), + log_det_jacobian=(lambda x: + tf.reduce_sum(tf.log(tf.sigmoid(x)) + tf.log(1. - tf.sigmoid(x)), + reduction_indices=[-1]))) + name="LogitNormalTransformedDistribution" +) +``` +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.__init__(base_dist_cls, transform, inverse, log_det_jacobian, name='ContinuousTransformedDistribution', **base_dist_args)` {#ContinuousTransformedDistribution.__init__} + +Construct a Transformed Distribution. + +##### Args: + + +* `base_dist_cls`: the base distribution class to transform. Must be a + subclass of `ContinuousDistribution`. +* `transform`: a callable that takes a `Tensor` sample from `base_dist` and + returns a `Tensor` of the same shape and type. `x => y`. +* `inverse`: a callable that computes the inverse of transform. `y => x`. If + None, users can only call `log_pdf` on values returned by `sample`. +* `log_det_jacobian`: a callable that takes a `Tensor` sample from `base_dist` + and returns the log of the determinant of the Jacobian of `transform`. +* `name`: The name for the distribution. +* `**base_dist_args`: kwargs to pass on to dist_cls on construction. + +##### Raises: + + +* `TypeError`: if `base_dist_cls` is not a subclass of + `ContinuousDistribution`. + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.base_distribution` {#ContinuousTransformedDistribution.base_distribution} + +Base distribution, p(x). + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.batch_shape(name='batch_shape')` {#ContinuousTransformedDistribution.batch_shape} + +Batch dimensions of this instance as a 1-D int32 `Tensor`. + +The product of the dimensions of the `batch_shape` is the number of +independent distributions of this kind the instance represents. + +##### Args: + + +* `name`: name to give to the op. + +##### Returns: + + `Tensor` `batch_shape` + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.cdf(value, name='cdf')` {#ContinuousTransformedDistribution.cdf} + +Cumulative distribution function. + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.dtype` {#ContinuousTransformedDistribution.dtype} + + + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.entropy(name='entropy')` {#ContinuousTransformedDistribution.entropy} + +Entropy of the distribution in nats. + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.event_shape(name='event_shape')` {#ContinuousTransformedDistribution.event_shape} + +Shape of a sample from a single distribution as a 1-D int32 `Tensor`. + +##### Args: + + +* `name`: name to give to the op. + +##### Returns: + + `Tensor` `event_shape` + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.get_batch_shape()` {#ContinuousTransformedDistribution.get_batch_shape} + +`TensorShape` available at graph construction time. + +Same meaning as `batch_shape`. May be only partially defined. + +##### Returns: + + batch shape + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.get_event_shape()` {#ContinuousTransformedDistribution.get_event_shape} + +`TensorShape` available at graph construction time. + +Same meaning as `event_shape`. May be only partially defined. + +##### Returns: + + event shape + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.inverse` {#ContinuousTransformedDistribution.inverse} + +Inverse function of transform, y => x. + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.is_reparameterized` {#ContinuousTransformedDistribution.is_reparameterized} + + + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.log_cdf(value, name='log_cdf')` {#ContinuousTransformedDistribution.log_cdf} + +Log CDF. + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.log_det_jacobian` {#ContinuousTransformedDistribution.log_det_jacobian} + +Function computing the log determinant of the Jacobian of transform. + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.log_likelihood(value, name='log_likelihood')` {#ContinuousTransformedDistribution.log_likelihood} + +Log likelihood of this distribution (same as log_pdf). + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.log_pdf(y, name='log_pdf')` {#ContinuousTransformedDistribution.log_pdf} + +Log pdf of observations in `y`. + +`log ( p(g(y)) / det|J(g(y))| )`, where `g` is the inverse of `transform`. + +##### Args: + + +* `y`: tensor of dtype `dtype`. +* `name`: The name to give this op. + +##### Returns: + + +* `log_pdf`: tensor of dtype `dtype`, the log-PDFs of `y`. + +##### Raises: + + +* `ValueError`: if `inverse` was not provided to the distribution and `y` was + not returned from `sample`. + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.mean(name='mean')` {#ContinuousTransformedDistribution.mean} + +Mean of the distribution. + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.mode(name='mode')` {#ContinuousTransformedDistribution.mode} + +Mode of the distribution. + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.name` {#ContinuousTransformedDistribution.name} + + + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.pdf(y, name='pdf')` {#ContinuousTransformedDistribution.pdf} + +The PDF of observations in `y`. + +`p(g(y)) / det|J(g(y))|`, where `g` is the inverse of `transform`. + +##### Args: + + +* `y`: `Tensor` of dtype `dtype`. +* `name`: The name to give this op. + +##### Returns: + + +* `pdf`: `Tensor` of dtype `dtype`, the pdf values of `y`. + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.sample(n, seed=None, name='sample')` {#ContinuousTransformedDistribution.sample} + +Sample `n` observations. + +Samples from the base distribution and then passes through the transform. + +##### Args: + + +* `n`: scalar, type int32, the number of observations to sample. +* `seed`: Python integer, the random seed. +* `name`: The name to give this op. + +##### Returns: + + +* `samples`: `[n, ...]`, a `Tensor` of `n` samples. + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.std(name='std')` {#ContinuousTransformedDistribution.std} + +Standard deviation of the distribution. + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.strict` {#ContinuousTransformedDistribution.strict} + + + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.strict_statistics` {#ContinuousTransformedDistribution.strict_statistics} + + + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.transform` {#ContinuousTransformedDistribution.transform} + +Function transforming x => y. + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.variance(name='variance')` {#ContinuousTransformedDistribution.variance} + +Variance of the distribution. + + + + ## Operators allowing for matrix-free methods ### Positive definite operators diff --git a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard0/tf.contrib.distributions.ContinuousTransformedDistribution.md b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard0/tf.contrib.distributions.ContinuousTransformedDistribution.md new file mode 100644 index 00000000000..1cda4145d6d --- /dev/null +++ b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard0/tf.contrib.distributions.ContinuousTransformedDistribution.md @@ -0,0 +1,309 @@ +A Transformed Distribution. + +A Transformed Distribution models `p(y)` given a base distribution `p(x)`, +an invertible transform, `y = f(x)`, and the determinant of the Jacobian of +`f(x)`. + +Shapes, type, and reparameterization are taken from the base distribution. + +#### Mathematical details + +* `p(x)` - probability distribution for random variable X +* `p(y)` - probability distribution for random variable Y +* `f` - transform +* `g` - inverse transform, `f(g(x)) = x` +* `J(x)` - Jacobian of f(x) + +A Transformed Distribution exposes `sample` and `pdf`: + + * `sample`: `y = f(x)`, after drawing a sample of X. + * `pdf`: `p(y) = p(x) / det|J(x)| = p(g(y)) / det|J(g(y))|` + +A simple example constructing a Log-Normal distribution from a Normal +distribution: + +``` +logit_normal = ContinuousTransformedDistribution( + base_dist=Normal(mu, sigma), + transform=lambda x: tf.sigmoid(x), + inverse=lambda y: tf.log(y) - tf.log(1. - y), + log_det_jacobian=(lambda x: + tf.reduce_sum(tf.log(tf.sigmoid(x)) + tf.log(1. - tf.sigmoid(x)), + reduction_indices=[-1]))) + name="LogitNormalTransformedDistribution" +) +``` +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.__init__(base_dist_cls, transform, inverse, log_det_jacobian, name='ContinuousTransformedDistribution', **base_dist_args)` {#ContinuousTransformedDistribution.__init__} + +Construct a Transformed Distribution. + +##### Args: + + +* `base_dist_cls`: the base distribution class to transform. Must be a + subclass of `ContinuousDistribution`. +* `transform`: a callable that takes a `Tensor` sample from `base_dist` and + returns a `Tensor` of the same shape and type. `x => y`. +* `inverse`: a callable that computes the inverse of transform. `y => x`. If + None, users can only call `log_pdf` on values returned by `sample`. +* `log_det_jacobian`: a callable that takes a `Tensor` sample from `base_dist` + and returns the log of the determinant of the Jacobian of `transform`. +* `name`: The name for the distribution. +* `**base_dist_args`: kwargs to pass on to dist_cls on construction. + +##### Raises: + + +* `TypeError`: if `base_dist_cls` is not a subclass of + `ContinuousDistribution`. + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.base_distribution` {#ContinuousTransformedDistribution.base_distribution} + +Base distribution, p(x). + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.batch_shape(name='batch_shape')` {#ContinuousTransformedDistribution.batch_shape} + +Batch dimensions of this instance as a 1-D int32 `Tensor`. + +The product of the dimensions of the `batch_shape` is the number of +independent distributions of this kind the instance represents. + +##### Args: + + +* `name`: name to give to the op. + +##### Returns: + + `Tensor` `batch_shape` + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.cdf(value, name='cdf')` {#ContinuousTransformedDistribution.cdf} + +Cumulative distribution function. + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.dtype` {#ContinuousTransformedDistribution.dtype} + + + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.entropy(name='entropy')` {#ContinuousTransformedDistribution.entropy} + +Entropy of the distribution in nats. + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.event_shape(name='event_shape')` {#ContinuousTransformedDistribution.event_shape} + +Shape of a sample from a single distribution as a 1-D int32 `Tensor`. + +##### Args: + + +* `name`: name to give to the op. + +##### Returns: + + `Tensor` `event_shape` + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.get_batch_shape()` {#ContinuousTransformedDistribution.get_batch_shape} + +`TensorShape` available at graph construction time. + +Same meaning as `batch_shape`. May be only partially defined. + +##### Returns: + + batch shape + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.get_event_shape()` {#ContinuousTransformedDistribution.get_event_shape} + +`TensorShape` available at graph construction time. + +Same meaning as `event_shape`. May be only partially defined. + +##### Returns: + + event shape + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.inverse` {#ContinuousTransformedDistribution.inverse} + +Inverse function of transform, y => x. + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.is_reparameterized` {#ContinuousTransformedDistribution.is_reparameterized} + + + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.log_cdf(value, name='log_cdf')` {#ContinuousTransformedDistribution.log_cdf} + +Log CDF. + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.log_det_jacobian` {#ContinuousTransformedDistribution.log_det_jacobian} + +Function computing the log determinant of the Jacobian of transform. + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.log_likelihood(value, name='log_likelihood')` {#ContinuousTransformedDistribution.log_likelihood} + +Log likelihood of this distribution (same as log_pdf). + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.log_pdf(y, name='log_pdf')` {#ContinuousTransformedDistribution.log_pdf} + +Log pdf of observations in `y`. + +`log ( p(g(y)) / det|J(g(y))| )`, where `g` is the inverse of `transform`. + +##### Args: + + +* `y`: tensor of dtype `dtype`. +* `name`: The name to give this op. + +##### Returns: + + +* `log_pdf`: tensor of dtype `dtype`, the log-PDFs of `y`. + +##### Raises: + + +* `ValueError`: if `inverse` was not provided to the distribution and `y` was + not returned from `sample`. + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.mean(name='mean')` {#ContinuousTransformedDistribution.mean} + +Mean of the distribution. + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.mode(name='mode')` {#ContinuousTransformedDistribution.mode} + +Mode of the distribution. + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.name` {#ContinuousTransformedDistribution.name} + + + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.pdf(y, name='pdf')` {#ContinuousTransformedDistribution.pdf} + +The PDF of observations in `y`. + +`p(g(y)) / det|J(g(y))|`, where `g` is the inverse of `transform`. + +##### Args: + + +* `y`: `Tensor` of dtype `dtype`. +* `name`: The name to give this op. + +##### Returns: + + +* `pdf`: `Tensor` of dtype `dtype`, the pdf values of `y`. + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.sample(n, seed=None, name='sample')` {#ContinuousTransformedDistribution.sample} + +Sample `n` observations. + +Samples from the base distribution and then passes through the transform. + +##### Args: + + +* `n`: scalar, type int32, the number of observations to sample. +* `seed`: Python integer, the random seed. +* `name`: The name to give this op. + +##### Returns: + + +* `samples`: `[n, ...]`, a `Tensor` of `n` samples. + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.std(name='std')` {#ContinuousTransformedDistribution.std} + +Standard deviation of the distribution. + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.strict` {#ContinuousTransformedDistribution.strict} + + + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.strict_statistics` {#ContinuousTransformedDistribution.strict_statistics} + + + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.transform` {#ContinuousTransformedDistribution.transform} + +Function transforming x => y. + + +- - - + +#### `tf.contrib.distributions.ContinuousTransformedDistribution.variance(name='variance')` {#ContinuousTransformedDistribution.variance} + +Variance of the distribution. + + diff --git a/tensorflow/g3doc/api_docs/python/index.md b/tensorflow/g3doc/api_docs/python/index.md index e9af0e65751..0fc608e893c 100644 --- a/tensorflow/g3doc/api_docs/python/index.md +++ b/tensorflow/g3doc/api_docs/python/index.md @@ -586,6 +586,7 @@ * [`Categorical`](../../api_docs/python/contrib.distributions.md#Categorical) * [`Chi2`](../../api_docs/python/contrib.distributions.md#Chi2) * [`ContinuousDistribution`](../../api_docs/python/contrib.distributions.md#ContinuousDistribution) + * [`ContinuousTransformedDistribution`](../../api_docs/python/contrib.distributions.md#ContinuousTransformedDistribution) * [`DirichletMultinomial`](../../api_docs/python/contrib.distributions.md#DirichletMultinomial) * [`DiscreteDistribution`](../../api_docs/python/contrib.distributions.md#DiscreteDistribution) * [`Exponential`](../../api_docs/python/contrib.distributions.md#Exponential) From d9cfbd63aefbeb48200d65841a401291b572d35b Mon Sep 17 00:00:00 2001 From: Derek Murray Date: Thu, 30 Jun 2016 16:44:14 -0800 Subject: [PATCH 25/36] Move POSIX-dependent tracing function from platform/default -> platform/posix. Change: 126364522 --- tensorflow/core/platform/default/tracing.cc | 17 --------- tensorflow/core/platform/posix/tracing.cc | 40 +++++++++++++++++++++ 2 files changed, 40 insertions(+), 17 deletions(-) create mode 100644 tensorflow/core/platform/posix/tracing.cc diff --git a/tensorflow/core/platform/default/tracing.cc b/tensorflow/core/platform/default/tracing.cc index 7910e97db9a..422564fb3e4 100644 --- a/tensorflow/core/platform/default/tracing.cc +++ b/tensorflow/core/platform/default/tracing.cc @@ -15,8 +15,6 @@ limitations under the License. #include "tensorflow/core/platform/tracing.h" -#include - namespace tensorflow { namespace port { @@ -26,21 +24,6 @@ void Tracing::RegisterEvent(EventCategory id, const char* name) { void Tracing::Initialize() {} -static bool TryGetEnv(const char* name, const char** value) { - *value = getenv(name); - return *value != nullptr && (*value)[0] != '\0'; -} - -const char* Tracing::LogDir() { - const char* dir; - if (TryGetEnv("TEST_TMPDIR", &dir)) return dir; - if (TryGetEnv("TMP", &dir)) return dir; - if (TryGetEnv("TMPDIR", &dir)) return dir; - dir = "/tmp"; - if (access(dir, R_OK | W_OK | X_OK) == 0) return dir; - return "."; // Default to current directory. -} - static bool DoInit() { Tracing::Initialize(); return true; diff --git a/tensorflow/core/platform/posix/tracing.cc b/tensorflow/core/platform/posix/tracing.cc new file mode 100644 index 00000000000..1d1aa53f2ca --- /dev/null +++ b/tensorflow/core/platform/posix/tracing.cc @@ -0,0 +1,40 @@ +/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/platform/tracing.h" + +#include +#include + +namespace tensorflow { +namespace port { + +static bool TryGetEnv(const char* name, const char** value) { + *value = getenv(name); + return *value != nullptr && (*value)[0] != '\0'; +} + +const char* Tracing::LogDir() { + const char* dir; + if (TryGetEnv("TEST_TMPDIR", &dir)) return dir; + if (TryGetEnv("TMP", &dir)) return dir; + if (TryGetEnv("TMPDIR", &dir)) return dir; + dir = "/tmp"; + if (access(dir, R_OK | W_OK | X_OK) == 0) return dir; + return "."; // Default to current directory. +} + +} // namespace port +} // namespace tensorflow From 6a61255cc3f22db55da68ef6d26a1aa477dc0cf3 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 30 Jun 2016 17:07:22 -0800 Subject: [PATCH 26/36] Allow to specify the list of "clean termination" exceptions when creating a Coordinator Change: 126366456 --- tensorflow/python/training/coordinator.py | 29 ++++++++++++------- .../python/training/coordinator_test.py | 10 +++++++ 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/tensorflow/python/training/coordinator.py b/tensorflow/python/training/coordinator.py index 5001b6acd9e..1785ff372b6 100644 --- a/tensorflow/python/training/coordinator.py +++ b/tensorflow/python/training/coordinator.py @@ -125,8 +125,21 @@ class Coordinator(object): ``` """ - def __init__(self): - """Create a new Coordinator.""" + def __init__(self, clean_stop_exception_types=None): + """Create a new Coordinator. + + Args: + clean_stop_exception_types: Optional tuple of Exception types that should + cause a clean stop of the coordinator. If an exception of one of these + types is reported to `request_stop(ex)` the coordinator will behave as + if `request_stop(None)` was called. Defaults to + `(tf.errors.OutOfRangeError,)` which is used by input queues to signal + the end of input. When feeding training data from a Python iterator it + is common to add `StopIteration` to this list. + """ + if clean_stop_exception_types is None: + clean_stop_exception_types = (errors.OutOfRangeError,) + self._clean_stop_exception_types = clean_stop_exception_types # Protects all attributes. self._lock = threading.Lock() # Event set when threads must stop. @@ -143,9 +156,8 @@ class Coordinator(object): reported to the users. If yes, it returns `ex` as is, otherwise it returns None. - The code returns None for exceptions that are used for control flow such as - the OutOfRangeError raised by the dequeue operations to indicate that a - queue was closed after its contents were dequeued. + The code returns None for exception types listed in + `_clean_stop_exception_types`. Args: ex: None, an `Exception`, or a Python `exc_info` tuple as returned by @@ -158,12 +170,7 @@ class Coordinator(object): ex2 = ex[1] else: ex2 = ex - # OutOfRangeError is used to indicate "end of input". We do not want to - # report an exception for it. TODO(touts): Likely also need to ignore - # some of the Aborted and Cancelled exceptions raised by queue ops after - # queues are closed, but this can only be done after these exceptions have - # been clearly identified. - if isinstance(ex2, (errors.OutOfRangeError)): + if isinstance(ex2, self._clean_stop_exception_types): # Ignore the exception. ex = None return ex diff --git a/tensorflow/python/training/coordinator_test.py b/tensorflow/python/training/coordinator_test.py index dac5fe59e9d..9e700cbe68c 100644 --- a/tensorflow/python/training/coordinator_test.py +++ b/tensorflow/python/training/coordinator_test.py @@ -132,6 +132,16 @@ class CoordinatorTest(tf.test.TestCase): t.start() coord.join(threads) + def testJoinIgnoresMyExceptionType(self): + coord = tf.train.Coordinator(clean_stop_exception_types=(ValueError,)) + threads = [ + threading.Thread(target=RaiseInN, + args=(coord, 0.01, ValueError("Clean stop"), True)) + ] + for t in threads: + t.start() + coord.join(threads) + def testJoinRaiseReportExceptionUsingHandler(self): coord = tf.train.Coordinator() threads = [ From e0469870d02d9bac9535b3862c84415f57fa085e Mon Sep 17 00:00:00 2001 From: Zongheng Yang Date: Thu, 30 Jun 2016 17:10:22 -0800 Subject: [PATCH 27/36] Automated rollback of change 126348349 Change: 126366720 --- tensorflow/core/kernels/BUILD | 1 - tensorflow/core/kernels/sparse_add_op.cc | 31 ++++--- tensorflow/core/ops/sparse_ops.cc | 54 ------------ .../python/kernel_tests/sparse_ops_test.py | 63 -------------- tensorflow/python/ops/sparse_grad.py | 12 --- tensorflow/python/ops/sparse_ops.py | 83 ------------------- 6 files changed, 15 insertions(+), 229 deletions(-) diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index 06c3c86c673..142f63c6b47 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -1562,7 +1562,6 @@ tf_kernel_libraries( "sparse_concat_op", "sparse_reduce_sum_op", "sparse_dense_binary_op_shared", - "sparse_sparse_binary_op_shared", "sparse_reorder_op", "sparse_reshape_op", "sparse_softmax", diff --git a/tensorflow/core/kernels/sparse_add_op.cc b/tensorflow/core/kernels/sparse_add_op.cc index bd91dfdce64..0cb77d785ad 100644 --- a/tensorflow/core/kernels/sparse_add_op.cc +++ b/tensorflow/core/kernels/sparse_add_op.cc @@ -54,32 +54,31 @@ class SparseAddOp : public OpKernel { b_values_t->shape().DebugString())); auto a_values = ctx->input(1).vec(); auto b_values = ctx->input(4).vec(); - OP_REQUIRES( - ctx, a_values.size() == a_nnz && b_values.size() == b_nnz, - errors::InvalidArgument("Expected ", a_nnz, " and ", b_nnz, - " non-empty input values, got ", - a_values.size(), " and ", b_values.size())); OP_REQUIRES_OK(ctx, ctx->input("a_shape", &a_shape)); OP_REQUIRES_OK(ctx, ctx->input("b_shape", &b_shape)); OP_REQUIRES(ctx, TensorShapeUtils::IsVector(a_shape->shape()) && TensorShapeUtils::IsVector(b_shape->shape()), errors::InvalidArgument( - "Input shapes should be a vector but received shapes ", + "Input shape should be a vector but received shapes ", a_shape->shape().DebugString(), " and ", b_shape->shape().DebugString())); + OP_REQUIRES( - ctx, a_shape->IsSameSize(*b_shape), - errors::InvalidArgument( - "Operands do not have the same ranks; got shapes: ", - a_shape->SummarizeValue(10), " and ", b_shape->SummarizeValue(10))); - const auto a_shape_flat = a_shape->flat(); - const auto b_shape_flat = b_shape->flat(); - for (int i = 0; i < a_shape->NumElements(); ++i) { - OP_REQUIRES(ctx, a_shape_flat(i) == b_shape_flat(i), + ctx, a_values.size() == a_nnz && b_values.size() == b_nnz, + errors::InvalidArgument("Expected ", a_nnz, " and ", b_nnz, + " non-empty input values, got ", + a_values.size(), " and ", b_values.size())); + + OP_REQUIRES(ctx, a_shape->dims() == b_shape->dims(), + errors::InvalidArgument( + "Ranks of input tensors must match, but saw ranks: ", + a_shape->dims(), " and ", b_shape->dims())); + for (int i = 0; i < a_shape->dims(); ++i) { + OP_REQUIRES(ctx, a_shape->dim_size(i) == b_shape->dim_size(i), errors::InvalidArgument( - "Operands' shapes do not match: got ", a_shape_flat(i), - " and ", b_shape_flat(i), " for dimension ", i)); + "Input shapes must match: got ", a_shape->dim_size(i), + " and ", b_shape->dim_size(i), " for dimension ", i)); } OP_REQUIRES_OK(ctx, ctx->input("thresh", &thresh_t)); diff --git a/tensorflow/core/ops/sparse_ops.cc b/tensorflow/core/ops/sparse_ops.cc index a39f2f70cba..d80a5b1f9bc 100644 --- a/tensorflow/core/ops/sparse_ops.cc +++ b/tensorflow/core/ops/sparse_ops.cc @@ -570,58 +570,4 @@ sp_shape: 1-D. Shape of the input SparseTensor. output: 1-D. The `NNZ` values for the result `SparseTensor`. )doc"); -REGISTER_OP("SparseSparseMaximum") - .Input("a_indices: int64") - .Input("a_values: T") - .Input("a_shape: int64") - .Input("b_indices: int64") - .Input("b_values: T") - .Input("b_shape: int64") - .Output("output_indices: int64") - .Output("output_values: T") - .Attr("T: realnumbertype") - .Doc(R"doc( -Returns the element-wise max of two SparseTensors. - -Assumes the two SparseTensors have the same shape, i.e., no broadcasting. - -a_indices: 2-D. `N x R` matrix with the indices of non-empty values in a - SparseTensor, in the canonical lexicographic ordering. -a_values: 1-D. `N` non-empty values corresponding to `a_indices`. -a_shape: 1-D. Shape of the input SparseTensor. -b_indices: counterpart to `a_indices` for the other operand. -b_values: counterpart to `a_values` for the other operand; must be of the same dtype. -b_shape: counterpart to `a_shape` for the other operand; the two shapes must be equal. - -output_indices: 2-D. The indices of the output SparseTensor. -output_values: 1-D. The values of the output SparseTensor. -)doc"); - -REGISTER_OP("SparseSparseMinimum") - .Input("a_indices: int64") - .Input("a_values: T") - .Input("a_shape: int64") - .Input("b_indices: int64") - .Input("b_values: T") - .Input("b_shape: int64") - .Output("output_indices: int64") - .Output("output_values: T") - .Attr("T: numbertype") - .Doc(R"doc( -Returns the element-wise min of two SparseTensors. - -Assumes the two SparseTensors have the same shape, i.e., no broadcasting. - -a_indices: 2-D. `N x R` matrix with the indices of non-empty values in a - SparseTensor, in the canonical lexicographic ordering. -a_values: 1-D. `N` non-empty values corresponding to `a_indices`. -a_shape: 1-D. Shape of the input SparseTensor. -b_indices: counterpart to `a_indices` for the other operand. -b_values: counterpart to `a_values` for the other operand; must be of the same dtype. -b_shape: counterpart to `a_shape` for the other operand; the two shapes must be equal. - -output_indices: 2-D. The indices of the output SparseTensor. -output_values: 1-D. The values of the output SparseTensor. -)doc"); - } // namespace tensorflow diff --git a/tensorflow/python/kernel_tests/sparse_ops_test.py b/tensorflow/python/kernel_tests/sparse_ops_test.py index a0394851d11..867bfc5b369 100644 --- a/tensorflow/python/kernel_tests/sparse_ops_test.py +++ b/tensorflow/python/kernel_tests/sparse_ops_test.py @@ -649,68 +649,5 @@ class SparseSoftmaxTest(test_util.TensorFlowTestCase): self.assertLess(err, 1e-4) -class SparseMinimumMaximumTest(test_util.TensorFlowTestCase): - - def _assertSparseTensorValueEqual(self, a, b): - self.assertAllEqual(a.indices, b.indices) - self.assertAllEqual(a.values, b.values) - self.assertAllEqual(a.shape, b.shape) - - def testBasic(self): - with self.test_session(use_gpu=False): - # 1-D, values at index 0. - sp_zero = ops.SparseTensor([[0]], [0], [7]) - sp_one = ops.SparseTensor([[0]], [1], [7]) - max_tf = tf.sparse_maximum(sp_zero, sp_one).eval() - min_tf = tf.sparse_minimum(sp_zero, sp_one).eval() - self._assertSparseTensorValueEqual(sp_one.eval(), max_tf) - self._assertSparseTensorValueEqual(sp_zero.eval(), min_tf) - - # Values at different indices. - sp_zero = ops.SparseTensor([[0]], [0], [7]) - sp_zero_2 = ops.SparseTensor([[1]], [0], [7]) - expected = ops.SparseTensor([[0], [1]], [0, 0], [7]) - max_tf = tf.sparse_maximum(sp_zero, sp_zero_2).eval() - min_tf = tf.sparse_minimum(sp_zero, sp_zero_2).eval() - self._assertSparseTensorValueEqual(expected.eval(), max_tf) - self._assertSparseTensorValueEqual(expected.eval(), min_tf) - - def testRandom(self): - np.random.seed(1618) - shapes = [(13,), (6, 8), (1, 7, 1)] - for shape in shapes: - for dtype in [np.int32, np.int64, np.float16, np.float32, np.float64]: - a_np = np.random.randn(*shape).astype(dtype) - b_np = np.random.randn(*shape).astype(dtype) - sp_a, unused_a_nnz = _sparsify(a_np, thresh=-.5) - sp_b, unused_b_nnz = _sparsify(b_np, thresh=-.5) - - with self.test_session(use_gpu=False): - maximum_tf = tf.sparse_maximum(sp_a, sp_b) - maximum_tf_densified = tf.sparse_tensor_to_dense(maximum_tf).eval() - minimum_tf = tf.sparse_minimum(sp_a, sp_b) - minimum_tf_densified = tf.sparse_tensor_to_dense(minimum_tf).eval() - - a_densified = tf.sparse_tensor_to_dense(sp_a).eval() - b_densified = tf.sparse_tensor_to_dense(sp_b).eval() - - self.assertAllEqual(np.maximum(a_densified, b_densified), - maximum_tf_densified) - self.assertAllEqual(np.minimum(a_densified, b_densified), - minimum_tf_densified) - - def testMismatchedShapes(self): - with self.test_session(use_gpu=False): - sp_zero = ops.SparseTensor([[0, 0]], [0], [1, 1]) - sp_one = ops.SparseTensor([[0]], [1], [2]) - with self.assertRaisesOpError("Operands do not have the same ranks"): - tf.sparse_maximum(sp_zero, sp_one).eval() - - sp_zero = ops.SparseTensor([[0]], [0], [1]) - sp_one = ops.SparseTensor([[0]], [1], [2]) - with self.assertRaisesOpError("Operands' shapes do not match"): - tf.sparse_maximum(sp_zero, sp_one).eval() - - if __name__ == "__main__": googletest.main() diff --git a/tensorflow/python/ops/sparse_grad.py b/tensorflow/python/ops/sparse_grad.py index 57350253ab8..93c026f2471 100644 --- a/tensorflow/python/ops/sparse_grad.py +++ b/tensorflow/python/ops/sparse_grad.py @@ -256,15 +256,3 @@ def _SparseSoftmaxGrad(op, grad): grad_x = sp_sum.values * sp_output.values return [None, grad_x, None] - - -@ops.RegisterGradient("SparseSparseMaximum") -def _SparseSparseMaximumGrad(unused_op, unused_grad): - raise NotImplementedError("Gradient for SparseSparseMaximum is currently not" - " implemented yet.") - - -@ops.RegisterGradient("SparseSparseMinimum") -def _SparseSparseMinimumGrad(unused_op, unused_grad): - raise NotImplementedError("Gradient for SparseSparseMinimum is currently not" - " implemented yet.") diff --git a/tensorflow/python/ops/sparse_ops.py b/tensorflow/python/ops/sparse_ops.py index b5877d27423..e0a922739bc 100644 --- a/tensorflow/python/ops/sparse_ops.py +++ b/tensorflow/python/ops/sparse_ops.py @@ -48,8 +48,6 @@ dimension, and dense along all other dimensions. @@sparse_add @@sparse_softmax @@sparse_tensor_dense_matmul -@@sparse_maximum -@@sparse_minimum """ from __future__ import absolute_import from __future__ import division @@ -1489,84 +1487,3 @@ def _SparseSoftmaxShape(op): # pylint: disable=invalid-name unused_shape_shape = op.inputs[2].get_shape().with_rank(1) nnz = values_shape[0] return [tensor_shape.vector(nnz)] - - -def sparse_maximum(sp_a, sp_b, name=None): - """Returns the element-wise max of two SparseTensors. - - Assumes the two SparseTensors have the same shape, i.e., no broadcasting. - Example: - - ```python - sp_zero = ops.SparseTensor([[0]], [0], [7]) - sp_one = ops.SparseTensor([[1]], [1], [7]) - res = tf.sparse_maximum(sp_zero, sp_one).eval() - # "res" should be equal to SparseTensor([[0], [1]], [0, 1], [7]). - ``` - - Args: - sp_a: a `SparseTensor` operand whose dtype is real, and indices - lexicographically ordered. - sp_b: the other `SparseTensor` operand with the same requirements (and the - same shape). - name: optional name of the operation. - Returns: - output: the output SparseTensor. - """ - with ops.op_scope([sp_a.indices, sp_a.values, sp_b.indices, sp_b.values], - name, "SparseSparseMaximum") as name: - out_indices, out_values = gen_sparse_ops.sparse_sparse_maximum(sp_a.indices, - sp_a.values, - sp_a.shape, - sp_b.indices, - sp_b.values, - sp_b.shape, - name=name) - return ops.SparseTensor(out_indices, out_values, sp_a.shape) - - -def sparse_minimum(sp_a, sp_b, name=None): - """Returns the element-wise min of two SparseTensors. - - Assumes the two SparseTensors have the same shape, i.e., no broadcasting. - Example: - - ```python - sp_zero = ops.SparseTensor([[0]], [0], [7]) - sp_one = ops.SparseTensor([[1]], [1], [7]) - res = tf.sparse_minimum(sp_zero, sp_one).eval() - # "res" should be equal to SparseTensor([[0], [1]], [0, 0], [7]). - ``` - - Args: - sp_a: a `SparseTensor` operand whose dtype is real, and indices - lexicographically ordered. - sp_b: the other `SparseTensor` operand with the same requirements (and the - same shape). - name: optional name of the operation. - Returns: - output: the output SparseTensor. - """ - with ops.op_scope([sp_a.indices, sp_a.values, sp_b.indices, sp_b.values], - name, "SparseSparseMinimum") as name: - out_indices, out_values = gen_sparse_ops.sparse_sparse_minimum(sp_a.indices, - sp_a.values, - sp_a.shape, - sp_b.indices, - sp_b.values, - sp_b.shape, - name=name) - return ops.SparseTensor(out_indices, out_values, sp_a.shape) - - -@ops.RegisterShape("SparseSparseMaximum") -@ops.RegisterShape("SparseSparseMinimum") -def _SparseSparseMaximumMinimumShape(op): # pylint: disable=invalid-name - """Shape function for SparseSparseMaximum and SparseSparseMinimum.""" - op.inputs[0].get_shape().assert_has_rank(2) # a_indices - op.inputs[1].get_shape().assert_has_rank(1) # a_values - op.inputs[2].get_shape().assert_has_rank(1) # a_shape - op.inputs[3].get_shape().assert_has_rank(2) # b_indices - op.inputs[4].get_shape().assert_has_rank(1) # b_values - op.inputs[5].get_shape().assert_has_rank(1) # b_shape - return [tensor_shape.unknown_shape(2), tensor_shape.unknown_shape(1)] From 3ff5ce17c4ab4578ba5e96fd823fbaa9be310e01 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 30 Jun 2016 17:26:22 -0800 Subject: [PATCH 28/36] Add a unit test to test DNN weights/biases lookup with correct names from saved checkpoints. Change: 126367935 --- .../estimators/dnn_linear_combined_test.py | 19 ++ .../kernels/sparse_sparse_binary_op_shared.cc | 230 ------------------ 2 files changed, 19 insertions(+), 230 deletions(-) delete mode 100644 tensorflow/core/kernels/sparse_sparse_binary_op_shared.cc diff --git a/tensorflow/contrib/learn/python/learn/estimators/dnn_linear_combined_test.py b/tensorflow/contrib/learn/python/learn/estimators/dnn_linear_combined_test.py index 00679a30b47..30750227681 100644 --- a/tensorflow/contrib/learn/python/learn/estimators/dnn_linear_combined_test.py +++ b/tensorflow/contrib/learn/python/learn/estimators/dnn_linear_combined_test.py @@ -471,6 +471,25 @@ class DNNLinearCombinedClassifierTest(tf.test.TestCase): self.assertNotIn('linear/feature_BUCKETIZED_weights', classifier.get_variable_names()) + def testDNNWeightsBiasesNames(self): + """Tests the names of DNN weights and biases in the checkpoints.""" + def _input_fn_train(): + # Create 4 rows, three (y = x), one (y=Not(x)) + target = tf.constant([[1], [1], [1], [0]]) + features = {'x': tf.ones(shape=[4, 1], dtype=tf.float32),} + return features, target + classifier = tf.contrib.learn.DNNLinearCombinedClassifier( + linear_feature_columns=[tf.contrib.layers.real_valued_column('x')], + dnn_feature_columns=[tf.contrib.layers.real_valued_column('x')], + dnn_hidden_units=[3, 3]) + + classifier.fit(input_fn=_input_fn_train, steps=5) + # hiddenlayer_0/weights,hiddenlayer_1/weights and dnn_logits/weights. + self.assertEquals(3, len(classifier.dnn_weights_)) + # hiddenlayer_0/biases, hiddenlayer_1/biases, dnn_logits/biases, + # centered_bias_weight. + self.assertEquals(4, len(classifier.dnn_bias_)) + class DNNLinearCombinedRegressorTest(tf.test.TestCase): diff --git a/tensorflow/core/kernels/sparse_sparse_binary_op_shared.cc b/tensorflow/core/kernels/sparse_sparse_binary_op_shared.cc deleted file mode 100644 index a8a116ebf58..00000000000 --- a/tensorflow/core/kernels/sparse_sparse_binary_op_shared.cc +++ /dev/null @@ -1,230 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -// SparseSparseBinaryOpShared is the shared code for binary coefficient-wise -// (cwise) operations of the following form: -// -// sparse_t sparse_t -> new sparse_t -// -// The output SparseTensor may store up to "a_nnz + b_nnz" elements. - -// IMPLEMENTATION DETAILS (not part of the interface specification). -// -// This kernel implements the "union" semantics on the non-zeros: namely, any -// non-zero from either side participate in the calculations, and any resultant -// zeros will NOT be excluded from the output storage. -// -// (In the future, we could always add a pruning op the prunes away the zeros, -// if desirable.) - -// See docs of all registered ops in ../ops/sparse_ops.cc. - -#define EIGEN_USE_THREADS - -#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/register_types.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_util.h" -#include "tensorflow/core/framework/types.h" -#include "tensorflow/core/kernels/cwise_ops.h" -#include "tensorflow/core/kernels/cwise_ops_common.h" -#include "tensorflow/core/util/sparse/sparse_tensor.h" - -namespace tensorflow { - -typedef Eigen::ThreadPoolDevice CPUDevice; - -namespace { -// Unions the sparse indices and outputs corresponding values: namely, if a -// non-zero appear in one side, it will participate in the calculation, where -// the counterpart on the other side is either a value or an implicit zero. -// -// On exit, outputs the augmented values in "{a,b}_augmented_values", and fills -// "entries_to_copy" with "(from_a?, index)" pairs. All three vectors have the -// same size. -// -// The input and output sparse tensors are assumed ordered in the canonical -// row-major order. -template -void UnionSparseIndicesAndValues( - typename TTypes::ConstMatrix a_indices_mat, - typename TTypes::ConstFlat a_values, int64 a_nnz, - typename TTypes::ConstMatrix b_indices_mat, - typename TTypes::ConstFlat b_values, int64 b_nnz, int num_dims, - std::vector *a_augmented_values, std::vector *b_augmented_values, - std::vector> *entries_to_copy) { - entries_to_copy->reserve(a_nnz + b_nnz); - a_augmented_values->reserve(a_nnz); - b_augmented_values->reserve(b_nnz); - - int64 i = 0, j = 0; - const T kZero = T(0); - while (i < a_nnz && j < b_nnz) { - switch (sparse::DimComparator::cmp(a_indices_mat, b_indices_mat, i, j, - num_dims)) { - case -1: - entries_to_copy->emplace_back(true, i); - a_augmented_values->push_back(a_values(i)); - b_augmented_values->push_back(kZero); - ++i; - break; - case 0: - entries_to_copy->emplace_back(true, i); - a_augmented_values->push_back(a_values(i)); - b_augmented_values->push_back(b_values(j)); - ++i; - ++j; - break; - case 1: - entries_to_copy->emplace_back(false, j); - a_augmented_values->push_back(kZero); - b_augmented_values->push_back(b_values(j)); - ++j; - break; - } - } - // Handles leftovers; at most one loop runs. - while (i < a_nnz) { - entries_to_copy->emplace_back(/* is_a */ true, i); - a_augmented_values->push_back(a_values(i++)); - b_augmented_values->push_back(kZero); - } - while (j < b_nnz) { - entries_to_copy->emplace_back(/* is_a */ false, j); - a_augmented_values->push_back(kZero); - b_augmented_values->push_back(b_values(j++)); - } -} -} // anonymous namespace - -// Device: CPUDevice. GPU kernel is not supported currently. -// T: dtype of the SparseTensor's. -// Functor: binary cwise operation to perform on the corresponding operand -// values. See cwise_ops.h for a list of possible functors to register with. -template -class SparseSparseBinaryOpShared : public OpKernel { - public: - explicit SparseSparseBinaryOpShared(OpKernelConstruction *ctx) - : OpKernel(ctx) {} - - void Compute(OpKernelContext *ctx) override { - const Tensor *a_indices_t, *a_values_t, *a_shape_t, *b_indices_t, - *b_values_t, *b_shape_t; - OP_REQUIRES_OK(ctx, ctx->input("a_indices", &a_indices_t)); - OP_REQUIRES_OK(ctx, ctx->input("a_values", &a_values_t)); - OP_REQUIRES_OK(ctx, ctx->input("a_shape", &a_shape_t)); - OP_REQUIRES_OK(ctx, ctx->input("b_indices", &b_indices_t)); - OP_REQUIRES_OK(ctx, ctx->input("b_values", &b_values_t)); - OP_REQUIRES_OK(ctx, ctx->input("b_shape", &b_shape_t)); - - // Validations. - OP_REQUIRES( - ctx, TensorShapeUtils::IsMatrix(a_indices_t->shape()) && - TensorShapeUtils::IsMatrix(b_indices_t->shape()), - errors::InvalidArgument("Inputs a_indices and b_indices should be " - "matrices but received shapes: ", - a_indices_t->shape().DebugString(), ", ", - b_indices_t->shape().DebugString())); - OP_REQUIRES(ctx, TensorShapeUtils::IsVector(a_values_t->shape()) && - TensorShapeUtils::IsVector(b_values_t->shape()), - errors::InvalidArgument( - "Inputs a_values and b_values should be vectors " - "but received shapes: ", - a_values_t->shape().DebugString(), " and ", - b_values_t->shape().DebugString())); - - const int64 a_nnz = a_indices_t->dim_size(0); - const int64 b_nnz = b_indices_t->dim_size(0); - const auto a_values = a_values_t->vec(); - const auto b_values = b_values_t->vec(); - - OP_REQUIRES( - ctx, a_values.size() == a_nnz && b_values.size() == b_nnz, - errors::InvalidArgument("Expected ", a_nnz, " and ", b_nnz, - " non-empty input values, got ", - a_values.size(), " and ", b_values.size())); - - OP_REQUIRES(ctx, TensorShapeUtils::IsVector(a_shape_t->shape()) && - TensorShapeUtils::IsVector(b_shape_t->shape()), - errors::InvalidArgument( - "Input shapes should be a vector but received shapes ", - a_shape_t->shape().DebugString(), " and ", - b_shape_t->shape().DebugString())); - OP_REQUIRES(ctx, a_shape_t->IsSameSize(*b_shape_t), - errors::InvalidArgument( - "Operands do not have the same ranks; got shapes: ", - a_shape_t->SummarizeValue(10), " and ", - b_shape_t->SummarizeValue(10))); - const auto a_shape = a_shape_t->flat(); - const auto b_shape = b_shape_t->flat(); - for (int i = 0; i < a_shape_t->NumElements(); ++i) { - OP_REQUIRES(ctx, a_shape(i) == b_shape(i), - errors::InvalidArgument("Operands' shapes do not match: got ", - a_shape(i), " and ", b_shape(i), - " for dimension ", i)); - } - - const int num_dims = a_indices_t->dim_size(1); - const auto a_indices_mat = a_indices_t->matrix(); - const auto b_indices_mat = b_indices_t->matrix(); - std::vector a_augmented_values, b_augmented_values; - std::vector> entries_to_copy; // from_a?, idx - UnionSparseIndicesAndValues(a_indices_mat, a_values, a_nnz, b_indices_mat, - b_values, b_nnz, num_dims, &a_augmented_values, - &b_augmented_values, &entries_to_copy); - - // Allocates and fills output tensors. - const int64 sum_nnz = a_augmented_values.size(); - Tensor *output_indices_t, *output_values_t; - OP_REQUIRES_OK(ctx, - ctx->allocate_output(0, TensorShape({sum_nnz, num_dims}), - &output_indices_t)); - OP_REQUIRES_OK( - ctx, ctx->allocate_output(1, TensorShape({sum_nnz}), &output_values_t)); - auto output_indices_mat = output_indices_t->matrix(); - - for (int64 i = 0; i < sum_nnz; ++i) { - const bool from_a = entries_to_copy[i].first; - const int64 idx = entries_to_copy[i].second; - output_indices_mat.chip<0>(i) = - from_a ? a_indices_mat.chip<0>(idx) : b_indices_mat.chip<0>(idx); - } - - // Performs the functor operation using Eigen. - using TensorMap = - Eigen::TensorMap, - Eigen::Aligned>; - auto a_augmented_values_t = TensorMap(a_augmented_values.data(), sum_nnz); - auto b_augmented_values_t = TensorMap(b_augmented_values.data(), sum_nnz); - output_values_t->flat().device(ctx->eigen_device()) = - a_augmented_values_t.binaryExpr(b_augmented_values_t, - typename Functor::func()); - } -}; - -#define REGISTER_KERNELS(T) \ - REGISTER_KERNEL_BUILDER( \ - Name("SparseSparseMinimum").Device(DEVICE_CPU).TypeConstraint("T"), \ - SparseSparseBinaryOpShared>) \ - \ - REGISTER_KERNEL_BUILDER( \ - Name("SparseSparseMaximum").Device(DEVICE_CPU).TypeConstraint("T"), \ - SparseSparseBinaryOpShared>) - -TF_CALL_REAL_NUMBER_TYPES(REGISTER_KERNELS); -#undef REGISTER_KERNELS - -} // namespace tensorflow From 1bcc32c99ca1fe8d08d02a27d9570233fdb92472 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 30 Jun 2016 17:54:25 -0800 Subject: [PATCH 29/36] Use learn.DNNClassifier for Iris examples. Use default graph and existing global step (if there is one) in Estimator, because they may have been created earlier when constructing an optimizer, as in the example iris_custom_decay_dnn. Change: 126369635 --- tensorflow/examples/skflow/BUILD | 4 ++ tensorflow/examples/skflow/examples_test.sh | 4 ++ .../examples/skflow/iris_custom_decay_dnn.py | 41 +++++++++++-------- tensorflow/examples/skflow/iris_run_config.py | 39 ++++++++++-------- .../skflow/iris_val_based_early_stopping.py | 8 ++-- .../examples/skflow/iris_with_pipeline.py | 31 +++++++++----- 6 files changed, 79 insertions(+), 48 deletions(-) diff --git a/tensorflow/examples/skflow/BUILD b/tensorflow/examples/skflow/BUILD index 5d6eae87459..7cac13df98f 100644 --- a/tensorflow/examples/skflow/BUILD +++ b/tensorflow/examples/skflow/BUILD @@ -231,7 +231,11 @@ sh_test( data = [ ":boston", ":iris", + ":iris_custom_decay_dnn", ":iris_custom_model", + ":iris_run_config", + ":iris_val_based_early_stopping", + ":iris_with_pipeline", ":text_classification", ":text_classification_builtin_rnn_model", ":text_classification_character_cnn", diff --git a/tensorflow/examples/skflow/examples_test.sh b/tensorflow/examples/skflow/examples_test.sh index da6b35c9bb3..f4010c915e3 100755 --- a/tensorflow/examples/skflow/examples_test.sh +++ b/tensorflow/examples/skflow/examples_test.sh @@ -49,6 +49,10 @@ function test() { test boston test iris test iris_custom_model +test iris_custom_decay_dnn +test iris_run_config +test iris_val_based_early_stopping +test iris_with_pipeline test text_classification --test_with_fake_data test text_classification_builtin_rnn_model --test_with_fake_data test text_classification_cnn --test_with_fake_data diff --git a/tensorflow/examples/skflow/iris_custom_decay_dnn.py b/tensorflow/examples/skflow/iris_custom_decay_dnn.py index c1e7d22d53a..1ce6a830e4b 100644 --- a/tensorflow/examples/skflow/iris_custom_decay_dnn.py +++ b/tensorflow/examples/skflow/iris_custom_decay_dnn.py @@ -17,24 +17,29 @@ from __future__ import print_function from sklearn import datasets, metrics from sklearn.cross_validation import train_test_split - import tensorflow as tf -iris = datasets.load_iris() -X_train, X_test, y_train, y_test = train_test_split(iris.data, - iris.target, - test_size=0.2, - random_state=42) -# setup exponential decay function -def exp_decay(global_step): - return tf.train.exponential_decay( - learning_rate=0.1, global_step=global_step, - decay_steps=100, decay_rate=0.001) -# use customized decay function in learning_rate -optimizer = tf.train.AdagradOptimizer(learning_rate=exp_decay) -classifier = tf.contrib.learn.DNNClassifier(hidden_units=[10, 20, 10], - n_classes=3, - optimizer=optimizer) -classifier.fit(X_train, y_train, steps=800) -score = metrics.accuracy_score(y_test, classifier.predict(X_test)) +def optimizer_exp_decay(): + global_step = tf.contrib.framework.get_or_create_global_step() + learning_rate = tf.train.exponential_decay( + learning_rate=0.1, global_step=global_step, + decay_steps=100, decay_rate=0.001) + return tf.train.AdagradOptimizer(learning_rate=learning_rate) + +def main(unused_argv): + iris = datasets.load_iris() + x_train, x_test, y_train, y_test = train_test_split( + iris.data, iris.target, test_size=0.2, random_state=42) + + classifier = tf.contrib.learn.DNNClassifier(hidden_units=[10, 20, 10], + n_classes=3, + optimizer=optimizer_exp_decay) + + classifier.fit(x_train, y_train, steps=800) + score = metrics.accuracy_score(y_test, classifier.predict(x_test)) + print('Accuracy: {0:f}'.format(score)) + + +if __name__ == '__main__': + tf.app.run() diff --git a/tensorflow/examples/skflow/iris_run_config.py b/tensorflow/examples/skflow/iris_run_config.py index dff0daf9e8c..c678c7c738c 100644 --- a/tensorflow/examples/skflow/iris_run_config.py +++ b/tensorflow/examples/skflow/iris_run_config.py @@ -16,24 +16,31 @@ from __future__ import division from __future__ import print_function from sklearn import datasets, metrics, cross_validation - -from tensorflow.contrib import learn +import tensorflow as tf -# Load dataset. -iris = datasets.load_iris() -X_train, X_test, y_train, y_test = cross_validation.train_test_split(iris.data, iris.target, - test_size=0.2, random_state=42) +def main(unused_argv): + # Load dataset. + iris = datasets.load_iris() + x_train, x_test, y_train, y_test = cross_validation.train_test_split( + iris.data, iris.target, test_size=0.2, random_state=42) -# You can define you configurations by providing a RunConfig object to -# estimator to control session configurations, e.g. num_cores and gpu_memory_fraction -run_config = learn.estimators.RunConfig(num_cores=3, gpu_memory_fraction=0.6) + # You can define you configurations by providing a RunConfig object to + # estimator to control session configurations, e.g. num_cores + # and gpu_memory_fraction + run_config = tf.contrib.learn.estimators.RunConfig( + num_cores=3, gpu_memory_fraction=0.6) -# Build 3 layer DNN with 10, 20, 10 units respectively. -classifier = learn.TensorFlowDNNClassifier(hidden_units=[10, 20, 10], - n_classes=3, steps=200, config=run_config) + # Build 3 layer DNN with 10, 20, 10 units respectively. + classifier = tf.contrib.learn.DNNClassifier(hidden_units=[10, 20, 10], + n_classes=3, + config=run_config) -# Fit and predict. -classifier.fit(X_train, y_train) -score = metrics.accuracy_score(y_test, classifier.predict(X_test)) -print('Accuracy: {0:f}'.format(score)) + # Fit and predict. + classifier.fit(x_train, y_train, steps=200) + score = metrics.accuracy_score(y_test, classifier.predict(x_test)) + print('Accuracy: {0:f}'.format(score)) + + +if __name__ == '__main__': + tf.app.run() diff --git a/tensorflow/examples/skflow/iris_val_based_early_stopping.py b/tensorflow/examples/skflow/iris_val_based_early_stopping.py index 72e0595544f..05dfa96a077 100644 --- a/tensorflow/examples/skflow/iris_val_based_early_stopping.py +++ b/tensorflow/examples/skflow/iris_val_based_early_stopping.py @@ -34,21 +34,23 @@ def main(unused_argv): x_val, y_val, early_stopping_rounds=200) # classifier with early stopping on training data - classifier1 = learn.TensorFlowDNNClassifier( + classifier1 = learn.DNNClassifier( hidden_units=[10, 20, 10], n_classes=3, model_dir='/tmp/iris_model/') classifier1.fit(x=x_train, y=y_train, steps=2000) score1 = metrics.accuracy_score(y_test, classifier1.predict(x_test)) # classifier with early stopping on validation data, save frequently for # monitor to pick up new checkpoints. - classifier2 = learn.TensorFlowDNNClassifier( + classifier2 = learn.DNNClassifier( hidden_units=[10, 20, 10], n_classes=3, model_dir='/tmp/iris_model_val/', config=tf.contrib.learn.RunConfig(save_checkpoints_secs=1)) classifier2.fit(x=x_train, y=y_train, steps=2000, monitors=[val_monitor]) score2 = metrics.accuracy_score(y_test, classifier2.predict(x_test)) # In many applications, the score is improved by using early stopping - print(score2 > score1) + print('score1: ', score1) + print('score2: ', score2) + print('score2 > score1: ', score2 > score1) if __name__ == '__main__': diff --git a/tensorflow/examples/skflow/iris_with_pipeline.py b/tensorflow/examples/skflow/iris_with_pipeline.py index 3ba5739250e..5535cd9e3bf 100644 --- a/tensorflow/examples/skflow/iris_with_pipeline.py +++ b/tensorflow/examples/skflow/iris_with_pipeline.py @@ -20,22 +20,31 @@ from sklearn.datasets import load_iris from sklearn import cross_validation from sklearn.preprocessing import StandardScaler from sklearn.metrics import accuracy_score +import tensorflow as tf + from tensorflow.contrib import learn -iris = load_iris() -X_train, X_test, y_train, y_test = cross_validation.train_test_split(iris.data, iris.target, - test_size=0.2, random_state=42) -# It's useful to scale to ensure Stochastic Gradient Descent will do the right thing -scaler = StandardScaler() +def main(unused_argv): + iris = load_iris() + x_train, x_test, y_train, y_test = cross_validation.train_test_split( + iris.data, iris.target, test_size=0.2, random_state=42) -# DNN classifier -DNNclassifier = learn.TensorFlowDNNClassifier(hidden_units=[10, 20, 10], n_classes=3, steps=200) + # It's useful to scale to ensure Stochastic Gradient Descent + # will do the right thing. + scaler = StandardScaler() -pipeline = Pipeline([('scaler', scaler), ('DNNclassifier', DNNclassifier)]) + # DNN classifier + classifier = learn.DNNClassifier(hidden_units=[10, 20, 10], n_classes=3) -pipeline.fit(X_train, y_train) + pipeline = Pipeline([('scaler', scaler), + ('DNNclassifier', classifier)]) -score = accuracy_score(y_test, pipeline.predict(X_test)) + pipeline.fit(x_train, y_train, DNNclassifier__steps=200) -print('Accuracy: {0:f}'.format(score)) + score = accuracy_score(y_test, pipeline.predict(x_test)) + print('Accuracy: {0:f}'.format(score)) + + +if __name__ == '__main__': + tf.app.run() From ccd57f57d879c95d77125c6b04781fdf952365da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dan=20Man=C3=A9?= Date: Thu, 30 Jun 2016 18:30:17 -0800 Subject: [PATCH 30/36] Autogenerated Change: Release TensorBoard at TAG: 21 Change: 126371321 --- WORKSPACE | 2 +- tensorflow/tensorboard/TAG | 2 +- .../tensorboard/dist/tf-tensorboard.html | 464 +++++++++++++++++- 3 files changed, 454 insertions(+), 14 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 36d382095b5..a0aaefa6f58 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -70,7 +70,7 @@ new_git_repository( name = "iron_a11y_keys_behavior", build_file = "bower.BUILD", remote = "https://github.com/polymerelements/iron-a11y-keys-behavior.git", - tag = "v1.1.5", + tag = "v1.1.6", ) new_git_repository( diff --git a/tensorflow/tensorboard/TAG b/tensorflow/tensorboard/TAG index aabe6ec3909..2bd5a0a98a3 100644 --- a/tensorflow/tensorboard/TAG +++ b/tensorflow/tensorboard/TAG @@ -1 +1 @@ -21 +22 diff --git a/tensorflow/tensorboard/dist/tf-tensorboard.html b/tensorflow/tensorboard/dist/tf-tensorboard.html index 77e224dd205..1c0297dbdb4 100644 --- a/tensorflow/tensorboard/dist/tf-tensorboard.html +++ b/tensorflow/tensorboard/dist/tf-tensorboard.html @@ -3963,6 +3963,7 @@ var tf; } return OpNodeImpl; }()); + graph_1.OpNodeImpl = OpNodeImpl; ; function createMetanode(name, opt) { if (opt === void 0) { opt = {}; } @@ -4171,6 +4172,7 @@ var tf; }; return MetanodeImpl; }()); + graph_1.MetanodeImpl = MetanodeImpl; ; function createMetaedge(v, w) { return new MetaedgeImpl(v, w); @@ -4527,6 +4529,7 @@ var tf; var parts = name.split(graph_1.NAMESPACE_DELIM); return name + graph_1.NAMESPACE_DELIM + '(' + parts[parts.length - 1] + ')'; } + graph_1.getStrictName = getStrictName; /** * For each op node (embedding or non-embedding), rename it if there is a * non-embedding node under its namespace. For example, assume node name 'A'. @@ -6494,6 +6497,7 @@ var tf; this.index[hierarchy.root.name] = this.root; this.buildSubhierarchy(hierarchy.root.name); this.root.expanded = true; + this.traceInputs = false; } RenderGraphInfo.prototype.computeScales = function () { this.deviceColorMap = d3.scale.ordinal() @@ -8788,10 +8792,333 @@ var tf; d3.rgb(fill).darker().toString(); } node_1.getStrokeForFill = getStrokeForFill; + /** + * Finds selected node and highlights all nodes which are providing direct + * or indirect input to the node and all edges connecting these nodes + * together and to the selected node. + * + * @param renderGraphInfo Information on the rendered state of the graph. + */ + function traceInputs(renderGraphInfo) { + // Reset all styling. + d3.selectAll('.input-highlight').classed('input-highlight', false); + d3.selectAll('.non-input').classed('non-input', false); + d3.selectAll('.input-parent').classed('input-parent', false); + d3.selectAll('.input-child').classed('input-child', false); + d3.selectAll('.input-edge-highlight').classed('input-edge-highlight', false); + d3.selectAll('.non-input-edge-highlight') + .classed('non-input-edge-highlight', false); + d3.selectAll('.input-highlight-selected') + .classed('input-highlight-selected', false); + // Extract currently selected node. Return if input tracing disabled or no + // node is selected. + var selectedNodeSelectorString = 'g.node.selected,g.op.selected'; + var node = d3.select(selectedNodeSelectorString); + var currentNode = undefined; + if (renderGraphInfo && renderGraphInfo.traceInputs && node && node[0] && + node[0][0]) { + currentNode = node[0][0]; + } + else { + return; + } + var nodeName = currentNode.getAttribute('data-name'); + var opNodes = _getAllContainedOpNodes(nodeName, renderGraphInfo); + var allTracedNodes = {}; + _.each(opNodes, function (nodeInstance) { + allTracedNodes = + traceAllInputsOfOpNode(renderGraphInfo, nodeInstance, allTracedNodes); + }); + d3.selectAll(selectedNodeSelectorString).classed({ + // Remove the input-highlight from the selected node. + 'input-highlight': false, + // Add input-highlight-selected class to selected node, which allows + // treating the selected not as a special case of an input node. + 'input-highlight-selected': true + }); + // Highlight all parent nodes of each OpNode as input parent to allow + // specific highlighting. + var highlightedNodes = Object.keys(allTracedNodes); + var visibleNodes = _findVisibleParentsFromOpNodes(renderGraphInfo, highlightedNodes); + _markParentsOfNodes(visibleNodes); + // Attach class to all non-input nodes and edges for styling. + d3.selectAll('g.node:not(.selected):not(.input-highlight)' + + ':not(.input-parent):not(.input-children)') + .classed('non-input', true) + .each(function (d) { + // Mark all nodes with the specified name as non-inputs. This + // results in Annotation nodes which are attached to inputs to be + // tagged as well. + var nodeName = d.node.name; + d3.selectAll("[data-name=\"" + nodeName + "\"]").classed('non-input', true); + }); + d3.selectAll('g.edge:not(.input-edge-highlight)') + .classed('non-input-edge-highlight', true); + } + node_1.traceInputs = traceInputs; + /** + * Recursively find all op nodes contained by the node identified by the + * provided name. + * @param nodeName The meta or op node of which the OpNode instances are + * required. + * @param renderGraphInfo The rendered graph information object. + * @returns {Array} An array of OpNodeImpl instances. + */ + function _getAllContainedOpNodes(nodeName, renderGraphInfo) { + var opNodes = []; + // Get current node. + var node = renderGraphInfo.getNodeByName(nodeName); + // If node is already OpNode then return the node plus its input embeddings. + if (node instanceof tf.graph.OpNodeImpl) { + return [node].concat(node.inEmbeddings); + } + // Otherwise, make recursive call for each node contained by the GroupNode. + var childNodeNames = node.metagraph.nodes(); + _.each(childNodeNames, function (childNodeName) { + opNodes = + opNodes.concat(_getAllContainedOpNodes(childNodeName, renderGraphInfo)); + }); + return opNodes; + } + node_1._getAllContainedOpNodes = _getAllContainedOpNodes; + function traceAllInputsOfOpNode(renderGraphInfo, startNode, allTracedNodes) { + // To prevent infinite loops due to cyclical relationships and improving + // performance by tracing OpNode which is input to 2+ nodes only once. + if (allTracedNodes[startNode.name]) { + return allTracedNodes; + } + else { + allTracedNodes[startNode.name] = true; + } + // Extract the inputs. + var inputs = startNode.inputs; + // Get visible parent. + var currentVisibleParent = getVisibleParent(renderGraphInfo, startNode); + // Mark as input node. + d3.select(".node[data-name=\"" + currentVisibleParent.name + "\"]") + .classed('input-highlight', true); + // Find the visible parent of each input. + var visibleInputs = {}; + _.each(inputs, function (nodeInstance) { + var resolvedNode = renderGraphInfo.getNodeByName(nodeInstance.name); + if (resolvedNode === undefined) { + // Node could not be found in rendered Hierarchy, which happens when + // tracing inputs of a SummaryNode. + return; + } + // Ensure node is resolved to OpNode if name collision with Metanode exists. + if (resolvedNode instanceof graph.MetanodeImpl) { + var resolvedNodeName = tf.graph.getStrictName(resolvedNode.name); + resolvedNode = renderGraphInfo.getNodeByName(resolvedNodeName); + } + var visibleParent = getVisibleParent(renderGraphInfo, resolvedNode); + // Append OpNode to visible parent entry. + var visibleInputsEntry = visibleInputs[visibleParent.name]; + if (visibleInputsEntry) { + visibleInputsEntry.opNodes.push(resolvedNode); + } + else { + visibleInputs[visibleParent.name] = { + visibleParent: visibleParent, + opNodes: [resolvedNode] + }; + } + }); + // Find all parents of the start node. + var startNodeParents = {}; + var indexedStartNodeParents = [currentVisibleParent]; + startNodeParents[currentVisibleParent.name] = { + traced: false, + index: 0, + connectionEndpoints: [] + }; + var currentNode = currentVisibleParent; + for (var index = 1; currentNode.name !== tf.graph.ROOT_NAME; index++) { + currentNode = currentNode.parentNode; + startNodeParents[currentNode.name] = { + traced: false, + index: index, + connectionEndpoints: [] + }; + indexedStartNodeParents[index] = currentNode; + } + // Find first mutual parent of each input node and highlight connection. + _.forOwn(visibleInputs, function (visibleParentInfo, key) { + var nodeInstance = visibleParentInfo.visibleParent; + // Make recursive call for each input-OpNode contained by the visible + // parent. + _.each(visibleParentInfo.opNodes, function (opNode) { + allTracedNodes = + traceAllInputsOfOpNode(renderGraphInfo, opNode, allTracedNodes); + }); + if (nodeInstance.name !== currentVisibleParent.name) { + _createVisibleTrace(nodeInstance, startNodeParents, indexedStartNodeParents); + } + }); + return allTracedNodes; + } + node_1.traceAllInputsOfOpNode = traceAllInputsOfOpNode; + /** + * Colors the edges to connect the passed node to the start node. This is + * done by: + * + * a) Finding the first (visible) common parent in the rendered + * hierarchy. + * NB: There are 2 types of connections: + * 1) Direct connections between node A + * and B, marked below as II, + * 2) Connections from any node A to its parent, A'. Marked below as I and III. + * For type 2 connection you need to know the inner-nested node, the + * direct parent, and the ultimate destination of the connection. + * + * A_parent B_parent + * +--------+ +---------+ + * | | | | + * | +--+ I| II |III+--+ | + * | |A +----------\x3e+B | | + * | +--+ | | +--+ | + * | | | | + * +--------+ +---------+ + * + * + * b) Highlighting the direct connection between the parents of A and B, + * called A_parent and B_parent, s.t. A_parent and B_parent are children of the + * mutual parent of A and B found in a), marked above as II. + * + * c) Highlighting the connection from A to A_parent and B to B_parent + * (through all layers of parents between A and A_parent and B and B_parent, + * respectively). Marked above as I and III. + * + * @param nodeInstance The instance of the node to use as destination node, B. + * @param startNodeParents Map of startNodeParent names to information objects + * about the parent. + * @param indexedStartNodeParents An array of all parents of the start node. + * This is required to find the child of the mutual parent which is a parent + * of the start node. + * @private + */ + function _createVisibleTrace(nodeInstance, startNodeParents, indexedStartNodeParents) { + var currentNode = nodeInstance; + var previousNode = nodeInstance; + // Ascend through parents until a mutual parent is found with the start + // node. + var destinationParentPairs = []; + while (!startNodeParents[currentNode.name]) { + if (previousNode.name !== currentNode.name) { + destinationParentPairs.push([previousNode, currentNode]); + } + previousNode = currentNode; + currentNode = currentNode.parentNode; + } + // Connection between nodes is drawn between the parents of each + // respective node, both of which share the mutual parent. + var startNodeIndex = startNodeParents[currentNode.name].index; + var startNodeName = indexedStartNodeParents[Math.max(startNodeIndex - 1, 0)].name; + var startNodeTopParentName = startNodeName; + var targetNodeTopParentName = previousNode.name; + var endNodeName = previousNode.name; + d3.selectAll("[data-edge=\"" + endNodeName + "--" + startNodeName + "\"]") + .classed('input-edge-highlight', true); + // Trace up the parents of the input. + _.each(destinationParentPairs, function (value) { + var inner = value[0]; + var outer = value[1]; + var edgeSelector = ("[data-edge=\"" + inner.name + "--" + startNodeTopParentName) + + ("~~" + outer.name + "~~OUT\"]"); + d3.selectAll(edgeSelector).classed('input-edge-highlight', true); + }); + // Trace up the parents of the start node. + for (var index = 1; index < startNodeIndex; index++) { + var inner = indexedStartNodeParents[index - 1]; + var outer = indexedStartNodeParents[index]; + var edgeSelector = ("[data-edge=\"" + targetNodeTopParentName + "~~" + outer.name) + + ("~~IN--" + inner.name + "\"]"); + d3.selectAll(edgeSelector).classed('input-edge-highlight', true); + } + } + /** + * Creates map { [name: string] -> Node } of all visible / rendered parents + * of the nodes identified by the node names passed in. + * + * @param renderGraphInfo The information on the rendered graph. + * @param nodeNames String array of node names. + * @returns {[nodeName: string]: Node} + * @private + */ + function _findVisibleParentsFromOpNodes(renderGraphInfo, nodeNames) { + var visibleParents = {}; + _.each(nodeNames, function (nodeName) { + var currentNode = renderGraphInfo.getNodeByName(nodeName); + var visibleParent = getVisibleParent(renderGraphInfo, currentNode); + visibleParents[visibleParent.name] = visibleParent; + }); + return visibleParents; + } + /** + * Traverse through the parents of all nodes in the list and mark each + * encountered node as input-parent. + * @param visibleNodes Map of input nodes, have to be visible/rendered when + * called. + * @private + */ + function _markParentsOfNodes(visibleNodes) { + _.forOwn(visibleNodes, function (nodeInstance) { + // Mark all parents of the node as input-parents. + var currentNode = nodeInstance; + while (currentNode.name !== tf.graph.ROOT_NAME) { + var renderedElement = d3.select(".node[data-name=\"" + currentNode.name + "\"]"); + // Only mark the element as a parent node to an input if it is not + // marked as input node itself. + if (renderedElement[0][0] && + !renderedElement.classed('input-highlight') && + !renderedElement.classed('selected') && + // OpNode only parent if start node is embedded node, in which case + // the OpNode should be faded as well. + !renderedElement.classed('op')) { + renderedElement.classed('input-parent', true); + } + currentNode = currentNode.parentNode; + } + }); + } + /** + * Find the parent of the passed in op node which is expanded. This is done + * by going through all parents until the parent's parent is expanded, thus + * finding the the first unexpanded parent which is rendered on the screen. + * @param renderGraphInfo The graph info object used to gain access to the + * render info of the parents. + * @param currentNode The node whose parent is to be found. + * @returns Node + */ + function getVisibleParent(renderGraphInfo, currentNode) { + var found = false; + var currentParent = currentNode; + while (!found) { + // Get parent element, to extract name. + currentNode = currentParent; + currentParent = currentNode.parentNode; + if (currentParent === undefined) { + found = true; + } + else { + var renderNode = renderGraphInfo.getRenderNodeByName(currentParent.name); + // Found if node is rendered on the screen (renderNode truthy), and + // the parent is either expanded (i.e. it is a metanode or seriesnode) + // or the parent is an OpNode in which case currentNode is an embedded + // node which has another OpNode as parent. + if (renderNode && + (renderNode.expanded || currentParent instanceof graph.OpNodeImpl)) { + found = true; + } + } + } // Close while loop. + return currentNode; + } + node_1.getVisibleParent = getVisibleParent; })(node = scene.node || (scene.node = {})); })(scene = graph.scene || (graph.scene = {})); })(graph = tf.graph || (tf.graph = {})); -})(tf || (tf = {})); // close module +})(tf || (tf = {})); // Close module.