From 30f9cb16f5a865e0b9afe789e6ee0195eb197279 Mon Sep 17 00:00:00 2001 From: Yifei Feng Date: Tue, 18 Oct 2016 19:27:12 -0700 Subject: [PATCH 001/139] Change test size of gradient_checker_test from small to medium. (cherry picked from commit ce6eeda24907b14e9fa0e40ebb60f5ce7ab05482) --- tensorflow/python/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index fcd478b1b36..7441799e62a 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -1385,7 +1385,7 @@ cuda_py_test( cuda_py_test( name = "gradient_checker_test", - size = "small", + size = "medium", srcs = ["ops/gradient_checker_test.py"], additional_deps = [ ":array_ops", From 8d9eb8e0a4f1142178b6c0ef2efaefca5a49f0ae Mon Sep 17 00:00:00 2001 From: Yifei Feng Date: Wed, 19 Oct 2016 11:38:11 -0700 Subject: [PATCH 002/139] Update release note with CUDA8 support. --- RELEASE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE.md b/RELEASE.md index 1e2148e87cc..552391a9bf2 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -2,6 +2,7 @@ ## Major Features and Improvements +* CUDA 8 support. * cuDNN 5 support. * HDFS Support. * Adds Fused LSTM support via cuDNN 5 in `tensorflow/contrib/cudnn_rnn`. From ddf49b96e05b25cb0f5d096538664965b31023c7 Mon Sep 17 00:00:00 2001 From: Yifei Feng Date: Wed, 19 Oct 2016 14:16:29 -0700 Subject: [PATCH 003/139] Update documentation for 0.11.0rc1 --- tensorflow/g3doc/api_docs/cc/ClassEnv.md | 36 ++++++++++++++++- .../g3doc/api_docs/cc/ClassEnvWrapper.md | 6 +++ .../api_docs/cc/ClassRandomAccessFile.md | 2 +- tensorflow/g3doc/api_docs/cc/ClassSession.md | 31 +-------------- tensorflow/g3doc/api_docs/cc/ClassTensor.md | 39 ++----------------- .../shard5/tf.train.Saver.md | 7 +++- tensorflow/g3doc/api_docs/python/state_ops.md | 7 +++- 7 files changed, 59 insertions(+), 69 deletions(-) diff --git a/tensorflow/g3doc/api_docs/cc/ClassEnv.md b/tensorflow/g3doc/api_docs/cc/ClassEnv.md index 0010c0fbb23..88a39a5ee86 100644 --- a/tensorflow/g3doc/api_docs/cc/ClassEnv.md +++ b/tensorflow/g3doc/api_docs/cc/ClassEnv.md @@ -90,6 +90,24 @@ Stores in *result the names of the children of the specified directory. The name Original contents of *results are dropped. +#### `virtual bool tensorflow::Env::MatchPath(const string &path, const string &pattern)=0` {#virtual_bool_tensorflow_Env_MatchPath} + +Returns true if the path matches the given pattern. The wildcards allowed in pattern are described below (GetMatchingPaths). + + + +#### `Status tensorflow::Env::GetMatchingPaths(const string &pattern, std::vector< string > *results)` {#Status_tensorflow_Env_GetMatchingPaths} + +Given a pattern, stores in *results the set of paths that matches that pattern. *results is cleared. + +pattern must match all of a name, not just a substring. pattern: { term } term: '*': matches any sequence of non-'/' characters '?': matches a single non-'/' character '[' [ '^' ] { match-list } ']': matches any single character (not) on the list c: matches character c (c != '*', '?', '\', '[') '\' c: matches character c character-range: c: matches character c (c != '\', '-', ']') '\' c: matches character c lo '-' hi: matches character c for lo <= c <= hi + +Typical return codes + +OK - no errors + +UNIMPLEMENTED - Some underlying functions (like GetChildren) are not implemented The default implementation uses a combination of GetChildren, MatchPath and IsDirectory. + #### `Status tensorflow::Env::DeleteFile(const string &fname)` {#Status_tensorflow_Env_DeleteFile} Deletes the named file. @@ -110,12 +128,28 @@ PERMISSION_DENIED - dirname or some descendant is not writable UNIMPLEMENTED - Some underlying functions (like Delete) are not implemented +#### `Status tensorflow::Env::RecursivelyCreateDir(const string &dirname)` {#Status_tensorflow_Env_RecursivelyCreateDir} + +Creates the specified directory and all the necessary subdirectories. Typical return codes. + + + +OK - successfully created the directory and sub directories, even if they were already created. + +PERMISSION_DENIED - dirname or some subdirectory is not writable. + #### `Status tensorflow::Env::CreateDir(const string &dirname)` {#Status_tensorflow_Env_CreateDir} -Creates the specified directory. +Creates the specified directory. Typical return codes. +OK - successfully created the directory. + +ALREADY_EXISTS - directory already exists. + +PERMISSION_DENIED - dirname is not writable. + #### `Status tensorflow::Env::DeleteDir(const string &dirname)` {#Status_tensorflow_Env_DeleteDir} Deletes the specified directory. diff --git a/tensorflow/g3doc/api_docs/cc/ClassEnvWrapper.md b/tensorflow/g3doc/api_docs/cc/ClassEnvWrapper.md index f0041f5be92..153dc8ca361 100644 --- a/tensorflow/g3doc/api_docs/cc/ClassEnvWrapper.md +++ b/tensorflow/g3doc/api_docs/cc/ClassEnvWrapper.md @@ -42,6 +42,12 @@ Returns the file system schemes registered for this Env . +#### `bool tensorflow::EnvWrapper::MatchPath(const string &path, const string &pattern) override` {#bool_tensorflow_EnvWrapper_MatchPath} + +Returns true if the path matches the given pattern. The wildcards allowed in pattern are described below (GetMatchingPaths). + + + #### `uint64 tensorflow::EnvWrapper::NowMicros() override` {#uint64_tensorflow_EnvWrapper_NowMicros} Returns the number of micro-seconds since some fixed point in time. Only useful for computing deltas of time. diff --git a/tensorflow/g3doc/api_docs/cc/ClassRandomAccessFile.md b/tensorflow/g3doc/api_docs/cc/ClassRandomAccessFile.md index 1a1526f66d5..1ff484c0834 100644 --- a/tensorflow/g3doc/api_docs/cc/ClassRandomAccessFile.md +++ b/tensorflow/g3doc/api_docs/cc/ClassRandomAccessFile.md @@ -18,7 +18,7 @@ A file abstraction for randomly reading the contents of a file. -#### `virtual Status tensorflow::RandomAccessFile::Read(uint64 offset, size_t n, StringPiece *result, char *scratch) const =0` {#virtual_Status_tensorflow_RandomAccessFile_Read} +#### `virtual Status tensorflow::RandomAccessFile::Read(uint64 offset, size_t n, StringPiece *result, char *scratch) const =0` {#virtual_Status_tensorflow_RandomAccessFile_Read} Reads up to `n` bytes from the file starting at `offset`. diff --git a/tensorflow/g3doc/api_docs/cc/ClassSession.md b/tensorflow/g3doc/api_docs/cc/ClassSession.md index 6829548530d..d2719d8ebbb 100644 --- a/tensorflow/g3doc/api_docs/cc/ClassSession.md +++ b/tensorflow/g3doc/api_docs/cc/ClassSession.md @@ -6,36 +6,7 @@ When a Session is created with a given target, a new Session object is bound to Example: -```c++ tensorflow::GraphDef graph; -// ... Create or load graph into "graph". - -// This example uses the default options which connects -// to a local runtime. -tensorflow::SessionOptions options; -std::unique_ptr -session(tensorflow::NewSession(options)); - -// Create the session with this graph. -tensorflow::Status s = session->Create(graph); -if (!s.ok()) { ... } - -// Run the graph and fetch the first output of the "output" -// operation, and also run to but do not return anything -// for the "update_state" operation. -std::vector outputs; -s = session->Run({}, {"output:0"}, {"update_state"}, &outputs); -if (!s.ok()) { ... } - -// Map the output as a flattened float tensor, and do something -// with it. -auto output_tensor = outputs[0].flat(); -if (output_tensor(0) > 0.5) { ... } - -// Close the session to release the resources associated with -// this session. -session->Close(); - -``` +{c++} tensorflow::GraphDef graph; // ... Create or load graph into "graph". // This example uses the default options which connects // to a local runtime. tensorflow::SessionOptions options; std::unique_ptr session(tensorflow::NewSession(options)); // Create the session with this graph. tensorflow::Status s = session->Create(graph); if (!s.ok()) { ... } // Run the graph and fetch the first output of the "output" // operation, and also run to but do not return anything // for the "update_state" operation. std::vector outputs; s = session->Run({}, {"output:0"}, {"update_state"}, &outputs); if (!s.ok()) { ... } // Map the output as a flattened float tensor, and do something // with it. auto output_tensor = outputs[0].flat(); if (output_tensor(0) > 0.5) { ... } // Close the session to release the resources associated with // this session. session->Close(); A Session allows concurrent calls to Run() , though a Session must be created / extended by a single thread. diff --git a/tensorflow/g3doc/api_docs/cc/ClassTensor.md b/tensorflow/g3doc/api_docs/cc/ClassTensor.md index e221a026935..78899def539 100644 --- a/tensorflow/g3doc/api_docs/cc/ClassTensor.md +++ b/tensorflow/g3doc/api_docs/cc/ClassTensor.md @@ -12,9 +12,7 @@ Creates a 1-dimensional, 0-element float tensor. The returned Tensor is not a scalar (shape {}), but is instead an empty one-dimensional Tensor (shape {0}, NumElements() == 0). Since it has no elements, it does not need to be assigned a value and is initialized by default ( IsInitialized() is true). If this is undesirable, consider creating a one-element scalar which does require initialization: -```c++ Tensor(DT_FLOAT, TensorShape({})) - -``` +{c++} Tensor(DT_FLOAT, TensorShape({})) #### `tensorflow::Tensor::Tensor(DataType type, const TensorShape &shape)` {#tensorflow_Tensor_Tensor} @@ -100,12 +98,6 @@ Convenience accessor for the tensor shape. -#### `size_t tensorflow::Tensor::BufferHash() const` {#size_t_tensorflow_Tensor_BufferHash} - - - - - #### `bool tensorflow::Tensor::IsInitialized() const` {#bool_tensorflow_Tensor_IsInitialized} If necessary, has this Tensor been initialized? @@ -184,15 +176,7 @@ Use these methods when you know the data type and the number of dimensions of th Example: -```c++ typedef float T; -Tensor my_mat(...built with Shape{rows: 3, cols: 5}...); -auto mat = my_mat.matrix(); // 2D Eigen::Tensor, 3 x 5. -auto mat = my_mat.tensor(); // 2D Eigen::Tensor, 3 x 5. -auto vec = my_mat.vec(); // CHECK fails as my_mat is 2D. -auto vec = my_mat.tensor(); // CHECK fails as my_mat is 2D. -auto mat = my_mat.matrix();// CHECK fails as type mismatch. - -``` +{c++} typedef float T; Tensor my_mat(...built with Shape{rows: 3, cols: 5}...); auto mat = my_mat.matrix(); // 2D Eigen::Tensor, 3 x 5. auto mat = my_mat.tensor(); // 2D Eigen::Tensor, 3 x 5. auto vec = my_mat.vec(); // CHECK fails as my_mat is 2D. auto vec = my_mat.tensor(); // CHECK fails as my_mat is 2D. auto mat = my_mat.matrix();// CHECK fails as type mismatch. #### `TTypes::Matrix tensorflow::Tensor::matrix()` {#TTypes_T_Matrix_tensorflow_Tensor_matrix} @@ -220,22 +204,7 @@ These methods allow you to access the data with the dimensions and sizes of your Example: -```c++ typedef float T; -Tensor my_ten(...built with Shape{planes: 4, rows: 3, cols: 5}...); -// 1D Eigen::Tensor, size 60: -auto flat = my_ten.flat(); -// 2D Eigen::Tensor 12 x 5: -auto inner = my_ten.flat_inner_dims(); -// 2D Eigen::Tensor 4 x 15: -auto outer = my_ten.shaped({4, 15}); -// CHECK fails, bad num elements: -auto outer = my_ten.shaped({4, 8}); -// 3D Eigen::Tensor 6 x 5 x 2: -auto weird = my_ten.shaped({6, 5, 2}); -// CHECK fails, type mismatch: -auto bad = my_ten.flat(); - -``` +{c++} typedef float T; Tensor my_ten(...built with Shape{planes: 4, rows: 3, cols: 5}...); // 1D Eigen::Tensor, size 60: auto flat = my_ten.flat(); // 2D Eigen::Tensor 12 x 5: auto inner = my_ten.flat_inner_dims(); // 2D Eigen::Tensor 4 x 15: auto outer = my_ten.shaped({4, 15}); // CHECK fails, bad num elements: auto outer = my_ten.shaped({4, 8}); // 3D Eigen::Tensor 6 x 5 x 2: auto weird = my_ten.shaped({6, 5, 2}); // CHECK fails, type mismatch: auto bad = my_ten.flat(); #### `TTypes::UnalignedFlat tensorflow::Tensor::unaligned_flat()` {#TTypes_T_UnalignedFlat_tensorflow_Tensor_unaligned_flat} @@ -379,7 +348,7 @@ NOTE: The underlying tensor buffer is refcounted, so the lifetime of the content REQUIRES: `DataTypeCanUseMemcpy(dtype())`. -#### `void tensorflow::Tensor::UnsafeCopyFromInternal(const Tensor &, const TensorShape &)` {#void_tensorflow_Tensor_UnsafeCopyFromInternal} +#### `void tensorflow::Tensor::UnsafeCopyFromInternal(const Tensor &, DataType dtype, const TensorShape &)` {#void_tensorflow_Tensor_UnsafeCopyFromInternal} diff --git a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard5/tf.train.Saver.md b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard5/tf.train.Saver.md index 58b112a979b..e758fc2f533 100644 --- a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard5/tf.train.Saver.md +++ b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard5/tf.train.Saver.md @@ -72,7 +72,7 @@ protocol buffer file in the call to `save()`. - - - -#### `tf.train.Saver.__init__(var_list=None, reshape=False, sharded=False, max_to_keep=5, keep_checkpoint_every_n_hours=10000.0, name=None, restore_sequentially=False, saver_def=None, builder=None, defer_build=False, allow_empty=False)` {#Saver.__init__} +#### `tf.train.Saver.__init__(var_list=None, reshape=False, sharded=False, max_to_keep=5, keep_checkpoint_every_n_hours=10000.0, name=None, restore_sequentially=False, saver_def=None, builder=None, defer_build=False, allow_empty=False, write_version=1)` {#Saver.__init__} Creates a `Saver`. @@ -140,6 +140,11 @@ checkpoints per device. * `allow_empty`: If `False` (default) raise an error if there are no variables in the graph. Otherwise, construct the saver anyway and make it a no-op. +* `write_version`: controls what format to use when saving checkpoints. It + also affects certain filepath matching logic. Defaults to V1 + currently, and will be switched to the more memory-efficient V2 format + in the future. If set to V2, the Saver is still able to restore from + old V1 checkpoints. ##### Raises: diff --git a/tensorflow/g3doc/api_docs/python/state_ops.md b/tensorflow/g3doc/api_docs/python/state_ops.md index 727481f24d0..aa49e7b32a8 100644 --- a/tensorflow/g3doc/api_docs/python/state_ops.md +++ b/tensorflow/g3doc/api_docs/python/state_ops.md @@ -1522,7 +1522,7 @@ protocol buffer file in the call to `save()`. - - - -#### `tf.train.Saver.__init__(var_list=None, reshape=False, sharded=False, max_to_keep=5, keep_checkpoint_every_n_hours=10000.0, name=None, restore_sequentially=False, saver_def=None, builder=None, defer_build=False, allow_empty=False)` {#Saver.__init__} +#### `tf.train.Saver.__init__(var_list=None, reshape=False, sharded=False, max_to_keep=5, keep_checkpoint_every_n_hours=10000.0, name=None, restore_sequentially=False, saver_def=None, builder=None, defer_build=False, allow_empty=False, write_version=1)` {#Saver.__init__} Creates a `Saver`. @@ -1590,6 +1590,11 @@ checkpoints per device. * `allow_empty`: If `False` (default) raise an error if there are no variables in the graph. Otherwise, construct the saver anyway and make it a no-op. +* `write_version`: controls what format to use when saving checkpoints. It + also affects certain filepath matching logic. Defaults to V1 + currently, and will be switched to the more memory-efficient V2 format + in the future. If set to V2, the Saver is still able to restore from + old V1 checkpoints. ##### Raises: From 4ee590e4e72b837c729406097ee9554b03d03771 Mon Sep 17 00:00:00 2001 From: Yifei Feng Date: Thu, 20 Oct 2016 14:29:28 -0700 Subject: [PATCH 004/139] Update setup instructions to cuda8. --- tensorflow/g3doc/get_started/os_setup.md | 41 ++++++++++++------------ 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/tensorflow/g3doc/get_started/os_setup.md b/tensorflow/g3doc/get_started/os_setup.md index 257b0755983..06fb37232a4 100644 --- a/tensorflow/g3doc/get_started/os_setup.md +++ b/tensorflow/g3doc/get_started/os_setup.md @@ -7,7 +7,7 @@ github source. The TensorFlow Python API supports Python 2.7 and Python 3.3+. -The GPU version works best with Cuda Toolkit 7.5 and +The GPU version works best with Cuda Toolkit 8.0 and cuDNN v5. Other versions are supported (Cuda toolkit >= 7.0 and cuDNN >= v3) only when installing from sources. Please see [Cuda installation](#optional-install-cuda-gpus-on-linux) for @@ -66,7 +66,7 @@ Then, select the correct binary to install: $ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.11.0rc1-cp27-none-linux_x86_64.whl # Ubuntu/Linux 64-bit, GPU enabled, Python 2.7 -# Requires CUDA toolkit 7.5 and CuDNN v5. For other versions, see "Install from sources" below. +# Requires CUDA toolkit 8.0 and CuDNN v5. For other versions, see "Install from sources" below. $ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.11.0rc1-cp27-none-linux_x86_64.whl # Mac OS X, CPU only, Python 2.7: @@ -79,14 +79,14 @@ $ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/gpu/tensorf $ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.11.0rc1-cp34-cp34m-linux_x86_64.whl # Ubuntu/Linux 64-bit, GPU enabled, Python 3.4 -# Requires CUDA toolkit 7.5 and CuDNN v5. For other versions, see "Install from sources" below. +# Requires CUDA toolkit 8.0 and CuDNN v5. For other versions, see "Install from sources" below. $ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.11.0rc1-cp34-cp34m-linux_x86_64.whl # Ubuntu/Linux 64-bit, CPU only, Python 3.5 $ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.11.0rc1-cp35-cp35m-linux_x86_64.whl # Ubuntu/Linux 64-bit, GPU enabled, Python 3.5 -# Requires CUDA toolkit 7.5 and CuDNN v5. For other versions, see "Install from sources" below. +# Requires CUDA toolkit 8.0 and CuDNN v5. For other versions, see "Install from sources" below. $ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.11.0rc1-cp35-cp35m-linux_x86_64.whl # Mac OS X, CPU only, Python 3.4 or 3.5: @@ -162,7 +162,7 @@ Now, install TensorFlow just as you would for a regular Pip installation. First (tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.11.0rc1-cp27-none-linux_x86_64.whl # Ubuntu/Linux 64-bit, GPU enabled, Python 2.7 -# Requires CUDA toolkit 7.5 and CuDNN v5. For other versions, see "Install from sources" below. +# Requires CUDA toolkit 8.0 and CuDNN v5. For other versions, see "Install from sources" below. (tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.11.0rc1-cp27-none-linux_x86_64.whl # Mac OS X, CPU only, Python 2.7: @@ -175,14 +175,14 @@ Now, install TensorFlow just as you would for a regular Pip installation. First (tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.11.0rc1-cp34-cp34m-linux_x86_64.whl # Ubuntu/Linux 64-bit, GPU enabled, Python 3.4 -# Requires CUDA toolkit 7.5 and CuDNN v5. For other versions, see "Install from sources" below. +# Requires CUDA toolkit 8.0 and CuDNN v5. For other versions, see "Install from sources" below. (tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.11.0rc1-cp34-cp34m-linux_x86_64.whl # Ubuntu/Linux 64-bit, CPU only, Python 3.5 (tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.11.0rc1-cp35-cp35m-linux_x86_64.whl # Ubuntu/Linux 64-bit, GPU enabled, Python 3.5 -# Requires CUDA toolkit 7.5 and CuDNN v5. For other versions, see "Install from sources" below. +# Requires CUDA toolkit 8.0 and CuDNN v5. For other versions, see "Install from sources" below. (tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.11.0rc1-cp35-cp35m-linux_x86_64.whl # Mac OS X, CPU only, Python 3.4 or 3.5: @@ -301,7 +301,7 @@ select the correct binary to install: (tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.11.0rc1-cp27-none-linux_x86_64.whl # Ubuntu/Linux 64-bit, GPU enabled, Python 2.7 -# Requires CUDA toolkit 7.5 and CuDNN v5. For other versions, see "Install from sources" below. +# Requires CUDA toolkit 8.0 and CuDNN v5. For other versions, see "Install from sources" below. (tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.11.0rc1-cp27-none-linux_x86_64.whl # Mac OS X, CPU only, Python 2.7: @@ -314,14 +314,14 @@ select the correct binary to install: (tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.11.0rc1-cp34-cp34m-linux_x86_64.whl # Ubuntu/Linux 64-bit, GPU enabled, Python 3.4 -# Requires CUDA toolkit 7.5 and CuDNN v5. For other versions, see "Install from sources" below. +# Requires CUDA toolkit 8.0 and CuDNN v5. For other versions, see "Install from sources" below. (tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.11.0rc1-cp34-cp34m-linux_x86_64.whl # Ubuntu/Linux 64-bit, CPU only, Python 3.5 (tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.11.0rc1-cp35-cp35m-linux_x86_64.whl # Ubuntu/Linux 64-bit, GPU enabled, Python 3.5 -# Requires CUDA toolkit 7.5 and CuDNN v5. For other versions, see "Install from sources" below. +# Requires CUDA toolkit 8.0 and CuDNN v5. For other versions, see "Install from sources" below. (tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.11.0rc1-cp35-cp35m-linux_x86_64.whl # Mac OS X, CPU only, Python 3.4 or 3.5: @@ -454,7 +454,7 @@ the Docker container. ### (Optional, Linux) Enable GPU Support If you installed the GPU version of TensorFlow, you must also install the Cuda -Toolkit 7.5 and cuDNN v5. Please see [Cuda +Toolkit 8.0 and cuDNN v5. Please see [Cuda installation](#optional-install-cuda-gpus-on-linux). You also need to set the `LD_LIBRARY_PATH` and `CUDA_HOME` environment @@ -594,7 +594,7 @@ https://developer.nvidia.com/cuda-gpus https://developer.nvidia.com/cuda-downloads -Install version 7.5 if using our binary releases. +Install version 8.0 if using our binary releases. Install the toolkit into e.g. `/usr/local/cuda` @@ -609,7 +609,7 @@ toolkit is installed in `/usr/local/cuda`, run the following commands (edited to reflect the cuDNN version you downloaded): ``` bash -tar xvzf cudnn-7.5-linux-x64-v5.1-ga.tgz +tar xvzf cudnn-8.0-linux-x64-v5.1-ga.tgz sudo cp cuda/include/cudnn.h /usr/local/cuda/include sudo cp cuda/lib64/libcudnn* /usr/local/cuda/lib64 sudo chmod a+r /usr/local/cuda/include/cudnn.h /usr/local/cuda/lib64/libcudnn* @@ -686,9 +686,9 @@ Once you have it downloaded locally, you can unzip and move the header and libraries to your local CUDA Toolkit folder: ```bash -$ sudo mv include/cudnn.h /Developer/NVIDIA/CUDA-7.5/include/ -$ sudo mv lib/libcudnn* /Developer/NVIDIA/CUDA-7.5/lib -$ sudo ln -s /Developer/NVIDIA/CUDA-7.5/lib/libcudnn* /usr/local/cuda/lib/ +$ sudo mv include/cudnn.h /Developer/NVIDIA/CUDA-8.0/include/ +$ sudo mv lib/libcudnn* /Developer/NVIDIA/CUDA-8.0/lib +$ sudo ln -s /Developer/NVIDIA/CUDA-8.0/lib/libcudnn* /usr/local/cuda/lib/ ``` To verify the CUDA installation, you can build and run deviceQuery to make sure @@ -702,8 +702,9 @@ $ popd $ ~/cuda-samples/bin/x86_64/darwin/release/deviceQuery ``` -If you want to compile tensorflow and have the XCode 7.3 installed, note that -Xcode 7.3 is not yet compatible with CUDA 7.5. You will need to download Xcode +If you want to compile tensorflow and have XCode 7.3 and CUDA 7.5 installed, note that +Xcode 7.3 is not yet compatible with CUDA 7.5. You can either upgrade to CUDA +8.0, or you will need to download Xcode 7.2 and select it as your default: ```bash @@ -734,8 +735,8 @@ No Google Cloud Platform support will be enabled for TensorFlow Do you wish to build TensorFlow with GPU support? [y/N] y GPU support will be enabled for TensorFlow Please specify which gcc nvcc should use as the host compiler. [Default is /usr/bin/gcc]: -Please specify the Cuda SDK version you want to use, e.g. 7.0. [Leave empty to use system default]: 7.5 -Please specify the location where CUDA 7.5 toolkit is installed. Refer to README.md for more details. [Default is /usr/local/cuda]: +Please specify the Cuda SDK version you want to use, e.g. 7.0. [Leave empty to use system default]: 8.0 +Please specify the location where CUDA 8.0 toolkit is installed. Refer to README.md for more details. [Default is /usr/local/cuda]: Please specify the cuDNN version you want to use. [Leave empty to use system default]: 5 Please specify the location where cuDNN 5 library is installed. Refer to README.md for more details. [Default is /usr/local/cuda]: Please specify a list of comma-separated Cuda compute capabilities you want to build with. From 2de7a69994772bba2bacca9447c8178bcd2b8c03 Mon Sep 17 00:00:00 2001 From: Yifei Feng Date: Thu, 20 Oct 2016 15:32:14 -0700 Subject: [PATCH 005/139] Regenerate docs for release. --- .../api_docs/cc/ClassRandomAccessFile.md | 2 +- tensorflow/g3doc/api_docs/cc/ClassSession.md | 31 ++++++++++++++++++- tensorflow/g3doc/api_docs/cc/ClassTensor.md | 31 +++++++++++++++++-- 3 files changed, 59 insertions(+), 5 deletions(-) diff --git a/tensorflow/g3doc/api_docs/cc/ClassRandomAccessFile.md b/tensorflow/g3doc/api_docs/cc/ClassRandomAccessFile.md index 1ff484c0834..1a1526f66d5 100644 --- a/tensorflow/g3doc/api_docs/cc/ClassRandomAccessFile.md +++ b/tensorflow/g3doc/api_docs/cc/ClassRandomAccessFile.md @@ -18,7 +18,7 @@ A file abstraction for randomly reading the contents of a file. -#### `virtual Status tensorflow::RandomAccessFile::Read(uint64 offset, size_t n, StringPiece *result, char *scratch) const =0` {#virtual_Status_tensorflow_RandomAccessFile_Read} +#### `virtual Status tensorflow::RandomAccessFile::Read(uint64 offset, size_t n, StringPiece *result, char *scratch) const =0` {#virtual_Status_tensorflow_RandomAccessFile_Read} Reads up to `n` bytes from the file starting at `offset`. diff --git a/tensorflow/g3doc/api_docs/cc/ClassSession.md b/tensorflow/g3doc/api_docs/cc/ClassSession.md index d2719d8ebbb..6829548530d 100644 --- a/tensorflow/g3doc/api_docs/cc/ClassSession.md +++ b/tensorflow/g3doc/api_docs/cc/ClassSession.md @@ -6,7 +6,36 @@ When a Session is created with a given target, a new Session object is bound to Example: -{c++} tensorflow::GraphDef graph; // ... Create or load graph into "graph". // This example uses the default options which connects // to a local runtime. tensorflow::SessionOptions options; std::unique_ptr session(tensorflow::NewSession(options)); // Create the session with this graph. tensorflow::Status s = session->Create(graph); if (!s.ok()) { ... } // Run the graph and fetch the first output of the "output" // operation, and also run to but do not return anything // for the "update_state" operation. std::vector outputs; s = session->Run({}, {"output:0"}, {"update_state"}, &outputs); if (!s.ok()) { ... } // Map the output as a flattened float tensor, and do something // with it. auto output_tensor = outputs[0].flat(); if (output_tensor(0) > 0.5) { ... } // Close the session to release the resources associated with // this session. session->Close(); +```c++ tensorflow::GraphDef graph; +// ... Create or load graph into "graph". + +// This example uses the default options which connects +// to a local runtime. +tensorflow::SessionOptions options; +std::unique_ptr +session(tensorflow::NewSession(options)); + +// Create the session with this graph. +tensorflow::Status s = session->Create(graph); +if (!s.ok()) { ... } + +// Run the graph and fetch the first output of the "output" +// operation, and also run to but do not return anything +// for the "update_state" operation. +std::vector outputs; +s = session->Run({}, {"output:0"}, {"update_state"}, &outputs); +if (!s.ok()) { ... } + +// Map the output as a flattened float tensor, and do something +// with it. +auto output_tensor = outputs[0].flat(); +if (output_tensor(0) > 0.5) { ... } + +// Close the session to release the resources associated with +// this session. +session->Close(); + +``` A Session allows concurrent calls to Run() , though a Session must be created / extended by a single thread. diff --git a/tensorflow/g3doc/api_docs/cc/ClassTensor.md b/tensorflow/g3doc/api_docs/cc/ClassTensor.md index 78899def539..b909bffe3a5 100644 --- a/tensorflow/g3doc/api_docs/cc/ClassTensor.md +++ b/tensorflow/g3doc/api_docs/cc/ClassTensor.md @@ -12,7 +12,9 @@ Creates a 1-dimensional, 0-element float tensor. The returned Tensor is not a scalar (shape {}), but is instead an empty one-dimensional Tensor (shape {0}, NumElements() == 0). Since it has no elements, it does not need to be assigned a value and is initialized by default ( IsInitialized() is true). If this is undesirable, consider creating a one-element scalar which does require initialization: -{c++} Tensor(DT_FLOAT, TensorShape({})) +```c++ Tensor(DT_FLOAT, TensorShape({})) + +``` #### `tensorflow::Tensor::Tensor(DataType type, const TensorShape &shape)` {#tensorflow_Tensor_Tensor} @@ -176,7 +178,15 @@ Use these methods when you know the data type and the number of dimensions of th Example: -{c++} typedef float T; Tensor my_mat(...built with Shape{rows: 3, cols: 5}...); auto mat = my_mat.matrix(); // 2D Eigen::Tensor, 3 x 5. auto mat = my_mat.tensor(); // 2D Eigen::Tensor, 3 x 5. auto vec = my_mat.vec(); // CHECK fails as my_mat is 2D. auto vec = my_mat.tensor(); // CHECK fails as my_mat is 2D. auto mat = my_mat.matrix();// CHECK fails as type mismatch. +```c++ typedef float T; +Tensor my_mat(...built with Shape{rows: 3, cols: 5}...); +auto mat = my_mat.matrix(); // 2D Eigen::Tensor, 3 x 5. +auto mat = my_mat.tensor(); // 2D Eigen::Tensor, 3 x 5. +auto vec = my_mat.vec(); // CHECK fails as my_mat is 2D. +auto vec = my_mat.tensor(); // CHECK fails as my_mat is 2D. +auto mat = my_mat.matrix();// CHECK fails as type mismatch. + +``` #### `TTypes::Matrix tensorflow::Tensor::matrix()` {#TTypes_T_Matrix_tensorflow_Tensor_matrix} @@ -204,7 +214,22 @@ These methods allow you to access the data with the dimensions and sizes of your Example: -{c++} typedef float T; Tensor my_ten(...built with Shape{planes: 4, rows: 3, cols: 5}...); // 1D Eigen::Tensor, size 60: auto flat = my_ten.flat(); // 2D Eigen::Tensor 12 x 5: auto inner = my_ten.flat_inner_dims(); // 2D Eigen::Tensor 4 x 15: auto outer = my_ten.shaped({4, 15}); // CHECK fails, bad num elements: auto outer = my_ten.shaped({4, 8}); // 3D Eigen::Tensor 6 x 5 x 2: auto weird = my_ten.shaped({6, 5, 2}); // CHECK fails, type mismatch: auto bad = my_ten.flat(); +```c++ typedef float T; +Tensor my_ten(...built with Shape{planes: 4, rows: 3, cols: 5}...); +// 1D Eigen::Tensor, size 60: +auto flat = my_ten.flat(); +// 2D Eigen::Tensor 12 x 5: +auto inner = my_ten.flat_inner_dims(); +// 2D Eigen::Tensor 4 x 15: +auto outer = my_ten.shaped({4, 15}); +// CHECK fails, bad num elements: +auto outer = my_ten.shaped({4, 8}); +// 3D Eigen::Tensor 6 x 5 x 2: +auto weird = my_ten.shaped({6, 5, 2}); +// CHECK fails, type mismatch: +auto bad = my_ten.flat(); + +``` #### `TTypes::UnalignedFlat tensorflow::Tensor::unaligned_flat()` {#TTypes_T_UnalignedFlat_tensorflow_Tensor_unaligned_flat} From 671f4c6519b4d82d3e97ab96825e0a748fae8e5d Mon Sep 17 00:00:00 2001 From: Yifei Feng Date: Thu, 20 Oct 2016 17:06:33 -0700 Subject: [PATCH 006/139] Temporarily remove tests that fail running with a single GPU. (#5100) * Temporarily remove tests that fail running with a single GPU. * Fix tags order according to buildifier --- tensorflow/contrib/training/BUILD | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/training/BUILD b/tensorflow/contrib/training/BUILD index 79901b6ee56..5f3368ee2c8 100644 --- a/tensorflow/contrib/training/BUILD +++ b/tensorflow/contrib/training/BUILD @@ -36,6 +36,7 @@ py_test( size = "medium", srcs = ["python/training/batch_sequences_with_states_test.py"], srcs_version = "PY2AND3", + tags = ["manual"], deps = [ ":training_py", "//tensorflow:tensorflow_py", @@ -60,7 +61,10 @@ py_test( size = "small", srcs = ["python/training/sampling_ops_threading_test.py"], srcs_version = "PY2AND3", - tags = ["notsan"], + tags = [ + "manual", + "notsan", + ], deps = [ ":training_py", "//tensorflow:tensorflow_py", @@ -73,6 +77,7 @@ py_test( size = "medium", srcs = ["python/training/bucket_ops_test.py"], srcs_version = "PY2AND3", + tags = ["manual"], deps = [ ":training_py", "//tensorflow:tensorflow_py", From f41daaa2d1c4c89424a688c8735c5f7fc58df4fd Mon Sep 17 00:00:00 2001 From: Shanqing Cai Date: Sun, 16 Oct 2016 15:42:47 -0800 Subject: [PATCH 007/139] Fixes and improvements to docker build script and dockerfiles 1) Clean up large Bazel build cache. Total filesystem size reduction as seen by du -sh /: devel image: 1.5 GB (Before: 2.9 GB; After: 1.4 GB) devel-gpu image: 2.3 GB (Before: 4.7 GB; After: 2.4 GB) 2) Using nvidia-docker for GPU docker build. 3) Upgrade Bazel version from 0.3.1 to 0.3.2. 4) Add missing libcurl3-dev build dependency to devel images. 5) Add scipy and sklearn to Dockerfile.devel-gpu to enhance consistency with other image types (e.g., Dockerfile.devel). 6) Remove the obsolete and unnecessary --recurse-submodules flag for git clone. Related to GH issues: https://github.com/tensorflow/tensorflow/issues/4116 and https://github.com/tensorflow/tensorflow/issues/4117 However, not using the "git clone --depth 1" suggested by issue #4117, because the size of the git repo is only reduced by about 50 MB by the "--depth 1" flag. This space saving is small compared to the space saving due to bazel cache removal. The complete history of the git repo can be useful for certain development purposes. Change: 136302103 --- tensorflow/tools/docker/Dockerfile.devel | 12 ++++++--- tensorflow/tools/docker/Dockerfile.devel-gpu | 14 +++++++--- .../docker/parameterized_docker_build.sh | 27 ++++++++++--------- 3 files changed, 33 insertions(+), 20 deletions(-) diff --git a/tensorflow/tools/docker/Dockerfile.devel b/tensorflow/tools/docker/Dockerfile.devel index 1c1f2e2059b..e13c7a011a8 100644 --- a/tensorflow/tools/docker/Dockerfile.devel +++ b/tensorflow/tools/docker/Dockerfile.devel @@ -6,6 +6,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ curl \ git \ + libcurl3-dev \ libfreetype6-dev \ libpng12-dev \ libzmq3-dev \ @@ -65,7 +66,7 @@ RUN echo "build --spawn_strategy=standalone --genrule_strategy=standalone" \ >>/root/.bazelrc ENV BAZELRC /root/.bazelrc # Install the most recent bazel release. -ENV BAZEL_VERSION 0.3.1 +ENV BAZEL_VERSION 0.3.2 WORKDIR / RUN mkdir /bazel && \ cd /bazel && \ @@ -78,7 +79,7 @@ RUN mkdir /bazel && \ # Download and build TensorFlow. -RUN git clone --recursive https://github.com/tensorflow/tensorflow.git && \ +RUN git clone https://github.com/tensorflow/tensorflow.git && \ cd tensorflow && \ git checkout r0.11 WORKDIR /tensorflow @@ -87,10 +88,13 @@ WORKDIR /tensorflow # more difficult to experiment with local changes. Instead, just add # the built directory to the path. -RUN ./configure && \ +RUN tensorflow/tools/ci_build/builds/configured CPU \ bazel build -c opt tensorflow/tools/pip_package:build_pip_package && \ bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/pip && \ - pip install --upgrade /tmp/pip/tensorflow-*.whl + pip install --upgrade /tmp/pip/tensorflow-*.whl && \ + rm -rf /tmp/pip && \ + rm -rf /root/.cache +# Clean up pip wheel and Bazel cache when done. # TensorBoard EXPOSE 6006 diff --git a/tensorflow/tools/docker/Dockerfile.devel-gpu b/tensorflow/tools/docker/Dockerfile.devel-gpu index 01e22412a34..8693059369c 100644 --- a/tensorflow/tools/docker/Dockerfile.devel-gpu +++ b/tensorflow/tools/docker/Dockerfile.devel-gpu @@ -6,6 +6,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ curl \ git \ + libcurl3-dev \ libfreetype6-dev \ libpng12-dev \ libzmq3-dev \ @@ -31,6 +32,8 @@ RUN pip --no-cache-dir install \ jupyter \ matplotlib \ numpy \ + scipy \ + sklearn \ && \ python -m ipykernel.kernelspec @@ -66,7 +69,7 @@ RUN echo "build --spawn_strategy=standalone --genrule_strategy=standalone" \ >>/root/.bazelrc ENV BAZELRC /root/.bazelrc # Install the most recent bazel release. -ENV BAZEL_VERSION 0.3.1 +ENV BAZEL_VERSION 0.3.2 WORKDIR / RUN mkdir /bazel && \ cd /bazel && \ @@ -79,7 +82,7 @@ RUN mkdir /bazel && \ # Download and build TensorFlow. -RUN git clone -b r0.11 --recursive --recurse-submodules https://github.com/tensorflow/tensorflow.git && \ +RUN git clone https://github.com/tensorflow/tensorflow.git && \ cd tensorflow && \ git checkout r0.11 WORKDIR /tensorflow @@ -89,10 +92,13 @@ ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH ENV TF_NEED_CUDA 1 ENV TF_CUDA_COMPUTE_CAPABILITIES=3.0,3.5,5.2 -RUN ./configure && \ +RUN tensorflow/tools/ci_build/builds/configured GPU \ bazel build -c opt --config=cuda tensorflow/tools/pip_package:build_pip_package && \ bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/pip && \ - pip install --upgrade /tmp/pip/tensorflow-*.whl + pip install --upgrade /tmp/pip/tensorflow-*.whl && \ + rm -rf /tmp/pip && \ + rm -rf /root/.cache +# Clean up pip wheel and Bazel cache when done. WORKDIR /root diff --git a/tensorflow/tools/docker/parameterized_docker_build.sh b/tensorflow/tools/docker/parameterized_docker_build.sh index 00f14f9cfb3..f781607d3cf 100755 --- a/tensorflow/tools/docker/parameterized_docker_build.sh +++ b/tensorflow/tools/docker/parameterized_docker_build.sh @@ -120,8 +120,10 @@ else fi if [[ ${TF_DOCKER_BUILD_TYPE} == "cpu" ]]; then - : + DOCKER_BINARY="docker" elif [[ ${TF_DOCKER_BUILD_TYPE} == "gpu" ]]; then + DOCKER_BINARY="nvidia-docker" + FINAL_TAG="${FINAL_TAG}-gpu" if [[ ${ORIG_DOCKERFILE} == *"."* ]]; then # There is already a dot in the tag, use "-" @@ -235,17 +237,18 @@ fi IMG="${USER}/tensorflow:${FINAL_TAG}" echo "Building docker image with image name and tag: ${IMG}" -docker build --no-cache -t "${IMG}" -f "${DOCKERFILE}" "${TMP_DIR}" +"${DOCKER_BINARY}" build --no-cache -t "${IMG}" -f "${DOCKERFILE}" "${TMP_DIR}" if [[ $? == "0" ]]; then - echo "docker build of ${IMG} succeeded" + echo "${DOCKER_BINARY} build of ${IMG} succeeded" else - die "FAIL: docker build of ${IMG} with Dockerfile ${DOCKERFILE} failed" + die "FAIL: ${DOCKER_BINARY} build of ${IMG} with Dockerfile ${DOCKERFILE} "\ +"failed" fi # Make sure that there is no other containers of the same image running # TODO(cais): Move to an earlier place. -if [[ ! -z $(docker ps | grep "${IMG}") ]]; then +if [[ ! -z $("${DOCKER_BINARY}" ps | grep "${IMG}") ]]; then die "ERROR: It appears that there are docker containers of the image "\ "${IMG} running. Please stop them before proceeding" fi @@ -258,7 +261,7 @@ echo " (Log file is at: ${DOCKER_RUN_LOG}" echo "" if [[ "${TF_DOCKER_BUILD_IS_DEVEL}" == "no" ]]; then - docker run --rm -p ${CONTAINER_PORT}:${CONTAINER_PORT} \ + "${DOCKER_BINARY}" run --rm -p ${CONTAINER_PORT}:${CONTAINER_PORT} \ -v ${TMP_DIR}/notebooks:/root/notebooks "${IMG}" \ 2>&1 > "${DOCKER_RUN_LOG}" & @@ -267,7 +270,7 @@ if [[ "${TF_DOCKER_BUILD_IS_DEVEL}" == "no" ]]; then while [[ -z ${CONTAINER_ID} ]]; do sleep 1 echo "Polling for container ID..." - CONTAINER_ID=$(docker ps | grep "${IMG}" | awk '{print $1}') + CONTAINER_ID=$("${DOCKER_BINARY}" ps | grep "${IMG}" | awk '{print $1}') done echo "ID of the running docker container: ${CONTAINER_ID}" @@ -293,10 +296,10 @@ if [[ "${TF_DOCKER_BUILD_IS_DEVEL}" == "no" ]]; then # Stop the running docker container sleep 1 - docker stop --time=0 ${CONTAINER_ID} + "${DOCKER_BINARY}" stop --time=0 ${CONTAINER_ID} else - docker run --rm -p ${CONTAINER_PORT}:${CONTAINER_PORT} \ + "${DOCKER_BINARY}" run --rm -p ${CONTAINER_PORT}:${CONTAINER_PORT} \ -v ${TMP_DIR}/notebooks:/root/notebooks "${IMG}" \ bash -c \ "cd /tensorflow; tensorflow/tools/ci_build/builds/test_tutorials.sh" @@ -324,9 +327,9 @@ fi # Apply the final image name and tag FINAL_IMG="${FINAL_IMAGE_NAME}:${FINAL_TAG}" -DOCKER_VER=$(docker version | grep Version | head -1 | awk '{print $NF}') +DOCKER_VER=$("${DOCKER_BINARY}" version | grep Version | head -1 | awk '{print $NF}') if [[ -z "${DOCKER_VER}" ]]; then - die "ERROR: Failed to determine docker version" + die "ERROR: Failed to determine ${DOCKER_BINARY} version" fi DOCKER_MAJOR_VER=$(echo "${DOCKER_VER}" | cut -d. -f 1) DOCKER_MINOR_VER=$(echo "${DOCKER_VER}" | cut -d. -f 2) @@ -337,7 +340,7 @@ if [[ "${DOCKER_MAJOR_VER}" -le 1 ]] && \ FORCE_TAG="--force" fi -docker tag ${FORCE_TAG} "${IMG}" "${FINAL_IMG}" || \ +"${DOCKER_BINARY}" tag ${FORCE_TAG} "${IMG}" "${FINAL_IMG}" || \ die "Failed to tag intermediate docker image ${IMG} as ${FINAL_IMG}" echo "" From 425f49e1ab6a32500a3c40b0a441d167ece70007 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Filip?= Date: Sat, 8 Oct 2016 04:04:43 +0200 Subject: [PATCH 008/139] Fix types in create_vocabulary (translate model) (#4793) --- tensorflow/models/rnn/translate/data_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensorflow/models/rnn/translate/data_utils.py b/tensorflow/models/rnn/translate/data_utils.py index 551c49d5323..e0e1d4bf058 100644 --- a/tensorflow/models/rnn/translate/data_utils.py +++ b/tensorflow/models/rnn/translate/data_utils.py @@ -26,6 +26,7 @@ import tarfile from six.moves import urllib from tensorflow.python.platform import gfile +import tensorflow as tf # Special vocabulary symbols - we always put them at the start. _PAD = b"_PAD" @@ -137,6 +138,7 @@ def create_vocabulary(vocabulary_path, data_path, max_vocabulary_size, counter += 1 if counter % 100000 == 0: print(" processing line %d" % counter) + line = tf.compat.as_bytes(line) tokens = tokenizer(line) if tokenizer else basic_tokenizer(line) for w in tokens: word = _DIGIT_RE.sub(b"0", w) if normalize_digits else w From 083c4dc9edbfce16c16413b350127f9ff3aad480 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 27 Sep 2016 10:48:51 -0800 Subject: [PATCH 009/139] Change StridedSlice to error on scalar input, in both the shape inference function and the kernel. Change: 134434589 --- .../contrib/cudnn_rnn/ops/cudnn_rnn_ops.cc | 5 +++- .../cudnn_rnn/ops/cudnn_rnn_ops_test.cc | 2 +- tensorflow/core/util/strided_slice_op.cc | 15 +++++++--- .../python/kernel_tests/slice_op_test.py | 28 +++++++++++++++++++ 4 files changed, 44 insertions(+), 6 deletions(-) diff --git a/tensorflow/contrib/cudnn_rnn/ops/cudnn_rnn_ops.cc b/tensorflow/contrib/cudnn_rnn/ops/cudnn_rnn_ops.cc index 49aa4d44952..ef8eeaead75 100644 --- a/tensorflow/contrib/cudnn_rnn/ops/cudnn_rnn_ops.cc +++ b/tensorflow/contrib/cudnn_rnn/ops/cudnn_rnn_ops.cc @@ -72,7 +72,10 @@ REGISTER_OP("CudnnRNNParamsSize") .Attr(kRNNInputModeAttrs) .Attr(kRNNDirectionAttrs) .Output("params_size: S") - .SetShapeFn(shape_inference::ScalarShape) + .SetShapeFn([](InferenceContext* c) { + c->set_output(0, c->Vector(1)); + return Status::OK(); + }) .Doc(strings::StrCat(R"doc( Return the params size that can be used by the Cudnn RNN model. Subsequent weight allocation and initialization should use this size. diff --git a/tensorflow/contrib/cudnn_rnn/ops/cudnn_rnn_ops_test.cc b/tensorflow/contrib/cudnn_rnn/ops/cudnn_rnn_ops_test.cc index 3a1afc8c599..95d45c0bb80 100644 --- a/tensorflow/contrib/cudnn_rnn/ops/cudnn_rnn_ops_test.cc +++ b/tensorflow/contrib/cudnn_rnn/ops/cudnn_rnn_ops_test.cc @@ -26,7 +26,7 @@ namespace tensorflow { TEST(CudnnRNNOpsTest, ParamsSize_ShapeFn) { ShapeInferenceTestOp op("CudnnRNNParamsSize"); - INFER_OK(op, "[1];[1];[1]", "[]"); + INFER_OK(op, "[1];[1];[1]", "[1]"); } TEST(CudnnRNNOpsTest, ForwardLstm_ShapeFn) { diff --git a/tensorflow/core/util/strided_slice_op.cc b/tensorflow/core/util/strided_slice_op.cc index 0f4695a58b1..96e5cda099b 100644 --- a/tensorflow/core/util/strided_slice_op.cc +++ b/tensorflow/core/util/strided_slice_op.cc @@ -94,8 +94,8 @@ struct StridedSliceDenseSpec { } // namespace template -static void BuildDenseSpec(const StridedSliceSparseSpec& sparse, - StridedSliceDenseSpec* dense) { +static Status TF_MUST_USE_RESULT BuildDenseSpec( + const StridedSliceSparseSpec& sparse, StridedSliceDenseSpec* dense) { // Build expanded begin, end, strides, begin_mask, end_mask // to remove any ellipsis dense->begin.resize(dense->dims); @@ -130,6 +130,12 @@ static void BuildDenseSpec(const StridedSliceSparseSpec& sparse, } else if ((1 << i) & sparse.new_axis_mask) { dense->final_shape_gather_indices.push_back(kNewAxis); } else { + if (full_index == dense->begin.size()) { + return errors::InvalidArgument("Index out of range using input dim ", + full_index, "; input has only ", + dense->dims, " dims"); + } + // Gather slicing spec into appropriate index dense->begin[full_index] = internal::SubtleMustCopy(begin_flat(i)); dense->end[full_index] = internal::SubtleMustCopy(end_flat(i)); @@ -154,6 +160,7 @@ static void BuildDenseSpec(const StridedSliceSparseSpec& sparse, } } } + return Status::OK(); } Status ValidateStridedSliceOp( @@ -233,9 +240,9 @@ Status ValidateStridedSliceOp( input_shape.dims(), 0, 0, *begin, *end, *strides}; if (begin_tensor.dtype() == DT_INT32) { - BuildDenseSpec(sparse_spec, &dense_spec); + TF_RETURN_IF_ERROR(BuildDenseSpec(sparse_spec, &dense_spec)); } else if (begin_tensor.dtype() == DT_INT64) { - BuildDenseSpec(sparse_spec, &dense_spec); + TF_RETURN_IF_ERROR(BuildDenseSpec(sparse_spec, &dense_spec)); } else { LOG(FATAL) << "begin must be either int32 or int64"; } diff --git a/tensorflow/python/kernel_tests/slice_op_test.py b/tensorflow/python/kernel_tests/slice_op_test.py index 27506126e0c..fd5c66c99d5 100644 --- a/tensorflow/python/kernel_tests/slice_op_test.py +++ b/tensorflow/python/kernel_tests/slice_op_test.py @@ -77,6 +77,34 @@ class SliceTest(tf.test.TestCase): slice_val = slice_t.eval() self.assertAllEqual(slice_val, inp[lo:hi]) + def testScalarInput(self): + input_val = 0 + with self.test_session() as sess: + # Test with constant input; shape inference fails. + with self.assertRaisesWithPredicateMatch(ValueError, "out of range"): + tf.constant(input_val)[:].get_shape() + + # Test evaluating with non-constant input; kernel execution fails. + input_t = tf.placeholder(tf.int32) + slice_t = input_t[:] + with self.assertRaisesWithPredicateMatch(tf.errors.InvalidArgumentError, + "out of range"): + sess.run([slice_t], feed_dict={input_t: input_val}) + + def testInvalidIndex(self): + input_val = [1, 2] + with self.test_session() as sess: + # Test with constant input; shape inference fails. + with self.assertRaisesWithPredicateMatch(ValueError, "out of range"): + tf.constant(input_val)[1:, 1:].get_shape() + + # Test evaluating with non-constant input; kernel execution fails. + input_t = tf.placeholder(tf.int32) + slice_t = input_t[1:, 1:] + with self.assertRaisesWithPredicateMatch(tf.errors.InvalidArgumentError, + "out of range"): + sess.run([slice_t], feed_dict={input_t: input_val}) + def _testSliceMatrixDim0(self, x, begin, size): with self.test_session(use_gpu=True): tf_ans = tf.slice(x, [begin, 0], [size, x.shape[1]]).eval() From b73effb7bf3cb4e72a53fcf1da2dd318c1cfc1aa Mon Sep 17 00:00:00 2001 From: Gunhan Gulsoy Date: Sat, 22 Oct 2016 23:05:17 -0700 Subject: [PATCH 010/139] Update version string to 0.11.rc2. --- README.md | 8 +-- tensorflow/contrib/cmake/setup.py | 2 +- tensorflow/core/public/version.h | 2 +- tensorflow/g3doc/get_started/os_setup.md | 64 +++++++++---------- tensorflow/tools/dist_test/server/Dockerfile | 2 +- .../tools/dist_test/server/Dockerfile.test | 2 +- tensorflow/tools/docker/Dockerfile | 2 +- tensorflow/tools/docker/Dockerfile.gpu | 2 +- tensorflow/tools/gcs_test/Dockerfile | 2 +- tensorflow/tools/pip_package/setup.py | 2 +- 10 files changed, 44 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index b3d8f6b33dd..03f9c0c61a1 100644 --- a/README.md +++ b/README.md @@ -33,10 +33,10 @@ and discussion.** People who are a little more adventurous can also try our nightly binaries: -* Linux CPU-only: [Python 2](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.11.0rc1-cp27-none-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave)) / [Python 3.4](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.11.0rc1-cp34-cp34m-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=cpu-slave/)) / [Python 3.5](https://ci.tensorflow.org/view/Nightly/job/nightly-python35-linux-cpu/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.11.0rc1-cp35-cp35m-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-python35-linux-cpu/)) -* Linux GPU: [Python 2](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-linux-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=gpu-linux/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.11.0rc1-cp27-none-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-linux-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=gpu-linux/)) / [Python 3.4](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-linux-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=gpu-linux/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.11.0rc1-cp34-cp34m-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-linux-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=gpu-linux/)) / [Python 3.5](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-linux-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3.5,label=gpu-linux/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.11.0rc1-cp35-cp35m-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-linux-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3.5,label=gpu-linux/)) -* Mac CPU-only: [Python 2](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=mac1-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.11.0rc1-py2-none-any.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=mac1-slave/)) / [Python 3](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=mac1-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.11.0rc1-py3-none-any.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=mac1-slave/)) -* Mac GPU: [Python 2](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-mac-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=gpu-mac/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.11.0rc1-py2-none-any.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-mac-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=gpu-mac/)) / [Python 3](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-mac-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=gpu-mac/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.11.0rc1-py3-none-any.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-mac-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=gpu-mac/)) +* Linux CPU-only: [Python 2](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.11.0rc2-cp27-none-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave)) / [Python 3.4](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.11.0rc2-cp34-cp34m-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=cpu-slave/)) / [Python 3.5](https://ci.tensorflow.org/view/Nightly/job/nightly-python35-linux-cpu/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.11.0rc2-cp35-cp35m-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-python35-linux-cpu/)) +* Linux GPU: [Python 2](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-linux-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=gpu-linux/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.11.0rc2-cp27-none-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-linux-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=gpu-linux/)) / [Python 3.4](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-linux-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=gpu-linux/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.11.0rc2-cp34-cp34m-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-linux-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=gpu-linux/)) / [Python 3.5](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-linux-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3.5,label=gpu-linux/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.11.0rc2-cp35-cp35m-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-linux-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3.5,label=gpu-linux/)) +* Mac CPU-only: [Python 2](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=mac1-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.11.0rc2-py2-none-any.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=mac1-slave/)) / [Python 3](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=mac1-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.11.0rc2-py3-none-any.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=mac1-slave/)) +* Mac GPU: [Python 2](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-mac-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=gpu-mac/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.11.0rc2-py2-none-any.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-mac-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=gpu-mac/)) / [Python 3](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-mac-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=gpu-mac/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.11.0rc2-py3-none-any.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-mac-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=gpu-mac/)) * [Android](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-android/TF_BUILD_CONTAINER_TYPE=ANDROID,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=NO_PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=android-slave/lastSuccessfulBuild/artifact/bazel-out/local_linux/bin/tensorflow/examples/android/tensorflow_demo.apk) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-android/TF_BUILD_CONTAINER_TYPE=ANDROID,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=NO_PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=android-slave/)) #### *Try your first TensorFlow program* diff --git a/tensorflow/contrib/cmake/setup.py b/tensorflow/contrib/cmake/setup.py index d71ec2abb5d..8a21e247da2 100644 --- a/tensorflow/contrib/cmake/setup.py +++ b/tensorflow/contrib/cmake/setup.py @@ -26,7 +26,7 @@ from setuptools import find_packages, setup, Command from setuptools.command.install import install as InstallCommandBase from setuptools.dist import Distribution -_VERSION = '0.11.0rc1-cmake-experimental' +_VERSION = '0.11.0rc2-cmake-experimental' REQUIRED_PACKAGES = [ 'numpy >= 1.11.0', diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h index 1e8ae0b1bf9..060682700cc 100644 --- a/tensorflow/core/public/version.h +++ b/tensorflow/core/public/version.h @@ -20,7 +20,7 @@ limitations under the License. #define TF_MAJOR_VERSION 0 #define TF_MINOR_VERSION 11 -#define TF_PATCH_VERSION 0rc1 +#define TF_PATCH_VERSION 0rc2 // TF_VERSION_SUFFIX is non-empty for pre-releases (e.g. "-alpha", "-alpha.1", // "-beta", "-rc", "-rc.1") diff --git a/tensorflow/g3doc/get_started/os_setup.md b/tensorflow/g3doc/get_started/os_setup.md index 06fb37232a4..2904965f619 100644 --- a/tensorflow/g3doc/get_started/os_setup.md +++ b/tensorflow/g3doc/get_started/os_setup.md @@ -63,37 +63,37 @@ Then, select the correct binary to install: ```bash # Ubuntu/Linux 64-bit, CPU only, Python 2.7 -$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.11.0rc1-cp27-none-linux_x86_64.whl +$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.11.0rc2-cp27-none-linux_x86_64.whl # Ubuntu/Linux 64-bit, GPU enabled, Python 2.7 # Requires CUDA toolkit 8.0 and CuDNN v5. For other versions, see "Install from sources" below. -$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.11.0rc1-cp27-none-linux_x86_64.whl +$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.11.0rc2-cp27-none-linux_x86_64.whl # Mac OS X, CPU only, Python 2.7: -$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-0.11.0rc1-py2-none-any.whl +$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-0.11.0rc2-py2-none-any.whl # Mac OS X, GPU enabled, Python 2.7: -$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/gpu/tensorflow-0.11.0rc1-py2-none-any.whl +$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/gpu/tensorflow-0.11.0rc2-py2-none-any.whl # Ubuntu/Linux 64-bit, CPU only, Python 3.4 -$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.11.0rc1-cp34-cp34m-linux_x86_64.whl +$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.11.0rc2-cp34-cp34m-linux_x86_64.whl # Ubuntu/Linux 64-bit, GPU enabled, Python 3.4 # Requires CUDA toolkit 8.0 and CuDNN v5. For other versions, see "Install from sources" below. -$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.11.0rc1-cp34-cp34m-linux_x86_64.whl +$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.11.0rc2-cp34-cp34m-linux_x86_64.whl # Ubuntu/Linux 64-bit, CPU only, Python 3.5 -$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.11.0rc1-cp35-cp35m-linux_x86_64.whl +$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.11.0rc2-cp35-cp35m-linux_x86_64.whl # Ubuntu/Linux 64-bit, GPU enabled, Python 3.5 # Requires CUDA toolkit 8.0 and CuDNN v5. For other versions, see "Install from sources" below. -$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.11.0rc1-cp35-cp35m-linux_x86_64.whl +$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.11.0rc2-cp35-cp35m-linux_x86_64.whl # Mac OS X, CPU only, Python 3.4 or 3.5: -$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-0.11.0rc1-py3-none-any.whl +$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-0.11.0rc2-py3-none-any.whl # Mac OS X, GPU enabled, Python 3.4 or 3.5: -$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/gpu/tensorflow-0.11.0rc1-py3-none-any.whl +$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/gpu/tensorflow-0.11.0rc2-py3-none-any.whl ``` Install TensorFlow: @@ -159,37 +159,37 @@ Now, install TensorFlow just as you would for a regular Pip installation. First ```bash # Ubuntu/Linux 64-bit, CPU only, Python 2.7 -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.11.0rc1-cp27-none-linux_x86_64.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.11.0rc2-cp27-none-linux_x86_64.whl # Ubuntu/Linux 64-bit, GPU enabled, Python 2.7 # Requires CUDA toolkit 8.0 and CuDNN v5. For other versions, see "Install from sources" below. -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.11.0rc1-cp27-none-linux_x86_64.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.11.0rc2-cp27-none-linux_x86_64.whl # Mac OS X, CPU only, Python 2.7: -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-0.11.0rc1-py2-none-any.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-0.11.0rc2-py2-none-any.whl # Mac OS X, GPU enabled, Python 2.7: -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/gpu/tensorflow-0.11.0rc1-py2-none-any.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/gpu/tensorflow-0.11.0rc2-py2-none-any.whl # Ubuntu/Linux 64-bit, CPU only, Python 3.4 -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.11.0rc1-cp34-cp34m-linux_x86_64.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.11.0rc2-cp34-cp34m-linux_x86_64.whl # Ubuntu/Linux 64-bit, GPU enabled, Python 3.4 # Requires CUDA toolkit 8.0 and CuDNN v5. For other versions, see "Install from sources" below. -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.11.0rc1-cp34-cp34m-linux_x86_64.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.11.0rc2-cp34-cp34m-linux_x86_64.whl # Ubuntu/Linux 64-bit, CPU only, Python 3.5 -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.11.0rc1-cp35-cp35m-linux_x86_64.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.11.0rc2-cp35-cp35m-linux_x86_64.whl # Ubuntu/Linux 64-bit, GPU enabled, Python 3.5 # Requires CUDA toolkit 8.0 and CuDNN v5. For other versions, see "Install from sources" below. -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.11.0rc1-cp35-cp35m-linux_x86_64.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.11.0rc2-cp35-cp35m-linux_x86_64.whl # Mac OS X, CPU only, Python 3.4 or 3.5: -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-0.11.0rc1-py3-none-any.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-0.11.0rc2-py3-none-any.whl # Mac OS X, GPU enabled, Python 3.4 or 3.5: -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/gpu/tensorflow-0.11.0rc1-py3-none-any.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/gpu/tensorflow-0.11.0rc2-py3-none-any.whl ``` Finally install TensorFlow: @@ -298,37 +298,37 @@ select the correct binary to install: ```bash # Ubuntu/Linux 64-bit, CPU only, Python 2.7 -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.11.0rc1-cp27-none-linux_x86_64.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.11.0rc2-cp27-none-linux_x86_64.whl # Ubuntu/Linux 64-bit, GPU enabled, Python 2.7 # Requires CUDA toolkit 8.0 and CuDNN v5. For other versions, see "Install from sources" below. -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.11.0rc1-cp27-none-linux_x86_64.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.11.0rc2-cp27-none-linux_x86_64.whl # Mac OS X, CPU only, Python 2.7: -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-0.11.0rc1-py2-none-any.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-0.11.0rc2-py2-none-any.whl # Mac OS X, GPU enabled, Python 2.7: -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/gpu/tensorflow-0.11.0rc1-py2-none-any.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/gpu/tensorflow-0.11.0rc2-py2-none-any.whl # Ubuntu/Linux 64-bit, CPU only, Python 3.4 -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.11.0rc1-cp34-cp34m-linux_x86_64.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.11.0rc2-cp34-cp34m-linux_x86_64.whl # Ubuntu/Linux 64-bit, GPU enabled, Python 3.4 # Requires CUDA toolkit 8.0 and CuDNN v5. For other versions, see "Install from sources" below. -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.11.0rc1-cp34-cp34m-linux_x86_64.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.11.0rc2-cp34-cp34m-linux_x86_64.whl # Ubuntu/Linux 64-bit, CPU only, Python 3.5 -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.11.0rc1-cp35-cp35m-linux_x86_64.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.11.0rc2-cp35-cp35m-linux_x86_64.whl # Ubuntu/Linux 64-bit, GPU enabled, Python 3.5 # Requires CUDA toolkit 8.0 and CuDNN v5. For other versions, see "Install from sources" below. -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.11.0rc1-cp35-cp35m-linux_x86_64.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.11.0rc2-cp35-cp35m-linux_x86_64.whl # Mac OS X, CPU only, Python 3.4 or 3.5: -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-0.11.0rc1-py3-none-any.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-0.11.0rc2-py3-none-any.whl # Mac OS X, GPU enabled, Python 3.4 or 3.5: -(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/gpu/tensorflow-0.11.0rc1-py3-none-any.whl +(tensorflow)$ export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/mac/gpu/tensorflow-0.11.0rc2-py3-none-any.whl ``` Finally install TensorFlow: @@ -396,7 +396,7 @@ code. code. We also have tags with `latest` replaced by a released version (e.g., -`0.11.0rc1-gpu`). +`0.11.0rc2-gpu`). With Docker the installation is as follows: @@ -785,7 +785,7 @@ $ bazel build -c opt --config=cuda //tensorflow/tools/pip_package:build_pip_pack $ bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_pkg # The name of the .whl file will depend on your platform. -$ sudo pip install /tmp/tensorflow_pkg/tensorflow-0.11.0rc1-py2-none-any.whl +$ sudo pip install /tmp/tensorflow_pkg/tensorflow-0.11.0rc2-py2-none-any.whl ``` ## Setting up TensorFlow for Development diff --git a/tensorflow/tools/dist_test/server/Dockerfile b/tensorflow/tools/dist_test/server/Dockerfile index 1653933b012..9d59abef2f1 100644 --- a/tensorflow/tools/dist_test/server/Dockerfile +++ b/tensorflow/tools/dist_test/server/Dockerfile @@ -36,7 +36,7 @@ RUN curl -O https://bootstrap.pypa.io/get-pip.py && \ # Install TensorFlow CPU version from nightly build RUN pip --no-cache-dir install \ - https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.11.0rc1-cp27-none-linux_x86_64.whl + https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.11.0rc2-cp27-none-linux_x86_64.whl # Copy files, including the GRPC server binary at # server/grpc_tensorflow_server.py diff --git a/tensorflow/tools/dist_test/server/Dockerfile.test b/tensorflow/tools/dist_test/server/Dockerfile.test index cbe6c388aff..1c75e79512d 100644 --- a/tensorflow/tools/dist_test/server/Dockerfile.test +++ b/tensorflow/tools/dist_test/server/Dockerfile.test @@ -42,7 +42,7 @@ RUN pip install --upgrade pandas==0.18.1 # Install TensorFlow CPU version. RUN pip --no-cache-dir install \ - https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.11.0rc1-cp27-none-linux_x86_64.whl + https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.11.0rc2-cp27-none-linux_x86_64.whl # Copy files, including the GRPC server binary at # server/grpc_tensorflow_server.py diff --git a/tensorflow/tools/docker/Dockerfile b/tensorflow/tools/docker/Dockerfile index 71dc41de836..289b7376333 100644 --- a/tensorflow/tools/docker/Dockerfile +++ b/tensorflow/tools/docker/Dockerfile @@ -32,7 +32,7 @@ RUN pip --no-cache-dir install \ && \ python -m ipykernel.kernelspec -ENV TENSORFLOW_VERSION 0.11.0rc1 +ENV TENSORFLOW_VERSION 0.11.0rc2 # --- DO NOT EDIT OR DELETE BETWEEN THE LINES --- # # These lines will be edited automatically by parameterized_docker_build.sh. # diff --git a/tensorflow/tools/docker/Dockerfile.gpu b/tensorflow/tools/docker/Dockerfile.gpu index d9e3d44e789..58e1ca74c69 100644 --- a/tensorflow/tools/docker/Dockerfile.gpu +++ b/tensorflow/tools/docker/Dockerfile.gpu @@ -32,7 +32,7 @@ RUN pip --no-cache-dir install \ && \ python -m ipykernel.kernelspec -ENV TENSORFLOW_VERSION 0.11.0rc1 +ENV TENSORFLOW_VERSION 0.11.0rc2 # --- DO NOT EDIT OR DELETE BETWEEN THE LINES --- # # These lines will be edited automatically by parameterized_docker_build.sh. # diff --git a/tensorflow/tools/gcs_test/Dockerfile b/tensorflow/tools/gcs_test/Dockerfile index 9057e885481..795f5f380b0 100644 --- a/tensorflow/tools/gcs_test/Dockerfile +++ b/tensorflow/tools/gcs_test/Dockerfile @@ -16,7 +16,7 @@ RUN ./install_google_cloud_sdk.bash --disable-prompts --install-dir=/var/gcloud # Install nightly TensorFlow pip RUN pip install \ - https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.11.0rc1-cp27-none-linux_x86_64.whl + https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.11.0rc2-cp27-none-linux_x86_64.whl # Copy test files RUN mkdir -p /gcs-smoke/python diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py index 027cec86726..cb3b9ea94bf 100644 --- a/tensorflow/tools/pip_package/setup.py +++ b/tensorflow/tools/pip_package/setup.py @@ -26,7 +26,7 @@ from setuptools import find_packages, setup, Command from setuptools.command.install import install as InstallCommandBase from setuptools.dist import Distribution -_VERSION = '0.11.0rc1' +_VERSION = '0.11.0rc2' REQUIRED_PACKAGES = [ 'numpy >= 1.11.0', From b5a169f3e307e18b959d4a7fd9d4315ac1eb802a Mon Sep 17 00:00:00 2001 From: tyfkda Date: Sat, 24 Sep 2016 13:27:26 +0900 Subject: [PATCH 011/139] Fix symbol duplication on Xcode/clang (#4504) Following linker error happened on Xcode: ``` duplicate symbol __Z14tf_git_versionv in: /Users/admin/tmp/tensorflow/tensorflow/contrib/ios_examples/simple/../../makefile/gen/lib/libtensorflow-core.a(version_info.o) duplicate symbol __Z19tf_compiler_versionv in: /Users/admin/tmp/tensorflow/tensorflow/contrib/ios_examples/simple/../../makefile/gen/lib/libtensorflow-core.a(version_info.o) ld: 2 duplicate symbols for architecture x86_64 clang: error: linker command failed with exit code 1 (use -v to see invocation) ``` These functions are defined in `tensorflow/core/util/version_info.cc`. It seems an object file for it is added twice in Makefile: (`$(wildcard tensorflow/core/util/*.cc)` and `$(tensorflow/core/util/version_info.cc) `) --- tensorflow/contrib/makefile/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensorflow/contrib/makefile/Makefile b/tensorflow/contrib/makefile/Makefile index 677326c548b..c100732e75d 100644 --- a/tensorflow/contrib/makefile/Makefile +++ b/tensorflow/contrib/makefile/Makefile @@ -426,6 +426,8 @@ $(wildcard tensorflow/core/platform/*/*/*.cc) \ $(wildcard tensorflow/core/util/*.cc) \ $(wildcard tensorflow/core/util/*/*.cc) \ tensorflow/core/util/version_info.cc +# Remove duplicates (for version_info.cc) +CORE_CC_ALL_SRCS := $(sort $(CORE_CC_ALL_SRCS)) CORE_CC_EXCLUDE_SRCS := \ $(wildcard tensorflow/core/*/*test.cc) \ $(wildcard tensorflow/core/*/*testutil*) \ From 2db795810b71dcdea17af43f8f5c61a0b4d50ddf Mon Sep 17 00:00:00 2001 From: gunan Date: Tue, 25 Oct 2016 12:32:03 -0700 Subject: [PATCH 012/139] Fix test_installation to skip using parallel_gpu_execute when not using GPU. (#5186) (cherry picked from commit 446b1cbb81058e921512ecd566ec883184c7bbd4) --- .../ci_build/builds/test_installation.sh | 42 ++++++++++--------- tensorflow/tools/ci_build/ci_build.sh | 2 +- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/tensorflow/tools/ci_build/builds/test_installation.sh b/tensorflow/tools/ci_build/builds/test_installation.sh index 5d7d6ec3f1a..014dc0cea6f 100755 --- a/tensorflow/tools/ci_build/builds/test_installation.sh +++ b/tensorflow/tools/ci_build/builds/test_installation.sh @@ -47,9 +47,8 @@ # TF_BUILD_BAZEL_CLEAN, if set to any non-empty and non-0 value, directs the # script to perform bazel clean prior to main build and test steps. # -# TF_BUILD_SERIAL_INSTALL_TESTS, if set to any non-empty and non-0 value, -# will force the Python install tests to run serially, overriding than the -# concurrent testing behavior. +# TF_BUILD_USE_GPU, if set to 1, limits the number of concurrent tests to +# the number stored in TF_GPU_COUNT and assigns each test to a different GPU. # # TF_GPU_COUNT, Set the number of GPUs in the system. We run only this many # concurrent tests when running GPU tests. @@ -411,21 +410,21 @@ SKIP_COUNTER=0 FAILED_TESTS="" FAILED_TEST_LOGS="" -N_JOBS=$(grep -c ^processor /proc/cpuinfo) -if [[ -z ${N_JOBS} ]]; then - # Try the Mac way of getting number of CPUs - N_JOBS=$(sysctl -n hw.ncpu) -fi - -if [[ -z ${N_JOBS} ]]; then - N_JOBS=8 - echo "Cannot determine the number of processors" - echo "Using default concurrent job counter ${N_JOBS}" -fi - -if [[ ! -z "${TF_BUILD_SERIAL_INSTALL_TESTS}" ]] && - [[ "${TF_BUILD_SERIAL_INSTALL_TESTS}" != "0" ]]; then +if [[ "${TF_BUILD_USE_GPU}" == "1" ]]; then N_JOBS=$TF_GPU_COUNT +else + N_JOBS=$(grep -c ^processor /proc/cpuinfo) + if [[ -z ${N_JOBS} ]]; then + # Try the Mac way of getting number of CPUs + N_JOBS=$(sysctl -n hw.ncpu) + fi + + # If still cannot determine the number of CPUs, pick 8. + if [[ -z ${N_JOBS} ]]; then + N_JOBS=8 + echo "Cannot determine the number of processors" + echo "Using default concurrent job counter ${N_JOBS}" + fi fi echo "Running Python tests-on-install with ${N_JOBS} concurrent jobs..." @@ -485,9 +484,14 @@ while true; do TEST_LOGS="${TEST_LOGS} ${TEST_LOG}" # Launch test asynchronously - "${SCRIPT_DIR}/../gpu_build/parallel_gpu_execute.sh" \ + if [[ "${TF_BUILD_USE_GPU}" == "1" ]]; then + "${SCRIPT_DIR}/../gpu_build/parallel_gpu_execute.sh" \ + "${SCRIPT_DIR}/py_test_delegate.sh" \ + "${PYTHON_BIN_PATH}" "${PY_TEST_DIR}/${TEST_BASENAME}" "${TEST_LOG}" & + else "${SCRIPT_DIR}/py_test_delegate.sh" \ - "${PYTHON_BIN_PATH}" "${PY_TEST_DIR}/${TEST_BASENAME}" "${TEST_LOG}" & + "${PYTHON_BIN_PATH}" "${PY_TEST_DIR}/${TEST_BASENAME}" "${TEST_LOG}" & + fi if [[ "${TEST_COUNTER}" -ge "${N_PAR_TESTS}" ]]; then # Run in exclusive mode diff --git a/tensorflow/tools/ci_build/ci_build.sh b/tensorflow/tools/ci_build/ci_build.sh index 0f165cd8647..e9271cf9eb9 100755 --- a/tensorflow/tools/ci_build/ci_build.sh +++ b/tensorflow/tools/ci_build/ci_build.sh @@ -105,7 +105,7 @@ BUILD_TAG="${BUILD_TAG:-tf_ci}" # Add extra params for cuda devices and libraries for GPU container. if [ "${CONTAINER_TYPE}" == "gpu" ]; then # GPU pip tests-on-install concurrency is limited to the number of GPUs. - GPU_EXTRA_PARAMS="${GPU_EXTRA_PARAMS} -e TF_BUILD_SERIAL_INSTALL_TESTS=1" + GPU_EXTRA_PARAMS="${GPU_EXTRA_PARAMS} -e TF_BUILD_USE_GPU=1" else GPU_EXTRA_PARAMS="" fi From 974a861a8082769cf3739c89b627bf690163a485 Mon Sep 17 00:00:00 2001 From: Shanqing Cai Date: Tue, 25 Oct 2016 22:26:18 -0400 Subject: [PATCH 013/139] Let build_server.sh take whl file URL as an input argument. (#5206) This make it possible to test OSS GRPC distributed runtime in dist_test/remote_test.sh against a release build. Usage example: 1. Build the server using a release whl file. (Obviously this means that the Linxu CPU PIP release build has to pass first.) $ export DOCKER_VERSION_TAG="0.11.0rc1" $ tensorflow/tools/dist_test/build_server.sh tensorflow/tf_grpc_test_server:${DOCKER_VERSION_TAG} http://ci.tensorflow.org/view/Release/job/release-matrix-cpu/TF_BUILD_CONTAINER_TYPE=CPU,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-${DOCKER_VERSION_TAG}-cp27-none-linux_x86_64.whl --test 2. Run remote_test.sh: $ export TF_DIST_DOCKER_NO_CACHE=1 $ export TF_DIST_SERVER_DOCKER_IMAGE="tensorflow/tf_grpc_test_server:${DOCKER_VERSION_TAG}" $ export TF_DIST_GCLOUD_PROJECT="my-project" $ export TF_DIST_GCLOUD_COMPUTE_ZONE="my-zone" $ export TF_DIST_CONTAINER_CLUSTER="my-cluster" $ export TF_DIST_GCLOUD_KEY_FILE="/path/to/my/key.json" $ tensorflow/tools/dist_test/remote_test.sh "http://ci.tensorflow.org/view/Release/job/release-matrix-cpu/TF_BUILD_CONTAINER_TYPE=CPU,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-${DOCKER_VERSION_TAG}-cp27-none-linux_x86_64.whl" (cherry picked from commit 7ba17e2c0dc0e1f1a92de0a3373b4f2197ad8077) --- tensorflow/tools/dist_test/build_server.sh | 39 +++++++++++++++---- tensorflow/tools/dist_test/server/Dockerfile | 7 ++-- .../tools/dist_test/server/Dockerfile.test | 7 ++-- 3 files changed, 39 insertions(+), 14 deletions(-) diff --git a/tensorflow/tools/dist_test/build_server.sh b/tensorflow/tools/dist_test/build_server.sh index 178fba8065f..878fabd248f 100755 --- a/tensorflow/tools/dist_test/build_server.sh +++ b/tensorflow/tools/dist_test/build_server.sh @@ -16,7 +16,14 @@ # # Builds the test server for distributed (GRPC) TensorFlow # -# Usage: build_server.sh [--test] +# Usage: build_server.sh [--test] +# +# Arguments: +# docker_image_name: Name of the docker image to build. +# E.g.: tensorflow/tf_grpc_test_server:0.11.0rc1 +# +# whl_url: URL from which the TensorFlow whl file will be downloaded. +# E.g.: https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.11.0rc1-cp27-none-linux_x86_64.whl # # The optional flag --test lets the script to use the Dockerfile for the # testing GRPC server. Without the flag, the script will build the non-test @@ -33,22 +40,35 @@ die() { } # Check arguments -if [[ $# != 1 ]] && [[ $# != 2 ]]; then - die "Usage: $0 [--test]" +if [[ $# -lt 2 ]]; then + die "Usage: $0 [--test]" fi DOCKER_IMG_NAME=$1 -shift +WHL_URL=$2 +shift 2 # Current script directory DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -DOCKER_FILE="${DIR}/server/Dockerfile" +BUILD_DIR=$(mktemp -d) +echo "" +echo "Using whl file URL: ${WHL_URL}" +echo "Building in temporary directory: ${BUILD_DIR}" + +cp -r ${DIR}/* "${BUILD_DIR}"/ || \ + die "Failed to copy files to ${BUILD_DIR}" + +DOCKER_FILE="${BUILD_DIR}/server/Dockerfile" if [[ $1 == "--test" ]]; then - DOCKER_FILE="${DIR}/server/Dockerfile.test" + DOCKER_FILE="${BUILD_DIR}/server/Dockerfile.test" fi echo "Using Docker file: ${DOCKER_FILE}" +# Download whl file into the build context directory. +wget -P "${BUILD_DIR}" ${WHL_URL} || \ + die "Failed to download tensorflow whl file from URL: ${WHL_URL}" + if [[ ! -f "${DOCKER_FILE}" ]]; then die "ERROR: Unable to find dockerfile: ${DOCKER_FILE}" fi @@ -56,5 +76,8 @@ echo "Dockerfile: ${DOCKER_FILE}" # Call docker build docker build --no-cache -t "${DOCKER_IMG_NAME}" \ - -f "${DOCKER_FILE}" \ - "${DIR}" + -f "${DOCKER_FILE}" "${BUILD_DIR}" || \ + die "Failed to build docker image: ${DOCKER_IMG_NAME}" + +# Clean up docker build context directory. +rm -rf "${BUILD_DIR}" diff --git a/tensorflow/tools/dist_test/server/Dockerfile b/tensorflow/tools/dist_test/server/Dockerfile index 9d59abef2f1..4b13b814e39 100644 --- a/tensorflow/tools/dist_test/server/Dockerfile +++ b/tensorflow/tools/dist_test/server/Dockerfile @@ -34,9 +34,10 @@ RUN curl -O https://bootstrap.pypa.io/get-pip.py && \ python get-pip.py && \ rm get-pip.py -# Install TensorFlow CPU version from nightly build -RUN pip --no-cache-dir install \ - https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.11.0rc2-cp27-none-linux_x86_64.whl +# Install TensorFlow wheel +COPY tensorflow-*.whl / +RUN pip install /tensorflow-*.whl && \ + rm -f /tensorflow-*.whl # Copy files, including the GRPC server binary at # server/grpc_tensorflow_server.py diff --git a/tensorflow/tools/dist_test/server/Dockerfile.test b/tensorflow/tools/dist_test/server/Dockerfile.test index 1c75e79512d..22438f39849 100644 --- a/tensorflow/tools/dist_test/server/Dockerfile.test +++ b/tensorflow/tools/dist_test/server/Dockerfile.test @@ -40,9 +40,10 @@ RUN curl -O https://bootstrap.pypa.io/get-pip.py && \ # Install python panda for the census wide&deep test RUN pip install --upgrade pandas==0.18.1 -# Install TensorFlow CPU version. -RUN pip --no-cache-dir install \ - https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.11.0rc2-cp27-none-linux_x86_64.whl +# Install TensorFlow wheel +COPY tensorflow-*.whl / +RUN pip install /tensorflow-*.whl && \ + rm -f /tensorflow-*.whl # Copy files, including the GRPC server binary at # server/grpc_tensorflow_server.py From d48f91d4dfbb964be2e7da6bf852db79f3cc0856 Mon Sep 17 00:00:00 2001 From: gunan Date: Wed, 26 Oct 2016 13:58:27 -0700 Subject: [PATCH 014/139] Fix: For mac non-docker GPU builds fix handling of test concurrency. (#5218) --- tensorflow/tools/ci_build/builds/test_installation.sh | 7 ++----- tensorflow/tools/ci_build/ci_build.sh | 6 ++---- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/tensorflow/tools/ci_build/builds/test_installation.sh b/tensorflow/tools/ci_build/builds/test_installation.sh index 014dc0cea6f..09b2ae51e5e 100755 --- a/tensorflow/tools/ci_build/builds/test_installation.sh +++ b/tensorflow/tools/ci_build/builds/test_installation.sh @@ -47,9 +47,6 @@ # TF_BUILD_BAZEL_CLEAN, if set to any non-empty and non-0 value, directs the # script to perform bazel clean prior to main build and test steps. # -# TF_BUILD_USE_GPU, if set to 1, limits the number of concurrent tests to -# the number stored in TF_GPU_COUNT and assigns each test to a different GPU. -# # TF_GPU_COUNT, Set the number of GPUs in the system. We run only this many # concurrent tests when running GPU tests. # @@ -410,7 +407,7 @@ SKIP_COUNTER=0 FAILED_TESTS="" FAILED_TEST_LOGS="" -if [[ "${TF_BUILD_USE_GPU}" == "1" ]]; then +if [[ "${IS_GPU}" == "1" ]]; then N_JOBS=$TF_GPU_COUNT else N_JOBS=$(grep -c ^processor /proc/cpuinfo) @@ -484,7 +481,7 @@ while true; do TEST_LOGS="${TEST_LOGS} ${TEST_LOG}" # Launch test asynchronously - if [[ "${TF_BUILD_USE_GPU}" == "1" ]]; then + if [[ "${IS_GPU}" == "1" ]]; then "${SCRIPT_DIR}/../gpu_build/parallel_gpu_execute.sh" \ "${SCRIPT_DIR}/py_test_delegate.sh" \ "${PYTHON_BIN_PATH}" "${PY_TEST_DIR}/${TEST_BASENAME}" "${TEST_LOG}" & diff --git a/tensorflow/tools/ci_build/ci_build.sh b/tensorflow/tools/ci_build/ci_build.sh index e9271cf9eb9..54587effc94 100755 --- a/tensorflow/tools/ci_build/ci_build.sh +++ b/tensorflow/tools/ci_build/ci_build.sh @@ -103,10 +103,8 @@ WORKSPACE="${WORKSPACE:-$(upsearch WORKSPACE)}" BUILD_TAG="${BUILD_TAG:-tf_ci}" # Add extra params for cuda devices and libraries for GPU container. -if [ "${CONTAINER_TYPE}" == "gpu" ]; then - # GPU pip tests-on-install concurrency is limited to the number of GPUs. - GPU_EXTRA_PARAMS="${GPU_EXTRA_PARAMS} -e TF_BUILD_USE_GPU=1" -else +# And clear them if we are not building for GPU. +if [ "${CONTAINER_TYPE}" != "gpu" ]; then GPU_EXTRA_PARAMS="" fi From 83aedd7e984aea80cd9397d383580f00b96c505d Mon Sep 17 00:00:00 2001 From: Yifei Feng Date: Fri, 28 Oct 2016 17:10:28 -0700 Subject: [PATCH 015/139] Revert "TF Checkpoint V2: add "write_version" arg to tf.train.Saver." (#5266) This reverts commit a1d0c8e59bec28643688852271859fb750fab1fc. --- tensorflow/python/BUILD | 1 - tensorflow/python/training/saver.py | 115 +++------------------------- 2 files changed, 12 insertions(+), 104 deletions(-) diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index 7441799e62a..34f583f7538 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -1537,7 +1537,6 @@ py_library( ":random_ops", ":sparse_ops", ":state_ops", - ":string_ops", ":training_ops_gen", ":variable_scope", ":variables", diff --git a/tensorflow/python/training/saver.py b/tensorflow/python/training/saver.py index 4572d507771..b1b40061468 100644 --- a/tensorflow/python/training/saver.py +++ b/tensorflow/python/training/saver.py @@ -48,7 +48,6 @@ from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import gen_io_ops from tensorflow.python.ops import io_ops from tensorflow.python.ops import state_ops -from tensorflow.python.ops import string_ops from tensorflow.python.ops import variables from tensorflow.python.platform import gfile from tensorflow.python.platform import tf_logging as logging @@ -141,6 +140,8 @@ class BaseSaverBuilder(object): Can be extended to create different Ops. """ + _CHECKPOINT_FORMAT_VERSION = saver_pb2.SaverDef.V1 + class SaveSpec(object): """Class used to describe tensor slices that need to be saved.""" @@ -210,8 +211,8 @@ class BaseSaverBuilder(object): validate_shape=restored_shapes is None and self.op.get_shape().is_fully_defined()) - def __init__(self, write_version=saver_pb2.SaverDef.V1): - self._write_version = write_version + def __init__(self): + pass def save_op(self, filename_tensor, saveables): """Create an Op to save 'saveables'. @@ -225,10 +226,6 @@ class BaseSaverBuilder(object): Returns: An Operation that save the variables. - - Raises: - RuntimeError: (implementation detail) if "self._write_version" is an - unexpected value. """ # pylint: disable=protected-access tensor_names = [] @@ -240,19 +237,11 @@ class BaseSaverBuilder(object): tensors.append(spec.tensor) tensor_slices.append(spec.slice_spec) - if self._write_version == saver_pb2.SaverDef.V1: - return io_ops._save( - filename=filename_tensor, - tensor_names=tensor_names, - tensors=tensors, - tensor_slices=tensor_slices) - elif self._write_version == saver_pb2.SaverDef.V2: - # "filename_tensor" is interpreted *NOT AS A FILENAME*, but as a prefix - # of a V2 checkpoint: e.g. "/fs/train/ckpt-/tmp/worker-". - return io_ops.save_v2(filename_tensor, tensor_names, tensor_slices, - tensors) - else: - raise RuntimeError("Unexpected write_version: " + self._write_version) + return io_ops._save( + filename=filename_tensor, + tensor_names=tensor_names, + tensors=tensors, + tensor_slices=tensor_slices) def restore_op(self, filename_tensor, saveable, preferred_shard): """Create ops to restore 'saveable'. @@ -309,76 +298,6 @@ class BaseSaverBuilder(object): save = self.save_op(filename_tensor, saveables) return control_flow_ops.with_dependencies([save], filename_tensor) - def _AddShardedSaveOpsForV2(self, checkpoint_prefix, per_device): - """Add ops to save the params per shard, for the V2 format. - - Note that the sharded save procedure for the V2 format is different from - V1: there is a special "merge" step that merges the small metadata produced - from each device. - - Args: - checkpoint_prefix: scalar String Tensor. Interpreted *NOT AS A - FILENAME*, but as a prefix of a V2 checkpoint; - per_device: A list of (device, BaseSaverBuilder.VarToSave) pairs, as - returned by _GroupByDevices(). - - Returns: - An op to save the variables, which, when evaluated, returns the prefix - "" only and does not include the sharded spec suffix. - """ - # IMPLEMENTATION DETAILS: most clients should skip. - # - # Suffix for any well-formed "checkpoint_prefix", when sharded. - # Transformations: - # * Users pass in "save_path" in save() and restore(). Say "myckpt". - # * checkpoint_prefix gets fed <_SHARDED_SUFFIX>. - # - # Example: - # During runtime, a temporary directory is first created, which contains - # files - # - # /myckpt_temp/ - # part-?????-of-?????{.index, .data-00000-of-00001} - # - # Before .save() finishes, they will be (hopefully, atomically) renamed to - # - # / - # myckpt{.index, .data-?????-of-?????} - # - # Users only need to interact with the user-specified prefix, which is - # "/myckpt" in this case. Save() and Restore() work with the - # prefix directly, instead of any physical pathname. (On failure and - # subsequent restore, an outdated and orphaned temporary directory can be - # safely removed.) - _SHARDED_SUFFIX = "_temp_%s/part" % uuid.uuid4().hex - tmp_checkpoint_prefix = string_ops.string_join( - [checkpoint_prefix, _SHARDED_SUFFIX]) - - num_shards = len(per_device) - sharded_saves = [] - sharded_prefixes = [] - num_shards_tensor = constant_op.constant(num_shards, name="num_shards") - last_device = None - for shard, (device, saveables) in enumerate(per_device): - last_device = device - with ops.device(device): - sharded_filename = self.sharded_filename(tmp_checkpoint_prefix, shard, - num_shards_tensor) - sharded_prefixes.append(sharded_filename) - sharded_saves.append(self._AddSaveOps(sharded_filename, saveables)) - - with ops.control_dependencies([x.op for x in sharded_saves]): - # Co-locates the merge step with the last device. - with ops.device(last_device): - # V2 format write path consists of a metadata merge step. Once merged, - # attempts to delete the temporary directory, "_temp". - merge_step = gen_io_ops.merge_v2_checkpoints( - sharded_prefixes, checkpoint_prefix, delete_old_dirs=True) - with ops.control_dependencies([merge_step]): - # Returns the prefix "" only. DOES NOT include the - # sharded spec suffix. - return array_ops.identity(checkpoint_prefix) - def _AddShardedSaveOps(self, filename_tensor, per_device): """Add ops to save the params per shard. @@ -390,9 +309,6 @@ class BaseSaverBuilder(object): Returns: An op to save the variables. """ - if self._write_version == saver_pb2.SaverDef.V2: - return self._AddShardedSaveOpsForV2(filename_tensor, per_device) - num_shards = len(per_device) sharded_saves = [] num_shards_tensor = constant_op.constant(num_shards, name="num_shards") @@ -722,7 +638,7 @@ class BaseSaverBuilder(object): max_to_keep=max_to_keep, sharded=sharded, keep_checkpoint_every_n_hours=keep_checkpoint_every_n_hours, - version=self._write_version) + version=self._CHECKPOINT_FORMAT_VERSION) def _GetCheckpointFilename(save_dir, latest_filename): @@ -980,8 +896,7 @@ class Saver(object): saver_def=None, builder=None, defer_build=False, - allow_empty=False, - write_version=saver_pb2.SaverDef.V1): + allow_empty=False): """Creates a `Saver`. The constructor adds ops to save and restore variables. @@ -1046,11 +961,6 @@ class Saver(object): allow_empty: If `False` (default) raise an error if there are no variables in the graph. Otherwise, construct the saver anyway and make it a no-op. - write_version: controls what format to use when saving checkpoints. It - also affects certain filepath matching logic. Defaults to V1 - currently, and will be switched to the more memory-efficient V2 format - in the future. If set to V2, the Saver is still able to restore from - old V1 checkpoints. Raises: TypeError: If `var_list` is invalid. @@ -1072,7 +982,6 @@ class Saver(object): self._is_built = False self._allow_empty = allow_empty self._is_empty = None - self._write_version = write_version if not defer_build: self.build() if self.saver_def: @@ -1085,7 +994,7 @@ class Saver(object): self._is_built = True if not self.saver_def: if self._builder is None: - self._builder = BaseSaverBuilder(self._write_version) + self._builder = BaseSaverBuilder() if self._var_list is None: # pylint: disable=protected-access self._var_list = variables._all_saveable_objects() From b5943bd1bcaa1510ab035a82f20adb6f7b2f7b05 Mon Sep 17 00:00:00 2001 From: Yifei Feng Date: Mon, 31 Oct 2016 10:55:37 -0700 Subject: [PATCH 016/139] Small header changes and a Doxygen tweak to shape the C++ API a bit better. (#5304) Change: 135747447 (cherry picked from commit 120c3f11f857ce6da7ae90b3eab943896499dca2) --- tensorflow/core/framework/tensor.h | 4 ++++ tensorflow/core/framework/tensor_shape.h | 14 ++++++++++++++ tensorflow/core/lib/core/status.h | 1 + tensorflow/core/platform/env.h | 5 +++++ tensorflow/core/platform/file_system.h | 4 ++++ 5 files changed, 28 insertions(+) diff --git a/tensorflow/core/framework/tensor.h b/tensorflow/core/framework/tensor.h index 5d75fc0fecc..82594b22fa9 100644 --- a/tensorflow/core/framework/tensor.h +++ b/tensorflow/core/framework/tensor.h @@ -466,6 +466,8 @@ class Tensor { // Implementation details +// START_SKIP_DOXYGEN + // Interface to access the raw ref-counted data buffer. class TensorBuffer : public core::RefCounted { public: @@ -659,6 +661,8 @@ inline Tensor& Tensor::operator=(Tensor&& other) { return *this; } +// END_SKIP_DOXYGEN + } // namespace tensorflow #endif // TENSORFLOW_CORE_FRAMEWORK_TENSOR_H_ diff --git a/tensorflow/core/framework/tensor_shape.h b/tensorflow/core/framework/tensor_shape.h index 19e73b9087b..19a36afc7f9 100644 --- a/tensorflow/core/framework/tensor_shape.h +++ b/tensorflow/core/framework/tensor_shape.h @@ -31,8 +31,19 @@ limitations under the License. namespace tensorflow { +// START_SKIP_DOXYGEN class TensorShapeIter; // Declared below +// END_SKIP_DOXYGEN +/// Represents the shape of a Tensor. +/// +/// A tensor's shape is denoted by its number of dimensions and a size for each +/// dimension. For example, a Tensor represented by a 3 x 4 matrix would have +/// a shape of 2-D, [3,4]. +/// +/// If you know the exact shape of your Tensor when you create the TensorShape +/// object, you can specify it then, or you can create a TensorShape with +/// zero dimensions and one element, and call AddDim() to add dimensions later. class TensorShape { public: /// \brief Construct a `TensorShape` from the provided sizes. @@ -220,11 +231,13 @@ class TensorShape { int64 num_elements_; }; +/// Represents the value of one dimension in a TensorShape. struct TensorShapeDim { explicit TensorShapeDim(int64 s) : size(s) {} int64 size; }; +// START_SKIP_DOXYGEN class TensorShapeIter { public: TensorShapeIter(const TensorShape* shape, int d) : shape_(shape), d_(d) {} @@ -243,6 +256,7 @@ class TensorShapeIter { const TensorShape* shape_; int d_; }; +// END_SKIP_DOXYGEN /// \brief Static helper routines for `TensorShape`. Includes a few common /// predicates on a tensor shape. diff --git a/tensorflow/core/lib/core/status.h b/tensorflow/core/lib/core/status.h index 3a3382dc84b..3b38f89d4f2 100644 --- a/tensorflow/core/lib/core/status.h +++ b/tensorflow/core/lib/core/status.h @@ -25,6 +25,7 @@ limitations under the License. namespace tensorflow { +/// Denotes success or failure of a call in Tensorflow. class Status { public: /// Create a success status. diff --git a/tensorflow/core/platform/env.h b/tensorflow/core/platform/env.h index 797e1eae7ce..4419f1ad9ce 100644 --- a/tensorflow/core/platform/env.h +++ b/tensorflow/core/platform/env.h @@ -343,6 +343,7 @@ class EnvWrapper : public Env { Env* target_; }; +/// Represents a thread used to run a Tensorflow function. class Thread { public: Thread() {} @@ -382,6 +383,8 @@ Status WriteBinaryProto(Env* env, const string& fname, Status ReadBinaryProto(Env* env, const string& fname, ::tensorflow::protobuf::MessageLite* proto); +// START_SKIP_DOXYGEN + namespace register_file_system { template @@ -394,6 +397,8 @@ struct Register { } // namespace register_file_system +// END_SKIP_DOXYGEN + } // namespace tensorflow // Register a FileSystem implementation for a scheme. Files with names that have diff --git a/tensorflow/core/platform/file_system.h b/tensorflow/core/platform/file_system.h index 3a30322191f..18b603d6362 100644 --- a/tensorflow/core/platform/file_system.h +++ b/tensorflow/core/platform/file_system.h @@ -98,6 +98,8 @@ class FileSystem { virtual Status IsDirectory(const string& fname); }; +// START_SKIP_DOXYGEN + #ifndef SWIG // Degenerate file system that provides no implementations. class NullFileSystem : public FileSystem { @@ -160,6 +162,8 @@ class NullFileSystem : public FileSystem { }; #endif +// END_SKIP_DOXYGEN + /// A file abstraction for randomly reading the contents of a file. class RandomAccessFile { public: From 5de9d3c392d5531eb3bbcefd007fcc25db7448cd Mon Sep 17 00:00:00 2001 From: Benoit Steiner Date: Mon, 31 Oct 2016 16:18:31 -0800 Subject: [PATCH 017/139] Added the ability to wait for queues to start running before returning from QueueRunner::Start(). This provides a reliable way to check the value of the status_ variable. Change: 137769682 --- tensorflow/cc/training/queue_runner.cc | 32 ++++++++++++++++++++- tensorflow/cc/training/queue_runner.h | 9 +++++- tensorflow/cc/training/queue_runner_test.cc | 17 +++++++++++ tensorflow/core/lib/core/blocking_counter.h | 16 ++++++++++- 4 files changed, 71 insertions(+), 3 deletions(-) diff --git a/tensorflow/cc/training/queue_runner.cc b/tensorflow/cc/training/queue_runner.cc index ed1d0a5da0c..6435e88d485 100644 --- a/tensorflow/cc/training/queue_runner.cc +++ b/tensorflow/cc/training/queue_runner.cc @@ -48,6 +48,7 @@ Status QueueRunner::Init(const QueueRunnerDef& queue_runner_def) { thread_pool_.reset(new thread::ThreadPool( Env::Default(), SanitizeThreadSuffix(queue_name_), runs_)); should_stop_ = false; + return Status::OK(); } @@ -57,11 +58,29 @@ QueueRunner::~QueueRunner() { Join(); } -Status QueueRunner::Start(Session* sess) { +Status QueueRunner::Start(Session* sess) { return Start(sess, 0); } + +Status QueueRunner::Start(Session* sess, int wait_for) { + counter_.reset(new BlockingCounter(runs_)); for (const string& enqueue_op : enqueue_op_names_) { thread_pool_->Schedule( std::bind(&QueueRunner::Run, this, sess, enqueue_op)); } + // Wait for up to 'wait_for' milliseconds. + if (wait_for > 0) { + if (!counter_->WaitFor(std::chrono::milliseconds(wait_for))) { + return Status(error::DEADLINE_EXCEEDED, + "Queues not fed before the timeout"); + } + // Check the status of the queue runner as well as the result of the enqueue + // operations. + mutex_lock l(mu_); + if (!enqueue_status_.ok()) { + return enqueue_status_; + } else { + return status_; + } + } return Status::OK(); } @@ -76,13 +95,23 @@ Status QueueRunner::Stop(Session* sess) { Status QueueRunner::Join() { thread_pool_.reset(); + mutex_lock l(mu_); return status_; } void QueueRunner::Run(Session* sess, const string& enqueue_op) { bool decremented = false; + bool first_iteration = true; while (!should_stop_.load()) { auto status = sess->Run({}, {}, {enqueue_op}, nullptr); + if (first_iteration) { + if (!status.ok()) { + mutex_lock l(mu_); + enqueue_status_ = status; + } + counter_->DecrementCount(); + first_iteration = false; + } if (status.ok()) { continue; } else if (queue_closed_exception_types_.count( @@ -114,6 +143,7 @@ void QueueRunner::Run(Session* sess, const string& enqueue_op) { // subsequent queues. Stop(sess); } + first_iteration = false; } if (!decremented) { diff --git a/tensorflow/cc/training/queue_runner.h b/tensorflow/cc/training/queue_runner.h index 9374fe36050..3affe45e551 100644 --- a/tensorflow/cc/training/queue_runner.h +++ b/tensorflow/cc/training/queue_runner.h @@ -21,6 +21,7 @@ limitations under the License. #include #include +#include "tensorflow/core/lib/core/blocking_counter.h" #include "tensorflow/core/lib/core/error_codes.pb.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/core/threadpool.h" @@ -46,6 +47,10 @@ class QueueRunner { // Starts the queue runner with the given session. Status Start(Session* sess); + // Starts the queue runner with the given session, and wait for up to the + // specified time (in milliseconds) for the queues to start to fill up. + Status Start(Session* sess, int wait_for); + // Requests to stop and runs the cancel op. Status Stop(Session* sess); @@ -78,7 +83,9 @@ class QueueRunner { mutex mu_; // TODO(yuefengz): implement c++ coordinator. int runs_ = 0; - Status status_; + Status status_ GUARDED_BY(mu_); + Status enqueue_status_ GUARDED_BY(mu_); + std::unique_ptr counter_; }; } // namespace tensorflow diff --git a/tensorflow/cc/training/queue_runner_test.cc b/tensorflow/cc/training/queue_runner_test.cc index 0d06c620566..df48b255430 100644 --- a/tensorflow/cc/training/queue_runner_test.cc +++ b/tensorflow/cc/training/queue_runner_test.cc @@ -317,5 +317,22 @@ TEST(QueueRunnerTest, EmptyEnqueueOps) { Code::INVALID_ARGUMENT); } +TEST(QueueRunnerTest, StartTimeout) { + GraphDef graph_def = BuildDoubleQueueGraph(); + SessionOptions options; + std::unique_ptr session(NewSession(options)); + TF_CHECK_OK(session->Create(graph_def)); + + QueueRunnerDef queue_runner_def = BuildQueueRunnerDef( + kQueueName1, {kEnqueueOp1}, kCloseOp1, kCancelOp1, {}); + + std::unique_ptr qr; + TF_EXPECT_OK(QueueRunner::New(queue_runner_def, &qr)); + // This will timeout since queue0 is not fed and queue1 is fetching data from + // queue0. + EXPECT_EQ(qr->Start(session.get(), 1).code(), Code::DEADLINE_EXCEEDED); + session->Close(); +} + } // namespace } // namespace tensorflow diff --git a/tensorflow/core/lib/core/blocking_counter.h b/tensorflow/core/lib/core/blocking_counter.h index b2411f5951f..5dab07dbef9 100644 --- a/tensorflow/core/lib/core/blocking_counter.h +++ b/tensorflow/core/lib/core/blocking_counter.h @@ -31,7 +31,7 @@ class BlockingCounter { DCHECK_EQ((initial_count << 1) >> 1, initial_count); } - ~BlockingCounter() { DCHECK_EQ(state_ >> 1, 0); } + ~BlockingCounter() {} inline void DecrementCount() { unsigned int v = state_.fetch_sub(2, std::memory_order_acq_rel) - 2; @@ -53,6 +53,20 @@ class BlockingCounter { cond_var_.wait(l); } } + // Wait for the specified time, return false iff the count has not dropped to + // zero before the timeout expired. + inline bool WaitFor(std::chrono::milliseconds ms) { + unsigned int v = state_.fetch_or(1, std::memory_order_acq_rel); + if ((v >> 1) == 0) return true; + mutex_lock l(mu_); + while (!notified_) { + const std::cv_status status = cond_var_.wait_for(l, ms); + if (status == std::cv_status::timeout) { + return false; + } + } + return true; + } private: mutex mu_; From a219b1dcfbd9eb55ee9176d123a82782b5191274 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 31 Oct 2016 16:38:30 -0800 Subject: [PATCH 018/139] Update generated Python Op docs. Change: 137771513 --- .../python/functions_and_classes/shard5/tf.train.Saver.md | 4 +++- .../shard7/tf.train.export_meta_graph.md | 4 +++- tensorflow/g3doc/api_docs/python/state_ops.md | 8 ++++++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard5/tf.train.Saver.md b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard5/tf.train.Saver.md index 8ce093840f0..cacfed94d2e 100644 --- a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard5/tf.train.Saver.md +++ b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard5/tf.train.Saver.md @@ -305,7 +305,7 @@ Builds saver_def. - - - -#### `tf.train.Saver.export_meta_graph(filename=None, collection_list=None, as_text=False, export_scope=None)` {#Saver.export_meta_graph} +#### `tf.train.Saver.export_meta_graph(filename=None, collection_list=None, as_text=False, export_scope=None, clear_devices=False)` {#Saver.export_meta_graph} Writes `MetaGraphDef` to save_path/filename. @@ -316,6 +316,8 @@ Writes `MetaGraphDef` to save_path/filename. * `collection_list`: List of string keys to collect. * `as_text`: If `True`, writes the meta_graph as an ASCII proto. * `export_scope`: Optional `string`. Name scope to remove. +* `clear_devices`: Whether or not to clear the device field for an `Operation` + or `Tensor` during export. ##### Returns: diff --git a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard7/tf.train.export_meta_graph.md b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard7/tf.train.export_meta_graph.md index fcd666bac0a..dd318197593 100644 --- a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard7/tf.train.export_meta_graph.md +++ b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard7/tf.train.export_meta_graph.md @@ -1,4 +1,4 @@ -### `tf.train.export_meta_graph(filename=None, meta_info_def=None, graph_def=None, saver_def=None, collection_list=None, as_text=False, graph=None, export_scope=None, **kwargs)` {#export_meta_graph} +### `tf.train.export_meta_graph(filename=None, meta_info_def=None, graph_def=None, saver_def=None, collection_list=None, as_text=False, graph=None, export_scope=None, clear_devices=False, **kwargs)` {#export_meta_graph} Returns `MetaGraphDef` proto. Optionally writes it to filename. @@ -22,6 +22,8 @@ a subgraph. the subgraph. The scope name will be striped from the node definitions for easy import later into new name scopes. If `None`, the whole graph is exported. graph_def and export_scope cannot both be specified. +* `clear_devices`: Whether or not to clear the device field for an `Operation` + or `Tensor` during export. * `**kwargs`: Optional keyed arguments. ##### Returns: diff --git a/tensorflow/g3doc/api_docs/python/state_ops.md b/tensorflow/g3doc/api_docs/python/state_ops.md index 0b5e396aa2d..2d01e520fec 100644 --- a/tensorflow/g3doc/api_docs/python/state_ops.md +++ b/tensorflow/g3doc/api_docs/python/state_ops.md @@ -1765,7 +1765,7 @@ Builds saver_def. - - - -#### `tf.train.Saver.export_meta_graph(filename=None, collection_list=None, as_text=False, export_scope=None)` {#Saver.export_meta_graph} +#### `tf.train.Saver.export_meta_graph(filename=None, collection_list=None, as_text=False, export_scope=None, clear_devices=False)` {#Saver.export_meta_graph} Writes `MetaGraphDef` to save_path/filename. @@ -1776,6 +1776,8 @@ Writes `MetaGraphDef` to save_path/filename. * `collection_list`: List of string keys to collect. * `as_text`: If `True`, writes the meta_graph as an ASCII proto. * `export_scope`: Optional `string`. Name scope to remove. +* `clear_devices`: Whether or not to clear the device field for an `Operation` + or `Tensor` during export. ##### Returns: @@ -3120,7 +3122,7 @@ Returns an Op that initializes all tables of the default graph. - - - -### `tf.train.export_meta_graph(filename=None, meta_info_def=None, graph_def=None, saver_def=None, collection_list=None, as_text=False, graph=None, export_scope=None, **kwargs)` {#export_meta_graph} +### `tf.train.export_meta_graph(filename=None, meta_info_def=None, graph_def=None, saver_def=None, collection_list=None, as_text=False, graph=None, export_scope=None, clear_devices=False, **kwargs)` {#export_meta_graph} Returns `MetaGraphDef` proto. Optionally writes it to filename. @@ -3144,6 +3146,8 @@ a subgraph. the subgraph. The scope name will be striped from the node definitions for easy import later into new name scopes. If `None`, the whole graph is exported. graph_def and export_scope cannot both be specified. +* `clear_devices`: Whether or not to clear the device field for an `Operation` + or `Tensor` during export. * `**kwargs`: Optional keyed arguments. ##### Returns: From f6d70256d598ff54f765b97b8c968fdd11e754bc Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 31 Oct 2016 16:40:40 -0800 Subject: [PATCH 019/139] Automated rollback of change 137715407 Change: 137771699 --- tensorflow/python/training/saver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/training/saver.py b/tensorflow/python/training/saver.py index 3a5c2ebb114..34893dbb9ff 100644 --- a/tensorflow/python/training/saver.py +++ b/tensorflow/python/training/saver.py @@ -899,7 +899,7 @@ class Saver(object): builder=None, defer_build=False, allow_empty=False, - write_version=saver_pb2.SaverDef.V2, + write_version=saver_pb2.SaverDef.V1, pad_step_number=False): """Creates a `Saver`. From df8e306db70bab53d7dab98dc9adbf54907e49f1 Mon Sep 17 00:00:00 2001 From: Sukriti Ramesh Date: Mon, 31 Oct 2016 16:50:48 -0800 Subject: [PATCH 020/139] Add support for clear devices in SavedModel. Change: 137772478 --- tensorflow/python/saved_model/builder.py | 22 +++++++++++++++--- .../python/saved_model/saved_model_test.py | 23 +++++++++++++++++++ 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/tensorflow/python/saved_model/builder.py b/tensorflow/python/saved_model/builder.py index 43b97cf70c6..24126c01935 100644 --- a/tensorflow/python/saved_model/builder.py +++ b/tensorflow/python/saved_model/builder.py @@ -249,11 +249,18 @@ class SavedModelBuilder(object): proto_meta_graph_def = self._saved_model.meta_graphs.add() proto_meta_graph_def.CopyFrom(meta_graph_def) + def _maybe_clear_devices(self, clear_devices): + if not clear_devices: + return + for node in ops.get_default_graph().as_graph_def().node: + node.device = "" + def add_meta_graph(self, tags, signature_def_map=None, assets_collection=None, - legacy_init_op=None): + legacy_init_op=None, + clear_devices=False): """Adds the current meta graph to the SavedModel. Creates a Saver in the current scope and uses the Saver to export the meta @@ -268,7 +275,9 @@ class SavedModelBuilder(object): that this collection should be a subset of the assets saved as part of the first meta graph in the SavedModel. legacy_init_op: Op or group of ops to execute after the restore op upon a - load. + load. + clear_devices: Set to true if the device info on the default graph should + be cleared. Raises: AssertionError: If the variables for the SavedModel have not been saved @@ -279,6 +288,8 @@ class SavedModelBuilder(object): "Variables and assets have not been saved yet. " "Please invoke `add_meta_graph_and_variables()` first.") + self._maybe_clear_devices(clear_devices) + # Save asset files and write them to disk, if any. self._save_and_write_assets(assets_collection) @@ -300,7 +311,8 @@ class SavedModelBuilder(object): tags, signature_def_map=None, assets_collection=None, - legacy_init_op=None): + legacy_init_op=None, + clear_devices=False): """Adds the current meta graph to the SavedModel and saves variables. Creates a Saver to save the variables from the provided session. Exports the @@ -318,11 +330,15 @@ class SavedModelBuilder(object): assets_collection: Assets collection to be saved with SavedModel. legacy_init_op: Op or group of ops to execute after the restore op upon a load. + clear_devices: Set to true if the device info on the default graph should + be cleared. """ if self._has_saved_variables: raise AssertionError("Variables and assets have already been saved. " "Please invoke `add_meta_graph()` instead.") + self._maybe_clear_devices(clear_devices) + # Save asset files and write them to disk, if any. self._save_and_write_assets(assets_collection) diff --git a/tensorflow/python/saved_model/saved_model_test.py b/tensorflow/python/saved_model/saved_model_test.py index a50620e113c..6f2132b4924 100644 --- a/tensorflow/python/saved_model/saved_model_test.py +++ b/tensorflow/python/saved_model/saved_model_test.py @@ -553,6 +553,29 @@ class SavedModelTest(tf.test.TestCase): tf.get_collection("init_op")[0].run() self.assertEqual(3, tf.get_collection("v")[2].eval()) + def testClearDevices(self): + export_dir = os.path.join(tf.test.get_temp_dir(), "test_clear_devices") + builder = saved_model_builder.SavedModelBuilder(export_dir) + + # Specify a device and save a variable. + tf.reset_default_graph() + with tf.Session( + target="", + config=config_pb2.ConfigProto(device_count={"CPU": 2})) as sess: + with sess.graph.device("/cpu:0"): + self._init_and_validate_variable(sess, "v", 42) + builder.add_meta_graph_and_variables( + sess, [tag_constants.TRAINING], clear_devices=True) + + # Save the SavedModel to disk. + builder.save() + + # Restore the graph with a single predefined tag whose variables were saved + # without any device information. + with self.test_session(graph=tf.Graph()) as sess: + loader.load(sess, [tag_constants.TRAINING], export_dir) + self.assertEqual(42, tf.get_collection(tf.GraphKeys.VARIABLES)[0].eval()) + if __name__ == "__main__": tf.test.main() From a74e4e72e42240c23fba3a9aae7015cc0c89ab6e Mon Sep 17 00:00:00 2001 From: Alexey Surkov Date: Mon, 31 Oct 2016 17:13:15 -0800 Subject: [PATCH 021/139] Also retry failed Close() in the WritableFile's destructor. Change: 137774256 --- .../platform/cloud/retrying_file_system.cc | 5 +++ .../cloud/retrying_file_system_test.cc | 37 +++++++++++++++++-- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/tensorflow/core/platform/cloud/retrying_file_system.cc b/tensorflow/core/platform/cloud/retrying_file_system.cc index f8a897a484d..1ff756106b6 100644 --- a/tensorflow/core/platform/cloud/retrying_file_system.cc +++ b/tensorflow/core/platform/cloud/retrying_file_system.cc @@ -88,6 +88,11 @@ class RetryingWritableFile : public WritableFile { : base_file_(std::move(base_file)), initial_delay_microseconds_(delay_microseconds) {} + ~RetryingWritableFile() { + // Makes sure the retrying version of Close() is called in the destructor. + Close(); + } + Status Append(const StringPiece& data) override { return CallWithRetries( std::bind(&WritableFile::Append, base_file_.get(), data), diff --git a/tensorflow/core/platform/cloud/retrying_file_system_test.cc b/tensorflow/core/platform/cloud/retrying_file_system_test.cc index f939776b757..06c16b90925 100644 --- a/tensorflow/core/platform/cloud/retrying_file_system_test.cc +++ b/tensorflow/core/platform/cloud/retrying_file_system_test.cc @@ -261,7 +261,8 @@ TEST(RetryingFileSystemTest, NewRandomAccessFile_NoRetriesForSomeErrors) { TEST(RetryingFileSystemTest, NewWritableFile_ImmediateSuccess) { // Configure the mock base random access file. - ExpectedCalls expected_file_calls({std::make_tuple("Sync", Status::OK())}); + ExpectedCalls expected_file_calls({std::make_tuple("Sync", Status::OK()), + std::make_tuple("Close", Status::OK())}); std::unique_ptr base_file( new MockWritableFile(expected_file_calls)); @@ -286,7 +287,8 @@ TEST(RetryingFileSystemTest, NewWritableFile_SuccessWith3rdTry) { ExpectedCalls expected_file_calls( {std::make_tuple("Sync", errors::Unavailable("Something is wrong")), std::make_tuple("Sync", errors::Unavailable("Something is wrong again")), - std::make_tuple("Sync", Status::OK())}); + std::make_tuple("Sync", Status::OK()), + std::make_tuple("Close", Status::OK())}); std::unique_ptr base_file( new MockWritableFile(expected_file_calls)); @@ -306,12 +308,38 @@ TEST(RetryingFileSystemTest, NewWritableFile_SuccessWith3rdTry) { TF_EXPECT_OK(writable_file->Sync()); } +TEST(RetryingFileSystemTest, NewWritableFile_SuccessWith3rdTry_ViaDestructor) { + // Configure the mock base random access file. + ExpectedCalls expected_file_calls( + {std::make_tuple("Close", errors::Unavailable("Something is wrong")), + std::make_tuple("Close", + errors::Unavailable("Something is wrong again")), + std::make_tuple("Close", Status::OK())}); + std::unique_ptr base_file( + new MockWritableFile(expected_file_calls)); + + // Configure the mock base file system. + ExpectedCalls expected_fs_calls( + {std::make_tuple("NewWritableFile", Status::OK())}); + std::unique_ptr base_fs( + new MockFileSystem(expected_fs_calls)); + base_fs->writable_file_to_return = std::move(base_file); + RetryingFileSystem fs(std::move(base_fs), 0); + + // Retrieve the wrapped writable file. + std::unique_ptr writable_file; + TF_EXPECT_OK(fs.NewWritableFile("filename.txt", &writable_file)); + + writable_file.reset(); // Trigger Close() via destructor. +} + TEST(RetryingFileSystemTest, NewAppendableFile_SuccessWith3rdTry) { // Configure the mock base random access file. ExpectedCalls expected_file_calls( {std::make_tuple("Sync", errors::Unavailable("Something is wrong")), std::make_tuple("Sync", errors::Unavailable("Something is wrong again")), - std::make_tuple("Sync", Status::OK())}); + std::make_tuple("Sync", Status::OK()), + std::make_tuple("Close", Status::OK())}); std::unique_ptr base_file( new MockWritableFile(expected_file_calls)); @@ -337,7 +365,8 @@ TEST(RetryingFileSystemTest, NewWritableFile_AllRetriesFailed) { {std::make_tuple("Sync", errors::Unavailable("Something is wrong")), std::make_tuple("Sync", errors::Unavailable("Something is wrong again")), std::make_tuple("Sync", errors::Unavailable("...and again")), - std::make_tuple("Sync", errors::Unavailable("And again"))}); + std::make_tuple("Sync", errors::Unavailable("And again")), + std::make_tuple("Close", Status::OK())}); std::unique_ptr base_file( new MockWritableFile(expected_file_calls)); From 8df7f120492b0b4a0af3f4e1c6a0ffe66ed63e37 Mon Sep 17 00:00:00 2001 From: Vijay Vasudevan Date: Mon, 31 Oct 2016 17:14:04 -0800 Subject: [PATCH 022/139] Make the DimensionHandle version of computing windowed output size public. Change: 137774322 --- tensorflow/core/framework/common_shape_fns.cc | 3 --- tensorflow/core/framework/common_shape_fns.h | 7 +++++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tensorflow/core/framework/common_shape_fns.cc b/tensorflow/core/framework/common_shape_fns.cc index f3a71a73f7e..cc6cd10a084 100644 --- a/tensorflow/core/framework/common_shape_fns.cc +++ b/tensorflow/core/framework/common_shape_fns.cc @@ -70,8 +70,6 @@ Status Get3dOutputSize(const std::array& input, namespace shape_inference { -namespace { - Status GetWindowedOutputSizeFromDims( shape_inference::InferenceContext* c, shape_inference::DimensionHandle input_size, @@ -97,7 +95,6 @@ Status GetWindowedOutputSizeFromDims( } return Status::OK(); } -} // namespace Status UnchangedShape(shape_inference::InferenceContext* c) { c->set_output(0, c->input(0)); diff --git a/tensorflow/core/framework/common_shape_fns.h b/tensorflow/core/framework/common_shape_fns.h index 2e711f5277b..176ea3519d8 100644 --- a/tensorflow/core/framework/common_shape_fns.h +++ b/tensorflow/core/framework/common_shape_fns.h @@ -96,6 +96,13 @@ Status Get3dOutputSize(const std::array& input, namespace shape_inference { +// Like GetWindowedOutputSize, but deals with DimensionHandles. +Status GetWindowedOutputSizeFromDims(InferenceContext* c, + DimensionHandle input_size, + DimensionOrConstant filter_size, + int64 stride, Padding padding_type, + DimensionHandle* output_size); + // Transfers shape of input(0) to output(0). Status UnchangedShape(shape_inference::InferenceContext* c); From b6ed997284f69e11224af9c158b5bc5a0ecfd550 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 31 Oct 2016 17:23:30 -0800 Subject: [PATCH 023/139] Update generated Python Op docs. Change: 137775024 --- .../python/functions_and_classes/shard5/tf.train.Saver.md | 2 +- tensorflow/g3doc/api_docs/python/state_ops.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard5/tf.train.Saver.md b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard5/tf.train.Saver.md index cacfed94d2e..e5be71361a5 100644 --- a/tensorflow/g3doc/api_docs/python/functions_and_classes/shard5/tf.train.Saver.md +++ b/tensorflow/g3doc/api_docs/python/functions_and_classes/shard5/tf.train.Saver.md @@ -72,7 +72,7 @@ protocol buffer file in the call to `save()`. - - - -#### `tf.train.Saver.__init__(var_list=None, reshape=False, sharded=False, max_to_keep=5, keep_checkpoint_every_n_hours=10000.0, name=None, restore_sequentially=False, saver_def=None, builder=None, defer_build=False, allow_empty=False, write_version=2, pad_step_number=False)` {#Saver.__init__} +#### `tf.train.Saver.__init__(var_list=None, reshape=False, sharded=False, max_to_keep=5, keep_checkpoint_every_n_hours=10000.0, name=None, restore_sequentially=False, saver_def=None, builder=None, defer_build=False, allow_empty=False, write_version=1, pad_step_number=False)` {#Saver.__init__} Creates a `Saver`. diff --git a/tensorflow/g3doc/api_docs/python/state_ops.md b/tensorflow/g3doc/api_docs/python/state_ops.md index 2d01e520fec..e0b6ad57671 100644 --- a/tensorflow/g3doc/api_docs/python/state_ops.md +++ b/tensorflow/g3doc/api_docs/python/state_ops.md @@ -1532,7 +1532,7 @@ protocol buffer file in the call to `save()`. - - - -#### `tf.train.Saver.__init__(var_list=None, reshape=False, sharded=False, max_to_keep=5, keep_checkpoint_every_n_hours=10000.0, name=None, restore_sequentially=False, saver_def=None, builder=None, defer_build=False, allow_empty=False, write_version=2, pad_step_number=False)` {#Saver.__init__} +#### `tf.train.Saver.__init__(var_list=None, reshape=False, sharded=False, max_to_keep=5, keep_checkpoint_every_n_hours=10000.0, name=None, restore_sequentially=False, saver_def=None, builder=None, defer_build=False, allow_empty=False, write_version=1, pad_step_number=False)` {#Saver.__init__} Creates a `Saver`. From d85fb4940a74663932e3029e2311435fd97f4fb0 Mon Sep 17 00:00:00 2001 From: Charles Nicholson Date: Mon, 31 Oct 2016 17:43:15 -0800 Subject: [PATCH 024/139] It seems extremely unlikely that the fragment being shaded lies /outside/ of the point sprite to which it belongs. Because of that, delete a point-on-side-of-line-segment test and simplify the analytial triangle renderer. Additionally, fix the "vampire teeth" effect when rendering the letters M, W, V in canvas labels by setting a very conservative miter limit. Change: 137776328 --- .../vz_projector/scatterPlotVisualizerCanvasLabels.ts | 1 + .../components/vz_projector/scatterPlotVisualizerSprites.ts | 5 +---- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tensorflow/tensorboard/components/vz_projector/scatterPlotVisualizerCanvasLabels.ts b/tensorflow/tensorboard/components/vz_projector/scatterPlotVisualizerCanvasLabels.ts index 072d21ca135..5e01b067e5a 100644 --- a/tensorflow/tensorboard/components/vz_projector/scatterPlotVisualizerCanvasLabels.ts +++ b/tensorflow/tensorboard/components/vz_projector/scatterPlotVisualizerCanvasLabels.ts @@ -81,6 +81,7 @@ export class ScatterPlotVisualizerCanvasLabels implements this.gc.lineWidth = 6; this.gc.textBaseline = 'middle'; + this.gc.miterLimit = 2; // Have extra space between neighboring labels. Don't pack too tightly. const labelMargin = 2; diff --git a/tensorflow/tensorboard/components/vz_projector/scatterPlotVisualizerSprites.ts b/tensorflow/tensorboard/components/vz_projector/scatterPlotVisualizerSprites.ts index cc03801073e..d187eed5656 100644 --- a/tensorflow/tensorboard/components/vz_projector/scatterPlotVisualizerSprites.ts +++ b/tensorflow/tensorboard/components/vz_projector/scatterPlotVisualizerSprites.ts @@ -80,10 +80,7 @@ const FRAGMENT_SHADER_POINT_TEST_CHUNK = ` vec3 p = vec3(spriteCoord, 0); float p_in_v0_v1 = cross(v1 - v0, p - v0).z; float p_in_v1_v2 = cross(v2 - v1, p - v1).z; - float p_in_v2_v0 = cross(v0 - v2, p - v2).z; - vec3 p_inside = vec3(p_in_v0_v1, p_in_v1_v2, p_in_v2_v0); - vec3 p_inside_norm = step(vec3(0, 0, 0), p_inside); - return all(bvec3(p_inside_norm)); + return (p_in_v0_v1 > 0.0) && (p_in_v1_v2 > 0.0); } `; From 1acc8cdfeda62cd977c8919136e5233330a63a2f Mon Sep 17 00:00:00 2001 From: Gunhan Gulsoy Date: Mon, 31 Oct 2016 21:27:34 -0800 Subject: [PATCH 025/139] Increase the size of retrying_file_system_test, as it flakily times out. Change: 137788506 --- tensorflow/core/platform/cloud/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/platform/cloud/BUILD b/tensorflow/core/platform/cloud/BUILD index 913a511d99b..8dcbf4e8997 100644 --- a/tensorflow/core/platform/cloud/BUILD +++ b/tensorflow/core/platform/cloud/BUILD @@ -201,7 +201,7 @@ tf_cc_test( tf_cc_test( name = "retrying_file_system_test", - size = "small", + size = "medium", srcs = ["retrying_file_system_test.cc"], deps = [ ":retrying_file_system", From def0c7f943de70675aad7adf88b43dbac2486b60 Mon Sep 17 00:00:00 2001 From: Gunhan Gulsoy Date: Mon, 31 Oct 2016 21:36:16 -0800 Subject: [PATCH 026/139] Disable flaky test_save_secs_saves_periodically. Change: 137788894 --- tensorflow/contrib/learn/python/learn/monitors_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/learn/python/learn/monitors_test.py b/tensorflow/contrib/learn/python/learn/monitors_test.py index ded0fb71ab7..0e5ce41f04a 100644 --- a/tensorflow/contrib/learn/python/learn/monitors_test.py +++ b/tensorflow/contrib/learn/python/learn/monitors_test.py @@ -542,7 +542,8 @@ class CheckpointSaverTest(tf.test.TestCase): self.assertEqual(1, tf.contrib.framework.load_variable( self.model_dir, self.global_step.name)) - def test_save_secs_saves_periodically(self): + # TODO(gunan): Reenable this test after b/32446874 is fixed. + def disabled_test_save_secs_saves_periodically(self): with self.graph.as_default(): monitor = learn.monitors.CheckpointSaver( self.model_dir, save_secs=2, scaffold=self.scaffold) From ca83167fb475162cf4f1e64e1452a9bf3f186ad1 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 1 Nov 2016 06:21:04 -0800 Subject: [PATCH 027/139] Increasing the limit of 20 colors for the Embedding Projector to 50. Some colors are duplicated if there are more than 20. Change: 137820753 --- .../tensorboard/components/vz_projector/data-provider.ts | 4 ++-- .../components/vz_projector/vz-projector-data-panel.ts | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tensorflow/tensorboard/components/vz_projector/data-provider.ts b/tensorflow/tensorboard/components/vz_projector/data-provider.ts index 6337aaa249b..b460e905554 100644 --- a/tensorflow/tensorboard/components/vz_projector/data-provider.ts +++ b/tensorflow/tensorboard/components/vz_projector/data-provider.ts @@ -18,7 +18,7 @@ import * as logging from './logging'; import {runAsyncTask} from './util'; /** Maximum number of colors supported in the color map. */ -const NUM_COLORS_COLOR_MAP = 20; +const NUM_COLORS_COLOR_MAP = 50; export const METADATA_MSG_ID = 'metadata'; export const TENSORS_MSG_ID = 'tensors'; @@ -278,4 +278,4 @@ export function retrieveSpriteAndMetadataInfo(metadataPath: string, metadata.spriteMetadata = spriteMetadata; callback(metadata); }); -} \ No newline at end of file +} diff --git a/tensorflow/tensorboard/components/vz_projector/vz-projector-data-panel.ts b/tensorflow/tensorboard/components/vz_projector/vz-projector-data-panel.ts index df8f7f0f3fd..c9d657a81bc 100644 --- a/tensorflow/tensorboard/components/vz_projector/vz-projector-data-panel.ts +++ b/tensorflow/tensorboard/components/vz_projector/vz-projector-data-panel.ts @@ -144,7 +144,9 @@ export class DataPanel extends DataPanelPolymer { } let desc = stats.tooManyUniqueValues ? 'gradient' : - stats.uniqueEntries.length + ' colors'; + stats.uniqueEntries.length + + ((stats.uniqueEntries.length > 20) ? ' non-unique' : '') + + ' colors'; return {name: stats.name, desc, map, items, thresholds}; }); if (metadataColorOption.length > 0) { From 0385e47dba67d26ad4c86cd88c873245f37cede6 Mon Sep 17 00:00:00 2001 From: Charles Nicholson Date: Tue, 1 Nov 2016 06:53:13 -0800 Subject: [PATCH 028/139] Fix picking bug in fragment shader. Since we're not doing a camera-relative depth sort, we can't do 1-bit alpha correctly. Revert to using discard in the picking shader until we do a global camera-space z sort (if ever). Aggregate [PointAccessor, PointAccessor, PointAccessor] as a new type, PointAccessors3D. Change: 137823236 --- tensorflow/tensorboard/components/vz_projector/data.ts | 1 + .../tensorboard/components/vz_projector/scatterPlot.ts | 7 +++---- .../vz_projector/scatterPlotVisualizerSprites.ts | 7 +++++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/tensorflow/tensorboard/components/vz_projector/data.ts b/tensorflow/tensorboard/components/vz_projector/data.ts index b8b7d884d90..d6496e64dd6 100644 --- a/tensorflow/tensorboard/components/vz_projector/data.ts +++ b/tensorflow/tensorboard/components/vz_projector/data.ts @@ -23,6 +23,7 @@ import {SpriteMetadata} from './data-provider'; export type DistanceFunction = (a: number[], b: number[]) => number; export type PointAccessor = (index: number) => number; +export type PointAccessors3D = [PointAccessor, PointAccessor, PointAccessor]; export interface PointMetadata { [key: string]: number | string; diff --git a/tensorflow/tensorboard/components/vz_projector/scatterPlot.ts b/tensorflow/tensorboard/components/vz_projector/scatterPlot.ts index 3b87915e81d..41a65d11828 100644 --- a/tensorflow/tensorboard/components/vz_projector/scatterPlot.ts +++ b/tensorflow/tensorboard/components/vz_projector/scatterPlot.ts @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -import {PointAccessor, DataSet} from './data'; +import {DataSet, PointAccessors3D} from './data'; import {HoverContext} from './hoverContext'; import {LabelRenderParams, RenderContext} from './renderContext'; import {ScatterPlotVisualizer} from './scatterPlotVisualizer'; @@ -103,7 +103,7 @@ export class ScatterPlot { private onCameraMoveListeners: OnCameraMoveListener[] = []; // Accessors for rendering and labeling the points. - private pointAccessors: [PointAccessor, PointAccessor, PointAccessor]; + private pointAccessors: PointAccessors3D; // Scaling functions for each axis. private xScale: d3.scale.Linear; @@ -718,8 +718,7 @@ export class ScatterPlot { this.renderer.render(this.scene, this.camera); } - setPointAccessors(pointAccessors: - [PointAccessor, PointAccessor, PointAccessor]) { + setPointAccessors(pointAccessors: PointAccessors3D) { this.pointAccessors = pointAccessors; } diff --git a/tensorflow/tensorboard/components/vz_projector/scatterPlotVisualizerSprites.ts b/tensorflow/tensorboard/components/vz_projector/scatterPlotVisualizerSprites.ts index d187eed5656..cdd1ce094d9 100644 --- a/tensorflow/tensorboard/components/vz_projector/scatterPlotVisualizerSprites.ts +++ b/tensorflow/tensorboard/components/vz_projector/scatterPlotVisualizerSprites.ts @@ -122,8 +122,11 @@ const FRAGMENT_SHADER_PICKING = ` if (isImage) { gl_FragColor = vec4(vColor, 1); } else { - float a = float(point_in_unit_circle(gl_PointCoord)); - gl_FragColor = vec4(vColor, a); + bool inside = point_in_unit_circle(gl_PointCoord); + if (!inside) { + discard; + } + gl_FragColor = vec4(vColor, 1); } }`; From 413998ed0c4e17179140335a46905cba5175bf2f Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 1 Nov 2016 08:35:26 -0800 Subject: [PATCH 029/139] Don't split nodes in TensorForest until they have accumulated more than 1 class. Change: 137833057 --- .../core/ops/training_ops_test.cc | 2 +- .../tensor_forest/core/ops/tree_utils.h | 22 ++++++++++++------- .../core/ops/update_fertile_slots_op.cc | 18 +++++++++++++++ .../update_fertile_slots_op_test.py | 22 ++++++++++++++++--- .../tensor_forest/python/tensor_forest.py | 1 + 5 files changed, 53 insertions(+), 12 deletions(-) diff --git a/tensorflow/contrib/tensor_forest/core/ops/training_ops_test.cc b/tensorflow/contrib/tensor_forest/core/ops/training_ops_test.cc index 451dcf2a94f..f8832a48fe2 100644 --- a/tensorflow/contrib/tensor_forest/core/ops/training_ops_test.cc +++ b/tensorflow/contrib/tensor_forest/core/ops/training_ops_test.cc @@ -26,7 +26,7 @@ namespace tensorflow { TEST(TrainingOpsTest, UpdateFertileSlots_ShapeFn) { ShapeInferenceTestOp op("UpdateFertileSlots"); - INFER_OK(op, "?;?;?;?;?;?;?", "[2,?];[2,?];[?];[?]"); + INFER_OK(op, "?;?;?;?;?;?;?;?", "[2,?];[2,?];[?];[?]"); } TEST(TrainingOpsTest, ScatterAddNdim_ShapeFn) { diff --git a/tensorflow/contrib/tensor_forest/core/ops/tree_utils.h b/tensorflow/contrib/tensor_forest/core/ops/tree_utils.h index b2f66acb6e4..7c7193f0f45 100644 --- a/tensorflow/contrib/tensor_forest/core/ops/tree_utils.h +++ b/tensorflow/contrib/tensor_forest/core/ops/tree_utils.h @@ -55,23 +55,29 @@ T Sum(Tensor counts) { // is stored in index 0, individual feature types start at index 1. DataColumnTypes FeatureSpec(int32 input_feature, const Tensor& spec); -// Given an Eigen::Tensor type, calculate the Gini impurity, which we use -// to determine the best split (lowest) and which nodes to allocate first -// (highest). -template -float WeightedGiniImpurity(const T& counts) { +// Given an Eigen::Tensor type, calculate the Gini impurity. +template +float RawWeightedGiniImpurity(const T& counts) { // Our split score is the Gini impurity times the number of examples // seen by the leaf. If c(i) denotes the i-th class count and c = sum_i c(i) // then // score = c * (1 - sum_i ( c(i) / c )^2 ) // = c - sum_i c(i)^2 / c - const auto smoothed = counts + counts.constant(1.0f); - const auto sum = smoothed.sum(); - const auto sum2 = smoothed.square().sum(); + const auto sum = counts.sum(); + const auto sum2 = counts.square().sum(); Eigen::Tensor ret = sum - (sum2 / sum); return ret(0); } +// Given an Eigen::Tensor type, calculate the smoothed Gini impurity, which we +// use to determine the best split (lowest) and which nodes to allocate first +// (highest). +template +float WeightedGiniImpurity(const T& counts) { + const auto smoothed = counts + counts.constant(1.0f); + return RawWeightedGiniImpurity(smoothed); +} + template float WeightedVariance(const T1& sums, const T2& squares, float count) { const auto e_x = sums / count; diff --git a/tensorflow/contrib/tensor_forest/core/ops/update_fertile_slots_op.cc b/tensorflow/contrib/tensor_forest/core/ops/update_fertile_slots_op.cc index 448869ab750..52b7f6d3b3a 100644 --- a/tensorflow/contrib/tensor_forest/core/ops/update_fertile_slots_op.cc +++ b/tensorflow/contrib/tensor_forest/core/ops/update_fertile_slots_op.cc @@ -48,6 +48,7 @@ REGISTER_OP("UpdateFertileSlots") .Input("accumulator_sums: float") .Input("node_to_accumulator: int32") .Input("stale_leaves: int32") + .Input("node_sums: float") .Output("node_to_accumulator_map_updates: int32") .Output("accumulator_to_node_map_updates: int32") .Output("accumulators_cleared: int32") @@ -84,6 +85,8 @@ node_to_accumulator: `node_to_accumulator[i]` is the accumulator slot used by fertile node i, or -1 if node i isn't fertile. stale_leaves:= A 1-d int32 tensor containing the indices of all leaves that have stopped accumulating statistics because they are too old. +node_sums: `node_sums[n][c]` records how many + training examples have class c and have ended up in node n. node_to_accumulator_map_updates:= A 2-d int32 tensor describing the changes that need to be applied to the node_to_accumulator map. Intended to be used with @@ -121,6 +124,7 @@ class UpdateFertileSlots : public OpKernel { const Tensor& accumulator_sums = context->input(4); const Tensor& node_to_accumulator = context->input(5); const Tensor& stale_leaves = context->input(6); + const Tensor& node_sums = context->input(7); OP_REQUIRES(context, finished.shape().dims() == 1, errors::InvalidArgument( @@ -204,6 +208,8 @@ class UpdateFertileSlots : public OpKernel { non_fertile_leaves, non_fertile_leaf_scores, eot, num_new_leaves, static_cast(accumulator_sums.shape().dim_size(1)), &leaf_heap); + const auto sums = node_sums.unaligned_flat(); + const int32 num_columns = node_sums.shape().dim_size(1); // Allocate leaves. std::unique_ptr values( leaf_heap.Extract()); @@ -218,6 +224,18 @@ class UpdateFertileSlots : public OpKernel { VLOG(1) << "No allocators left."; break; } + // For classification, don't make a node fertile until it is unpure. + if (!regression_) { + // Add 1 here because index 0 contains the sum of the weights across + // classes. + Eigen::array offsets = {node.first * num_columns + 1}; + Eigen::array extents = {num_columns - 1}; + const auto node_counts = sums.slice(offsets, extents); + // TODO(thomaswc): Implement a faster check for pure nodes. + if (tensorforest::RawWeightedGiniImpurity(node_counts) == 0) { + continue; + } + } VLOG(1) << "setting node " << node.first << " to accumulator " << accumulator; ++num_accumulators_allocated; diff --git a/tensorflow/contrib/tensor_forest/python/kernel_tests/update_fertile_slots_op_test.py b/tensorflow/contrib/tensor_forest/python/kernel_tests/update_fertile_slots_op_test.py index 1506f65e163..35faeed95c1 100644 --- a/tensorflow/contrib/tensor_forest/python/kernel_tests/update_fertile_slots_op_test.py +++ b/tensorflow/contrib/tensor_forest/python/kernel_tests/update_fertile_slots_op_test.py @@ -40,6 +40,8 @@ class UpdateFertileSlotsTest(test_util.TensorFlowTestCase): self.total_counts = [[80., 40., 40.]] self.ops = training_ops.Load() self.stale_leaves = [] + self.node_sums = [[3, 1, 2], [4, 2, 2], [5, 2, 3], [6, 1, 5], [7, 5, 2], + [8, 4, 4], [9, 7, 2]] def testSimple(self): with self.test_session(): @@ -47,7 +49,7 @@ class UpdateFertileSlotsTest(test_util.TensorFlowTestCase): accumulators_allocated) = self.ops.update_fertile_slots( self.finished, self.non_fertile_leaves, self.non_fertile_leaf_scores, self.end_of_tree, self.total_counts, self.node_map, - self.stale_leaves) + self.stale_leaves, self.node_sums) self.assertAllEqual([[2, 4], [-1, 0]], n2a_map_updates.eval()) self.assertAllEqual([[0], [4]], a2n_map_updates.eval()) @@ -60,13 +62,27 @@ class UpdateFertileSlotsTest(test_util.TensorFlowTestCase): accumulators_allocated) = self.ops.update_fertile_slots( [], self.non_fertile_leaves, self.non_fertile_leaf_scores, self.end_of_tree, self.total_counts, self.node_map, - self.stale_leaves) + self.stale_leaves, self.node_sums) self.assertAllEqual((2, 0), n2a_map_updates.eval().shape) self.assertAllEqual((2, 0), a2n_map_updates.eval().shape) self.assertAllEqual([], accumulators_cleared.eval()) self.assertAllEqual([], accumulators_allocated.eval()) + def testPureCounts(self): + with self.test_session(): + self.node_sums[4] = [10, 0, 10] + (n2a_map_updates, a2n_map_updates, accumulators_cleared, + accumulators_allocated) = self.ops.update_fertile_slots( + self.finished, self.non_fertile_leaves, self.non_fertile_leaf_scores, + self.end_of_tree, self.total_counts, self.node_map, + self.stale_leaves, self.node_sums) + + self.assertAllEqual([[2, 3], [-1, 0]], n2a_map_updates.eval()) + self.assertAllEqual([[0], [3]], a2n_map_updates.eval()) + self.assertAllEqual([], accumulators_cleared.eval()) + self.assertAllEqual([0], accumulators_allocated.eval()) + def testBadInput(self): del self.non_fertile_leaf_scores[-1] with self.test_session(): @@ -76,7 +92,7 @@ class UpdateFertileSlotsTest(test_util.TensorFlowTestCase): (n2a_map_updates, _, _, _) = self.ops.update_fertile_slots( self.finished, self.non_fertile_leaves, self.non_fertile_leaf_scores, self.end_of_tree, self.total_counts, - self.node_map, self.stale_leaves) + self.node_map, self.stale_leaves, self.node_sums) self.assertAllEqual((2, 0), n2a_map_updates.eval().shape) diff --git a/tensorflow/contrib/tensor_forest/python/tensor_forest.py b/tensorflow/contrib/tensor_forest/python/tensor_forest.py index 17d469739f9..4e4984115ad 100644 --- a/tensorflow/contrib/tensor_forest/python/tensor_forest.py +++ b/tensorflow/contrib/tensor_forest/python/tensor_forest.py @@ -780,6 +780,7 @@ class RandomTreeGraphs(object): self.variables.accumulator_sums, self.variables.node_to_accumulator_map, stale, + self.variables.node_sums, regression=self.params.regression)) # Ensure end_of_tree doesn't get updated until UpdateFertileSlots has From a964d5f3c8a6ad924379a19e9924afae7c428d62 Mon Sep 17 00:00:00 2001 From: Dan Smilkov Date: Tue, 1 Nov 2016 08:45:02 -0800 Subject: [PATCH 030/139] Increase initial zoom in. Change: 137834159 --- tensorflow/tensorboard/components/vz_projector/scatterPlot.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/tensorboard/components/vz_projector/scatterPlot.ts b/tensorflow/tensorboard/components/vz_projector/scatterPlot.ts index 41a65d11828..844ffa02694 100644 --- a/tensorflow/tensorboard/components/vz_projector/scatterPlot.ts +++ b/tensorflow/tensorboard/components/vz_projector/scatterPlot.ts @@ -42,7 +42,7 @@ const ORTHO_CAMERA_FRUSTUM_HALF_EXTENT = 1.2; const SHIFT_KEY = 16; const CTRL_KEY = 17; -const START_CAMERA_POS_3D = new THREE.Vector3(0.6, 1.0, 1.85); +const START_CAMERA_POS_3D = new THREE.Vector3(0.45, 0.9, 1.6); const START_CAMERA_TARGET_3D = new THREE.Vector3(0, 0, 0); const START_CAMERA_POS_2D = new THREE.Vector3(0, 0, 1); const START_CAMERA_TARGET_2D = new THREE.Vector3(0, 0, 0); From 6d4956edb02c5a2d0a9bdeb919a4c4c0e55fd882 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 1 Nov 2016 09:13:26 -0800 Subject: [PATCH 031/139] Allow passing constant_value_as_shape from call_cpp_shape_fn to the C++ shape inference. Delegate to C++ shape function for Reshape. Fix reshape to handle attempting to infer unknown dim when product of known elements is 0. Change: 137837591 --- tensorflow/core/framework/shape_inference.cc | 13 ++++++-- tensorflow/core/framework/shape_inference.h | 2 +- tensorflow/core/ops/array_ops.cc | 2 +- tensorflow/core/ops/array_ops_test.cc | 8 +++++ tensorflow/python/framework/common_shapes.py | 29 ++++++++++++----- .../python/framework/cpp_shape_inference.cc | 32 +++++++++++++------ .../python/framework/cpp_shape_inference.h | 4 ++- .../python/kernel_tests/reshape_op_test.py | 2 +- tensorflow/python/ops/array_ops.py | 4 +++ 9 files changed, 72 insertions(+), 24 deletions(-) diff --git a/tensorflow/core/framework/shape_inference.cc b/tensorflow/core/framework/shape_inference.cc index 4aa32f6a841..f6475e07366 100644 --- a/tensorflow/core/framework/shape_inference.cc +++ b/tensorflow/core/framework/shape_inference.cc @@ -31,11 +31,20 @@ InferenceContext::InferenceContext( const NodeDef* node_def, const OpDef& op_def, const std::vector& input_shapes, const std::vector& input_tensors, - const std::vector& input_tensors_as_shapes, + const std::vector& input_tensors_as_shapes, const std::vector& input_handle_shapes, const std::vector& input_handle_dtypes) : node_def_(*CHECK_NOTNULL(node_def)) { - PreInputInit(op_def, input_tensors, input_tensors_as_shapes); + std::vector input_tensors_as_shape_handles; + for (const TensorShapeProto& p : input_tensors_as_shapes) { + ShapeHandle shape; + construction_status_.Update(MakeShapeFromShapeProto(p, &shape)); + if (!construction_status_.ok()) { + return; + } + input_tensors_as_shape_handles.push_back(shape); + } + PreInputInit(op_def, input_tensors, input_tensors_as_shape_handles); if (!construction_status_.ok()) return; for (const TensorShapeProto& p : input_shapes) { ShapeHandle shape; diff --git a/tensorflow/core/framework/shape_inference.h b/tensorflow/core/framework/shape_inference.h index e02490efd92..1a8107ef00d 100644 --- a/tensorflow/core/framework/shape_inference.h +++ b/tensorflow/core/framework/shape_inference.h @@ -164,7 +164,7 @@ class InferenceContext { InferenceContext(const NodeDef* node_def, const OpDef& op_def, const std::vector& input_shapes, const std::vector& input_tensors, - const std::vector& input_tensors_as_shapes, + const std::vector& input_tensors_as_shapes, const std::vector& input_handle_shapes, const std::vector& input_handle_dtypes); diff --git a/tensorflow/core/ops/array_ops.cc b/tensorflow/core/ops/array_ops.cc index a7fb3375c8f..0d3e6fa94ad 100644 --- a/tensorflow/core/ops/array_ops.cc +++ b/tensorflow/core/ops/array_ops.cc @@ -151,7 +151,7 @@ Status SetOutputShapeForReshape(InferenceContext* c) { TF_RETURN_IF_ERROR(c->Multiply(known_elems, dim, &known_elems)); } } - if (!too_many_unknown) { + if (!too_many_unknown && c->Value(known_elems) != 0) { DimensionHandle inferred_dim; TF_RETURN_IF_ERROR(c->Divide(num_in_elems, c->Value(known_elems), true /* evenly_divisible */, &inferred_dim)); diff --git a/tensorflow/core/ops/array_ops_test.cc b/tensorflow/core/ops/array_ops_test.cc index 7f7861384cb..4e10d72816e 100644 --- a/tensorflow/core/ops/array_ops_test.cc +++ b/tensorflow/core/ops/array_ops_test.cc @@ -718,6 +718,14 @@ TEST(ArrayOpsTest, Reshape_ShapeFn) { INFER_ERROR( "Cannot reshape a tensor with 2 elements to shape [] (1 elements)", op, "[1,2];[0]"); + + // Reshaping a tensor with no elements. + new_shape = test::AsTensor({-1}); + INFER_OK(op, "[0];[1]", "[0]"); + new_shape = test::AsTensor({-1, 6}); + INFER_OK(op, "[0,2];[1]", "[0,6]"); + new_shape = test::AsTensor({0, -1}); + INFER_OK(op, "[0,2];[1]", "[0,?]"); } TEST(ArrayOpsTest, QuantizedReshape_ShapeFn) { diff --git a/tensorflow/python/framework/common_shapes.py b/tensorflow/python/framework/common_shapes.py index 09afe56b191..c8867b4b1b0 100644 --- a/tensorflow/python/framework/common_shapes.py +++ b/tensorflow/python/framework/common_shapes.py @@ -552,7 +552,9 @@ def broadcast_shape(shape_x, shape_y): return tensor_shape.TensorShape(return_dims) -def call_cpp_shape_fn(op, input_tensors_needed=None, +def call_cpp_shape_fn(op, + input_tensors_needed=None, + input_tensors_as_shapes_needed=None, debug_python_shape_fn=None): """A shape function that delegates to the registered C++ shape function. @@ -560,6 +562,8 @@ def call_cpp_shape_fn(op, input_tensors_needed=None, op: the node in the graph for which to compute output shapes. input_tensors_needed: a list of input tensor indices for which to compute the input tensor's value and pass to the C++ shape function. + input_tensors_as_shapes_needed: a list of input tensor indices for which to + compute the constant_value_as_shape and pass to the C++ shape function. debug_python_shape_fn: For testing only during migration to using call_cpp_shape_fn. Do not submit calls that set this, as the comparison is slow. If non-None, the python shape function; @@ -594,16 +598,25 @@ def call_cpp_shape_fn(op, input_tensors_needed=None, input_tensors = [None for i in input_shapes] if input_tensors_needed: for idx in input_tensors_needed: - input_tensors[idx] = tensor_util.constant_value(op.inputs[idx]) - if input_tensors[idx] is not None: - input_tensors[idx] = np.asarray(input_tensors[idx]) + v = tensor_util.constant_value(op.inputs[idx]) + if v is not None: + input_tensors[idx] = np.asarray(v) + + serialized_unknown_shape = ( + tensor_shape.TensorShape(None).as_proto().SerializeToString()) + arr = [serialized_unknown_shape for i in input_shapes] + if input_tensors_as_shapes_needed: + for idx in input_tensors_as_shapes_needed: + s = tensor_util.constant_value_as_shape(op.inputs[idx]) + if s is not None: + arr[idx] = s.as_proto().SerializeToString() + input_tensors_as_shapes = arr try: with errors.raise_exception_on_not_ok_status() as status: - output_shapes = pywrap_tensorflow.RunCppShapeInference(node_def_str, - input_shapes, - input_tensors, - status) + output_shapes = pywrap_tensorflow.RunCppShapeInference( + node_def_str, input_shapes, input_tensors, input_tensors_as_shapes, + status) except errors.InvalidArgumentError as err: raise ValueError(err.message) diff --git a/tensorflow/python/framework/cpp_shape_inference.cc b/tensorflow/python/framework/cpp_shape_inference.cc index 57b85e81187..7620b52f9fc 100644 --- a/tensorflow/python/framework/cpp_shape_inference.cc +++ b/tensorflow/python/framework/cpp_shape_inference.cc @@ -50,6 +50,7 @@ Status RunCppShapeInferenceImpl( const string& serialized_node_def, const std::vector& input_serialized_shapes, const std::vector& input_constant_tensor_values, + const std::vector& input_constant_tensor_as_shape_values, std::vector* output_tensor_shape_protos) { tensorflow::NodeDef node; if (!node.ParseFromString(serialized_node_def)) { @@ -87,10 +88,9 @@ Status RunCppShapeInferenceImpl( } // Convert input tensor values; - const int num_input_tensors = input_constant_tensor_values.size(); - std::vector input_tensor_values(num_input_tensors); + std::vector input_tensor_values(input_constant_tensor_values.size()); std::vector input_tensors; - for (int i = 0; i < num_input_tensors; ++i) { + for (int i = 0; i < input_constant_tensor_values.size(); ++i) { auto* py_val = input_constant_tensor_values[i]; if (py_val == Py_None) { input_tensors.push_back(nullptr); @@ -101,11 +101,21 @@ Status RunCppShapeInferenceImpl( } } + // Convert input tensor-as-shape values; + std::vector input_tensor_as_shapes_protos( + input_constant_tensor_as_shape_values.size()); + for (int i = 0; i < input_constant_tensor_as_shape_values.size(); ++i) { + if (!input_tensor_as_shapes_protos[i].ParseFromString( + input_constant_tensor_as_shape_values[i])) { + return errors::InvalidArgument( + "Error parsing shape proto during cpp shape inference"); + } + } + // Run shape inference. tensorflow::shape_inference::InferenceContext c( &node, op_reg_data->op_def, input_shapes, input_tensors, - {} /* input_tensors_as_shapes */, input_handle_shapes, - input_handle_dtypes); + input_tensor_as_shapes_protos, input_handle_shapes, input_handle_dtypes); TF_RETURN_IF_ERROR(c.construction_status()); TF_RETURN_IF_ERROR(c.Run(op_reg_data->shape_inference_fn)); @@ -130,16 +140,17 @@ Status RunCppShapeInferenceImpl( std::vector RunCppShapeInference( const string& serialized_node_def, const std::vector& input_serialized_shapes, - PyObject* input_constant_tensor_values, TF_Status* out_status) { + PyObject* input_constant_tensor_values, + const std::vector& input_constant_tensor_as_shape_values, + TF_Status* out_status) { if (!PyList_Check(input_constant_tensor_values)) { TF_SetStatus(out_status, TF_INVALID_ARGUMENT, "Invalid python value"); return std::vector(); } std::vector input_constant_tensor_values_v; - int num_input_constant_tensor_values = - PyList_Size(input_constant_tensor_values); - for (int i = 0; i < num_input_constant_tensor_values; ++i) { + int cnt = PyList_Size(input_constant_tensor_values); + for (int i = 0; i < cnt; ++i) { input_constant_tensor_values_v.push_back( PyList_GetItem(input_constant_tensor_values, i)); } @@ -147,7 +158,8 @@ std::vector RunCppShapeInference( std::vector output_tensor_shape_protos; tensorflow::Status status = RunCppShapeInferenceImpl( serialized_node_def, input_serialized_shapes, - input_constant_tensor_values_v, &output_tensor_shape_protos); + input_constant_tensor_values_v, input_constant_tensor_as_shape_values, + &output_tensor_shape_protos); Set_TF_Status_from_Status(out_status, status); return status.ok() ? output_tensor_shape_protos : std::vector(); diff --git a/tensorflow/python/framework/cpp_shape_inference.h b/tensorflow/python/framework/cpp_shape_inference.h index f91af8e1a84..b4893829939 100644 --- a/tensorflow/python/framework/cpp_shape_inference.h +++ b/tensorflow/python/framework/cpp_shape_inference.h @@ -44,7 +44,9 @@ namespace swig { std::vector RunCppShapeInference( const string& serialized_node_def, const std::vector& input_serialized_shapes, - PyObject* input_constant_tensor_values, TF_Status* out_status); + PyObject* input_constant_tensor_values, + const std::vector& input_constant_tensor_as_shape_values, + TF_Status* out_status); } // namespace swig } // namespace tensorflow diff --git a/tensorflow/python/kernel_tests/reshape_op_test.py b/tensorflow/python/kernel_tests/reshape_op_test.py index a68f722244e..8e62be107be 100644 --- a/tensorflow/python/kernel_tests/reshape_op_test.py +++ b/tensorflow/python/kernel_tests/reshape_op_test.py @@ -100,7 +100,7 @@ class ReshapeTest(tf.test.TestCase): def testErrors(self): y = tf.constant(0.0, shape=[23, 29, 31]) - with self.assertRaisesRegexp(ValueError, "isn't divisible by 17"): + with self.assertRaisesRegexp(ValueError, "must be evenly divisible by 17"): tf.reshape(y, [17, -1]) z = tf.constant(0.0, shape=[32, 128]) diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py index 6474d54f66f..533b5fcb982 100644 --- a/tensorflow/python/ops/array_ops.py +++ b/tensorflow/python/ops/array_ops.py @@ -1774,6 +1774,10 @@ ops.RegisterShape("Bitcast")(common_shapes.call_cpp_shape_fn) @ops.RegisterShape("Reshape") +def _DelegateReshapeShape(op): + return common_shapes.call_cpp_shape_fn(op, input_tensors_as_shapes_needed=[1]) + + def _ReshapeShape(op): """Shape function for Reshape op.""" input_shape = op.inputs[0].get_shape() From 39f5cc1caa396e1e8a13341847be602fb2609023 Mon Sep 17 00:00:00 2001 From: Patrick Nguyen Date: Tue, 1 Nov 2016 09:57:27 -0800 Subject: [PATCH 032/139] Seal tf.python_io's interface. Change: 137843740 --- tensorflow/python/lib/io/python_io.py | 7 +++++-- tensorflow/python/lib/io/tf_record.py | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/lib/io/python_io.py b/tensorflow/python/lib/io/python_io.py index 7ffb79ef9ae..a4c682aa28a 100644 --- a/tensorflow/python/lib/io/python_io.py +++ b/tensorflow/python/lib/io/python_io.py @@ -21,6 +21,8 @@ suitable if fast sharding or other non-sequential access is desired. @@TFRecordWriter @@tf_record_iterator +@@TFRecordCompressionType +@@TFRecordOptions - - - @@ -49,7 +51,8 @@ from __future__ import print_function # pylint: disable=wildcard-import from tensorflow.python.lib.io.tf_record import * # pylint: enable=wildcard-import -from tensorflow.python.util.all_util import make_all +from tensorflow.python.util.all_util import remove_undocumented +_allowed_symbols = [] -__all__ = make_all(__name__) +remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/python/lib/io/tf_record.py b/tensorflow/python/lib/io/tf_record.py index c07ff5c2d3d..9dc3ac52c2a 100644 --- a/tensorflow/python/lib/io/tf_record.py +++ b/tensorflow/python/lib/io/tf_record.py @@ -25,6 +25,7 @@ from tensorflow.python.util import compat class TFRecordCompressionType(object): + """The type of compression for the record.""" NONE = 0 ZLIB = 1 GZIP = 2 From 155994b58ab116c2fb76a0f7ee7f069a5546ada4 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 1 Nov 2016 10:11:34 -0800 Subject: [PATCH 033/139] Load convolution node definitions to transfer graphs to SOC Change: 137845970 --- .../core/kernels/hexagon/graph_transferer.cc | 85 +++++++++++++++++-- .../core/kernels/hexagon/graph_transferer.h | 11 ++- .../kernels/hexagon/graph_transferer_test.cc | 67 +++++++++++++-- 3 files changed, 148 insertions(+), 15 deletions(-) diff --git a/tensorflow/core/kernels/hexagon/graph_transferer.cc b/tensorflow/core/kernels/hexagon/graph_transferer.cc index bf91c7678f7..eed6d578614 100644 --- a/tensorflow/core/kernels/hexagon/graph_transferer.cc +++ b/tensorflow/core/kernels/hexagon/graph_transferer.cc @@ -27,6 +27,12 @@ static constexpr bool DBG = false; static constexpr const char* const INPUTS_NODE_PREFIX = "inputs_for_"; static constexpr const char* const OUTPUTS_NODE_PREFIX = "outputs_for_"; static constexpr const char* const DATA_NODE_PREFIX = "data_for_op_"; +static constexpr const char* const CONST_SHAPE_PREFIX = "const_shape_"; +static constexpr const char* const PADDING_PREFIX = "NN_PAD_"; +static constexpr const char* const PADDING_ATTR_NAME = "padding"; +static constexpr const char* const STRIDES_ATTR_NAME = "strides"; +static constexpr const char* const PADDING_VALID_STR = "VALID"; +static constexpr const char* const PADDING_SAME_STR = "SAME"; void GraphTransferer::LoadGraphFromProto(const GraphDef& graph_def) { ImportGraphDefOptions opts; @@ -63,6 +69,11 @@ GraphTransferer::GetConstNodeParams() const { return const_node_transfer_params_list_; } +const std::vector& +GraphTransferer::GetOpNodeParams() const { + return node_transfer_params_list_; +} + int GraphTransferer::CacheNode(const Node& node) { if (node_name_to_id_cache_map_.count(node.name()) > 0) { if (DBG) { @@ -107,6 +118,8 @@ void GraphTransferer::RegisterNode(const ShapeRefiner& shape_refiner, } } else if (node.IsConstant()) { RegisterConstantNode(shape_refiner, node); + } else if (HasPaddingAndStrides(node)) { + RegisterNodeWithPaddingAndStrides(shape_refiner, node); } else { // TODO(satok): register params for nodes which are supported by SOC if (DBG) { @@ -134,8 +147,6 @@ void GraphTransferer::RegisterConstantNode(const ShapeRefiner& shape_refiner, CHECK(context->ValueKnown(num_elements_dim)); const int64 num_output_elements = context->Value(num_elements_dim); const int data_size = max_bytes_per_data * num_output_elements; - const int rank = context->Rank(shape_handle); - CHECK(rank == 0); const std::array shape = BuildShapeArray(shape_handle, context); const_node_transfer_params_list_.emplace_back( @@ -146,6 +157,46 @@ void GraphTransferer::RegisterConstantNode(const ShapeRefiner& shape_refiner, data_size}); } +int GraphTransferer::RegisterConstantShape(const std::vector& shape) { + // TODO(satok): Handle non-4dim strides + CHECK(shape.size() == 4); + const string shape_name = + std::string(CONST_SHAPE_PREFIX) + std::to_string(shape.at(0)) + 'x' + + std::to_string(shape.at(1)) + 'x' + std::to_string(shape.at(2)) + 'x' + + std::to_string(shape.at(3)); + if (node_name_to_id_cache_map_.count(shape_name) <= 0) { + node_name_cache_list_.emplace_back(nullptr); + const int id = node_name_cache_list_.size() - 1; + node_name_to_id_cache_map_.emplace(shape_name, id); + const_node_transfer_params_list_.emplace_back(ConstNodeTransferParams{ + shape_name, id, {{shape[0], shape[1], shape[2], shape[3]}}, "", 0}); + } + return node_name_to_id_cache_map_[shape_name]; +} + +bool GraphTransferer::HasPaddingAndStrides(const Node& node) { + return node.def().attr().count(PADDING_ATTR_NAME) > 0 && + node.def().attr().count(STRIDES_ATTR_NAME) > 0; +} + +void GraphTransferer::RegisterNodeWithPaddingAndStrides( + const ShapeRefiner& shape_refiner, const Node& node) { + CHECK(node_name_to_id_cache_map_.count(node.name()) == 1); + const int id = node_name_to_id_cache_map_[node.name()]; + shape_inference::InferenceContext* context = shape_refiner.GetContext(&node); + CHECK(node.def().attr().count(PADDING_ATTR_NAME) > 0); + // TODO(satok): Use context->GetAttr(...) instead? + Padding padding; + context->GetAttr(PADDING_ATTR_NAME, &padding); + CHECK(node.def().attr().count(STRIDES_ATTR_NAME) > 0); + std::vector strides; + context->GetAttr(STRIDES_ATTR_NAME, &strides); + const int stride_id = RegisterConstantShape(strides); + std::vector extra_inputs{stride_id, 0}; + AppendNodeParams(node.name(), id, node.type_string(), padding, + node.num_inputs(), extra_inputs, node.num_outputs()); +} + bool GraphTransferer::RegisterNodeIfAllInputsAreCached( const ShapeRefiner& shape_refiner, const Node& node, const bool only_register_const_node) { @@ -161,14 +212,21 @@ bool GraphTransferer::RegisterNodeIfAllInputsAreCached( void GraphTransferer::AppendNodeParams(const string& name, const int id, const string& type, - const string& padding, + const Padding& padding, const int inputs_size, + const std::vector& extra_inputs, const int outputs_size) { + // TODO(satok): register inputs + // TODO(satok): register outputs + // TODO(satok): store padding as Padding? node_transfer_params_list_.emplace_back(NodeTransferParams{ - name, id, type, padding, - string(INPUTS_NODE_PREFIX) + std::to_string(inputs_size), inputs_size, - string(OUTPUTS_NODE_PREFIX) + std::to_string(outputs_size), - outputs_size}); + name, id, type, + string(PADDING_PREFIX) + + string(padding == VALID ? PADDING_VALID_STR : PADDING_SAME_STR), + string(INPUTS_NODE_PREFIX) + std::to_string(id), + inputs_size + static_cast(extra_inputs.size()), + string(OUTPUTS_NODE_PREFIX) + std::to_string(id), + static_cast(outputs_size)}); } /* static */ std::array @@ -205,6 +263,7 @@ GraphTransferer::BuildShapeArray( void GraphTransferer::DumpNodeTransferParams() const { // TODO(satok): Dump all params + LOG(INFO) << "*** Const Nodes ***"; for (const ConstNodeTransferParams& params : const_node_transfer_params_list_) { LOG(INFO) << "[ " << params.id << " \"" << params.name << "\" (Const)"; @@ -214,6 +273,18 @@ void GraphTransferer::DumpNodeTransferParams() const { LOG(INFO) << " data_size: " << params.data_size << " bytes" << " ]"; } + LOG(INFO) << "******"; + LOG(INFO) << "*** Op Nodes ***"; + for (const NodeTransferParams& params : node_transfer_params_list_) { + LOG(INFO) << "[ " << params.id << " \"" << params.name; + LOG(INFO) << " type: " << params.type; + LOG(INFO) << " padding: " << params.padding; + LOG(INFO) << " inputs: " << params.inputs_name + << ", size = " << params.inputs_size; + LOG(INFO) << " outputs: " << params.outputs_name + << ", size = " << params.outputs_size << " ]"; + } + LOG(INFO) << "******"; } } // namespace tensorflow diff --git a/tensorflow/core/kernels/hexagon/graph_transferer.h b/tensorflow/core/kernels/hexagon/graph_transferer.h index 3e3ee3d49ba..99e1952f607 100644 --- a/tensorflow/core/kernels/hexagon/graph_transferer.h +++ b/tensorflow/core/kernels/hexagon/graph_transferer.h @@ -25,6 +25,7 @@ limitations under the License. #include "tensorflow/core/graph/graph.h" #include "tensorflow/core/platform/macros.h" #include "tensorflow/core/platform/protobuf.h" +#include "tensorflow/core/util/padding.h" namespace tensorflow { @@ -67,17 +68,25 @@ class GraphTransferer { // Return const node parameters for transfer const std::vector& GetConstNodeParams() const; + // Return op node parameters for transfer + const std::vector& GetOpNodeParams() const; + private: int CacheNode(const Node& node); bool AreAllInputsCached(const Node& node) const; void RegisterConstantNode(const ShapeRefiner& shape_refiner, const Node& node); + int RegisterConstantShape(const std::vector& shape); + bool HasPaddingAndStrides(const Node& node); + void RegisterNodeWithPaddingAndStrides(const ShapeRefiner& shape_refiner, + const Node& node); void RegisterNode(const ShapeRefiner& shape_refiner, const Node& node); bool RegisterNodeIfAllInputsAreCached(const ShapeRefiner& shape_refiner, const Node& node, const bool only_register_const_node); void AppendNodeParams(const string& name, const int id, const string& type, - const string& padding, const int inputs_size, + const Padding& padding, const int inputs_size, + const std::vector& extra_inputs, const int outputs_size); static std::array BuildShapeArray( const shape_inference::ShapeHandle& shape_handle, diff --git a/tensorflow/core/kernels/hexagon/graph_transferer_test.cc b/tensorflow/core/kernels/hexagon/graph_transferer_test.cc index 4c386f4a623..272305717d3 100644 --- a/tensorflow/core/kernels/hexagon/graph_transferer_test.cc +++ b/tensorflow/core/kernels/hexagon/graph_transferer_test.cc @@ -16,7 +16,9 @@ limitations under the License. #include #include "tensorflow/cc/ops/const_op.h" +#include "tensorflow/cc/ops/nn_ops.h" #include "tensorflow/cc/ops/standard_ops.h" +#include "tensorflow/core/framework/tensor_testutil.h" #include "tensorflow/core/graph/graph_def_builder.h" #include "tensorflow/core/kernels/hexagon/graph_transferer.h" #include "tensorflow/core/lib/core/status.h" @@ -40,12 +42,30 @@ class GraphTransfererTest : public ::testing::Test { std::unique_ptr _session; }; -static GraphDef CreateSmallGraphDef() { +static GraphDef CreateAddGraphDef() { Scope root = Scope::NewRootScope(); ops::Output node_a = ops::Const(root.WithOpName(NAME_A), 1); ops::Output node_b = ops::Const(root.WithOpName(NAME_B), 2); - ops::Add(root.WithOpName("a_plus_b"), node_a, node_b); + ops::Output node_add = ops::Add(root.WithOpName("a_plus_b"), node_a, node_b); + GraphDef def; + TF_CHECK_OK(root.ToGraphDef(&def)); + return def; +} +static GraphDef CreateConvGraphDef() { + Scope root = Scope::NewRootScope(); + Tensor input_data(DT_FLOAT, TensorShape({1, 1, 1, 1})); + test::FillIota(&input_data, 1.0f); + ops::Output input = + ops::Const(root.WithOpName("input"), ops::Input::Initializer(input_data)); + const int stride = 1; + Tensor filter_data(DT_FLOAT, TensorShape({1, 1, 1, 1})); + test::FillIota(&filter_data, 1.0f); + ops::Output filter = ops::Const(root.WithOpName("filter"), + ops::Input::Initializer(filter_data)); + const std::vector padding{0, 0, 0, 0}; + ops::Output conv = ops::Conv2D(root.WithOpName("conv"), input, filter, + {1, stride, stride, 1}, "SAME"); GraphDef def; TF_CHECK_OK(root.ToGraphDef(&def)); return def; @@ -62,17 +82,29 @@ static const GraphTransferer::ConstNodeTransferParams* FindConstNodeParams( return nullptr; } -TEST_F(GraphTransfererTest, LoadGraph) { - GraphDef def = CreateSmallGraphDef(); +static const GraphTransferer::NodeTransferParams* FindOpNodeParams( + const GraphTransferer& gt, const string& name) { + for (const GraphTransferer::NodeTransferParams& params : + gt.GetOpNodeParams()) { + if (params.name == name) { + return ¶ms; + } + } + return nullptr; +} + +TEST_F(GraphTransfererTest, LoadAddGraph) { + GraphDef def = CreateAddGraphDef(); _session->Create(def); GraphTransferer gt; gt.LoadGraphFromProto(def); - ASSERT_EQ(2, gt.GetConstNodeParams().size()); + const int const_node_count = gt.GetConstNodeParams().size(); + ASSERT_EQ(2, const_node_count); const GraphTransferer::ConstNodeTransferParams* params_a = FindConstNodeParams(gt, NAME_A); ASSERT_TRUE(params_a != nullptr); - EXPECT_TRUE(params_a->id > 0 && params_a->id <= 2); + EXPECT_TRUE(params_a->id > 0 && params_a->id <= const_node_count); EXPECT_EQ(NAME_A, params_a->name); EXPECT_EQ(1, params_a->shape[0]); EXPECT_EQ(1, params_a->shape[1]); @@ -83,7 +115,7 @@ TEST_F(GraphTransfererTest, LoadGraph) { const GraphTransferer::ConstNodeTransferParams* params_b = FindConstNodeParams(gt, NAME_B); ASSERT_TRUE(params_b != nullptr); - EXPECT_TRUE(params_b->id > 0 && params_b->id <= 2); + EXPECT_TRUE(params_b->id > 0 && params_b->id <= const_node_count); EXPECT_EQ(1, params_b->shape[0]); EXPECT_EQ(1, params_b->shape[1]); EXPECT_EQ(1, params_b->shape[2]); @@ -91,4 +123,25 @@ TEST_F(GraphTransfererTest, LoadGraph) { EXPECT_EQ(10, params_b->data_size); } +TEST_F(GraphTransfererTest, LoadConvGraph) { + GraphDef def = CreateConvGraphDef(); + _session->Create(def); + + GraphTransferer gt; + gt.LoadGraphFromProto(def); + const int const_node_count = gt.GetConstNodeParams().size(); + ASSERT_EQ(3, const_node_count); + const int op_node_count = gt.GetOpNodeParams().size(); + ASSERT_EQ(1, op_node_count); + const GraphTransferer::NodeTransferParams* params_conv = + FindOpNodeParams(gt, "conv"); + ASSERT_TRUE(params_conv != nullptr); + const int id = params_conv->id; + EXPECT_TRUE(id > 0 && id <= (const_node_count + op_node_count)); + EXPECT_EQ("Conv2D", params_conv->type); + EXPECT_EQ(4, params_conv->inputs_size); + EXPECT_EQ(1, params_conv->outputs_size); + EXPECT_EQ("NN_PAD_SAME", params_conv->padding); +} + } // namespace tensorflow From 26f20568e42cdc1e9a522392c11ef6d85924e57f Mon Sep 17 00:00:00 2001 From: Dan Smilkov Date: Tue, 1 Nov 2016 10:23:07 -0800 Subject: [PATCH 034/139] Rename tab "projections" to "embeddings" in TensorBoard Change: 137847545 --- tensorflow/tensorboard/components/tf_globals/globals.ts | 2 +- .../components/tf_tensorboard/tf-tensorboard.html | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tensorflow/tensorboard/components/tf_globals/globals.ts b/tensorflow/tensorboard/components/tf_globals/globals.ts index e985941d9cc..36a7072a7ff 100644 --- a/tensorflow/tensorboard/components/tf_globals/globals.ts +++ b/tensorflow/tensorboard/components/tf_globals/globals.ts @@ -22,7 +22,7 @@ module TF.Globals { ['scalars', 'images', 'audio', 'graphs', 'distributions', 'histograms']; if (PROJECTOR_LAUNCHED) { - TABS.push('projections'); + TABS.push('embeddings'); } // If true, TensorBoard stores its hash in the URI state. diff --git a/tensorflow/tensorboard/components/tf_tensorboard/tf-tensorboard.html b/tensorflow/tensorboard/components/tf_tensorboard/tf-tensorboard.html index bdbb915a7a0..3d1f2ecf0bb 100644 --- a/tensorflow/tensorboard/components/tf_tensorboard/tf-tensorboard.html +++ b/tensorflow/tensorboard/components/tf_tensorboard/tf-tensorboard.html @@ -125,7 +125,7 @@ allows the user to toggle between various dashboards. > -