TensorFlow: Removal of large assets and small other fixes.

Changes:

- Remove all large assets from the repoistory, incuding the other 50MiB
  model protobuf and a lot of images in our g3doc directory. We will
  maintain these assets externally for now.  g3doc images may be
  broken for a little bit, but the website will be fine, which
  is the important resource. By @vrv and @petewarden.  Updates
  READMES to reflect the external model resources.

- Fix to saver's latest_checkpoint function by Zhifeng

- Made protos visibility public by @vrv

- Updates to docs by @mrry, Andy

- Embed tensorboard resource for summary icon by Daniel

- More updates to backwars compat by @josh11b

Base CL: 108194981
This commit is contained in:
Vijay Vasudevan 2015-11-18 16:41:37 -08:00
parent ab34d55ce7
commit f7918e1dcd
22 changed files with 184 additions and 2066 deletions

View File

@ -301,7 +301,7 @@ tf_proto_library(
go_api_version = 2,
java_api_version = 2,
py_api_version = 2, # TODO(irving): Handle 3
visibility = ["//tensorflow:internal"],
visibility = ["//visibility:public"],
)
cc_library(

View File

@ -499,4 +499,43 @@ Status OpDefCompatible(const OpDef& old_op, const OpDef& new_op) {
return Status::OK();
}
Status OpDefAddedDefaultsUnchanged(const OpDef& old_op,
const OpDef& penultimate_op,
const OpDef& new_op) {
AttrMap new_attrs, old_attrs;
FillAttrMap(old_op, &old_attrs);
FillAttrMap(new_op, &new_attrs);
for (const auto& penultimate_attr : penultimate_op.attr()) {
const OpDef::AttrDef* old_attr =
gtl::FindPtrOrNull(old_attrs, penultimate_attr.name());
if (old_attr != nullptr) continue; // attr wasn't added
const OpDef::AttrDef* new_attr =
gtl::FindPtrOrNull(new_attrs, penultimate_attr.name());
// These shouldn't happen if the op passed OpDefCompatible().
if (new_attr == nullptr) {
return errors::InvalidArgument("Missing attr '", penultimate_attr.name(),
"' in op: ", SummarizeOpDef(new_op));
}
if (!penultimate_attr.has_default_value() ||
!new_attr->has_default_value()) {
return errors::InvalidArgument("Missing default for attr '",
penultimate_attr.name(), "' in op: ",
SummarizeOpDef(new_op));
}
// Actually test that the attr's default value hasn't changed.
if (!AreAttrValuesEqual(penultimate_attr.default_value(),
new_attr->default_value())) {
return errors::InvalidArgument(
"Can't change default value for attr '", penultimate_attr.name(),
"' from ", SummarizeAttrValue(penultimate_attr.default_value()),
" in op: ", SummarizeOpDef(new_op));
}
}
return Status::OK();
}
} // namespace tensorflow

View File

@ -32,6 +32,13 @@ string SummarizeOpDef(const OpDef& op_def);
// REQUIRES: old_op and new_op must pass validation.
Status OpDefCompatible(const OpDef& old_op, const OpDef& new_op);
// Returns an error if any attr in penultimate_op that is not in old_op
// has a different default value in new_op. In general it is not safe
// to change the default for an attr that has been added to an op.
Status OpDefAddedDefaultsUnchanged(const OpDef& old_op,
const OpDef& penultimate_op,
const OpDef& new_op);
} // namespace tensorflow
#endif // TENSORFLOW_FRAMEWORK_OP_DEF_UTIL_H_

View File

@ -996,6 +996,10 @@ Status Partition(const PartitionOptions& opts, Graph* g,
if (!edge->IsControlEdge() &&
IsRefType(src->output_type(edge->src_output()))) {
AddNodeAttr("_start_time", recv_start_time, recv);
if (real_recv != recv) {
AddNodeAttr("_start_time", recv_start_time, real_recv);
}
// If src is of ref type and the edge is not a control edge, dst has
// read semantics and therefore we must control the recv.
ref_recvs.push_back(real_recv);

View File

@ -5,9 +5,7 @@ This folder contains a simple camera-based demo application utilizing Tensorflow
## Description
This demo uses a Google Inception model to classify camera frames in real-time,
displaying the top results in an overlay on the camera image. See
[`assets/imagenet_comp_graph_label_strings.txt`](assets/imagenet_comp_graph_label_strings.txt)
for the possible classifications.
displaying the top results in an overlay on the camera image.
## To build/install/run
@ -21,7 +19,21 @@ installed the NDK and SDK. Otherwise an error such as:
"The external label '//external:android/sdk' is not bound to anything" will
be reported.
To build the APK, run this from your workspace root:
The TensorFlow `GraphDef` that contains the model definition and weights
is not packaged in the repo because of its size. Instead, you must
first download the file to the `assets` directory in the source tree:
```bash
$ wget https://storage.googleapis.com/download.tensorflow.org/models/inception5h.zip -O tensorflow/examples/android/assets/inception5h.zip
$ unzip tensorflow/examples/android/assets/inception5h.zip -d tensorflow/examples/android/assets/
```
The labels file describing the possible classification will also be in the
assets directory.
Then, after editing your WORKSPACE file, you must build the APK. Run this from
your workspace root:
```bash
$ bazel build //tensorflow/examples/android:tensorflow_demo -c opt --copt=-mfpu=neon

View File

@ -6,15 +6,26 @@ to recognize objects in images.
## Description
This demo uses a Google Inception model to classify image files that are passed
in on the command line. See
[`googlenet_labels.txt`](data/googlenet_labels.txt)
for the possible classifications, which are the 1,000 categories used in the
Imagenet competition.
in on the command line.
## To build/install/run
As long as you've managed to build the main TensorFlow framework, you should
have everything you need to run this example installed already.
The TensorFlow `GraphDef` that contains the model definition and weights
is not packaged in the repo because of its size. Instead, you must
first download the file to the `data` directory in the source tree:
```bash
$ wget https://storage.googleapis.com/download.tensorflow.org/models/inception5h.zip -O tensorflow/examples/label_image/data/inception5h.zip
$ unzip tensorflow/examples/label_image/data/inception5h.zip -d tensorflow/examples/label_image/data/
```
Then, as long as you've managed to build the main TensorFlow framework, you
should have everything you need to run this example installed already.
Once extracted, see the labels file in the data directory for the possible
classifications, which are the 1,000 categories used in the Imagenet
competition.
To build it, run this command:

File diff suppressed because it is too large Load Diff

View File

@ -14,7 +14,8 @@
// customize it to use your own models or images by changing the file names at
// the top of the main() function.
//
// The googlenet_graph.pb file included by default is created from Inception.
// The model file specified in the README is a pre-trained version of
// an Inception (GoogleNet) model.
#include <fstream>
@ -48,12 +49,13 @@ using tensorflow::int32;
TF_DEFINE_string(image,
"tensorflow/examples/label_image/data/grace_hopper.jpg",
"The image to classify (JPEG or PNG).");
TF_DEFINE_string(graph,
"tensorflow/examples/label_image/data/googlenet_graph.pb",
"The location of the GraphDef file containing the protobuf"
" definition of the network.");
TF_DEFINE_string(
graph, "tensorflow/examples/label_image/data/tensorflow_inception_graph.pb",
"The location of the GraphDef file containing the protobuf"
" definition of the network.");
TF_DEFINE_string(labels,
"tensorflow/examples/label_image/data/googlenet_labels.txt",
"tensorflow/examples/label_image/data/"
"imagenet_comp_graph_label_strings.txt",
"A text file containing the labels of all the categories, one"
" per line.");
TF_DEFINE_int32(input_width, 224, "Width of the image the network expects.");

View File

@ -5,7 +5,7 @@ TensorFlow computation graphs are powerful but complicated. The graph visualizat
![Visualization of a TensorFlow graph](./graph_vis_animation.gif "Visualization of a TensorFlow graph")
*Visualization of a TensorFlow graph.*
To see your own graph, run TensorBoard pointing it to the log directory of the job, click on the graph tab on the top pane and select the appropriate run using the menu at the upper left corner. For in depth information on how to run TensorBoard and make sure you are logging all the necessary information, see [Summaries and TensorBoard](../../how_tos/summaries_and_tensorboard/index.md).
To see your own graph, run TensorBoard pointing it to the log directory of the job, click on the graph tab on the top pane and select the appropriate run using the menu at the upper left corner. For in depth information on how to run TensorBoard and make sure you are logging all the necessary information, see [TensorBoard: Visualizing Learning](../../how_tos/summaries_and_tensorboard/index.md).
## Name scoping and nodes

View File

@ -107,4 +107,4 @@ not contain any data relevant to that tab, a message will be displayed
indicating how to serialize data that is applicable to that tab.
For in depth information on how to use the *graph* tab to visualize your graph,
see [TensorBoard: Visualizing your graph](../../how_tos/graph_viz/index.md).
see [TensorBoard: Graph Visualization](../../how_tos/graph_viz/index.md).

View File

@ -21,7 +21,7 @@ def maybe_download(filename, work_directory):
if not os.path.exists(filepath):
filepath, _ = urllib.request.urlretrieve(SOURCE_URL + filename, filepath)
statinfo = os.stat(filepath)
print('Succesfully downloaded', filename, statinfo.st_size, 'bytes.')
print('Successfully downloaded', filename, statinfo.st_size, 'bytes.')
return filepath

View File

@ -295,6 +295,11 @@ def batch(tensor_list, batch_size, num_threads=1, capacity=32,
output will have shape `[batch_size, x, y, z]`. The `capacity` argument
controls the how long the prefetching is allowed to grow the queues.
*N.B.:* You must ensure that either (i) the `shapes` argument is
passed, or (ii) all of the tensors in `tensor_list` must have
fully-defined shapes. `ValueError` will be raised if neither of
these conditions holds.
Args:
tensor_list: The list of tensors to enqueue.
batch_size: The new batch size pulled from the queue.
@ -307,6 +312,10 @@ def batch(tensor_list, batch_size, num_threads=1, capacity=32,
Returns:
A list of tensors with the same number and types as `tensor_list`.
Raises:
ValueError: If the `shapes` are not specified, and cannot be
inferred from the elements of `tensor_list`.
"""
with ops.op_scope(tensor_list, name, "batch") as name:
tensor_list = _validate(tensor_list)
@ -356,6 +365,11 @@ def batch_join(tensor_list_list, batch_size, capacity=32, enqueue_many=False,
The `capacity` argument controls the how long the prefetching is allowed to
grow the queues.
*N.B.:* You must ensure that either (i) the `shapes` argument is
passed, or (ii) all of the tensors in `tensor_list_list` must have
fully-defined shapes. `ValueError` will be raised if neither of
these conditions holds.
Args:
tensor_list_list: A list of tuples of tensors to enqueue.
batch_size: An integer. The new batch size pulled from the queue.
@ -369,6 +383,10 @@ def batch_join(tensor_list_list, batch_size, capacity=32, enqueue_many=False,
Returns:
A list of tensors with the same number and types as
`tensor_list_list[i]`.
Raises:
ValueError: If the `shapes` are not specified, and cannot be
inferred from the elements of `tensor_list_list`.
"""
with ops.op_scope(_flatten(tensor_list_list), name, "batch_join") as name:
tensor_list_list = _validate_join(tensor_list_list)
@ -421,6 +439,11 @@ def shuffle_batch(tensor_list, batch_size, capacity, min_after_dequeue,
min_after_dequeue=10000)
```
*N.B.:* You must ensure that either (i) the `shapes` argument is
passed, or (ii) all of the tensors in `tensor_list` must have
fully-defined shapes. `ValueError` will be raised if neither of
these conditions holds.
Args:
tensor_list: The list of tensors to enqueue.
batch_size: The new batch size pulled from the queue.
@ -436,6 +459,10 @@ def shuffle_batch(tensor_list, batch_size, capacity, min_after_dequeue,
Returns:
A list of tensors with the same number and types as `tensor_list`.
Raises:
ValueError: If the `shapes` are not specified, and cannot be
inferred from the elements of `tensor_list`.
"""
with ops.op_scope(tensor_list, name, "shuffle_batch") as name:
tensor_list = _validate(tensor_list)
@ -489,6 +516,11 @@ def shuffle_batch_join(tensor_list_list, batch_size, capacity,
The `capacity` argument controls the how long the prefetching is allowed to
grow the queues.
*N.B.:* You must ensure that either (i) the `shapes` argument is
passed, or (ii) all of the tensors in `tensor_list_list` must have
fully-defined shapes. `ValueError` will be raised if neither of
these conditions holds.
Args:
tensor_list_list: A list of tuples of tensors to enqueue.
batch_size: An integer. The new batch size pulled from the queue.
@ -504,6 +536,10 @@ def shuffle_batch_join(tensor_list_list, batch_size, capacity,
Returns:
A list of tensors with the same number and types as `tensor_list_list[i]`.
Raises:
ValueError: If the `shapes` are not specified, and cannot be
inferred from the elements of `tensor_list_list`.
"""
with ops.op_scope(
_flatten(tensor_list_list), name, "shuffle_batch_join") as name:

View File

@ -884,9 +884,9 @@ def latest_checkpoint(checkpoint_dir, latest_filename=None):
# Pick the latest checkpoint based on checkpoint state.
ckpt = get_checkpoint_state(checkpoint_dir, latest_filename)
if ckpt and ckpt.model_checkpoint_path:
checkpoint_full_path = os.path.join(
checkpoint_pattern = os.path.join(
checkpoint_dir, ckpt.model_checkpoint_path)
if gfile.Exists(checkpoint_full_path):
return checkpoint_full_path
if gfile.Glob(checkpoint_pattern):
return checkpoint_pattern
return None

View File

@ -310,6 +310,10 @@ class SaveRestoreShardedTest(tf.test.TestCase):
self.assertEqual(10, v0.eval())
self.assertEqual(20, v1.eval())
self.assertEqual(
tf.train.latest_checkpoint(self.get_temp_dir()),
os.path.join(self.get_temp_dir(), "sharded-?????-of-00002"))
def testSaverDef(self):
with self.test_session():
v0 = tf.Variable(123, name="v0")

View File

@ -100,11 +100,10 @@ function annotationToClassName(annotationType: render.AnnotationType) {
function buildShape(aGroup, a: render.Annotation, sceneBehavior) {
if (a.annotationType === tf.graph.render.AnnotationType.SUMMARY) {
let image = scene.selectOrCreateChild(aGroup, "image");
image.attr({
"xlink:href": sceneBehavior.resolveUrl("../../lib/svg/summary-icon.svg"),
"height": "12px",
"width": "12px",
let summary = scene.selectOrCreateChild(aGroup, "use");
summary.attr({
"class": "summary",
"xlink:href": "#summary-icon",
"cursor": "pointer"
});
} else {
@ -197,7 +196,7 @@ function update(aGroup, d: render.RenderNodeInformation, a: render.Annotation,
// If there is an image, we adjust the location of the image to be vertically
// centered with the node and horizontally centered between the arrow and the
// text label.
aGroup.select("image").transition().attr({
aGroup.select("use.summary").transition().attr({
x: d.x + a.dx - 3,
y: d.y + a.dy - 6
});

View File

@ -162,6 +162,14 @@ svg.icon {
clear: both;
}
</style>
<svg style="display:none;">
<defs>
<!-- Summary icon. -->
<svg id="summary-icon" fill="#848484" height="12" viewBox="0 0 24 24" width="12">
<path d="M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z" />
</svg>
</defs>
</svg>
<div class="allcontrols">
<div class="control-holder">
<paper-icon-button id="fit" icon="aspect-ratio" class="fit-button" on-click="fit" alt="Fit to screen">
@ -301,9 +309,8 @@ svg.icon {
</tr>
<tr>
<td>
<svg class="image-icon">
<image id="summary-icon" width="24" height="24" x="0" y="0"
class="image-icon"/>
<svg class="image-icon" viewBox="0 0 12 12" width="24" height="24">
<use x="0" y="0" class="image-icon" xlink:href="#summary-icon"/>
</svg>
</td>
<td>Summary</td>
@ -357,11 +364,6 @@ svg.icon {
(function() { // Private scope.
Polymer({
is: 'tf-graph-controls',
ready: function() {
// Set the url to download the summary icon.
d3.select(this.$['summary-icon'])
.attr('xlink:href', this.resolveUrl('../../lib/svg/summary-icon.svg'));
},
properties: {
// Public API.
hasStats: {

View File

@ -12,8 +12,9 @@
</svg>
</template>
<template is="dom-if" if="[[_isSummary(node, summary)]]">
<img height$="[[height]]"
src="[[resolveUrl('../../lib/svg/summary-icon.svg')]]" />
<svg width$="[[height]]" height$="[[height]]" viewBox="0 0 12 12">
<use x="0" y="0" xlink:href="#summary-icon" />
</svg>
</template>
<template is="dom-if" if="[[_isRegularOp(node, const, summary)]]">
<svg height$="[[height]]"

View File

@ -1,3 +0,0 @@
<svg fill="#848484" height="24" viewBox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
<path d="M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z"/>
</svg>

Before

Width:  |  Height:  |  Size: 244 B

View File

@ -296,10 +296,14 @@
// 16 byte alignment on all platforms where vectorization might be enabled. In theory we could always
// enable alignment, but it can be a cause of problems on some platforms, so we just disable it in
// certain common platform (compiler+architecture combinations) to avoid these problems.
// Only static alignment is really problematic (relies on nonstandard compiler extensions that don't
// work everywhere, for example don't work on GCC/ARM), try to keep heap alignment even
// when we have to disable static alignment.
#if EIGEN_COMP_GNUC && !(EIGEN_ARCH_i386_OR_x86_64 || EIGEN_ARCH_PPC || EIGEN_ARCH_IA64)
// Only static alignment is really problematic (relies on nonstandard compiler extensions),
// try to keep heap alignment even when we have to disable static alignment.
#if EIGEN_COMP_GNUC && !(EIGEN_ARCH_i386_OR_x86_64 || EIGEN_ARCH_ARM_OR_ARM64 || EIGEN_ARCH_PPC || EIGEN_ARCH_IA64)
#define EIGEN_GCC_AND_ARCH_DOESNT_WANT_STACK_ALIGNMENT 1
#elif EIGEN_ARCH_ARM_OR_ARM64 && EIGEN_COMP_GNUC_STRICT && EIGEN_GNUC_AT_MOST(4, 6)
// Old versions of GCC on ARM, at least 4.4, were once seen to have buggy static alignment support.
// Not sure which version fixed it, hopefully it doesn't affect 4.7, which is still somewhat in use.
// 4.8 and newer seem definitely unaffected.
#define EIGEN_GCC_AND_ARCH_DOESNT_WANT_STACK_ALIGNMENT 1
#else
#define EIGEN_GCC_AND_ARCH_DOESNT_WANT_STACK_ALIGNMENT 0

View File

@ -760,11 +760,15 @@ struct GpuDevice {
GpuDevice()
: stream_(perftools::gputools::MachineManager::singleton()->stream_for_device(0)),
allocator_(nullptr),
stream_exec_(stream_->parent()) {}
stream_exec_(stream_->parent()),
device_descr_(&(stream_exec_->GetDeviceDescription())) {}
GpuDevice(perftools::gputools::Stream* stream,
const Allocator* alloc = nullptr)
: stream_(stream), allocator_(alloc), stream_exec_(stream_->parent()) { }
: stream_(stream),
allocator_(alloc),
stream_exec_(stream_->parent()),
device_descr_(&(stream_exec_->GetDeviceDescription())) {}
EIGEN_STRONG_INLINE perftools::gputools::Stream* stream() const {
return stream_;
@ -873,28 +877,25 @@ struct GpuDevice {
stream_->BlockHostUntilDone();
}
// A gpu::DeviceDescription is cached inside a StreamExecutor, so these calls
// aren't expensive/wasteful.
EIGEN_DEVICE_FUNC inline int getNumCudaMultiProcessors() const {
return stream_exec_->GetDeviceDescription().core_count();
return device_descr_->core_count();
}
EIGEN_DEVICE_FUNC inline int maxCudaThreadsPerBlock() const {
return stream_exec_->GetDeviceDescription().threads_per_block_limit();
return device_descr_->threads_per_block_limit();
}
EIGEN_DEVICE_FUNC inline int maxCudaThreadsPerMultiProcessor() const {
return stream_exec_->GetDeviceDescription().threads_per_core_limit();
return device_descr_->threads_per_core_limit();
}
EIGEN_DEVICE_FUNC inline int sharedMemPerBlock() const {
return stream_exec_->GetDeviceDescription().shared_memory_per_block();
return device_descr_->shared_memory_per_block();
}
EIGEN_DEVICE_FUNC inline int majorDeviceVersion() const {
int major, minor;
if (stream_exec_->GetDeviceDescription().cuda_compute_capability(&major,
&minor)) {
if (device_descr_->cuda_compute_capability(&major, &minor)) {
return major;
} else {
return 0;
@ -906,6 +907,7 @@ struct GpuDevice {
private:
perftools::gputools::Stream* stream_;
perftools::gputools::StreamExecutor* stream_exec_;
const perftools::gputools::DeviceDescription* device_descr_;
const Allocator* allocator_;
};

View File

@ -115,7 +115,7 @@ namespace {
}
template <typename T>
template <typename T, bool div_gt_one = false>
struct TensorIntDivisor {
public:
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorIntDivisor() {
@ -166,7 +166,7 @@ struct TensorIntDivisor {
// Optimized version for signed 32 bit integers.
// Derived from Hacker's Delight.
template <>
class TensorIntDivisor<int32_t> {
class TensorIntDivisor<int32_t, true> {
public:
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorIntDivisor() {
magic = 0;
@ -225,15 +225,15 @@ private:
};
template <typename T>
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T operator / (const T& numerator, const TensorIntDivisor<T>& divisor) {
template <typename T, bool div_gt_one>
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T operator / (const T& numerator, const TensorIntDivisor<T, div_gt_one>& divisor) {
return divisor.divide(numerator);
}
#else
// Reverse to the old code since gcudacc doesn't support the code above.
template <typename T>
template <typename T, bool div_gt_one = false>
struct TensorIntDivisor {
public:
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorIntDivisor() {
@ -285,7 +285,7 @@ struct TensorIntDivisor {
// Optimized version for signed 32 bit integers.
// Derived from Hacker's Delight.
template <>
class TensorIntDivisor<int> {
class TensorIntDivisor<int, true> {
public:
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorIntDivisor() {
magic = 0;
@ -344,8 +344,8 @@ private:
};
template <typename T>
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T operator / (const T& numerator, const TensorIntDivisor<T>& divisor) {
template <typename T, bool div_gt_one>
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T operator / (const T& numerator, const TensorIntDivisor<T, div_gt_one>& divisor) {
return divisor.divide(numerator);
}