Make a better doc generator.

Change: 146177492

tensorflow/python/framework/random_seed.py

@@ -39,7 +39,7 @@ def get_seed(op_seed):
   graph, or for only specific operations.
 
   For details on how the graph-level seed interacts with op seeds, see
-  [`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed).
+  @{set_random_seed}.
 
   Args:
     op_seed: integer.

tensorflow/tools/common/public_api.py

@@ -40,16 +40,28 @@ class PublicAPIVisitor(object):
   # Each entry maps a module path to a name to ignore in traversal.
   _do_not_descend_map = {
       # TODO(drpng): This can be removed once sealed off.
-      '': ['platform', 'pywrap_tensorflow'],
+      '': ['platform', 'pywrap_tensorflow', 'user_ops'],
 
       # Some implementations have this internal module that we shouldn't expose.
       'flags': ['cpp_flags'],
 
       # Everything below here is legitimate.
-      'app': 'flags',  # It'll stay, but it's not officially part of the API
+      'app': ['flags'],  # It'll stay, but it's not officially part of the API.
       'test': ['mock'],  # Imported for compatibility between py2/3.
   }
 
+  @property
+  def do_not_descend_map(self):
+    """A map from parents to symbols that should not be descended into.
+
+    This map can be edited, but it should not be edited once traversal has
+    begun.
+
+    Returns:
+      The map marking symbols to not explore.
+    """
+    return self._do_not_descend_map
+
   def _isprivate(self, name):
     """Return whether a name is private."""
     return name.startswith('_')
@@ -61,6 +73,8 @@ class PublicAPIVisitor(object):
 
   def __call__(self, path, parent, children):
     """Visitor interface, see `traverse` for details."""
+
+    # Avoid long waits in cases of pretty unambiguous failure.
     if inspect.ismodule(parent) and len(path.split('.')) > 10:
       raise RuntimeError('Modules nested too deep:\n%s\n\nThis is likely a '
                          'problem with an accidental public import.' % path)

tensorflow/tools/docs/BUILD

@@ -16,6 +16,74 @@ py_binary(
     deps = ["//tensorflow:tensorflow_py"],
 )
 
+py_library(
+    name = "doc_generator_visitor",
+    srcs = [
+        "doc_generator_visitor.py",
+    ],
+    srcs_version = "PY2AND3",
+)
+
+py_test(
+    name = "doc_generator_visitor_test",
+    size = "small",
+    srcs = [
+        "doc_generator_visitor_test.py",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":doc_generator_visitor",
+        "//tensorflow/python:platform_test",
+    ],
+)
+
+py_library(
+    name = "parser",
+    srcs = [
+        "parser.py",
+    ],
+    srcs_version = "PY2AND3",
+)
+
+py_test(
+    name = "parser_test",
+    size = "small",
+    srcs = [
+        "parser_test.py",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":parser",
+        "//tensorflow/python:platform_test",
+    ],
+)
+
+py_binary(
+    name = "generate",
+    srcs = ["generate.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow:tensorflow_py",
+        "//tensorflow/tools/common:public_api",
+        "//tensorflow/tools/common:traverse",
+        "//tensorflow/tools/docs:doc_generator_visitor",
+        "//tensorflow/tools/docs:parser",
+    ],
+)
+
+py_test(
+    name = "generate_test",
+    size = "small",
+    srcs = [
+        "generate_test.py",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":generate",
+        "//tensorflow/python:platform_test",
+    ],
+)
+
 filegroup(
     name = "doxy_config",
     srcs = ["tf-doxy_for_md-config"],

tensorflow/tools/docs/doc_generator_visitor.py

@@ -0,0 +1,149 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""A `traverse` visitor for processing documentation."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import inspect
+
+
+class DocGeneratorVisitor(object):
+  """A visitor that generates docs for a python object when __call__ed."""
+
+  def __init__(self):
+    self._index = {}
+    self._tree = {}
+
+  @property
+  def index(self):
+    """A map from fully qualified names to objects to be documented.
+
+    The index is filled when the visitor is passed to `traverse`.
+
+    Returns:
+      The index filled by traversal.
+    """
+    return self._index
+
+  @property
+  def tree(self):
+    """A map from fully qualified names to all its child names for traversal.
+
+    The full name to member names map is filled when the visitor is passed to
+    `traverse`.
+
+    Returns:
+      The full name to member name map filled by traversal.
+    """
+    return self._tree
+
+  def __call__(self, parent_name, parent, children):
+    """Visitor interface, see `tensorflow/tools/common:traverse` for details.
+
+    This method is called for each symbol found in a traversal using
+    `tensorflow/tools/common:traverse`. It should not be called directly in
+    user code.
+
+    Args:
+      parent_name: The fully qualified name of a symbol found during traversal.
+      parent: The Python object referenced by `parent_name`.
+      children: A list of `(name, py_object)` pairs enumerating, in alphabetical
+        order, the children (as determined by `inspect.getmembers`) of `parent`.
+        `name` is the local name of `py_object` in `parent`.
+
+    Raises:
+      RuntimeError: If this visitor is called with a `parent` that is not a
+        class or module.
+    """
+    self._index[parent_name] = parent
+    self._tree[parent_name] = []
+
+    if inspect.ismodule(parent):
+      print('module %s: %r' % (parent_name, parent))
+    elif inspect.isclass(parent):
+      print('class %s: %r' % (parent_name, parent))
+    else:
+      raise RuntimeError('Unexpected type in visitor -- %s: %r' %
+                         (parent_name, parent))
+
+    for name, child in children:
+      full_name = '.'.join([parent_name, name]) if parent_name else name
+      self._index[full_name] = child
+      self._tree[parent_name].append(name)
+
+  def find_duplicates(self):
+    """Compute data structures containing information about duplicates.
+
+    Find duplicates in `index` and decide on one to be the "master" name.
+
+    Returns a map `duplicate_of` from aliases to their master name (the master
+    name itself has no entry in this map), and a map `duplicates` from master
+    names to a lexicographically sorted list of all aliases for that name (incl.
+    the master name).
+
+    Returns:
+      A tuple `(duplicate_of, duplicates)` as described above.
+    """
+    # Maps the id of a symbol to its fully qualified name. For symbols that have
+    # several aliases, this map contains the first one found.
+    # We use id(py_object) to get a hashable value for py_object. Note all
+    # objects in _index are in memory at the same time so this is safe.
+    reverse_index = {}
+
+    # Make a preliminary duplicates map. For all sets of duplicate names, it
+    # maps the first name found to a list of all duplicate names.
+    raw_duplicates = {}
+    for full_name, py_object in self._index.iteritems():
+      # We cannot use the duplicate mechanism for constants, since e.g.,
+      # id(c1) == id(c2) with c1=1, c2=1. This is unproblematic since constants
+      # have no usable docstring and won't be documented automatically.
+      if (inspect.ismodule(py_object) or
+          inspect.isclass(py_object) or
+          inspect.isfunction(py_object) or
+          inspect.isroutine(py_object) or
+          inspect.ismethod(py_object) or
+          isinstance(py_object, property)):
+        object_id = id(py_object)
+        if object_id in reverse_index:
+          master_name = reverse_index[object_id]
+          if master_name in raw_duplicates:
+            raw_duplicates[master_name].append(full_name)
+          else:
+            raw_duplicates[master_name] = [master_name, full_name]
+        else:
+          reverse_index[object_id] = full_name
+
+    # Decide on master names, rewire duplicates and make a duplicate_of map
+    # mapping all non-master duplicates to the master name. The master symbol
+    # does not have an entry in this map.
+    duplicate_of = {}
+    # Duplicates maps the main symbols to the set of all duplicates of that
+    # symbol (incl. itself).
+    duplicates = {}
+    for names in raw_duplicates.values():
+      names = sorted(names)
+
+      # Choose the lexicographically first name with the minimum number of
+      # submodules. This will prefer highest level namespace for any symbol.
+      master_name = min(names, key=lambda name: name.count('.'))
+
+      duplicates[master_name] = names
+      for name in names:
+        if name != master_name:
+          duplicate_of[name] = master_name
+
+    return duplicate_of, duplicates

tensorflow/tools/docs/doc_generator_visitor_test.py

@@ -0,0 +1,107 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for tools.docs.doc_generator_visitor."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.platform import googletest
+from tensorflow.tools.docs import doc_generator_visitor
+
+
+class DocGeneratorVisitorTest(googletest.TestCase):
+
+  def test_call_module(self):
+    visitor = doc_generator_visitor.DocGeneratorVisitor()
+    visitor(
+        'doc_generator_visitor', doc_generator_visitor,
+        [('DocGeneratorVisitor', doc_generator_visitor.DocGeneratorVisitor)])
+
+    self.assertEqual({'doc_generator_visitor': ['DocGeneratorVisitor']},
+                     visitor.tree)
+    self.assertEqual({
+        'doc_generator_visitor': doc_generator_visitor,
+        'doc_generator_visitor.DocGeneratorVisitor':
+        doc_generator_visitor.DocGeneratorVisitor,
+    }, visitor.index)
+
+  def test_call_class(self):
+    visitor = doc_generator_visitor.DocGeneratorVisitor()
+    visitor(
+        'DocGeneratorVisitor', doc_generator_visitor.DocGeneratorVisitor,
+        [('index', doc_generator_visitor.DocGeneratorVisitor.index)])
+
+    self.assertEqual({'DocGeneratorVisitor': ['index']},
+                     visitor.tree)
+    self.assertEqual({
+        'DocGeneratorVisitor': doc_generator_visitor.DocGeneratorVisitor,
+        'DocGeneratorVisitor.index':
+        doc_generator_visitor.DocGeneratorVisitor.index
+    }, visitor.index)
+
+  def test_call_raises(self):
+    visitor = doc_generator_visitor.DocGeneratorVisitor()
+    with self.assertRaises(RuntimeError):
+      visitor('non_class_or_module', 'non_class_or_module_object', [])
+
+  def test_duplicates(self):
+    visitor = doc_generator_visitor.DocGeneratorVisitor()
+    visitor(
+        'submodule.DocGeneratorVisitor',
+        doc_generator_visitor.DocGeneratorVisitor,
+        [('index', doc_generator_visitor.DocGeneratorVisitor.index),
+         ('index2', doc_generator_visitor.DocGeneratorVisitor.index)])
+    visitor(
+        'submodule2.DocGeneratorVisitor',
+        doc_generator_visitor.DocGeneratorVisitor,
+        [('index', doc_generator_visitor.DocGeneratorVisitor.index),
+         ('index2', doc_generator_visitor.DocGeneratorVisitor.index)])
+    visitor(
+        'DocGeneratorVisitor2',
+        doc_generator_visitor.DocGeneratorVisitor,
+        [('index', doc_generator_visitor.DocGeneratorVisitor.index),
+         ('index2', doc_generator_visitor.DocGeneratorVisitor.index)])
+
+    duplicate_of, duplicates = visitor.find_duplicates()
+
+    # The shorter path should be master, or if equal, the lexicographically
+    # first will be.
+    self.assertEqual(
+        {'DocGeneratorVisitor2': sorted(['submodule.DocGeneratorVisitor',
+                                         'submodule2.DocGeneratorVisitor',
+                                         'DocGeneratorVisitor2']),
+         'DocGeneratorVisitor2.index': sorted([
+             'submodule.DocGeneratorVisitor.index',
+             'submodule.DocGeneratorVisitor.index2',
+             'submodule2.DocGeneratorVisitor.index',
+             'submodule2.DocGeneratorVisitor.index2',
+             'DocGeneratorVisitor2.index',
+             'DocGeneratorVisitor2.index2'
+         ]),
+        }, duplicates)
+    self.assertEqual({
+        'submodule.DocGeneratorVisitor': 'DocGeneratorVisitor2',
+        'submodule.DocGeneratorVisitor.index': 'DocGeneratorVisitor2.index',
+        'submodule.DocGeneratorVisitor.index2': 'DocGeneratorVisitor2.index',
+        'submodule2.DocGeneratorVisitor': 'DocGeneratorVisitor2',
+        'submodule2.DocGeneratorVisitor.index': 'DocGeneratorVisitor2.index',
+        'submodule2.DocGeneratorVisitor.index2': 'DocGeneratorVisitor2.index',
+        'DocGeneratorVisitor2.index2': 'DocGeneratorVisitor2.index'
+    }, duplicate_of)
+
+
+if __name__ == '__main__':
+  googletest.main()

tensorflow/tools/docs/generate.py

@@ -0,0 +1,232 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Generate docs for the TensorFlow Python API."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import argparse
+import inspect
+import os
+
+import tensorflow as tf
+
+from tensorflow.tools.common import public_api
+from tensorflow.tools.common import traverse
+from tensorflow.tools.docs import doc_generator_visitor
+from tensorflow.tools.docs import parser
+
+
+def write_docs(output_dir, base_dir, duplicate_of, duplicates, index, tree):
+  """Write previously extracted docs to disk.
+
+  Write a docs page for each symbol in `index` to a tree of docs at
+  `output_dir`.
+
+  Symbols with multiple aliases will have only one page written about them,
+  which is referenced for all aliases. `duplicate_of` and `duplicates` are used
+  to determine which docs pages to write.
+
+  Args:
+    output_dir: Directory to write documentation markdown files to. Will be
+      created if it doesn't exist.
+    base_dir: Base directory of the code being documented. This prefix is
+      stripped from all file paths that are part of the documentation.
+    duplicate_of: A `dict` mapping fully qualified names to "master" names. This
+      is used to resolve "@{symbol}" references to the "master" name.
+    duplicates: A `dict` mapping fully qualified names to a set of all
+      aliases of this name. This is used to automatically generate a list of all
+      aliases for each name.
+    index: A `dict` mapping fully qualified names to the corresponding Python
+      objects. Used to produce docs for child objects, and to check the validity
+      of "@{symbol}" references.
+    tree: A `dict` mapping a fully qualified name to the names of all its
+      members. Used to populate the members section of a class or module page.
+  """
+  # Make output_dir.
+  try:
+    if not os.path.exists(output_dir):
+      os.makedirs(output_dir)
+  except OSError as e:
+    print('Creating output dir "%s" failed: %s' % (output_dir, e))
+    raise
+
+  # Parse and write Markdown pages, resolving cross-links (@{symbol}).
+  for full_name, py_object in index.iteritems():
+
+    if full_name in duplicate_of:
+      print('Not writing docs for %s, duplicate of %s.' % (
+          full_name, duplicate_of[full_name]))
+      continue
+
+    # Methods and some routines are documented only as part of their class.
+    if not (inspect.ismodule(py_object) or
+            inspect.isclass(py_object) or
+            inspect.isfunction(py_object)):
+      print('Not writing docs for %s, not a class, module, or function.' % (
+          full_name))
+      continue
+
+    print('Writing docs for %s (%r).' % (full_name, py_object))
+
+    # Generate docs for `py_object`, resolving references.
+    markdown = parser.generate_markdown(full_name, py_object,
+                                        duplicate_of=duplicate_of,
+                                        duplicates=duplicates,
+                                        index=index,
+                                        tree=tree,
+                                        base_dir=base_dir)
+
+    # TODO(deannarubin): use _tree to generate sidebar information.
+
+    path = os.path.join(output_dir, parser.documentation_path(full_name))
+    directory = os.path.dirname(path)
+    try:
+      if not os.path.exists(directory):
+        os.makedirs(directory)
+      with open(path, 'w') as f:
+        f.write(markdown)
+    except OSError as e:
+      print('Cannot write documentation for %s to %s: %s' % (full_name,
+                                                             directory, e))
+      raise
+    # TODO(deannarubin): write sidebar file?
+
+  # Write a global index containing all full names with links.
+  with open(os.path.join(output_dir, 'full_index.md'), 'w') as f:
+    f.write(parser.generate_global_index('TensorFlow', 'tensorflow',
+                                         index, duplicate_of))
+
+
+def extract():
+  """Extract docs from tf namespace and write them to disk."""
+  visitor = doc_generator_visitor.DocGeneratorVisitor()
+  api_visitor = public_api.PublicAPIVisitor(visitor)
+
+  # Access something in contrib so tf.contrib is properly loaded (it's hidden
+  # behind lazy loading)
+  # TODO(wicke): Enable contrib traversal once contrib is sealed.
+  # _ = tf.contrib.__name__
+
+  # Exclude some libaries in contrib from the documentation altogether.
+  # TODO(wicke): Shrink this list.
+  api_visitor.do_not_descend_map.update({
+      'contrib': [
+          'compiler',
+          'factorization',
+          'grid_rnn',
+          'labeled_tensor',
+          'ndlstm',
+          'quantization',
+          'session_bundle',
+          'slim',
+          'solvers',
+          'specs',
+          'tensor_forest',
+          'tensorboard',
+          'testing',
+          'tfprof',
+          'training',
+      ],
+      'contrib.bayesflow': [
+          'entropy', 'monte_carlo',
+          'special_math', 'stochastic_gradient_estimators',
+          'stochastic_graph', 'stochastic_tensor',
+          'stochastic_variables', 'variational_inference'
+      ],
+      'contrib.distributions': ['bijector'],
+      'contrib.graph_editor': [
+          'edit',
+          'match',
+          'reroute',
+          'subgraph',
+          'transform',
+          'select',
+          'util'
+      ],
+      'contrib.layers': [
+          'feature_column',
+          'summaries'
+      ],
+      'contrib.learn': [
+          'datasets',
+          'graph_actions',
+          'io',
+          'models',
+          'monitors',
+          'ops',
+          'preprocessing',
+          'utils',
+      ],
+      'contrib.util': ['loader'],
+  })
+
+  traverse.traverse(tf, api_visitor)
+
+  return visitor
+
+
+def write(output_dir, base_dir, visitor):
+  """Write documentation for an index in a `DocGeneratorVisitor` to disk.
+
+  This function will create `output_dir` if it doesn't exist, and write
+  the documentation contained in `visitor`.
+
+  Args:
+    output_dir: The directory to write documentation to. Must not exist.
+    base_dir: The base dir of the library `visitor` has traversed. This is used
+      to compute relative paths for file references.
+    visitor: A `DocGeneratorVisitor` that has traversed a library located at
+      `base_dir`.
+  """
+  duplicate_of, duplicates = visitor.find_duplicates()
+  write_docs(output_dir, os.path.abspath(base_dir),
+             duplicate_of, duplicates, visitor.index, visitor.tree)
+
+
+if __name__ == '__main__':
+  argument_parser = argparse.ArgumentParser()
+  argument_parser.add_argument(
+      '--output_dir',
+      type=str,
+      default=None,
+      required=True,
+      help='Directory to write docs to. Must not exist.'
+  )
+
+  # This doc generator works on the TensorFlow codebase. Since this script lives
+  # at tensorflow/tools/docs, we can compute the base directory (three levels
+  # up), which is valid unless we're trying to apply this to a different code
+  # base, or are moving the script around.
+  script_dir = os.path.dirname(inspect.getfile(inspect.currentframe()))
+  default_base_dir = os.path.join(script_dir, '..', '..', '..')
+
+  argument_parser.add_argument(
+      '--base_dir',
+      type=str,
+      default=default_base_dir,
+      help=('Base directory to to strip from file names referenced in docs. '
+            'Defaults to three directories up from the location of this file.')
+  )
+
+  flags, _ = argument_parser.parse_known_args()
+
+  if os.path.exists(flags.output_dir):
+    raise RuntimeError('output_dir %s exists.\n'
+                       'Cowardly refusing to wipe it, please do that yourself.'
+                       % flags.output_dir)
+
+  write(flags.output_dir, flags.base_dir, extract())

tensorflow/tools/docs/generate_test.py

@@ -0,0 +1,114 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for doc generator traversal."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+import sys
+import tempfile
+
+from tensorflow.python.platform import googletest
+from tensorflow.tools.docs import generate
+
+
+def test_function():
+  """Docstring for test_function."""
+  pass
+
+
+class TestClass(object):
+  """Docstring for TestClass itself."""
+
+  class ChildClass(object):
+    """Docstring for a child class."""
+
+    class GrandChildClass(object):
+      """Docstring for a child of a child class."""
+      pass
+
+
+class GenerateTest(googletest.TestCase):
+
+  def test_extraction(self):
+    try:
+      generate.extract()
+    except RuntimeError:
+      print('*****************************************************************')
+      print('If this test fails, you have most likely introduced an unsealed')
+      print('module. Make sure to use remove_undocumented or similar utilities')
+      print('to avoid leaking symbols. See below for more information on the')
+      print('failure.')
+      print('*****************************************************************')
+      raise
+
+  def test_write(self):
+    module = sys.modules[__name__]
+
+    index = {
+        '': sys,  # Can be any module, this test doesn't care about content.
+        'TestModule': module,
+        'test_function': test_function,
+        'TestModule.test_function': test_function,
+        'TestModule.TestClass': TestClass,
+        'TestModule.TestClass.ChildClass': TestClass.ChildClass,
+        'TestModule.TestClass.ChildClass.GrandChildClass':
+        TestClass.ChildClass.GrandChildClass,
+    }
+
+    tree = {
+        '': ['TestModule', 'test_function'],
+        'TestModule': ['test_function', 'TestClass'],
+        'TestModule.TestClass': ['ChildClass'],
+        'TestModule.TestClass.ChildClass': ['GrandChildClass'],
+        'TestModule.TestClass.ChildClass.GrandChildClass': []
+    }
+
+    duplicate_of = {
+        'TestModule.test_function': 'test_function'
+    }
+
+    duplicates = {
+        'test_function': ['test_function', 'TestModule.test_function']
+    }
+
+    output_dir = tempfile.mkdtemp()
+    base_dir = os.path.dirname(__file__)
+
+    generate.write_docs(output_dir, base_dir,
+                        duplicate_of, duplicates,
+                        index, tree)
+
+    # Make sure that the right files are written to disk.
+    self.assertTrue(os.path.exists(os.path.join(output_dir, 'index.md')))
+    self.assertTrue(os.path.exists(os.path.join(output_dir, 'full_index.md')))
+    self.assertTrue(os.path.exists(os.path.join(output_dir, 'TestModule.md')))
+    self.assertTrue(os.path.exists(os.path.join(
+        output_dir, 'test_function.md')))
+    self.assertTrue(os.path.exists(os.path.join(
+        output_dir, 'TestModule/TestClass.md')))
+    self.assertTrue(os.path.exists(os.path.join(
+        output_dir, 'TestModule/TestClass/ChildClass.md')))
+    self.assertTrue(os.path.exists(os.path.join(
+        output_dir, 'TestModule/TestClass/ChildClass/GrandChildClass.md')))
+    # Make sure that duplicates are not written
+    self.assertFalse(os.path.exists(os.path.join(
+        output_dir, 'TestModule/test_function.md')))
+
+
+if __name__ == '__main__':
+  googletest.main()

tensorflow/tools/docs/parser.py

@@ -0,0 +1,634 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Turn Python docstrings into Markdown for TensorFlow documentation."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import functools
+import inspect
+import os
+import re
+
+
+# A regular expression capturing a python indentifier.
+IDENTIFIER_RE = '[a-zA-Z_][a-zA-Z0-9_]*'
+
+
+def documentation_path(full_name):
+  """Returns the file path for the documentation for the given API symbol.
+
+  Given the fully qualified name of a library symbol, compute the path to which
+  to write the documentation for that symbol (relative to a base directory).
+  Documentation files are organized into directories that mirror the python
+  module/class structure. The path for the top-level module (whose full name is
+  '') is 'index.md'.
+
+  Args:
+    full_name: Fully qualified name of a library symbol.
+
+  Returns:
+    The file path to which to write the documentation for `full_name`.
+  """
+  # The main page is special, since it has no name in here.
+  if not full_name:
+    dirs = ['index']
+  else:
+    dirs = full_name.split('.')
+
+  return os.path.join(*dirs) + '.md'
+
+
+def _get_raw_docstring(py_object):
+  """Get the docs for a given python object.
+
+  Args:
+    py_object: A python object to retrieve the docs for (class, function/method,
+      or module).
+
+  Returns:
+    The docstring, or the empty string if no docstring was found.
+  """
+  # For object instances, inspect.getdoc does give us the docstring of their
+  # type, which is not what we want. Only return the docstring if it is useful.
+  if (inspect.isclass(py_object) or inspect.ismethod(py_object) or
+      inspect.isfunction(py_object) or inspect.ismodule(py_object) or
+      isinstance(py_object, property)):
+    return inspect.getdoc(py_object) or ''
+  else:
+    return ''
+
+
+def _get_brief_docstring(py_object):
+  """Gets the one line docstring of a python object."""
+  return _get_raw_docstring(py_object).split('\n')[0]
+
+
+def _reference_to_link(ref_full_name, relative_path_to_root, duplicate_of):
+  """Resolve a "@{symbol}" reference to a relative path, respecting duplicates.
+
+  The input to this function should already be stripped of the '@' and '{}', and
+  its output is only the link, not the full Markdown.
+
+  Args:
+    ref_full_name: The fully qualified name of the symbol to link to.
+    relative_path_to_root: The relative path from the location of the current
+      document to the root of the API documentation.
+    duplicate_of: A map from duplicate full names to master names.
+
+  Returns:
+    A relative path that links from the documentation page of `from_full_name`
+    to the documentation page of `ref_full_name`.
+  """
+  master_name = duplicate_of.get(ref_full_name, ref_full_name)
+  ref_path = documentation_path(master_name)
+  return os.path.join(relative_path_to_root, ref_path)
+
+
+def _markdown_link(link_text, ref_full_name, relative_path_to_root,
+                   duplicate_of):
+  """Resolve a "@{symbol}" reference to a Markdown link, respecting duplicates.
+
+  The input to this function should already be stripped of the '@' and '{}'.
+  This function returns a Markdown link. It is assumed that this is a code
+  reference, so the link text will always be rendered as code (using backticks).
+
+  `link_text` should refer to a library symbol. You can either refer to it with
+  or without the `tf.` prefix.
+
+  Args:
+    link_text: The text of the Markdown link.
+    ref_full_name: The fully qualified name of the symbol to link to
+      (may optionally include 'tf.').
+    relative_path_to_root: The relative path from the location of the current
+      document to the root of the API documentation.
+    duplicate_of: A map from duplicate full names to master names.
+
+  Returns:
+    A markdown link from the documentation page of `from_full_name`
+    to the documentation page of `ref_full_name`.
+  """
+  if ref_full_name.startswith('tf.'):
+    ref_full_name = ref_full_name[3:]
+
+  return '[`%s`](%s)' % (
+      link_text,
+      _reference_to_link(ref_full_name, relative_path_to_root, duplicate_of))
+
+
+def replace_references(string, relative_path_to_root, duplicate_of):
+  """Replace "@{symbol}" references with links to symbol's documentation page.
+
+  This functions finds all occurrences of "@{symbol}" in `string` and replaces
+  them with markdown links to the documentation page for "symbol".
+
+  `relative_path_to_root` is the relative path from the document that contains
+  the "@{symbol}" reference to the root of the API documentation that is linked
+  to. If the containing page is part of the same API docset,
+  `relative_path_to_root` can be set to
+  `os.path.dirname(documentation_path(name))`, where `name` is the python name
+  of the object whose documentation page the reference lives on.
+
+  Args:
+    string: A string in which "@{symbol}" references should be replaced.
+    relative_path_to_root: The relative path from the contianing document to the
+      root of the API documentation that is being linked to.
+    duplicate_of: A map from duplicate names to preferred names of API symbols.
+
+  Returns:
+    `string`, with "@{symbol}" references replaced by Markdown links.
+  """
+  full_name_re = '%s(.%s)*' % (IDENTIFIER_RE, IDENTIFIER_RE)
+  symbol_reference_re = re.compile(r'@\{(' + full_name_re + r')\}')
+  match = symbol_reference_re.search(string)
+  while match:
+    symbol_name = match.group(1)
+    link_text = _markdown_link(symbol_name, symbol_name,
+                               relative_path_to_root, duplicate_of)
+
+    # Remove only the '@symbol' part of the match, and replace with the link.
+    string = string[:match.start()] + link_text + string[match.end():]
+    match = symbol_reference_re.search(string,
+                                       pos=match.start() + len(link_text))
+  return string
+
+
+def _md_docstring(py_object, relative_path_to_root, duplicate_of):
+  """Get the docstring from an object and make it into nice Markdown.
+
+  For links within the same set of docs, the `relative_path_to_root` for a
+  docstring on the page for `full_name` can be set to
+
+  ```python
+  relative_path_to_root = os.path.relpath(
+    os.path.dirname(documentation_path(full_name)) or '.', '.')
+  ```
+
+  Args:
+    py_object: A python object to retrieve the docs for (class, function/method,
+      or module).
+    relative_path_to_root: The relative path from the location of the current
+      document to the root of the API documentation. This is used to compute
+      links for "@symbol" references.
+    duplicate_of: A map from duplicate symbol names to master names. Used to
+      resolve "@symbol" references.
+
+  Returns:
+    The docstring, or the empty string if no docstring was found.
+  """
+  # TODO(wicke): If this is a partial, use the .func docstring and add a note.
+  raw_docstring = _get_raw_docstring(py_object)
+  raw_lines = raw_docstring.split('\n')
+
+  # Define regular expressions used during parsing below.
+  symbol_list_item_re = re.compile(r'^  (%s): ' % IDENTIFIER_RE)
+  section_re = re.compile(r'^(\w+):\s*$')
+
+  # Translate docstring line by line.
+  in_special_section = False
+  lines = []
+
+  def is_section_start(i):
+    # Previous line is empty, line i is "Word:", and next line is indented.
+    return (i > 0 and not raw_lines[i-1].strip() and
+            re.match(section_re, raw_lines[i]) and
+            len(raw_lines) > i+1 and raw_lines[i+1].startswith('  '))
+
+  for i, line in enumerate(raw_lines):
+    if not in_special_section and is_section_start(i):
+      in_special_section = True
+      lines.append('#### ' + section_re.sub(r'\1:', line))
+      lines.append('')
+      continue
+
+    # If the next line starts a new section, this one ends. Add an extra line.
+    if in_special_section and is_section_start(i+1):
+      in_special_section = False
+      lines.append('')
+
+    if in_special_section:
+      # Translate symbols in 'Args:', 'Parameters:', 'Raises:', etc. sections.
+      lines.append(symbol_list_item_re.sub(r'* <b>`\1`</b>: ', line))
+    else:
+      lines.append(line)
+
+  docstring = '\n'.join(lines)
+
+  # TODO(deannarubin): Improve formatting for devsite
+  # TODO(deannarubin): Interpret @compatibility and other formatting notes.
+
+  return replace_references(docstring, relative_path_to_root, duplicate_of)
+
+
+def _get_arg_spec(func):
+  """Extracts signature information from a function or functools.partial object.
+
+  For functions, uses `inspect.getargspec`. For `functools.partial` objects,
+  corrects the signature of the underlying function to take into account the
+  removed arguments.
+
+  Args:
+    func: A function whose signature to extract.
+
+  Returns:
+    An `ArgSpec` namedtuple `(args, varargs, keywords, defaults)`, as returned
+    by `inspect.getargspec`.
+  """
+  # getargspec does not work for functools.partial objects directly.
+  if isinstance(func, functools.partial):
+    argspec = inspect.getargspec(func.func)
+    # Remove the args from the original function that have been used up.
+    first_default_arg = (
+        len(argspec.args or []) - len(argspec.defaults or []))
+    partial_args = len(func.args)
+    argspec_args = []
+
+    if argspec.args:
+      argspec_args = list(argspec.args[partial_args:])
+
+    argspec_defaults = list(argspec.defaults or ())
+    if argspec.defaults and partial_args > first_default_arg:
+      argspec_defaults = list(argspec.defaults[partial_args-first_default_arg:])
+
+    first_default_arg = max(0, first_default_arg - partial_args)
+    for kwarg in func.keywords:
+      if kwarg in argspec.args:
+        i = argspec_args.index(kwarg)
+        argspec_args.pop(i)
+        if i >= first_default_arg:
+          argspec_defaults.pop(i-first_default_arg)
+        else:
+          first_default_arg -= 1
+    return inspect.ArgSpec(args=argspec_args,
+                           varargs=argspec.varargs,
+                           keywords=argspec.keywords,
+                           defaults=tuple(argspec_defaults))
+  else:  # Regular function or method, getargspec will work fine.
+    return inspect.getargspec(func)
+
+
+def _generate_signature(func):
+  """Given a function, returns a string representing its args.
+
+  This function produces a string representing the arguments to a python
+  function, including surrounding parentheses. It uses inspect.getargspec, which
+  does not generalize well to Python 3.x, which is more flexible in how *args
+  and **kwargs are handled. This is not a problem in TF, since we have to remain
+  compatible to Python 2.7 anyway.
+
+  This function uses `__name__` for callables if it is available. This can lead
+  to poor results for functools.partial and other callable objects.
+
+  The returned string is Python code, so if it is included in a Markdown
+  document, it should be typeset as code (using backticks), or escaped.
+
+  Args:
+    func: A function of method to extract the signature for (anything
+      `inspect.getargspec` will accept).
+
+  Returns:
+    A string representing the signature of `func` as python code.
+  """
+
+  # This produces poor signatures for decorated functions.
+  # TODO(wicke): We need to use something like the decorator module to fix it.
+
+  args_list = []
+
+  argspec = _get_arg_spec(func)
+  first_arg_with_default = (
+      len(argspec.args or []) - len(argspec.defaults or []))
+
+  # Python documentation skips `self` when printing method signatures.
+  first_arg = 1 if inspect.ismethod(func) and 'self' in argspec.args[:1] else 0
+
+  # Add all args without defaults.
+  for arg in argspec.args[first_arg:first_arg_with_default]:
+    args_list.append(arg)
+
+  # Add all args with defaults.
+  if argspec.defaults:
+    for arg, default in zip(
+        argspec.args[first_arg_with_default:], argspec.defaults):
+      # Some callables don't have __name__, fall back to including their repr.
+      # TODO(wicke): This could be improved at least for common cases.
+      if callable(default) and hasattr(default, '__name__'):
+        args_list.append('%s=%s' % (arg, default.__name__))
+      else:
+        args_list.append('%s=%r' % (arg, default))
+
+  # Add *args and *kwargs.
+  if argspec.varargs:
+    args_list.append('*' + argspec.varargs)
+  if argspec.keywords:
+    args_list.append('**' + argspec.keywords)
+
+  return '(%s)' % ', '.join(args_list)
+
+
+def _generate_markdown_for_function(full_name, duplicate_names,
+                                    function, duplicate_of):
+  """Generate Markdown docs for a function or method.
+
+  This function creates a documentation page for a function. It uses the
+  function name (incl. signature) as the title, followed by a list of duplicate
+  names (if there are any), and the Markdown formatted docstring of the
+  function.
+
+  Args:
+    full_name: The preferred name of the function. Used in the title. Must not
+      be present in `duplicate_of` (master names never are).
+    duplicate_names: A sorted list of alternative names (incl. `full_name`).
+    function: The python object referenced by `full_name`.
+    duplicate_of: A map of duplicate full names to master names. Used to resolve
+      @{symbol} references in the docstring.
+
+  Returns:
+    A string that can be written to a documentation file for this function.
+  """
+  # TODO(wicke): Make sure this works for partials.
+  relative_path = os.path.relpath(
+      os.path.dirname(documentation_path(full_name)) or '.', '.')
+  docstring = _md_docstring(function, relative_path, duplicate_of)
+  signature = _generate_signature(function)
+
+  if duplicate_names:
+    aliases = '\n'.join(['### `%s`' % (name + signature)
+                         for name in duplicate_names])
+    aliases += '\n\n'
+  else:
+    aliases = ''
+
+  return '#`%s%s`\n\n%s%s' % (full_name, signature, aliases, docstring)
+
+
+def _generate_markdown_for_class(full_name, duplicate_names, py_class,
+                                 duplicate_of, index, tree):
+  """Generate Markdown docs for a class.
+
+  This function creates a documentation page for a class. It uses the
+  class name as the title, followed by a list of duplicate
+  names (if there are any), the Markdown formatted docstring of the
+  class, a list of links to all child class docs, a list of all properties
+  including their docstrings, a list of all methods incl. their docstrings, and
+  a list of all class member names (public fields).
+
+  Args:
+    full_name: The preferred name of the class. Used in the title. Must not
+      be present in `duplicate_of` (master names never are).
+    duplicate_names: A sorted list of alternative names (incl. `full_name`).
+    py_class: The python object referenced by `full_name`.
+    duplicate_of: A map of duplicate full names to master names. Used to resolve
+      @{symbol} references in the docstrings.
+    index: A map from full names to python object references.
+    tree: A map from full names to the names of all documentable child objects.
+
+  Returns:
+    A string that can be written to a documentation file for this class.
+  """
+  relative_path = os.path.relpath(
+      os.path.dirname(documentation_path(full_name)) or '.', '.')
+  docstring = _md_docstring(py_class, relative_path, duplicate_of)
+  if duplicate_names:
+    aliases = '\n'.join(['### `class %s`' % name for name in duplicate_names])
+    aliases += '\n\n'
+  else:
+    aliases = ''
+
+  docs = '# `%s`\n\n%s%s\n\n' % (full_name, aliases, docstring)
+
+  field_names = []
+  properties = []
+  methods = []
+  class_links = []
+  for member in tree[full_name]:
+    child_name = '.'.join([full_name, member])
+    child = index[child_name]
+
+    if isinstance(child, property):
+      properties.append((member, child))
+    elif inspect.isclass(child):
+      class_links.append(_markdown_link('class ' + member, child_name,
+                                        relative_path, duplicate_of))
+    elif inspect.ismethod(child) or inspect.isfunction(child):
+      methods.append((member, child))
+    else:
+      # TODO(wicke): We may want to also remember the object itself.
+      field_names.append(member)
+
+  if class_links:
+    docs += '## Child Classes\n'
+    docs += '\n\n'.join(sorted(class_links))
+    docs += '\n\n'
+
+  if properties:
+    docs += '## Properties\n\n'
+    for property_name, prop in sorted(properties, key=lambda x: x[0]):
+      docs += '### `%s`\n\n%s\n\n' % (
+          property_name, _md_docstring(prop, relative_path, duplicate_of))
+    docs += '\n\n'
+
+  if methods:
+    docs += '## Methods\n\n'
+    for method_name, method in sorted(methods, key=lambda x: x[0]):
+      method_signature = method_name + _generate_signature(method)
+      docs += '### `%s`\n\n%s\n\n' % (method_signature,
+                                      _md_docstring(method, relative_path,
+                                                    duplicate_of))
+    docs += '\n\n'
+
+  if field_names:
+    docs += '## Class Members\n\n'
+    # TODO(wicke): Document the value of the members, at least for basic types.
+    docs += '\n\n'.join(sorted(field_names))
+    docs += '\n\n'
+
+  return docs
+
+
+def _generate_markdown_for_module(full_name, duplicate_names, module,
+                                  duplicate_of, index, tree):
+  """Generate Markdown docs for a module.
+
+  This function creates a documentation page for a module. It uses the
+  module name as the title, followed by a list of duplicate
+  names (if there are any), the Markdown formatted docstring of the
+  class, and a list of links to all members of this module.
+
+  Args:
+    full_name: The preferred name of the module. Used in the title. Must not
+      be present in `duplicate_of` (master names never are).
+    duplicate_names: A sorted list of alternative names (incl. `full_name`).
+    module: The python object referenced by `full_name`.
+    duplicate_of: A map of duplicate full names to master names. Used to resolve
+      @{symbol} references in the docstrings.
+    index: A map from full names to python object references.
+    tree: A map from full names to the names of all documentable child objects.
+
+  Returns:
+    A string that can be written to a documentation file for this module.
+  """
+  relative_path = os.path.relpath(
+      os.path.dirname(documentation_path(full_name)) or '.', '.')
+  docstring = _md_docstring(module, relative_path, duplicate_of)
+  if duplicate_names:
+    aliases = '\n'.join(['### Module `%s`' % name for name in duplicate_names])
+    aliases += '\n\n'
+  else:
+    aliases = ''
+
+  member_names = tree.get(full_name, [])
+
+  # Make links to all members.
+  member_links = []
+  for name in member_names:
+    member_full_name = full_name + '.' + name if full_name else name
+    member = index[member_full_name]
+
+    if inspect.isclass(member):
+      link_text = 'class ' + name
+    elif inspect.isfunction(member):
+      link_text = name + _generate_signature(member)
+    else:
+      link_text = name
+
+    member_links.append(_markdown_link(link_text, member_full_name,
+                                       relative_path, duplicate_of))
+
+  # TODO(deannarubin): Make this list into a table and add the brief docstring.
+  # (use _get_brief_docstring)
+
+  return '# Module `%s`\n\n%s%s\n\n## Members\n\n%s' % (
+      full_name, aliases, docstring, '\n\n'.join(member_links))
+
+
+def generate_markdown(full_name, py_object,
+                      duplicate_of, duplicates,
+                      index, tree, base_dir):
+  """Generate Markdown docs for a given object that's part of the TF API.
+
+  This function uses _md_docstring to obtain the docs pertaining to
+  `object`.
+
+  This function resolves '@symbol' references in the docstrings into links to
+  the appropriate location. It also adds a list of alternative names for the
+  symbol automatically.
+
+  It assumes that the docs for each object live in a file given by
+  `documentation_path`, and that relative links to files within the
+  documentation are resolvable.
+
+  The output is Markdown that can be written to file and published.
+
+  Args:
+    full_name: The fully qualified name (excl. "tf.") of the symbol to be
+      documented.
+    py_object: The Python object to be documented. Its documentation is sourced
+      from `py_object`'s docstring.
+    duplicate_of: A `dict` mapping fully qualified names to "master" names. This
+      is used to resolve "@{symbol}" references to the "master" name.
+    duplicates: A `dict` mapping fully qualified names to a set of all
+      aliases of this name. This is used to automatically generate a list of all
+      aliases for each name.
+    index: A `dict` mapping fully qualified names to the corresponding Python
+      objects. Used to produce docs for child objects, and to check the validity
+      of "@{symbol}" references.
+    tree: A `dict` mapping a fully qualified name to the names of all its
+      members. Used to populate the members section of a class or module page.
+    base_dir: A base path that is stripped from file locations written to the
+      docs.
+
+  Returns:
+    A string containing the Markdown docs for `py_object`.
+
+  Raises:
+    RuntimeError: If an object is encountered for which we don't know how
+      to make docs.
+  """
+
+  # Which other aliases exist for the object referenced by full_name?
+  master_name = duplicate_of.get(full_name, full_name)
+  duplicate_names = duplicates.get(master_name, [full_name])
+
+  # TODO(wicke): Once other pieces are ready, enable this also for partials.
+  if (inspect.ismethod(py_object) or inspect.isfunction(py_object) or
+      # Some methods in classes from extensions come in as routines.
+      inspect.isroutine(py_object)):
+    markdown = _generate_markdown_for_function(master_name, duplicate_names,
+                                               py_object, duplicate_of)
+  elif inspect.isclass(py_object):
+    markdown = _generate_markdown_for_class(master_name, duplicate_names,
+                                            py_object, duplicate_of,
+                                            index, tree)
+  elif inspect.ismodule(py_object):
+    markdown = _generate_markdown_for_module(master_name, duplicate_names,
+                                             py_object, duplicate_of,
+                                             index, tree)
+  else:
+    raise RuntimeError('Cannot make docs for object %s: %r' % (full_name,
+                                                               py_object))
+
+  # Every page gets a note on the bottom about where this object is defined
+  # TODO(wicke): If py_object is decorated, get the decorated object instead.
+  # TODO(wicke): Only use decorators that support this in TF.
+
+  try:
+    path = os.path.relpath(inspect.getfile(py_object), base_dir)
+
+    # TODO(wicke): If this is a generated file, point to the source instead.
+
+    # Never include links outside this code base.
+    if not path.startswith('..'):
+      # TODO(wicke): Make this a link to github.
+      markdown += '\n\ndefined in %s\n\n' % path
+  except TypeError:  # getfile throws TypeError if py_object is a builtin.
+    markdown += '\n\nthis is an alias for a Python built-in.'
+
+  return markdown
+
+
+def generate_global_index(library_name, root_name, index, duplicate_of):
+  """Given a dict of full names to python objects, generate an index page.
+
+  The index page generated contains a list of links for all symbols in `index`
+  that have their own documentation page.
+
+  Args:
+    library_name: The name for the documented library to use in the title.
+    root_name: The name to use for the root module.
+    index: A dict mapping full names to python objects.
+    duplicate_of: A map of duplicate names to preferred names.
+
+  Returns:
+    A string containing an index page as Markdown.
+  """
+  symbol_links = []
+  for full_name, py_object in index.iteritems():
+    index_name = full_name or root_name
+    if (inspect.ismodule(py_object) or inspect.isfunction(py_object) or
+        inspect.isclass(py_object)):
+      symbol_links.append((index_name,
+                           _markdown_link(index_name, full_name,
+                                          '.', duplicate_of)))
+
+  lines = ['# All symbols in %s' % library_name, '']
+  for _, link in sorted(symbol_links, key=lambda x: x[0]):
+    lines.append('*  %s' % link)
+
+  # TODO(deannarubin): Make this list into a table and add the brief docstring.
+  # (use _get_brief_docstring)
+
+  return '\n'.join(lines)

tensorflow/tools/docs/parser_test.py

@@ -0,0 +1,349 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for documentation parser."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import functools
+import inspect
+import os
+import sys
+
+from tensorflow.python.platform import googletest
+from tensorflow.tools.docs import parser
+
+
+def test_function_for_markdown_reference(unused_arg):
+  """Docstring with reference to @{test_function}."""
+  pass
+
+
+def test_function(unused_arg, unused_kwarg='default'):
+  """Docstring for test function."""
+  pass
+
+
+def test_function_with_args_kwargs(unused_arg, *unused_args, **unused_kwargs):
+  """Docstring for second test function."""
+  pass
+
+
+def test_function_with_fancy_docstring(arg):
+  """Function with a fancy docstring.
+
+  Args:
+    arg: An argument.
+
+  Returns:
+    arg: the input, and
+    arg: the input, again.
+  """
+  return arg, arg
+
+
+class TestClass(object):
+  """Docstring for TestClass itself."""
+
+  def a_method(self, arg='default'):
+    """Docstring for a method."""
+    pass
+
+  class ChildClass(object):
+    """Docstring for a child class."""
+    pass
+
+  @property
+  def a_property(self):
+    """Docstring for a property."""
+    pass
+
+  CLASS_MEMBER = 'a class member'
+
+
+class ParserTest(googletest.TestCase):
+
+  def test_documentation_path(self):
+    self.assertEqual('test.md', parser.documentation_path('test'))
+    self.assertEqual('test/module.md', parser.documentation_path('test.module'))
+
+  def test_documentation_path_empty(self):
+    self.assertEqual('index.md', parser.documentation_path(''))
+
+  def test_replace_references(self):
+    string = 'A @{reference}, another @{tf.reference}, and a @{third}.'
+    duplicate_of = {'third': 'fourth'}
+    result = parser.replace_references(string, '../..', duplicate_of)
+    self.assertEqual(
+        'A [`reference`](../../reference.md), another '
+        '[`tf.reference`](../../reference.md), '
+        'and a [`third`](../../fourth.md).',
+        result)
+
+  def test_generate_markdown_for_class(self):
+
+    index = {
+        'TestClass': TestClass,
+        'TestClass.a_method': TestClass.a_method,
+        'TestClass.a_property': TestClass.a_property,
+        'TestClass.ChildClass': TestClass.ChildClass,
+        'TestClass.CLASS_MEMBER': TestClass.CLASS_MEMBER
+    }
+
+    tree = {
+        'TestClass': ['a_method', 'a_property', 'ChildClass', 'CLASS_MEMBER']
+    }
+
+    docs = parser.generate_markdown(full_name='TestClass', py_object=TestClass,
+                                    duplicate_of={}, duplicates={},
+                                    index=index, tree=tree, base_dir='/')
+
+    # Make sure all required docstrings are present.
+    self.assertTrue(inspect.getdoc(TestClass) in docs)
+    self.assertTrue(inspect.getdoc(TestClass.a_method) in docs)
+    self.assertTrue(inspect.getdoc(TestClass.a_property) in docs)
+
+    # Make sure that the signature is extracted properly and omits self.
+    self.assertTrue('a_method(arg=\'default\')' in docs)
+
+    # Make sure there is a link to the child class and it points the right way.
+    self.assertTrue('[`class ChildClass`](./TestClass/ChildClass.md)' in docs)
+
+    # Make sure CLASS_MEMBER is mentioned.
+    self.assertTrue('CLASS_MEMBER' in docs)
+
+    # Make sure this file is contained as the definition location.
+    self.assertTrue(os.path.relpath(__file__, '/') in docs)
+
+  def test_generate_markdown_for_module(self):
+    module = sys.modules[__name__]
+
+    index = {
+        'TestModule': module,
+        'TestModule.test_function': test_function,
+        'TestModule.test_function_with_args_kwargs':
+        test_function_with_args_kwargs,
+        'TestModule.TestClass': TestClass,
+    }
+
+    tree = {
+        'TestModule': ['TestClass', 'test_function',
+                       'test_function_with_args_kwargs']
+    }
+
+    docs = parser.generate_markdown(full_name='TestModule', py_object=module,
+                                    duplicate_of={}, duplicates={},
+                                    index=index, tree=tree, base_dir='/')
+
+    # Make sure all required docstrings are present.
+    self.assertTrue(inspect.getdoc(module) in docs)
+
+    # Make sure that links to the members are there (not asserting on exact link
+    # text for functions).
+    self.assertTrue('./TestModule/test_function.md' in docs)
+    self.assertTrue('./TestModule/test_function_with_args_kwargs.md' in docs)
+
+    # Make sure there is a link to the child class and it points the right way.
+    self.assertTrue('[`class TestClass`](./TestModule/TestClass.md)' in docs)
+
+    # Make sure this file is contained as the definition location.
+    self.assertTrue(os.path.relpath(__file__, '/') in docs)
+
+  def test_generate_markdown_for_function(self):
+    index = {
+        'test_function': test_function
+    }
+
+    tree = {
+        '': ['test_function']
+    }
+
+    docs = parser.generate_markdown(full_name='test_function',
+                                    py_object=test_function,
+                                    duplicate_of={}, duplicates={},
+                                    index=index, tree=tree, base_dir='/')
+
+    # Make sure docstring shows up.
+    self.assertTrue(inspect.getdoc(test_function) in docs)
+
+    # Make sure the extracted signature is good.
+    self.assertTrue(
+        'test_function(unused_arg, unused_kwarg=\'default\')' in docs)
+
+    # Make sure this file is contained as the definition location.
+    self.assertTrue(os.path.relpath(__file__, '/') in docs)
+
+  def test_generate_markdown_for_function_with_kwargs(self):
+    index = {
+        'test_function_with_args_kwargs': test_function_with_args_kwargs
+    }
+
+    tree = {
+        '': ['test_function_with_args_kwargs']
+    }
+
+    docs = parser.generate_markdown(full_name='test_function_with_args_kwargs',
+                                    py_object=test_function_with_args_kwargs,
+                                    duplicate_of={}, duplicates={},
+                                    index=index, tree=tree, base_dir='/')
+
+    # Make sure docstring shows up.
+    self.assertTrue(inspect.getdoc(test_function_with_args_kwargs) in docs)
+
+    # Make sure the extracted signature is good.
+    self.assertTrue(
+        'test_function_with_args_kwargs(unused_arg,'
+        ' *unused_args, **unused_kwargs)' in docs)
+
+  def test_references_replaced_in_generated_markdown(self):
+    index = {
+        'test_function_for_markdown_reference':
+        test_function_for_markdown_reference
+    }
+
+    tree = {
+        '': ['test_function_for_markdown_reference']
+    }
+
+    docs = parser.generate_markdown(
+        full_name='test_function_for_markdown_reference',
+        py_object=test_function_for_markdown_reference,
+        duplicate_of={}, duplicates={},
+        index=index, tree=tree, base_dir='/')
+
+    # Make sure docstring shows up and is properly processed.
+    expected_docs = parser.replace_references(
+        inspect.getdoc(test_function_for_markdown_reference),
+        relative_path_to_root='.', duplicate_of={})
+
+    self.assertTrue(expected_docs in docs)
+
+  def test_docstring_special_section(self):
+    index = {
+        'test_function': test_function_with_fancy_docstring
+    }
+
+    tree = {
+        '': 'test_function'
+    }
+
+    docs = parser.generate_markdown(
+        full_name='test_function',
+        py_object=test_function_with_fancy_docstring,
+        duplicate_of={}, duplicates={},
+        index=index, tree=tree, base_dir='/')
+
+    expected = '\n'.join([
+        'Function with a fancy docstring.',
+        '',
+        '#### Args:',
+        '',
+        '* <b>`arg`</b>: An argument.',
+        '',
+        '',
+        '#### Returns:',
+        '',
+        '* <b>`arg`</b>: the input, and',
+        '* <b>`arg`</b>: the input, again.',
+        ''])
+    self.assertTrue(expected in docs)
+
+  def test_generate_index(self):
+    module = sys.modules[__name__]
+
+    index = {
+        'TestModule': module,
+        'test_function': test_function,
+        'TestModule.test_function': test_function,
+        'TestModule.TestClass': TestClass,
+        'TestModule.TestClass.a_method': TestClass.a_method,
+        'TestModule.TestClass.a_property': TestClass.a_property,
+        'TestModule.TestClass.ChildClass': TestClass.ChildClass,
+    }
+
+    duplicate_of = {
+        'TestModule.test_function': 'test_function'
+    }
+
+    docs = parser.generate_global_index('TestLibrary', 'test',
+                                        index=index,
+                                        duplicate_of=duplicate_of)
+
+    # Make sure duplicates and non-top-level symbols are in the index, but
+    # methods and properties are not.
+    self.assertTrue('a_method' not in docs)
+    self.assertTrue('a_property' not in docs)
+    self.assertTrue('TestModule.TestClass' in docs)
+    self.assertTrue('TestModule.TestClass.ChildClass' in docs)
+    self.assertTrue('TestModule.test_function' in docs)
+    # Leading backtick to make sure it's included top-level.
+    # This depends on formatting, but should be stable.
+    self.assertTrue('`test_function' in docs)
+
+  def test_argspec_for_functoos_partial(self):
+
+    # pylint: disable=unused-argument
+    def test_function_for_partial1(arg1, arg2, kwarg1=1, kwarg2=2):
+      pass
+
+    def test_function_for_partial2(arg1, arg2, *my_args, **my_kwargs):
+      pass
+    # pylint: enable=unused-argument
+
+    # pylint: disable=protected-access
+    # Make sure everything works for regular functions.
+    expected = inspect.ArgSpec(['arg1', 'arg2', 'kwarg1', 'kwarg2'], None, None,
+                               (1, 2))
+    self.assertEqual(expected, parser._get_arg_spec(test_function_for_partial1))
+
+    # Make sure doing nothing works.
+    expected = inspect.ArgSpec(['arg1', 'arg2', 'kwarg1', 'kwarg2'], None, None,
+                               (1, 2))
+    partial = functools.partial(test_function_for_partial1)
+    self.assertEqual(expected, parser._get_arg_spec(partial))
+
+    # Make sure setting args from the front works.
+    expected = inspect.ArgSpec(['arg2', 'kwarg1', 'kwarg2'], None, None, (1, 2))
+    partial = functools.partial(test_function_for_partial1, 1)
+    self.assertEqual(expected, parser._get_arg_spec(partial))
+
+    expected = inspect.ArgSpec(['kwarg2',], None, None, (2,))
+    partial = functools.partial(test_function_for_partial1, 1, 2, 3)
+    self.assertEqual(expected, parser._get_arg_spec(partial))
+
+    # Make sure setting kwargs works.
+    expected = inspect.ArgSpec(['arg1', 'arg2', 'kwarg2'], None, None, (2,))
+    partial = functools.partial(test_function_for_partial1, kwarg1=0)
+    self.assertEqual(expected, parser._get_arg_spec(partial))
+
+    expected = inspect.ArgSpec(['arg1', 'arg2', 'kwarg1'], None, None, (1,))
+    partial = functools.partial(test_function_for_partial1, kwarg2=0)
+    self.assertEqual(expected, parser._get_arg_spec(partial))
+
+    expected = inspect.ArgSpec(['arg1'], None, None, ())
+    partial = functools.partial(test_function_for_partial1,
+                                arg2=0, kwarg1=0, kwarg2=0)
+    self.assertEqual(expected, parser._get_arg_spec(partial))
+
+    # Make sure *args, *kwargs is accounted for.
+    expected = inspect.ArgSpec([], 'my_args', 'my_kwargs', ())
+    partial = functools.partial(test_function_for_partial2, 0, 1)
+    self.assertEqual(expected, parser._get_arg_spec(partial))
+
+    # pylint: enable=protected-access
+
+if __name__ == '__main__':
+  googletest.main()