experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Use tf. for all symbols.

Browse Source 
Change base_dir to exclude tensorflow/.

Improve the doc generator's handling of defaults in argument lists. Use an index to look up the preferred name for objects that are passed around as defaults which has a canonical TensorFlow name. Use ast and codegen to reproduce what was in the actual code for everything else.

Also simplify @{} replace logic.
Change: 146750326
This commit is contained in:
Martin Wicke 2017-02-06 22:19:07 -08:00 committed by TensorFlower Gardener
parent 8b5a8df5e3
commit 3414c1ebab
9 changed files with 215 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
tensorflow/python/framework/random_seed.py
View File

@ -39,7 +39,7 @@ def get_seed(op_seed):
graph, or for only specific operations.
For details on how the graph-level seed interacts with op seeds, see
@{set_random_seed}.
@{tf.set_random_seed}.
Args:
op_seed: integer.

1
tensorflow/tools/docs/BUILD
View File

@ -52,6 +52,7 @@ py_test(
"parser_test.py",
],
srcs_version = "PY2AND3",
tags = ["manual"],
deps = [
":parser",
"//tensorflow/python:platform_test",

100
tensorflow/tools/docs/doc_generator_visitor.py
View File

@ -26,9 +26,24 @@ import six
class DocGeneratorVisitor(object):
"""A visitor that generates docs for a python object when __call__ed."""
def __init__(self):
def __init__(self, root_name=''):
"""Make a visitor.
As this visitor is starting its traversal at a module or class, it will not
be old the name of that object during traversal. `root_name` is the name it
should use for that object, effectively prefixing all names with
"root_name.".
Args:
root_name: The name of the root module/class.
"""
self._root_name = root_name or ''
self._prefix = (root_name + '.') if root_name else ''
self._index = {}
self._tree = {}
self._reverse_index = None
self._duplicates = None
self._duplicate_of = None
@property
def index(self):
@ -53,6 +68,52 @@ class DocGeneratorVisitor(object):
"""
return self._tree
@property
def reverse_index(self):
"""A map from `id(object)` to the preferred fully qualified name.
This map only contains non-primitive objects (no numbers or strings) present
in `index` (for primitive objects, `id()` doesn't quite do the right thing).
It is computed when it, `duplicate_of`, or `duplicates` are first accessed.
Returns:
The `id(object)` to full name map.
"""
self._maybe_find_duplicates()
return self._reverse_index
@property
def duplicate_of(self):
"""A map from duplicate full names to a preferred fully qualified name.
This map only contains names that are not themself a preferred name.
It is computed when it, `reverse_index`, or `duplicates` are first accessed.
Returns:
The map from duplicate name to preferred name.
"""
self._maybe_find_duplicates()
return self._duplicate_of
@property
def duplicates(self):
"""A map from preferred full names to a list of all names for this symbol.
This function returns a map from preferred (master) name for a symbol to a
lexicographically sorted list of all aliases for that name (incl. the master
name). Symbols without duplicate names do not appear in this map.
It is computed when it, `reverse_index`, or `duplicate_of` are first
accessed.
Returns:
The map from master name to list of all duplicate names.
"""
self._maybe_find_duplicates()
return self._duplicates
def __call__(self, parent_name, parent, children):
"""Visitor interface, see `tensorflow/tools/common:traverse` for details.
@ -71,6 +132,7 @@ class DocGeneratorVisitor(object):
RuntimeError: If this visitor is called with a `parent` that is not a
class or module.
"""
parent_name = self._prefix + parent_name if parent_name else self._root_name
self._index[parent_name] = parent
self._tree[parent_name] = []
@ -87,19 +149,23 @@ class DocGeneratorVisitor(object):
self._index[full_name] = child
self._tree[parent_name].append(name)
def find_duplicates(self):
def _maybe_find_duplicates(self):
"""Compute data structures containing information about duplicates.
Find duplicates in `index` and decide on one to be the "master" name.
Returns a map `duplicate_of` from aliases to their master name (the master
name itself has no entry in this map), and a map `duplicates` from master
names to a lexicographically sorted list of all aliases for that name (incl.
the master name).
Computes a reverse_index mapping each object id to its master name.
Returns:
A tuple `(duplicate_of, duplicates)` as described above.
Also computes a map `duplicate_of` from aliases to their master name (the
master name itself has no entry in this map), and a map `duplicates` from
master names to a lexicographically sorted list of all aliases for that name
(incl. the master name).
All these are computed and set as fields if they haven't aready.
"""
if self._reverse_index is not None:
return
# Maps the id of a symbol to its fully qualified name. For symbols that have
# several aliases, this map contains the first one found.
# We use id(py_object) to get a hashable value for py_object. Note all
@ -110,15 +176,12 @@ class DocGeneratorVisitor(object):
# maps the first name found to a list of all duplicate names.
raw_duplicates = {}
for full_name, py_object in six.iteritems(self._index):
# We cannot use the duplicate mechanism for constants, since e.g.,
# We cannot use the duplicate mechanism for some constants, since e.g.,
# id(c1) == id(c2) with c1=1, c2=1. This is unproblematic since constants
# have no usable docstring and won't be documented automatically.
if (inspect.ismodule(py_object) or
inspect.isclass(py_object) or
inspect.isfunction(py_object) or
inspect.isroutine(py_object) or
inspect.ismethod(py_object) or
isinstance(py_object, property)):
if py_object is not None and not isinstance(
py_object, six.integer_types + six.string_types +
(six.binary_type, six.text_type, float, complex, bool)):
object_id = id(py_object)
if object_id in reverse_index:
master_name = reverse_index[object_id]
@ -148,4 +211,9 @@ class DocGeneratorVisitor(object):
if name != master_name:
duplicate_of[name] = master_name
return duplicate_of, duplicates
# Set the reverse index to the canonical name.
reverse_index[id(self._index[master_name])] = master_name
self._duplicate_of = duplicate_of
self._duplicates = duplicates
self._reverse_index = reverse_index

12
tensorflow/tools/docs/doc_generator_visitor_test.py
View File

@ -75,8 +75,6 @@ class DocGeneratorVisitorTest(googletest.TestCase):
[('index', doc_generator_visitor.DocGeneratorVisitor.index),
('index2', doc_generator_visitor.DocGeneratorVisitor.index)])
duplicate_of, duplicates = visitor.find_duplicates()
# The shorter path should be master, or if equal, the lexicographically
# first will be.
self.assertEqual(
@ -91,7 +89,7 @@ class DocGeneratorVisitorTest(googletest.TestCase):
'DocGeneratorVisitor2.index',
'DocGeneratorVisitor2.index2'
]),
}, duplicates)
}, visitor.duplicates)
self.assertEqual({
'submodule.DocGeneratorVisitor': 'DocGeneratorVisitor2',
'submodule.DocGeneratorVisitor.index': 'DocGeneratorVisitor2.index',
@ -100,8 +98,12 @@ class DocGeneratorVisitorTest(googletest.TestCase):
'submodule2.DocGeneratorVisitor.index': 'DocGeneratorVisitor2.index',
'submodule2.DocGeneratorVisitor.index2': 'DocGeneratorVisitor2.index',
'DocGeneratorVisitor2.index2': 'DocGeneratorVisitor2.index'
}, duplicate_of)
}, visitor.duplicate_of)
self.assertEqual({
id(doc_generator_visitor.DocGeneratorVisitor): 'DocGeneratorVisitor2',
id(doc_generator_visitor.DocGeneratorVisitor.index):
'DocGeneratorVisitor2.index',
}, visitor.reverse_index)
if __name__ == '__main__':
googletest.main()

25
tensorflow/tools/docs/generate.py
View File

@ -31,7 +31,8 @@ from tensorflow.tools.docs import doc_generator_visitor
from tensorflow.tools.docs import parser
def write_docs(output_dir, base_dir, duplicate_of, duplicates, index, tree):
def write_docs(output_dir, base_dir, duplicate_of, duplicates, index, tree,
reverse_index):
"""Write previously extracted docs to disk.
Write a docs page for each symbol in `index` to a tree of docs at
@ -56,6 +57,7 @@ def write_docs(output_dir, base_dir, duplicate_of, duplicates, index, tree):
of "@{symbol}" references.
tree: A `dict` mapping a fully qualified name to the names of all its
members. Used to populate the members section of a class or module page.
reverse_index: A `dict` mapping object ids to fully qualified names.
"""
# Make output_dir.
try:
@ -89,6 +91,7 @@ def write_docs(output_dir, base_dir, duplicate_of, duplicates, index, tree):
duplicates=duplicates,
index=index,
tree=tree,
reverse_index=reverse_index,
base_dir=base_dir)
# TODO(deannarubin): use _tree to generate sidebar information.
@ -107,14 +110,13 @@ def write_docs(output_dir, base_dir, duplicate_of, duplicates, index, tree):
# TODO(deannarubin): write sidebar file?
# Write a global index containing all full names with links.
with open(os.path.join(output_dir, 'full_index.md'), 'w') as f:
f.write(parser.generate_global_index('TensorFlow', 'tensorflow',
index, duplicate_of))
with open(os.path.join(output_dir, 'index.md'), 'w') as f:
f.write(parser.generate_global_index('TensorFlow', index, duplicate_of))
def extract():
"""Extract docs from tf namespace and write them to disk."""
visitor = doc_generator_visitor.DocGeneratorVisitor()
visitor = doc_generator_visitor.DocGeneratorVisitor('tf')
api_visitor = public_api.PublicAPIVisitor(visitor)
# Access something in contrib so tf.contrib is properly loaded (it's hidden
@ -193,9 +195,9 @@ def write(output_dir, base_dir, visitor):
visitor: A `DocGeneratorVisitor` that has traversed a library located at
`base_dir`.
"""
duplicate_of, duplicates = visitor.find_duplicates()
write_docs(output_dir, os.path.abspath(base_dir),
duplicate_of, duplicates, visitor.index, visitor.tree)
visitor.duplicate_of, visitor.duplicates,
visitor.index, visitor.tree, visitor.reverse_index)
if __name__ == '__main__':
@ -209,11 +211,12 @@ if __name__ == '__main__':
)
# This doc generator works on the TensorFlow codebase. Since this script lives
# at tensorflow/tools/docs, we can compute the base directory (three levels
# up), which is valid unless we're trying to apply this to a different code
# base, or are moving the script around.
# at tensorflow/tools/docs, and all code is defined somewhere inside
# tensorflow/, we can compute the base directory (two levels up), which is
# valid unless we're trying to apply this to a different code base, or are
# moving the script around.
script_dir = os.path.dirname(inspect.getfile(inspect.currentframe()))
default_base_dir = os.path.join(script_dir, '..', '..', '..')
default_base_dir = os.path.join(script_dir, '..', '..')
argument_parser.add_argument(
'--base_dir',

45
tensorflow/tools/docs/generate_test.py
View File

@ -60,30 +60,30 @@ class GenerateTest(googletest.TestCase):
module = sys.modules[__name__]
index = {
'': sys, # Can be any module, this test doesn't care about content.
'TestModule': module,
'test_function': test_function,
'TestModule.test_function': test_function,
'TestModule.TestClass': TestClass,
'TestModule.TestClass.ChildClass': TestClass.ChildClass,
'TestModule.TestClass.ChildClass.GrandChildClass':
'tf': sys, # Can be any module, this test doesn't care about content.
'tf.TestModule': module,
'tf.test_function': test_function,
'tf.TestModule.test_function': test_function,
'tf.TestModule.TestClass': TestClass,
'tf.TestModule.TestClass.ChildClass': TestClass.ChildClass,
'tf.TestModule.TestClass.ChildClass.GrandChildClass':
TestClass.ChildClass.GrandChildClass,
}
tree = {
'': ['TestModule', 'test_function'],
'TestModule': ['test_function', 'TestClass'],
'TestModule.TestClass': ['ChildClass'],
'TestModule.TestClass.ChildClass': ['GrandChildClass'],
'TestModule.TestClass.ChildClass.GrandChildClass': []
'tf': ['TestModule', 'test_function'],
'tf.TestModule': ['test_function', 'TestClass'],
'tf.TestModule.TestClass': ['ChildClass'],
'tf.TestModule.TestClass.ChildClass': ['GrandChildClass'],
'tf.TestModule.TestClass.ChildClass.GrandChildClass': []
}
duplicate_of = {
'TestModule.test_function': 'test_function'
'tf.TestModule.test_function': 'tf.test_function'
}
duplicates = {
'test_function': ['test_function', 'TestModule.test_function']
'tf.test_function': ['tf.test_function', 'tf.TestModule.test_function']
}
output_dir = tempfile.mkdtemp()
@ -91,23 +91,24 @@ class GenerateTest(googletest.TestCase):
generate.write_docs(output_dir, base_dir,
duplicate_of, duplicates,
index, tree)
index, tree, reverse_index={})
# Make sure that the right files are written to disk.
self.assertTrue(os.path.exists(os.path.join(output_dir, 'index.md')))
self.assertTrue(os.path.exists(os.path.join(output_dir, 'full_index.md')))
self.assertTrue(os.path.exists(os.path.join(output_dir, 'TestModule.md')))
self.assertTrue(os.path.exists(os.path.join(output_dir, 'tf.md')))
self.assertTrue(os.path.exists(os.path.join(
output_dir, 'test_function.md')))
output_dir, 'tf/TestModule.md')))
self.assertTrue(os.path.exists(os.path.join(
output_dir, 'TestModule/TestClass.md')))
output_dir, 'tf/test_function.md')))
self.assertTrue(os.path.exists(os.path.join(
output_dir, 'TestModule/TestClass/ChildClass.md')))
output_dir, 'tf/TestModule/TestClass.md')))
self.assertTrue(os.path.exists(os.path.join(
output_dir, 'TestModule/TestClass/ChildClass/GrandChildClass.md')))
output_dir, 'tf/TestModule/TestClass/ChildClass.md')))
self.assertTrue(os.path.exists(os.path.join(
output_dir, 'tf/TestModule/TestClass/ChildClass/GrandChildClass.md')))
# Make sure that duplicates are not written
self.assertFalse(os.path.exists(os.path.join(
output_dir, 'TestModule/test_function.md')))
output_dir, 'tf/TestModule/test_function.md')))
if __name__ == '__main__':

3
tensorflow/tools/docs/make_py_guides.py
View File

@ -35,7 +35,6 @@ def _md_files_in_dir(input_dir):
def _main(input_dir, output_dir):
"""Convert all the files in `input_dir` and write results to `output_dir`."""
visitor = generate.extract()
duplicate_of, unused_duplicates = visitor.find_duplicates()
# Make output_dir.
try:
@ -53,7 +52,7 @@ def _main(input_dir, output_dir):
print('Processing %s...' % base_name)
md_string = open(full_path).read()
output = parser.replace_references(
md_string, relative_path_to_root, duplicate_of)
md_string, relative_path_to_root, visitor.duplicate_of)
open(os.path.join(output_dir, base_name), 'w').write(output)
print('Done.')

121
tensorflow/tools/docs/parser.py
View File

@ -18,11 +18,13 @@ from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import ast
import functools
import inspect
import os
import re
import codegen
import six
# A regular expression capturing a python indentifier.
@ -35,8 +37,7 @@ def documentation_path(full_name):
Given the fully qualified name of a library symbol, compute the path to which
to write the documentation for that symbol (relative to a base directory).
Documentation files are organized into directories that mirror the python
module/class structure. The path for the top-level module (whose full name is
'') is 'index.md'.
module/class structure.
Args:
full_name: Fully qualified name of a library symbol.
@ -44,12 +45,7 @@ def documentation_path(full_name):
Returns:
The file path to which to write the documentation for `full_name`.
"""
# The main page is special, since it has no name in here.
if not full_name:
dirs = ['index']
else:
dirs = full_name.split('.')
dirs = full_name.split('.')
return os.path.join(*dirs) + '.md'
@ -107,13 +103,11 @@ def _markdown_link(link_text, ref_full_name, relative_path_to_root,
This function returns a Markdown link. It is assumed that this is a code
reference, so the link text will always be rendered as code (using backticks).
`link_text` should refer to a library symbol. You can either refer to it with
or without the `tf.` prefix.
`link_text` should refer to a library symbol, starting with 'tf.'.
Args:
link_text: The text of the Markdown link.
ref_full_name: The fully qualified name of the symbol to link to
(may optionally include 'tf.').
ref_full_name: The fully qualified name of the symbol to link to.
relative_path_to_root: The relative path from the location of the current
document to the root of the API documentation.
duplicate_of: A map from duplicate full names to master names.
@ -122,9 +116,6 @@ def _markdown_link(link_text, ref_full_name, relative_path_to_root,
A markdown link from the documentation page of `from_full_name`
to the documentation page of `ref_full_name`.
"""
if ref_full_name.startswith('tf.'):
ref_full_name = ref_full_name[3:]
return '[`%s`](%s)' % (
link_text,
_reference_to_link(ref_full_name, relative_path_to_root, duplicate_of))
@ -154,17 +145,11 @@ def replace_references(string, relative_path_to_root, duplicate_of):
"""
full_name_re = '%s(.%s)*' % (IDENTIFIER_RE, IDENTIFIER_RE)
symbol_reference_re = re.compile(r'@\{(' + full_name_re + r')\}')
match = symbol_reference_re.search(string)
while match:
symbol_name = match.group(1)
link_text = _markdown_link(symbol_name, symbol_name,
relative_path_to_root, duplicate_of)
# Remove only the '@symbol' part of the match, and replace with the link.
string = string[:match.start()] + link_text + string[match.end():]
match = symbol_reference_re.search(string,
pos=match.start() + len(link_text))
return string
return re.sub(symbol_reference_re,
lambda match: _markdown_link(match.group(1), match.group(1), # pylint: disable=g-long-lambda
relative_path_to_root,
duplicate_of),
string)
def _md_docstring(py_object, relative_path_to_root, duplicate_of):
@ -281,7 +266,12 @@ def _get_arg_spec(func):
return inspect.getargspec(func)
def _generate_signature(func):
def _remove_first_line_indent(string):
indent = len(re.match(r'^\s*', string).group(0))
return '\n'.join([line[indent:] for line in string.split('\n')])
def _generate_signature(func, reverse_index):
"""Given a function, returns a string representing its args.
This function produces a string representing the arguments to a python
@ -297,8 +287,8 @@ def _generate_signature(func):
document, it should be typeset as code (using backticks), or escaped.
Args:
func: A function of method to extract the signature for (anything
`inspect.getargspec` will accept).
func: A function, method, or functools.partial to extract the signature for.
reverse_index: A map from object ids to canonical full names to use.
Returns:
A string representing the signature of `func` as python code.
@ -322,14 +312,42 @@ def _generate_signature(func):
# Add all args with defaults.
if argspec.defaults:
for arg, default in zip(
argspec.args[first_arg_with_default:], argspec.defaults):
# Some callables don't have __name__, fall back to including their repr.
# TODO(wicke): This could be improved at least for common cases.
if callable(default) and hasattr(default, '__name__'):
args_list.append('%s=%s' % (arg, default.__name__))
source = _remove_first_line_indent(inspect.getsource(func))
func_ast = ast.parse(source)
ast_defaults = func_ast.body[0].args.defaults
for arg, default, ast_default in zip(
argspec.args[first_arg_with_default:], argspec.defaults, ast_defaults):
if id(default) in reverse_index:
default_text = reverse_index[id(default)]
else:
args_list.append('%s=%r' % (arg, default))
default_text = codegen.to_source(ast_default)
if default_text != repr(default):
# This may be an internal name. If so, handle the ones we know about.
# TODO(wicke): This should be replaced with a lookup in the index.
# TODO(wicke): (replace first ident with tf., check if in index)
internal_names = {
'ops.GraphKeys': 'tf.GraphKeys',
'_ops.GraphKeys': 'tf.GraphKeys',
'init_ops.zeros_initializer': 'tf.zeros_initializer',
'init_ops.ones_initializer': 'tf.ones_initializer',
'saver_pb2.SaverDef': 'tf.SaverDef',
}
full_name_re = '^%s(.%s)+' % (IDENTIFIER_RE, IDENTIFIER_RE)
match = re.match(full_name_re, default_text)
if match:
lookup_text = default_text
for internal_name, public_name in six.iteritems(internal_names):
if match.group(0).startswith(internal_name):
lookup_text = public_name + default_text[len(internal_name):]
break
if default_text is lookup_text:
print('Using default arg, failed lookup: %s, repr: %r' % (
default_text, default))
else:
default_text = lookup_text
args_list.append('%s=%s' % (arg, default_text))
# Add *args and *kwargs.
if argspec.varargs:
@ -341,7 +359,7 @@ def _generate_signature(func):
def _generate_markdown_for_function(full_name, duplicate_names,
function, duplicate_of):
function, duplicate_of, reverse_index):
"""Generate Markdown docs for a function or method.
This function creates a documentation page for a function. It uses the
@ -356,6 +374,7 @@ def _generate_markdown_for_function(full_name, duplicate_names,
function: The python object referenced by `full_name`.
duplicate_of: A map of duplicate full names to master names. Used to resolve
@{symbol} references in the docstring.
reverse_index: A map from object ids in the index to full names.
Returns:
A string that can be written to a documentation file for this function.
@ -364,7 +383,7 @@ def _generate_markdown_for_function(full_name, duplicate_names,
relative_path = os.path.relpath(
os.path.dirname(documentation_path(full_name)) or '.', '.')
docstring = _md_docstring(function, relative_path, duplicate_of)
signature = _generate_signature(function)
signature = _generate_signature(function, reverse_index)
if duplicate_names:
aliases = '\n'.join(['### `%s`' % (name + signature)
@ -377,7 +396,7 @@ def _generate_markdown_for_function(full_name, duplicate_names,
def _generate_markdown_for_class(full_name, duplicate_names, py_class,
duplicate_of, index, tree):
duplicate_of, index, tree, reverse_index):
"""Generate Markdown docs for a class.
This function creates a documentation page for a class. It uses the
@ -396,6 +415,7 @@ def _generate_markdown_for_class(full_name, duplicate_names, py_class,
@{symbol} references in the docstrings.
index: A map from full names to python object references.
tree: A map from full names to the names of all documentable child objects.
reverse_index: A map from object ids in the index to full names.
Returns:
A string that can be written to a documentation file for this class.
@ -445,7 +465,8 @@ def _generate_markdown_for_class(full_name, duplicate_names, py_class,
if methods:
docs += '## Methods\n\n'
for method_name, method in sorted(methods, key=lambda x: x[0]):
method_signature = method_name + _generate_signature(method)
method_signature = method_name + _generate_signature(method,
reverse_index)
docs += '### `%s`\n\n%s\n\n' % (method_signature,
_md_docstring(method, relative_path,
duplicate_of))
@ -502,7 +523,7 @@ def _generate_markdown_for_module(full_name, duplicate_names, module,
if inspect.isclass(member):
link_text = 'class ' + name
elif inspect.isfunction(member):
link_text = name + _generate_signature(member)
link_text = name + '(...)'
else:
link_text = name
@ -522,7 +543,7 @@ _CODE_URL_PREFIX = (
def generate_markdown(full_name, py_object,
duplicate_of, duplicates,
index, tree, base_dir):
index, tree, reverse_index, base_dir):
"""Generate Markdown docs for a given object that's part of the TF API.
This function uses _md_docstring to obtain the docs pertaining to
@ -539,7 +560,7 @@ def generate_markdown(full_name, py_object,
The output is Markdown that can be written to file and published.
Args:
full_name: The fully qualified name (excl. "tf.") of the symbol to be
full_name: The fully qualified name of the symbol to be
documented.
py_object: The Python object to be documented. Its documentation is sourced
from `py_object`'s docstring.
@ -553,6 +574,7 @@ def generate_markdown(full_name, py_object,
of "@{symbol}" references.
tree: A `dict` mapping a fully qualified name to the names of all its
members. Used to populate the members section of a class or module page.
reverse_index: A `dict` mapping objects in the index to full names.
base_dir: A base path that is stripped from file locations written to the
docs.
@ -573,11 +595,12 @@ def generate_markdown(full_name, py_object,
# Some methods in classes from extensions come in as routines.
inspect.isroutine(py_object)):
markdown = _generate_markdown_for_function(master_name, duplicate_names,
py_object, duplicate_of)
py_object, duplicate_of,
reverse_index)
elif inspect.isclass(py_object):
markdown = _generate_markdown_for_class(master_name, duplicate_names,
py_object, duplicate_of,
index, tree)
index, tree, reverse_index)
elif inspect.ismodule(py_object):
markdown = _generate_markdown_for_module(master_name, duplicate_names,
py_object, duplicate_of,
@ -605,7 +628,7 @@ def generate_markdown(full_name, py_object,
return markdown
def generate_global_index(library_name, root_name, index, duplicate_of):
def generate_global_index(library_name, index, duplicate_of):
"""Given a dict of full names to python objects, generate an index page.
The index page generated contains a list of links for all symbols in `index`
@ -613,7 +636,6 @@ def generate_global_index(library_name, root_name, index, duplicate_of):
Args:
library_name: The name for the documented library to use in the title.
root_name: The name to use for the root module.
index: A dict mapping full names to python objects.
duplicate_of: A map of duplicate names to preferred names.
@ -622,11 +644,10 @@ def generate_global_index(library_name, root_name, index, duplicate_of):
"""
symbol_links = []
for full_name, py_object in six.iteritems(index):
index_name = full_name or root_name
if (inspect.ismodule(py_object) or inspect.isfunction(py_object) or
inspect.isclass(py_object)):
symbol_links.append((index_name,
_markdown_link(index_name, full_name,
symbol_links.append((full_name,
_markdown_link(full_name, full_name,
'.', duplicate_of)))
lines = ['# All symbols in %s' % library_name, '']

26
tensorflow/tools/docs/parser_test.py
View File

@ -80,16 +80,13 @@ class ParserTest(googletest.TestCase):
self.assertEqual('test.md', parser.documentation_path('test'))
self.assertEqual('test/module.md', parser.documentation_path('test.module'))
def test_documentation_path_empty(self):
self.assertEqual('index.md', parser.documentation_path(''))
def test_replace_references(self):
string = 'A @{reference}, another @{tf.reference}, and a @{third}.'
string = 'A @{reference}, another @{reference}, and a @{third}.'
duplicate_of = {'third': 'fourth'}
result = parser.replace_references(string, '../..', duplicate_of)
self.assertEqual(
'A [`reference`](../../reference.md), another '
'[`tf.reference`](../../reference.md), '
'[`reference`](../../reference.md), '
'and a [`third`](../../fourth.md).',
result)
@ -109,7 +106,8 @@ class ParserTest(googletest.TestCase):
docs = parser.generate_markdown(full_name='TestClass', py_object=TestClass,
duplicate_of={}, duplicates={},
index=index, tree=tree, base_dir='/')
index=index, tree=tree, reverse_index={},
base_dir='/')
# Make sure all required docstrings are present.
self.assertTrue(inspect.getdoc(TestClass) in docs)
@ -146,7 +144,8 @@ class ParserTest(googletest.TestCase):
docs = parser.generate_markdown(full_name='TestModule', py_object=module,
duplicate_of={}, duplicates={},
index=index, tree=tree, base_dir='/')
index=index, tree=tree, reverse_index={},
base_dir='/')
# Make sure all required docstrings are present.
self.assertTrue(inspect.getdoc(module) in docs)
@ -174,7 +173,8 @@ class ParserTest(googletest.TestCase):
docs = parser.generate_markdown(full_name='test_function',
py_object=test_function,
duplicate_of={}, duplicates={},
index=index, tree=tree, base_dir='/')
index=index, tree=tree, reverse_index={},
base_dir='/')
# Make sure docstring shows up.
self.assertTrue(inspect.getdoc(test_function) in docs)
@ -198,7 +198,8 @@ class ParserTest(googletest.TestCase):
docs = parser.generate_markdown(full_name='test_function_with_args_kwargs',
py_object=test_function_with_args_kwargs,
duplicate_of={}, duplicates={},
index=index, tree=tree, base_dir='/')
index=index, tree=tree, reverse_index={},
base_dir='/')
# Make sure docstring shows up.
self.assertTrue(inspect.getdoc(test_function_with_args_kwargs) in docs)
@ -222,7 +223,7 @@ class ParserTest(googletest.TestCase):
full_name='test_function_for_markdown_reference',
py_object=test_function_for_markdown_reference,
duplicate_of={}, duplicates={},
index=index, tree=tree, base_dir='/')
index=index, tree=tree, reverse_index={}, base_dir='/')
# Make sure docstring shows up and is properly processed.
expected_docs = parser.replace_references(
@ -244,7 +245,7 @@ class ParserTest(googletest.TestCase):
full_name='test_function',
py_object=test_function_with_fancy_docstring,
duplicate_of={}, duplicates={},
index=index, tree=tree, base_dir='/')
index=index, tree=tree, reverse_index={}, base_dir='/')
expected = '\n'.join([
'Function with a fancy docstring.',
@ -278,8 +279,7 @@ class ParserTest(googletest.TestCase):
'TestModule.test_function': 'test_function'
}
docs = parser.generate_global_index('TestLibrary', 'test',
index=index,
docs = parser.generate_global_index('TestLibrary', index=index,
duplicate_of=duplicate_of)
# Make sure duplicates and non-top-level symbols are in the index, but