experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Make a better doc generator.

Browse Source 
Change: 146177492
This commit is contained in:
Martin Wicke 2017-01-31 15:55:47 -08:00 committed by TensorFlower Gardener
parent e2127701a5
commit d33692e9cc
9 changed files with 1670 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
tensorflow/python/framework/random_seed.py
View File

@ -39,7 +39,7 @@ def get_seed(op_seed):
graph, or for only specific operations.
For details on how the graph-level seed interacts with op seeds, see
[`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed).
@{set_random_seed}.
Args:
op_seed: integer.

18
tensorflow/tools/common/public_api.py
View File

@ -40,16 +40,28 @@ class PublicAPIVisitor(object):
# Each entry maps a module path to a name to ignore in traversal.
_do_not_descend_map = {
# TODO(drpng): This can be removed once sealed off.
'': ['platform', 'pywrap_tensorflow'],
'': ['platform', 'pywrap_tensorflow', 'user_ops'],
# Some implementations have this internal module that we shouldn't expose.
'flags': ['cpp_flags'],
# Everything below here is legitimate.
'app': 'flags', # It'll stay, but it's not officially part of the API
'app': ['flags'], # It'll stay, but it's not officially part of the API.
'test': ['mock'], # Imported for compatibility between py2/3.
}
@property
def do_not_descend_map(self):
"""A map from parents to symbols that should not be descended into.
This map can be edited, but it should not be edited once traversal has
begun.
Returns:
The map marking symbols to not explore.
"""
return self._do_not_descend_map
def _isprivate(self, name):
"""Return whether a name is private."""
return name.startswith('_')
@ -61,6 +73,8 @@ class PublicAPIVisitor(object):
def __call__(self, path, parent, children):
"""Visitor interface, see `traverse` for details."""
# Avoid long waits in cases of pretty unambiguous failure.
if inspect.ismodule(parent) and len(path.split('.')) > 10:
raise RuntimeError('Modules nested too deep:\n%s\n\nThis is likely a '
'problem with an accidental public import.' % path)

68
tensorflow/tools/docs/BUILD
View File

@ -16,6 +16,74 @@ py_binary(
deps = ["//tensorflow:tensorflow_py"],
)
py_library(
name = "doc_generator_visitor",
srcs = [
"doc_generator_visitor.py",
],
srcs_version = "PY2AND3",
)
py_test(
name = "doc_generator_visitor_test",
size = "small",
srcs = [
"doc_generator_visitor_test.py",
],
srcs_version = "PY2AND3",
deps = [
":doc_generator_visitor",
"//tensorflow/python:platform_test",
],
)
py_library(
name = "parser",
srcs = [
"parser.py",
],
srcs_version = "PY2AND3",
)
py_test(
name = "parser_test",
size = "small",
srcs = [
"parser_test.py",
],
srcs_version = "PY2AND3",
deps = [
":parser",
"//tensorflow/python:platform_test",
],
)
py_binary(
name = "generate",
srcs = ["generate.py"],
srcs_version = "PY2AND3",
deps = [
"//tensorflow:tensorflow_py",
"//tensorflow/tools/common:public_api",
"//tensorflow/tools/common:traverse",
"//tensorflow/tools/docs:doc_generator_visitor",
"//tensorflow/tools/docs:parser",
],
)
py_test(
name = "generate_test",
size = "small",
srcs = [
"generate_test.py",
],
srcs_version = "PY2AND3",
deps = [
":generate",
"//tensorflow/python:platform_test",
],
)
filegroup(
name = "doxy_config",
srcs = ["tf-doxy_for_md-config"],

149
tensorflow/tools/docs/doc_generator_visitor.py Normal file
View File

@ -0,0 +1,149 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""A `traverse` visitor for processing documentation."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import inspect
class DocGeneratorVisitor(object):
"""A visitor that generates docs for a python object when __call__ed."""
def __init__(self):
self._index = {}
self._tree = {}
@property
def index(self):
"""A map from fully qualified names to objects to be documented.
The index is filled when the visitor is passed to `traverse`.
Returns:
The index filled by traversal.
"""
return self._index
@property
def tree(self):
"""A map from fully qualified names to all its child names for traversal.
The full name to member names map is filled when the visitor is passed to
`traverse`.
Returns:
The full name to member name map filled by traversal.
"""
return self._tree
def __call__(self, parent_name, parent, children):
"""Visitor interface, see `tensorflow/tools/common:traverse` for details.
This method is called for each symbol found in a traversal using
`tensorflow/tools/common:traverse`. It should not be called directly in
user code.
Args:
parent_name: The fully qualified name of a symbol found during traversal.
parent: The Python object referenced by `parent_name`.
children: A list of `(name, py_object)` pairs enumerating, in alphabetical
order, the children (as determined by `inspect.getmembers`) of `parent`.
`name` is the local name of `py_object` in `parent`.
Raises:
RuntimeError: If this visitor is called with a `parent` that is not a
class or module.
"""
self._index[parent_name] = parent
self._tree[parent_name] = []
if inspect.ismodule(parent):
print('module %s: %r' % (parent_name, parent))
elif inspect.isclass(parent):
print('class %s: %r' % (parent_name, parent))
else:
raise RuntimeError('Unexpected type in visitor -- %s: %r' %
(parent_name, parent))
for name, child in children:
full_name = '.'.join([parent_name, name]) if parent_name else name
self._index[full_name] = child
self._tree[parent_name].append(name)
def find_duplicates(self):
"""Compute data structures containing information about duplicates.
Find duplicates in `index` and decide on one to be the "master" name.
Returns a map `duplicate_of` from aliases to their master name (the master
name itself has no entry in this map), and a map `duplicates` from master
names to a lexicographically sorted list of all aliases for that name (incl.
the master name).
Returns:
A tuple `(duplicate_of, duplicates)` as described above.
"""
# Maps the id of a symbol to its fully qualified name. For symbols that have
# several aliases, this map contains the first one found.
# We use id(py_object) to get a hashable value for py_object. Note all
# objects in _index are in memory at the same time so this is safe.
reverse_index = {}
# Make a preliminary duplicates map. For all sets of duplicate names, it
# maps the first name found to a list of all duplicate names.
raw_duplicates = {}
for full_name, py_object in self._index.iteritems():
# We cannot use the duplicate mechanism for constants, since e.g.,
# id(c1) == id(c2) with c1=1, c2=1. This is unproblematic since constants
# have no usable docstring and won't be documented automatically.
if (inspect.ismodule(py_object) or
inspect.isclass(py_object) or
inspect.isfunction(py_object) or
inspect.isroutine(py_object) or
inspect.ismethod(py_object) or
isinstance(py_object, property)):
object_id = id(py_object)
if object_id in reverse_index:
master_name = reverse_index[object_id]
if master_name in raw_duplicates:
raw_duplicates[master_name].append(full_name)
else:
raw_duplicates[master_name] = [master_name, full_name]
else:
reverse_index[object_id] = full_name
# Decide on master names, rewire duplicates and make a duplicate_of map
# mapping all non-master duplicates to the master name. The master symbol
# does not have an entry in this map.
duplicate_of = {}
# Duplicates maps the main symbols to the set of all duplicates of that
# symbol (incl. itself).
duplicates = {}
for names in raw_duplicates.values():
names = sorted(names)
# Choose the lexicographically first name with the minimum number of
# submodules. This will prefer highest level namespace for any symbol.
master_name = min(names, key=lambda name: name.count('.'))
duplicates[master_name] = names
for name in names:
if name != master_name:
duplicate_of[name] = master_name
return duplicate_of, duplicates

107
tensorflow/tools/docs/doc_generator_visitor_test.py Normal file
View File

@ -0,0 +1,107 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for tools.docs.doc_generator_visitor."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from tensorflow.python.platform import googletest
from tensorflow.tools.docs import doc_generator_visitor
class DocGeneratorVisitorTest(googletest.TestCase):
def test_call_module(self):
visitor = doc_generator_visitor.DocGeneratorVisitor()
visitor(
'doc_generator_visitor', doc_generator_visitor,
[('DocGeneratorVisitor', doc_generator_visitor.DocGeneratorVisitor)])
self.assertEqual({'doc_generator_visitor': ['DocGeneratorVisitor']},
visitor.tree)
self.assertEqual({
'doc_generator_visitor': doc_generator_visitor,
'doc_generator_visitor.DocGeneratorVisitor':
doc_generator_visitor.DocGeneratorVisitor,
}, visitor.index)
def test_call_class(self):
visitor = doc_generator_visitor.DocGeneratorVisitor()
visitor(
'DocGeneratorVisitor', doc_generator_visitor.DocGeneratorVisitor,
[('index', doc_generator_visitor.DocGeneratorVisitor.index)])
self.assertEqual({'DocGeneratorVisitor': ['index']},
visitor.tree)
self.assertEqual({
'DocGeneratorVisitor': doc_generator_visitor.DocGeneratorVisitor,
'DocGeneratorVisitor.index':
doc_generator_visitor.DocGeneratorVisitor.index
}, visitor.index)
def test_call_raises(self):
visitor = doc_generator_visitor.DocGeneratorVisitor()
with self.assertRaises(RuntimeError):
visitor('non_class_or_module', 'non_class_or_module_object', [])
def test_duplicates(self):
visitor = doc_generator_visitor.DocGeneratorVisitor()
visitor(
'submodule.DocGeneratorVisitor',
doc_generator_visitor.DocGeneratorVisitor,
[('index', doc_generator_visitor.DocGeneratorVisitor.index),
('index2', doc_generator_visitor.DocGeneratorVisitor.index)])
visitor(
'submodule2.DocGeneratorVisitor',
doc_generator_visitor.DocGeneratorVisitor,
[('index', doc_generator_visitor.DocGeneratorVisitor.index),
('index2', doc_generator_visitor.DocGeneratorVisitor.index)])
visitor(
'DocGeneratorVisitor2',
doc_generator_visitor.DocGeneratorVisitor,
[('index', doc_generator_visitor.DocGeneratorVisitor.index),
('index2', doc_generator_visitor.DocGeneratorVisitor.index)])
duplicate_of, duplicates = visitor.find_duplicates()
# The shorter path should be master, or if equal, the lexicographically
# first will be.
self.assertEqual(
{'DocGeneratorVisitor2': sorted(['submodule.DocGeneratorVisitor',
'submodule2.DocGeneratorVisitor',
'DocGeneratorVisitor2']),
'DocGeneratorVisitor2.index': sorted([
'submodule.DocGeneratorVisitor.index',
'submodule.DocGeneratorVisitor.index2',
'submodule2.DocGeneratorVisitor.index',
'submodule2.DocGeneratorVisitor.index2',
'DocGeneratorVisitor2.index',
'DocGeneratorVisitor2.index2'
]),
}, duplicates)
self.assertEqual({
'submodule.DocGeneratorVisitor': 'DocGeneratorVisitor2',
'submodule.DocGeneratorVisitor.index': 'DocGeneratorVisitor2.index',
'submodule.DocGeneratorVisitor.index2': 'DocGeneratorVisitor2.index',
'submodule2.DocGeneratorVisitor': 'DocGeneratorVisitor2',
'submodule2.DocGeneratorVisitor.index': 'DocGeneratorVisitor2.index',
'submodule2.DocGeneratorVisitor.index2': 'DocGeneratorVisitor2.index',
'DocGeneratorVisitor2.index2': 'DocGeneratorVisitor2.index'
}, duplicate_of)
if __name__ == '__main__':
googletest.main()

232
tensorflow/tools/docs/generate.py Normal file
View File

@ -0,0 +1,232 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Generate docs for the TensorFlow Python API."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import argparse
import inspect
import os
import tensorflow as tf
from tensorflow.tools.common import public_api
from tensorflow.tools.common import traverse
from tensorflow.tools.docs import doc_generator_visitor
from tensorflow.tools.docs import parser
def write_docs(output_dir, base_dir, duplicate_of, duplicates, index, tree):
"""Write previously extracted docs to disk.
Write a docs page for each symbol in `index` to a tree of docs at
`output_dir`.
Symbols with multiple aliases will have only one page written about them,
which is referenced for all aliases. `duplicate_of` and `duplicates` are used
to determine which docs pages to write.
Args:
output_dir: Directory to write documentation markdown files to. Will be
created if it doesn't exist.
base_dir: Base directory of the code being documented. This prefix is
stripped from all file paths that are part of the documentation.
duplicate_of: A `dict` mapping fully qualified names to "master" names. This
is used to resolve "@{symbol}" references to the "master" name.
duplicates: A `dict` mapping fully qualified names to a set of all
aliases of this name. This is used to automatically generate a list of all
aliases for each name.
index: A `dict` mapping fully qualified names to the corresponding Python
objects. Used to produce docs for child objects, and to check the validity
of "@{symbol}" references.
tree: A `dict` mapping a fully qualified name to the names of all its
members. Used to populate the members section of a class or module page.
"""
# Make output_dir.
try:
if not os.path.exists(output_dir):
os.makedirs(output_dir)
except OSError as e:
print('Creating output dir "%s" failed: %s' % (output_dir, e))
raise
# Parse and write Markdown pages, resolving cross-links (@{symbol}).
for full_name, py_object in index.iteritems():
if full_name in duplicate_of:
print('Not writing docs for %s, duplicate of %s.' % (
full_name, duplicate_of[full_name]))
continue
# Methods and some routines are documented only as part of their class.
if not (inspect.ismodule(py_object) or
inspect.isclass(py_object) or
inspect.isfunction(py_object)):
print('Not writing docs for %s, not a class, module, or function.' % (
full_name))
continue
print('Writing docs for %s (%r).' % (full_name, py_object))
# Generate docs for `py_object`, resolving references.
markdown = parser.generate_markdown(full_name, py_object,
duplicate_of=duplicate_of,
duplicates=duplicates,
index=index,
tree=tree,
base_dir=base_dir)
# TODO(deannarubin): use _tree to generate sidebar information.
path = os.path.join(output_dir, parser.documentation_path(full_name))
directory = os.path.dirname(path)
try:
if not os.path.exists(directory):
os.makedirs(directory)
with open(path, 'w') as f:
f.write(markdown)
except OSError as e:
print('Cannot write documentation for %s to %s: %s' % (full_name,
directory, e))
raise
# TODO(deannarubin): write sidebar file?
# Write a global index containing all full names with links.
with open(os.path.join(output_dir, 'full_index.md'), 'w') as f:
f.write(parser.generate_global_index('TensorFlow', 'tensorflow',
index, duplicate_of))
def extract():
"""Extract docs from tf namespace and write them to disk."""
visitor = doc_generator_visitor.DocGeneratorVisitor()
api_visitor = public_api.PublicAPIVisitor(visitor)
# Access something in contrib so tf.contrib is properly loaded (it's hidden
# behind lazy loading)
# TODO(wicke): Enable contrib traversal once contrib is sealed.
# _ = tf.contrib.__name__
# Exclude some libaries in contrib from the documentation altogether.
# TODO(wicke): Shrink this list.
api_visitor.do_not_descend_map.update({
'contrib': [
'compiler',
'factorization',
'grid_rnn',
'labeled_tensor',
'ndlstm',
'quantization',
'session_bundle',
'slim',
'solvers',
'specs',
'tensor_forest',
'tensorboard',
'testing',
'tfprof',
'training',
],
'contrib.bayesflow': [
'entropy', 'monte_carlo',
'special_math', 'stochastic_gradient_estimators',
'stochastic_graph', 'stochastic_tensor',
'stochastic_variables', 'variational_inference'
],
'contrib.distributions': ['bijector'],
'contrib.graph_editor': [
'edit',
'match',
'reroute',
'subgraph',
'transform',
'select',
'util'
],
'contrib.layers': [
'feature_column',
'summaries'
],
'contrib.learn': [
'datasets',
'graph_actions',
'io',
'models',
'monitors',
'ops',
'preprocessing',
'utils',
],
'contrib.util': ['loader'],
})
traverse.traverse(tf, api_visitor)
return visitor
def write(output_dir, base_dir, visitor):
"""Write documentation for an index in a `DocGeneratorVisitor` to disk.
This function will create `output_dir` if it doesn't exist, and write
the documentation contained in `visitor`.
Args:
output_dir: The directory to write documentation to. Must not exist.
base_dir: The base dir of the library `visitor` has traversed. This is used
to compute relative paths for file references.
visitor: A `DocGeneratorVisitor` that has traversed a library located at
`base_dir`.
"""
duplicate_of, duplicates = visitor.find_duplicates()
write_docs(output_dir, os.path.abspath(base_dir),
duplicate_of, duplicates, visitor.index, visitor.tree)
if __name__ == '__main__':
argument_parser = argparse.ArgumentParser()
argument_parser.add_argument(
'--output_dir',
type=str,
default=None,
required=True,
help='Directory to write docs to. Must not exist.'
)
# This doc generator works on the TensorFlow codebase. Since this script lives
# at tensorflow/tools/docs, we can compute the base directory (three levels
# up), which is valid unless we're trying to apply this to a different code
# base, or are moving the script around.
script_dir = os.path.dirname(inspect.getfile(inspect.currentframe()))
default_base_dir = os.path.join(script_dir, '..', '..', '..')
argument_parser.add_argument(
'--base_dir',
type=str,
default=default_base_dir,
help=('Base directory to to strip from file names referenced in docs. '
'Defaults to three directories up from the location of this file.')
)
flags, _ = argument_parser.parse_known_args()
if os.path.exists(flags.output_dir):
raise RuntimeError('output_dir %s exists.\n'
'Cowardly refusing to wipe it, please do that yourself.'
% flags.output_dir)
write(flags.output_dir, flags.base_dir, extract())

114
tensorflow/tools/docs/generate_test.py Normal file
View File

@ -0,0 +1,114 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for doc generator traversal."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
import sys
import tempfile
from tensorflow.python.platform import googletest
from tensorflow.tools.docs import generate
def test_function():
"""Docstring for test_function."""
pass
class TestClass(object):
"""Docstring for TestClass itself."""
class ChildClass(object):
"""Docstring for a child class."""
class GrandChildClass(object):
"""Docstring for a child of a child class."""
pass
class GenerateTest(googletest.TestCase):
def test_extraction(self):
try:
generate.extract()
except RuntimeError:
print('*****************************************************************')
print('If this test fails, you have most likely introduced an unsealed')
print('module. Make sure to use remove_undocumented or similar utilities')
print('to avoid leaking symbols. See below for more information on the')
print('failure.')
print('*****************************************************************')
raise
def test_write(self):
module = sys.modules[__name__]
index = {
'': sys, # Can be any module, this test doesn't care about content.
'TestModule': module,
'test_function': test_function,
'TestModule.test_function': test_function,
'TestModule.TestClass': TestClass,
'TestModule.TestClass.ChildClass': TestClass.ChildClass,
'TestModule.TestClass.ChildClass.GrandChildClass':
TestClass.ChildClass.GrandChildClass,
}
tree = {
'': ['TestModule', 'test_function'],
'TestModule': ['test_function', 'TestClass'],
'TestModule.TestClass': ['ChildClass'],
'TestModule.TestClass.ChildClass': ['GrandChildClass'],
'TestModule.TestClass.ChildClass.GrandChildClass': []
}
duplicate_of = {
'TestModule.test_function': 'test_function'
}
duplicates = {
'test_function': ['test_function', 'TestModule.test_function']
}
output_dir = tempfile.mkdtemp()
base_dir = os.path.dirname(__file__)
generate.write_docs(output_dir, base_dir,
duplicate_of, duplicates,
index, tree)
# Make sure that the right files are written to disk.
self.assertTrue(os.path.exists(os.path.join(output_dir, 'index.md')))
self.assertTrue(os.path.exists(os.path.join(output_dir, 'full_index.md')))
self.assertTrue(os.path.exists(os.path.join(output_dir, 'TestModule.md')))
self.assertTrue(os.path.exists(os.path.join(
output_dir, 'test_function.md')))
self.assertTrue(os.path.exists(os.path.join(
output_dir, 'TestModule/TestClass.md')))
self.assertTrue(os.path.exists(os.path.join(
output_dir, 'TestModule/TestClass/ChildClass.md')))
self.assertTrue(os.path.exists(os.path.join(
output_dir, 'TestModule/TestClass/ChildClass/GrandChildClass.md')))
# Make sure that duplicates are not written
self.assertFalse(os.path.exists(os.path.join(
output_dir, 'TestModule/test_function.md')))
if __name__ == '__main__':
googletest.main()

634
tensorflow/tools/docs/parser.py Normal file
View File

@ -0,0 +1,634 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Turn Python docstrings into Markdown for TensorFlow documentation."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import functools
import inspect
import os
import re
# A regular expression capturing a python indentifier.
IDENTIFIER_RE = '[a-zA-Z_][a-zA-Z0-9_]*'
def documentation_path(full_name):
"""Returns the file path for the documentation for the given API symbol.
Given the fully qualified name of a library symbol, compute the path to which
to write the documentation for that symbol (relative to a base directory).
Documentation files are organized into directories that mirror the python
module/class structure. The path for the top-level module (whose full name is
'') is 'index.md'.
Args:
full_name: Fully qualified name of a library symbol.
Returns:
The file path to which to write the documentation for `full_name`.
"""
# The main page is special, since it has no name in here.
if not full_name:
dirs = ['index']
else:
dirs = full_name.split('.')
return os.path.join(*dirs) + '.md'
def _get_raw_docstring(py_object):
"""Get the docs for a given python object.
Args:
py_object: A python object to retrieve the docs for (class, function/method,
or module).
Returns:
The docstring, or the empty string if no docstring was found.
"""
# For object instances, inspect.getdoc does give us the docstring of their
# type, which is not what we want. Only return the docstring if it is useful.
if (inspect.isclass(py_object) or inspect.ismethod(py_object) or
inspect.isfunction(py_object) or inspect.ismodule(py_object) or
isinstance(py_object, property)):
return inspect.getdoc(py_object) or ''
else:
return ''
def _get_brief_docstring(py_object):
"""Gets the one line docstring of a python object."""
return _get_raw_docstring(py_object).split('\n')[0]
def _reference_to_link(ref_full_name, relative_path_to_root, duplicate_of):
"""Resolve a "@{symbol}" reference to a relative path, respecting duplicates.
The input to this function should already be stripped of the '@' and '{}', and
its output is only the link, not the full Markdown.
Args:
ref_full_name: The fully qualified name of the symbol to link to.
relative_path_to_root: The relative path from the location of the current
document to the root of the API documentation.
duplicate_of: A map from duplicate full names to master names.
Returns:
A relative path that links from the documentation page of `from_full_name`
to the documentation page of `ref_full_name`.
"""
master_name = duplicate_of.get(ref_full_name, ref_full_name)
ref_path = documentation_path(master_name)
return os.path.join(relative_path_to_root, ref_path)
def _markdown_link(link_text, ref_full_name, relative_path_to_root,
duplicate_of):
"""Resolve a "@{symbol}" reference to a Markdown link, respecting duplicates.
The input to this function should already be stripped of the '@' and '{}'.
This function returns a Markdown link. It is assumed that this is a code
reference, so the link text will always be rendered as code (using backticks).
`link_text` should refer to a library symbol. You can either refer to it with
or without the `tf.` prefix.
Args:
link_text: The text of the Markdown link.
ref_full_name: The fully qualified name of the symbol to link to
(may optionally include 'tf.').
relative_path_to_root: The relative path from the location of the current
document to the root of the API documentation.
duplicate_of: A map from duplicate full names to master names.
Returns:
A markdown link from the documentation page of `from_full_name`
to the documentation page of `ref_full_name`.
"""
if ref_full_name.startswith('tf.'):
ref_full_name = ref_full_name[3:]
return '[`%s`](%s)' % (
link_text,
_reference_to_link(ref_full_name, relative_path_to_root, duplicate_of))
def replace_references(string, relative_path_to_root, duplicate_of):
"""Replace "@{symbol}" references with links to symbol's documentation page.
This functions finds all occurrences of "@{symbol}" in `string` and replaces
them with markdown links to the documentation page for "symbol".
`relative_path_to_root` is the relative path from the document that contains
the "@{symbol}" reference to the root of the API documentation that is linked
to. If the containing page is part of the same API docset,
`relative_path_to_root` can be set to
`os.path.dirname(documentation_path(name))`, where `name` is the python name
of the object whose documentation page the reference lives on.
Args:
string: A string in which "@{symbol}" references should be replaced.
relative_path_to_root: The relative path from the contianing document to the
root of the API documentation that is being linked to.
duplicate_of: A map from duplicate names to preferred names of API symbols.
Returns:
`string`, with "@{symbol}" references replaced by Markdown links.
"""
full_name_re = '%s(.%s)*' % (IDENTIFIER_RE, IDENTIFIER_RE)
symbol_reference_re = re.compile(r'@\{(' + full_name_re + r')\}')
match = symbol_reference_re.search(string)
while match:
symbol_name = match.group(1)
link_text = _markdown_link(symbol_name, symbol_name,
relative_path_to_root, duplicate_of)
# Remove only the '@symbol' part of the match, and replace with the link.
string = string[:match.start()] + link_text + string[match.end():]
match = symbol_reference_re.search(string,
pos=match.start() + len(link_text))
return string
def _md_docstring(py_object, relative_path_to_root, duplicate_of):
"""Get the docstring from an object and make it into nice Markdown.
For links within the same set of docs, the `relative_path_to_root` for a
docstring on the page for `full_name` can be set to
```python
relative_path_to_root = os.path.relpath(
os.path.dirname(documentation_path(full_name)) or '.', '.')
```
Args:
py_object: A python object to retrieve the docs for (class, function/method,
or module).
relative_path_to_root: The relative path from the location of the current
document to the root of the API documentation. This is used to compute
links for "@symbol" references.
duplicate_of: A map from duplicate symbol names to master names. Used to
resolve "@symbol" references.
Returns:
The docstring, or the empty string if no docstring was found.
"""
# TODO(wicke): If this is a partial, use the .func docstring and add a note.
raw_docstring = _get_raw_docstring(py_object)
raw_lines = raw_docstring.split('\n')
# Define regular expressions used during parsing below.
symbol_list_item_re = re.compile(r'^ (%s): ' % IDENTIFIER_RE)
section_re = re.compile(r'^(\w+):\s*$')
# Translate docstring line by line.
in_special_section = False
lines = []
def is_section_start(i):
# Previous line is empty, line i is "Word:", and next line is indented.
return (i > 0 and not raw_lines[i-1].strip() and
re.match(section_re, raw_lines[i]) and
len(raw_lines) > i+1 and raw_lines[i+1].startswith(' '))
for i, line in enumerate(raw_lines):
if not in_special_section and is_section_start(i):
in_special_section = True
lines.append('#### ' + section_re.sub(r'\1:', line))
lines.append('')
continue
# If the next line starts a new section, this one ends. Add an extra line.
if in_special_section and is_section_start(i+1):
in_special_section = False
lines.append('')
if in_special_section:
# Translate symbols in 'Args:', 'Parameters:', 'Raises:', etc. sections.
lines.append(symbol_list_item_re.sub(r'* <b>`\1`</b>: ', line))
else:
lines.append(line)
docstring = '\n'.join(lines)
# TODO(deannarubin): Improve formatting for devsite
# TODO(deannarubin): Interpret @compatibility and other formatting notes.
return replace_references(docstring, relative_path_to_root, duplicate_of)
def _get_arg_spec(func):
"""Extracts signature information from a function or functools.partial object.
For functions, uses `inspect.getargspec`. For `functools.partial` objects,
corrects the signature of the underlying function to take into account the
removed arguments.
Args:
func: A function whose signature to extract.
Returns:
An `ArgSpec` namedtuple `(args, varargs, keywords, defaults)`, as returned
by `inspect.getargspec`.
"""
# getargspec does not work for functools.partial objects directly.
if isinstance(func, functools.partial):
argspec = inspect.getargspec(func.func)
# Remove the args from the original function that have been used up.
first_default_arg = (
len(argspec.args or []) - len(argspec.defaults or []))
partial_args = len(func.args)
argspec_args = []
if argspec.args:
argspec_args = list(argspec.args[partial_args:])
argspec_defaults = list(argspec.defaults or ())
if argspec.defaults and partial_args > first_default_arg:
argspec_defaults = list(argspec.defaults[partial_args-first_default_arg:])
first_default_arg = max(0, first_default_arg - partial_args)
for kwarg in func.keywords:
if kwarg in argspec.args:
i = argspec_args.index(kwarg)
argspec_args.pop(i)
if i >= first_default_arg:
argspec_defaults.pop(i-first_default_arg)
else:
first_default_arg -= 1
return inspect.ArgSpec(args=argspec_args,
varargs=argspec.varargs,
keywords=argspec.keywords,
defaults=tuple(argspec_defaults))
else: # Regular function or method, getargspec will work fine.
return inspect.getargspec(func)
def _generate_signature(func):
"""Given a function, returns a string representing its args.
This function produces a string representing the arguments to a python
function, including surrounding parentheses. It uses inspect.getargspec, which
does not generalize well to Python 3.x, which is more flexible in how *args
and **kwargs are handled. This is not a problem in TF, since we have to remain
compatible to Python 2.7 anyway.
This function uses `__name__` for callables if it is available. This can lead
to poor results for functools.partial and other callable objects.
The returned string is Python code, so if it is included in a Markdown
document, it should be typeset as code (using backticks), or escaped.
Args:
func: A function of method to extract the signature for (anything
`inspect.getargspec` will accept).
Returns:
A string representing the signature of `func` as python code.
"""
# This produces poor signatures for decorated functions.
# TODO(wicke): We need to use something like the decorator module to fix it.
args_list = []
argspec = _get_arg_spec(func)
first_arg_with_default = (
len(argspec.args or []) - len(argspec.defaults or []))
# Python documentation skips `self` when printing method signatures.
first_arg = 1 if inspect.ismethod(func) and 'self' in argspec.args[:1] else 0
# Add all args without defaults.
for arg in argspec.args[first_arg:first_arg_with_default]:
args_list.append(arg)
# Add all args with defaults.
if argspec.defaults:
for arg, default in zip(
argspec.args[first_arg_with_default:], argspec.defaults):
# Some callables don't have __name__, fall back to including their repr.
# TODO(wicke): This could be improved at least for common cases.
if callable(default) and hasattr(default, '__name__'):
args_list.append('%s=%s' % (arg, default.__name__))
else:
args_list.append('%s=%r' % (arg, default))
# Add *args and *kwargs.
if argspec.varargs:
args_list.append('*' + argspec.varargs)
if argspec.keywords:
args_list.append('**' + argspec.keywords)
return '(%s)' % ', '.join(args_list)
def _generate_markdown_for_function(full_name, duplicate_names,
function, duplicate_of):
"""Generate Markdown docs for a function or method.
This function creates a documentation page for a function. It uses the
function name (incl. signature) as the title, followed by a list of duplicate
names (if there are any), and the Markdown formatted docstring of the
function.
Args:
full_name: The preferred name of the function. Used in the title. Must not
be present in `duplicate_of` (master names never are).
duplicate_names: A sorted list of alternative names (incl. `full_name`).
function: The python object referenced by `full_name`.
duplicate_of: A map of duplicate full names to master names. Used to resolve
@{symbol} references in the docstring.
Returns:
A string that can be written to a documentation file for this function.
"""
# TODO(wicke): Make sure this works for partials.
relative_path = os.path.relpath(
os.path.dirname(documentation_path(full_name)) or '.', '.')
docstring = _md_docstring(function, relative_path, duplicate_of)
signature = _generate_signature(function)
if duplicate_names:
aliases = '\n'.join(['### `%s`' % (name + signature)
for name in duplicate_names])
aliases += '\n\n'
else:
aliases = ''
return '#`%s%s`\n\n%s%s' % (full_name, signature, aliases, docstring)
def _generate_markdown_for_class(full_name, duplicate_names, py_class,
duplicate_of, index, tree):
"""Generate Markdown docs for a class.
This function creates a documentation page for a class. It uses the
class name as the title, followed by a list of duplicate
names (if there are any), the Markdown formatted docstring of the
class, a list of links to all child class docs, a list of all properties
including their docstrings, a list of all methods incl. their docstrings, and
a list of all class member names (public fields).
Args:
full_name: The preferred name of the class. Used in the title. Must not
be present in `duplicate_of` (master names never are).
duplicate_names: A sorted list of alternative names (incl. `full_name`).
py_class: The python object referenced by `full_name`.
duplicate_of: A map of duplicate full names to master names. Used to resolve
@{symbol} references in the docstrings.
index: A map from full names to python object references.
tree: A map from full names to the names of all documentable child objects.
Returns:
A string that can be written to a documentation file for this class.
"""
relative_path = os.path.relpath(
os.path.dirname(documentation_path(full_name)) or '.', '.')
docstring = _md_docstring(py_class, relative_path, duplicate_of)
if duplicate_names:
aliases = '\n'.join(['### `class %s`' % name for name in duplicate_names])
aliases += '\n\n'
else:
aliases = ''
docs = '# `%s`\n\n%s%s\n\n' % (full_name, aliases, docstring)
field_names = []
properties = []
methods = []
class_links = []
for member in tree[full_name]:
child_name = '.'.join([full_name, member])
child = index[child_name]
if isinstance(child, property):
properties.append((member, child))
elif inspect.isclass(child):
class_links.append(_markdown_link('class ' + member, child_name,
relative_path, duplicate_of))
elif inspect.ismethod(child) or inspect.isfunction(child):
methods.append((member, child))
else:
# TODO(wicke): We may want to also remember the object itself.
field_names.append(member)
if class_links:
docs += '## Child Classes\n'
docs += '\n\n'.join(sorted(class_links))
docs += '\n\n'
if properties:
docs += '## Properties\n\n'
for property_name, prop in sorted(properties, key=lambda x: x[0]):
docs += '### `%s`\n\n%s\n\n' % (
property_name, _md_docstring(prop, relative_path, duplicate_of))
docs += '\n\n'
if methods:
docs += '## Methods\n\n'
for method_name, method in sorted(methods, key=lambda x: x[0]):
method_signature = method_name + _generate_signature(method)
docs += '### `%s`\n\n%s\n\n' % (method_signature,
_md_docstring(method, relative_path,
duplicate_of))
docs += '\n\n'
if field_names:
docs += '## Class Members\n\n'
# TODO(wicke): Document the value of the members, at least for basic types.
docs += '\n\n'.join(sorted(field_names))
docs += '\n\n'
return docs
def _generate_markdown_for_module(full_name, duplicate_names, module,
duplicate_of, index, tree):
"""Generate Markdown docs for a module.
This function creates a documentation page for a module. It uses the
module name as the title, followed by a list of duplicate
names (if there are any), the Markdown formatted docstring of the
class, and a list of links to all members of this module.
Args:
full_name: The preferred name of the module. Used in the title. Must not
be present in `duplicate_of` (master names never are).
duplicate_names: A sorted list of alternative names (incl. `full_name`).
module: The python object referenced by `full_name`.
duplicate_of: A map of duplicate full names to master names. Used to resolve
@{symbol} references in the docstrings.
index: A map from full names to python object references.
tree: A map from full names to the names of all documentable child objects.
Returns:
A string that can be written to a documentation file for this module.
"""
relative_path = os.path.relpath(
os.path.dirname(documentation_path(full_name)) or '.', '.')
docstring = _md_docstring(module, relative_path, duplicate_of)
if duplicate_names:
aliases = '\n'.join(['### Module `%s`' % name for name in duplicate_names])
aliases += '\n\n'
else:
aliases = ''
member_names = tree.get(full_name, [])
# Make links to all members.
member_links = []
for name in member_names:
member_full_name = full_name + '.' + name if full_name else name
member = index[member_full_name]
if inspect.isclass(member):
link_text = 'class ' + name
elif inspect.isfunction(member):
link_text = name + _generate_signature(member)
else:
link_text = name
member_links.append(_markdown_link(link_text, member_full_name,
relative_path, duplicate_of))
# TODO(deannarubin): Make this list into a table and add the brief docstring.
# (use _get_brief_docstring)
return '# Module `%s`\n\n%s%s\n\n## Members\n\n%s' % (
full_name, aliases, docstring, '\n\n'.join(member_links))
def generate_markdown(full_name, py_object,
duplicate_of, duplicates,
index, tree, base_dir):
"""Generate Markdown docs for a given object that's part of the TF API.
This function uses _md_docstring to obtain the docs pertaining to
`object`.
This function resolves '@symbol' references in the docstrings into links to
the appropriate location. It also adds a list of alternative names for the
symbol automatically.
It assumes that the docs for each object live in a file given by
`documentation_path`, and that relative links to files within the
documentation are resolvable.
The output is Markdown that can be written to file and published.
Args:
full_name: The fully qualified name (excl. "tf.") of the symbol to be
documented.
py_object: The Python object to be documented. Its documentation is sourced
from `py_object`'s docstring.
duplicate_of: A `dict` mapping fully qualified names to "master" names. This
is used to resolve "@{symbol}" references to the "master" name.
duplicates: A `dict` mapping fully qualified names to a set of all
aliases of this name. This is used to automatically generate a list of all
aliases for each name.
index: A `dict` mapping fully qualified names to the corresponding Python
objects. Used to produce docs for child objects, and to check the validity
of "@{symbol}" references.
tree: A `dict` mapping a fully qualified name to the names of all its
members. Used to populate the members section of a class or module page.
base_dir: A base path that is stripped from file locations written to the
docs.
Returns:
A string containing the Markdown docs for `py_object`.
Raises:
RuntimeError: If an object is encountered for which we don't know how
to make docs.
"""
# Which other aliases exist for the object referenced by full_name?
master_name = duplicate_of.get(full_name, full_name)
duplicate_names = duplicates.get(master_name, [full_name])
# TODO(wicke): Once other pieces are ready, enable this also for partials.
if (inspect.ismethod(py_object) or inspect.isfunction(py_object) or
# Some methods in classes from extensions come in as routines.
inspect.isroutine(py_object)):
markdown = _generate_markdown_for_function(master_name, duplicate_names,
py_object, duplicate_of)
elif inspect.isclass(py_object):
markdown = _generate_markdown_for_class(master_name, duplicate_names,
py_object, duplicate_of,
index, tree)
elif inspect.ismodule(py_object):
markdown = _generate_markdown_for_module(master_name, duplicate_names,
py_object, duplicate_of,
index, tree)
else:
raise RuntimeError('Cannot make docs for object %s: %r' % (full_name,
py_object))
# Every page gets a note on the bottom about where this object is defined
# TODO(wicke): If py_object is decorated, get the decorated object instead.
# TODO(wicke): Only use decorators that support this in TF.
try:
path = os.path.relpath(inspect.getfile(py_object), base_dir)
# TODO(wicke): If this is a generated file, point to the source instead.
# Never include links outside this code base.
if not path.startswith('..'):
# TODO(wicke): Make this a link to github.
markdown += '\n\ndefined in %s\n\n' % path
except TypeError: # getfile throws TypeError if py_object is a builtin.
markdown += '\n\nthis is an alias for a Python built-in.'
return markdown
def generate_global_index(library_name, root_name, index, duplicate_of):
"""Given a dict of full names to python objects, generate an index page.
The index page generated contains a list of links for all symbols in `index`
that have their own documentation page.
Args:
library_name: The name for the documented library to use in the title.
root_name: The name to use for the root module.
index: A dict mapping full names to python objects.
duplicate_of: A map of duplicate names to preferred names.
Returns:
A string containing an index page as Markdown.
"""
symbol_links = []
for full_name, py_object in index.iteritems():
index_name = full_name or root_name
if (inspect.ismodule(py_object) or inspect.isfunction(py_object) or
inspect.isclass(py_object)):
symbol_links.append((index_name,
_markdown_link(index_name, full_name,
'.', duplicate_of)))
lines = ['# All symbols in %s' % library_name, '']
for _, link in sorted(symbol_links, key=lambda x: x[0]):
lines.append('* %s' % link)
# TODO(deannarubin): Make this list into a table and add the brief docstring.
# (use _get_brief_docstring)
return '\n'.join(lines)

349
tensorflow/tools/docs/parser_test.py Normal file
View File

@ -0,0 +1,349 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for documentation parser."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import functools
import inspect
import os
import sys
from tensorflow.python.platform import googletest
from tensorflow.tools.docs import parser
def test_function_for_markdown_reference(unused_arg):
"""Docstring with reference to @{test_function}."""
pass
def test_function(unused_arg, unused_kwarg='default'):
"""Docstring for test function."""
pass
def test_function_with_args_kwargs(unused_arg, *unused_args, **unused_kwargs):
"""Docstring for second test function."""
pass
def test_function_with_fancy_docstring(arg):
"""Function with a fancy docstring.
Args:
arg: An argument.
Returns:
arg: the input, and
arg: the input, again.
"""
return arg, arg
class TestClass(object):
"""Docstring for TestClass itself."""
def a_method(self, arg='default'):
"""Docstring for a method."""
pass
class ChildClass(object):
"""Docstring for a child class."""
pass
@property
def a_property(self):
"""Docstring for a property."""
pass
CLASS_MEMBER = 'a class member'
class ParserTest(googletest.TestCase):
def test_documentation_path(self):
self.assertEqual('test.md', parser.documentation_path('test'))
self.assertEqual('test/module.md', parser.documentation_path('test.module'))
def test_documentation_path_empty(self):
self.assertEqual('index.md', parser.documentation_path(''))
def test_replace_references(self):
string = 'A @{reference}, another @{tf.reference}, and a @{third}.'
duplicate_of = {'third': 'fourth'}
result = parser.replace_references(string, '../..', duplicate_of)
self.assertEqual(
'A [`reference`](../../reference.md), another '
'[`tf.reference`](../../reference.md), '
'and a [`third`](../../fourth.md).',
result)
def test_generate_markdown_for_class(self):
index = {
'TestClass': TestClass,
'TestClass.a_method': TestClass.a_method,
'TestClass.a_property': TestClass.a_property,
'TestClass.ChildClass': TestClass.ChildClass,
'TestClass.CLASS_MEMBER': TestClass.CLASS_MEMBER
}
tree = {
'TestClass': ['a_method', 'a_property', 'ChildClass', 'CLASS_MEMBER']
}
docs = parser.generate_markdown(full_name='TestClass', py_object=TestClass,
duplicate_of={}, duplicates={},
index=index, tree=tree, base_dir='/')
# Make sure all required docstrings are present.
self.assertTrue(inspect.getdoc(TestClass) in docs)
self.assertTrue(inspect.getdoc(TestClass.a_method) in docs)
self.assertTrue(inspect.getdoc(TestClass.a_property) in docs)
# Make sure that the signature is extracted properly and omits self.
self.assertTrue('a_method(arg=\'default\')' in docs)
# Make sure there is a link to the child class and it points the right way.
self.assertTrue('[`class ChildClass`](./TestClass/ChildClass.md)' in docs)
# Make sure CLASS_MEMBER is mentioned.
self.assertTrue('CLASS_MEMBER' in docs)
# Make sure this file is contained as the definition location.
self.assertTrue(os.path.relpath(__file__, '/') in docs)
def test_generate_markdown_for_module(self):
module = sys.modules[__name__]
index = {
'TestModule': module,
'TestModule.test_function': test_function,
'TestModule.test_function_with_args_kwargs':
test_function_with_args_kwargs,
'TestModule.TestClass': TestClass,
}
tree = {
'TestModule': ['TestClass', 'test_function',
'test_function_with_args_kwargs']
}
docs = parser.generate_markdown(full_name='TestModule', py_object=module,
duplicate_of={}, duplicates={},
index=index, tree=tree, base_dir='/')
# Make sure all required docstrings are present.
self.assertTrue(inspect.getdoc(module) in docs)
# Make sure that links to the members are there (not asserting on exact link
# text for functions).
self.assertTrue('./TestModule/test_function.md' in docs)
self.assertTrue('./TestModule/test_function_with_args_kwargs.md' in docs)
# Make sure there is a link to the child class and it points the right way.
self.assertTrue('[`class TestClass`](./TestModule/TestClass.md)' in docs)
# Make sure this file is contained as the definition location.
self.assertTrue(os.path.relpath(__file__, '/') in docs)
def test_generate_markdown_for_function(self):
index = {
'test_function': test_function
}
tree = {
'': ['test_function']
}
docs = parser.generate_markdown(full_name='test_function',
py_object=test_function,
duplicate_of={}, duplicates={},
index=index, tree=tree, base_dir='/')
# Make sure docstring shows up.
self.assertTrue(inspect.getdoc(test_function) in docs)
# Make sure the extracted signature is good.
self.assertTrue(
'test_function(unused_arg, unused_kwarg=\'default\')' in docs)
# Make sure this file is contained as the definition location.
self.assertTrue(os.path.relpath(__file__, '/') in docs)
def test_generate_markdown_for_function_with_kwargs(self):
index = {
'test_function_with_args_kwargs': test_function_with_args_kwargs
}
tree = {
'': ['test_function_with_args_kwargs']
}
docs = parser.generate_markdown(full_name='test_function_with_args_kwargs',
py_object=test_function_with_args_kwargs,
duplicate_of={}, duplicates={},
index=index, tree=tree, base_dir='/')
# Make sure docstring shows up.
self.assertTrue(inspect.getdoc(test_function_with_args_kwargs) in docs)
# Make sure the extracted signature is good.
self.assertTrue(
'test_function_with_args_kwargs(unused_arg,'
' *unused_args, **unused_kwargs)' in docs)
def test_references_replaced_in_generated_markdown(self):
index = {
'test_function_for_markdown_reference':
test_function_for_markdown_reference
}
tree = {
'': ['test_function_for_markdown_reference']
}
docs = parser.generate_markdown(
full_name='test_function_for_markdown_reference',
py_object=test_function_for_markdown_reference,
duplicate_of={}, duplicates={},
index=index, tree=tree, base_dir='/')
# Make sure docstring shows up and is properly processed.
expected_docs = parser.replace_references(
inspect.getdoc(test_function_for_markdown_reference),
relative_path_to_root='.', duplicate_of={})
self.assertTrue(expected_docs in docs)
def test_docstring_special_section(self):
index = {
'test_function': test_function_with_fancy_docstring
}
tree = {
'': 'test_function'
}
docs = parser.generate_markdown(
full_name='test_function',
py_object=test_function_with_fancy_docstring,
duplicate_of={}, duplicates={},
index=index, tree=tree, base_dir='/')
expected = '\n'.join([
'Function with a fancy docstring.',
'',
'#### Args:',
'',
'* <b>`arg`</b>: An argument.',
'',
'',
'#### Returns:',
'',
'* <b>`arg`</b>: the input, and',
'* <b>`arg`</b>: the input, again.',
''])
self.assertTrue(expected in docs)
def test_generate_index(self):
module = sys.modules[__name__]
index = {
'TestModule': module,
'test_function': test_function,
'TestModule.test_function': test_function,
'TestModule.TestClass': TestClass,
'TestModule.TestClass.a_method': TestClass.a_method,
'TestModule.TestClass.a_property': TestClass.a_property,
'TestModule.TestClass.ChildClass': TestClass.ChildClass,
}
duplicate_of = {
'TestModule.test_function': 'test_function'
}
docs = parser.generate_global_index('TestLibrary', 'test',
index=index,
duplicate_of=duplicate_of)
# Make sure duplicates and non-top-level symbols are in the index, but
# methods and properties are not.
self.assertTrue('a_method' not in docs)
self.assertTrue('a_property' not in docs)
self.assertTrue('TestModule.TestClass' in docs)
self.assertTrue('TestModule.TestClass.ChildClass' in docs)
self.assertTrue('TestModule.test_function' in docs)
# Leading backtick to make sure it's included top-level.
# This depends on formatting, but should be stable.
self.assertTrue('`test_function' in docs)
def test_argspec_for_functoos_partial(self):
# pylint: disable=unused-argument
def test_function_for_partial1(arg1, arg2, kwarg1=1, kwarg2=2):
pass
def test_function_for_partial2(arg1, arg2, *my_args, **my_kwargs):
pass
# pylint: enable=unused-argument
# pylint: disable=protected-access
# Make sure everything works for regular functions.
expected = inspect.ArgSpec(['arg1', 'arg2', 'kwarg1', 'kwarg2'], None, None,
(1, 2))
self.assertEqual(expected, parser._get_arg_spec(test_function_for_partial1))
# Make sure doing nothing works.
expected = inspect.ArgSpec(['arg1', 'arg2', 'kwarg1', 'kwarg2'], None, None,
(1, 2))
partial = functools.partial(test_function_for_partial1)
self.assertEqual(expected, parser._get_arg_spec(partial))
# Make sure setting args from the front works.
expected = inspect.ArgSpec(['arg2', 'kwarg1', 'kwarg2'], None, None, (1, 2))
partial = functools.partial(test_function_for_partial1, 1)
self.assertEqual(expected, parser._get_arg_spec(partial))
expected = inspect.ArgSpec(['kwarg2',], None, None, (2,))
partial = functools.partial(test_function_for_partial1, 1, 2, 3)
self.assertEqual(expected, parser._get_arg_spec(partial))
# Make sure setting kwargs works.
expected = inspect.ArgSpec(['arg1', 'arg2', 'kwarg2'], None, None, (2,))
partial = functools.partial(test_function_for_partial1, kwarg1=0)
self.assertEqual(expected, parser._get_arg_spec(partial))
expected = inspect.ArgSpec(['arg1', 'arg2', 'kwarg1'], None, None, (1,))
partial = functools.partial(test_function_for_partial1, kwarg2=0)
self.assertEqual(expected, parser._get_arg_spec(partial))
expected = inspect.ArgSpec(['arg1'], None, None, ())
partial = functools.partial(test_function_for_partial1,
arg2=0, kwarg1=0, kwarg2=0)
self.assertEqual(expected, parser._get_arg_spec(partial))
# Make sure *args, *kwargs is accounted for.
expected = inspect.ArgSpec([], 'my_args', 'my_kwargs', ())
partial = functools.partial(test_function_for_partial2, 0, 1)
self.assertEqual(expected, parser._get_arg_spec(partial))
# pylint: enable=protected-access
if __name__ == '__main__':
googletest.main()