From d33692e9ccb8974946d4b0142e7b1046ffd05cb1 Mon Sep 17 00:00:00 2001 From: Martin Wicke Date: Tue, 31 Jan 2017 15:55:47 -0800 Subject: [PATCH] Make a better doc generator. Change: 146177492 --- tensorflow/python/framework/random_seed.py | 2 +- tensorflow/tools/common/public_api.py | 18 +- tensorflow/tools/docs/BUILD | 68 ++ .../tools/docs/doc_generator_visitor.py | 149 ++++ .../tools/docs/doc_generator_visitor_test.py | 107 +++ tensorflow/tools/docs/generate.py | 232 +++++++ tensorflow/tools/docs/generate_test.py | 114 ++++ tensorflow/tools/docs/parser.py | 634 ++++++++++++++++++ tensorflow/tools/docs/parser_test.py | 349 ++++++++++ 9 files changed, 1670 insertions(+), 3 deletions(-) create mode 100644 tensorflow/tools/docs/doc_generator_visitor.py create mode 100644 tensorflow/tools/docs/doc_generator_visitor_test.py create mode 100644 tensorflow/tools/docs/generate.py create mode 100644 tensorflow/tools/docs/generate_test.py create mode 100644 tensorflow/tools/docs/parser.py create mode 100644 tensorflow/tools/docs/parser_test.py diff --git a/tensorflow/python/framework/random_seed.py b/tensorflow/python/framework/random_seed.py index 25696eb0bbe..27df2888393 100644 --- a/tensorflow/python/framework/random_seed.py +++ b/tensorflow/python/framework/random_seed.py @@ -39,7 +39,7 @@ def get_seed(op_seed): graph, or for only specific operations. For details on how the graph-level seed interacts with op seeds, see - [`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed). + @{set_random_seed}. Args: op_seed: integer. diff --git a/tensorflow/tools/common/public_api.py b/tensorflow/tools/common/public_api.py index 5d70cb7b767..a7ac8cb22ad 100644 --- a/tensorflow/tools/common/public_api.py +++ b/tensorflow/tools/common/public_api.py @@ -40,16 +40,28 @@ class PublicAPIVisitor(object): # Each entry maps a module path to a name to ignore in traversal. _do_not_descend_map = { # TODO(drpng): This can be removed once sealed off. - '': ['platform', 'pywrap_tensorflow'], + '': ['platform', 'pywrap_tensorflow', 'user_ops'], # Some implementations have this internal module that we shouldn't expose. 'flags': ['cpp_flags'], # Everything below here is legitimate. - 'app': 'flags', # It'll stay, but it's not officially part of the API + 'app': ['flags'], # It'll stay, but it's not officially part of the API. 'test': ['mock'], # Imported for compatibility between py2/3. } + @property + def do_not_descend_map(self): + """A map from parents to symbols that should not be descended into. + + This map can be edited, but it should not be edited once traversal has + begun. + + Returns: + The map marking symbols to not explore. + """ + return self._do_not_descend_map + def _isprivate(self, name): """Return whether a name is private.""" return name.startswith('_') @@ -61,6 +73,8 @@ class PublicAPIVisitor(object): def __call__(self, path, parent, children): """Visitor interface, see `traverse` for details.""" + + # Avoid long waits in cases of pretty unambiguous failure. if inspect.ismodule(parent) and len(path.split('.')) > 10: raise RuntimeError('Modules nested too deep:\n%s\n\nThis is likely a ' 'problem with an accidental public import.' % path) diff --git a/tensorflow/tools/docs/BUILD b/tensorflow/tools/docs/BUILD index 4c37c291d04..75b0c507692 100644 --- a/tensorflow/tools/docs/BUILD +++ b/tensorflow/tools/docs/BUILD @@ -16,6 +16,74 @@ py_binary( deps = ["//tensorflow:tensorflow_py"], ) +py_library( + name = "doc_generator_visitor", + srcs = [ + "doc_generator_visitor.py", + ], + srcs_version = "PY2AND3", +) + +py_test( + name = "doc_generator_visitor_test", + size = "small", + srcs = [ + "doc_generator_visitor_test.py", + ], + srcs_version = "PY2AND3", + deps = [ + ":doc_generator_visitor", + "//tensorflow/python:platform_test", + ], +) + +py_library( + name = "parser", + srcs = [ + "parser.py", + ], + srcs_version = "PY2AND3", +) + +py_test( + name = "parser_test", + size = "small", + srcs = [ + "parser_test.py", + ], + srcs_version = "PY2AND3", + deps = [ + ":parser", + "//tensorflow/python:platform_test", + ], +) + +py_binary( + name = "generate", + srcs = ["generate.py"], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow:tensorflow_py", + "//tensorflow/tools/common:public_api", + "//tensorflow/tools/common:traverse", + "//tensorflow/tools/docs:doc_generator_visitor", + "//tensorflow/tools/docs:parser", + ], +) + +py_test( + name = "generate_test", + size = "small", + srcs = [ + "generate_test.py", + ], + srcs_version = "PY2AND3", + deps = [ + ":generate", + "//tensorflow/python:platform_test", + ], +) + filegroup( name = "doxy_config", srcs = ["tf-doxy_for_md-config"], diff --git a/tensorflow/tools/docs/doc_generator_visitor.py b/tensorflow/tools/docs/doc_generator_visitor.py new file mode 100644 index 00000000000..79f7f35ffb5 --- /dev/null +++ b/tensorflow/tools/docs/doc_generator_visitor.py @@ -0,0 +1,149 @@ +# Copyright 2015 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""A `traverse` visitor for processing documentation.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import inspect + + +class DocGeneratorVisitor(object): + """A visitor that generates docs for a python object when __call__ed.""" + + def __init__(self): + self._index = {} + self._tree = {} + + @property + def index(self): + """A map from fully qualified names to objects to be documented. + + The index is filled when the visitor is passed to `traverse`. + + Returns: + The index filled by traversal. + """ + return self._index + + @property + def tree(self): + """A map from fully qualified names to all its child names for traversal. + + The full name to member names map is filled when the visitor is passed to + `traverse`. + + Returns: + The full name to member name map filled by traversal. + """ + return self._tree + + def __call__(self, parent_name, parent, children): + """Visitor interface, see `tensorflow/tools/common:traverse` for details. + + This method is called for each symbol found in a traversal using + `tensorflow/tools/common:traverse`. It should not be called directly in + user code. + + Args: + parent_name: The fully qualified name of a symbol found during traversal. + parent: The Python object referenced by `parent_name`. + children: A list of `(name, py_object)` pairs enumerating, in alphabetical + order, the children (as determined by `inspect.getmembers`) of `parent`. + `name` is the local name of `py_object` in `parent`. + + Raises: + RuntimeError: If this visitor is called with a `parent` that is not a + class or module. + """ + self._index[parent_name] = parent + self._tree[parent_name] = [] + + if inspect.ismodule(parent): + print('module %s: %r' % (parent_name, parent)) + elif inspect.isclass(parent): + print('class %s: %r' % (parent_name, parent)) + else: + raise RuntimeError('Unexpected type in visitor -- %s: %r' % + (parent_name, parent)) + + for name, child in children: + full_name = '.'.join([parent_name, name]) if parent_name else name + self._index[full_name] = child + self._tree[parent_name].append(name) + + def find_duplicates(self): + """Compute data structures containing information about duplicates. + + Find duplicates in `index` and decide on one to be the "master" name. + + Returns a map `duplicate_of` from aliases to their master name (the master + name itself has no entry in this map), and a map `duplicates` from master + names to a lexicographically sorted list of all aliases for that name (incl. + the master name). + + Returns: + A tuple `(duplicate_of, duplicates)` as described above. + """ + # Maps the id of a symbol to its fully qualified name. For symbols that have + # several aliases, this map contains the first one found. + # We use id(py_object) to get a hashable value for py_object. Note all + # objects in _index are in memory at the same time so this is safe. + reverse_index = {} + + # Make a preliminary duplicates map. For all sets of duplicate names, it + # maps the first name found to a list of all duplicate names. + raw_duplicates = {} + for full_name, py_object in self._index.iteritems(): + # We cannot use the duplicate mechanism for constants, since e.g., + # id(c1) == id(c2) with c1=1, c2=1. This is unproblematic since constants + # have no usable docstring and won't be documented automatically. + if (inspect.ismodule(py_object) or + inspect.isclass(py_object) or + inspect.isfunction(py_object) or + inspect.isroutine(py_object) or + inspect.ismethod(py_object) or + isinstance(py_object, property)): + object_id = id(py_object) + if object_id in reverse_index: + master_name = reverse_index[object_id] + if master_name in raw_duplicates: + raw_duplicates[master_name].append(full_name) + else: + raw_duplicates[master_name] = [master_name, full_name] + else: + reverse_index[object_id] = full_name + + # Decide on master names, rewire duplicates and make a duplicate_of map + # mapping all non-master duplicates to the master name. The master symbol + # does not have an entry in this map. + duplicate_of = {} + # Duplicates maps the main symbols to the set of all duplicates of that + # symbol (incl. itself). + duplicates = {} + for names in raw_duplicates.values(): + names = sorted(names) + + # Choose the lexicographically first name with the minimum number of + # submodules. This will prefer highest level namespace for any symbol. + master_name = min(names, key=lambda name: name.count('.')) + + duplicates[master_name] = names + for name in names: + if name != master_name: + duplicate_of[name] = master_name + + return duplicate_of, duplicates diff --git a/tensorflow/tools/docs/doc_generator_visitor_test.py b/tensorflow/tools/docs/doc_generator_visitor_test.py new file mode 100644 index 00000000000..bbaa1c6474c --- /dev/null +++ b/tensorflow/tools/docs/doc_generator_visitor_test.py @@ -0,0 +1,107 @@ +# Copyright 2015 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for tools.docs.doc_generator_visitor.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.platform import googletest +from tensorflow.tools.docs import doc_generator_visitor + + +class DocGeneratorVisitorTest(googletest.TestCase): + + def test_call_module(self): + visitor = doc_generator_visitor.DocGeneratorVisitor() + visitor( + 'doc_generator_visitor', doc_generator_visitor, + [('DocGeneratorVisitor', doc_generator_visitor.DocGeneratorVisitor)]) + + self.assertEqual({'doc_generator_visitor': ['DocGeneratorVisitor']}, + visitor.tree) + self.assertEqual({ + 'doc_generator_visitor': doc_generator_visitor, + 'doc_generator_visitor.DocGeneratorVisitor': + doc_generator_visitor.DocGeneratorVisitor, + }, visitor.index) + + def test_call_class(self): + visitor = doc_generator_visitor.DocGeneratorVisitor() + visitor( + 'DocGeneratorVisitor', doc_generator_visitor.DocGeneratorVisitor, + [('index', doc_generator_visitor.DocGeneratorVisitor.index)]) + + self.assertEqual({'DocGeneratorVisitor': ['index']}, + visitor.tree) + self.assertEqual({ + 'DocGeneratorVisitor': doc_generator_visitor.DocGeneratorVisitor, + 'DocGeneratorVisitor.index': + doc_generator_visitor.DocGeneratorVisitor.index + }, visitor.index) + + def test_call_raises(self): + visitor = doc_generator_visitor.DocGeneratorVisitor() + with self.assertRaises(RuntimeError): + visitor('non_class_or_module', 'non_class_or_module_object', []) + + def test_duplicates(self): + visitor = doc_generator_visitor.DocGeneratorVisitor() + visitor( + 'submodule.DocGeneratorVisitor', + doc_generator_visitor.DocGeneratorVisitor, + [('index', doc_generator_visitor.DocGeneratorVisitor.index), + ('index2', doc_generator_visitor.DocGeneratorVisitor.index)]) + visitor( + 'submodule2.DocGeneratorVisitor', + doc_generator_visitor.DocGeneratorVisitor, + [('index', doc_generator_visitor.DocGeneratorVisitor.index), + ('index2', doc_generator_visitor.DocGeneratorVisitor.index)]) + visitor( + 'DocGeneratorVisitor2', + doc_generator_visitor.DocGeneratorVisitor, + [('index', doc_generator_visitor.DocGeneratorVisitor.index), + ('index2', doc_generator_visitor.DocGeneratorVisitor.index)]) + + duplicate_of, duplicates = visitor.find_duplicates() + + # The shorter path should be master, or if equal, the lexicographically + # first will be. + self.assertEqual( + {'DocGeneratorVisitor2': sorted(['submodule.DocGeneratorVisitor', + 'submodule2.DocGeneratorVisitor', + 'DocGeneratorVisitor2']), + 'DocGeneratorVisitor2.index': sorted([ + 'submodule.DocGeneratorVisitor.index', + 'submodule.DocGeneratorVisitor.index2', + 'submodule2.DocGeneratorVisitor.index', + 'submodule2.DocGeneratorVisitor.index2', + 'DocGeneratorVisitor2.index', + 'DocGeneratorVisitor2.index2' + ]), + }, duplicates) + self.assertEqual({ + 'submodule.DocGeneratorVisitor': 'DocGeneratorVisitor2', + 'submodule.DocGeneratorVisitor.index': 'DocGeneratorVisitor2.index', + 'submodule.DocGeneratorVisitor.index2': 'DocGeneratorVisitor2.index', + 'submodule2.DocGeneratorVisitor': 'DocGeneratorVisitor2', + 'submodule2.DocGeneratorVisitor.index': 'DocGeneratorVisitor2.index', + 'submodule2.DocGeneratorVisitor.index2': 'DocGeneratorVisitor2.index', + 'DocGeneratorVisitor2.index2': 'DocGeneratorVisitor2.index' + }, duplicate_of) + + +if __name__ == '__main__': + googletest.main() diff --git a/tensorflow/tools/docs/generate.py b/tensorflow/tools/docs/generate.py new file mode 100644 index 00000000000..4189deaec9c --- /dev/null +++ b/tensorflow/tools/docs/generate.py @@ -0,0 +1,232 @@ +# Copyright 2015 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Generate docs for the TensorFlow Python API.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import inspect +import os + +import tensorflow as tf + +from tensorflow.tools.common import public_api +from tensorflow.tools.common import traverse +from tensorflow.tools.docs import doc_generator_visitor +from tensorflow.tools.docs import parser + + +def write_docs(output_dir, base_dir, duplicate_of, duplicates, index, tree): + """Write previously extracted docs to disk. + + Write a docs page for each symbol in `index` to a tree of docs at + `output_dir`. + + Symbols with multiple aliases will have only one page written about them, + which is referenced for all aliases. `duplicate_of` and `duplicates` are used + to determine which docs pages to write. + + Args: + output_dir: Directory to write documentation markdown files to. Will be + created if it doesn't exist. + base_dir: Base directory of the code being documented. This prefix is + stripped from all file paths that are part of the documentation. + duplicate_of: A `dict` mapping fully qualified names to "master" names. This + is used to resolve "@{symbol}" references to the "master" name. + duplicates: A `dict` mapping fully qualified names to a set of all + aliases of this name. This is used to automatically generate a list of all + aliases for each name. + index: A `dict` mapping fully qualified names to the corresponding Python + objects. Used to produce docs for child objects, and to check the validity + of "@{symbol}" references. + tree: A `dict` mapping a fully qualified name to the names of all its + members. Used to populate the members section of a class or module page. + """ + # Make output_dir. + try: + if not os.path.exists(output_dir): + os.makedirs(output_dir) + except OSError as e: + print('Creating output dir "%s" failed: %s' % (output_dir, e)) + raise + + # Parse and write Markdown pages, resolving cross-links (@{symbol}). + for full_name, py_object in index.iteritems(): + + if full_name in duplicate_of: + print('Not writing docs for %s, duplicate of %s.' % ( + full_name, duplicate_of[full_name])) + continue + + # Methods and some routines are documented only as part of their class. + if not (inspect.ismodule(py_object) or + inspect.isclass(py_object) or + inspect.isfunction(py_object)): + print('Not writing docs for %s, not a class, module, or function.' % ( + full_name)) + continue + + print('Writing docs for %s (%r).' % (full_name, py_object)) + + # Generate docs for `py_object`, resolving references. + markdown = parser.generate_markdown(full_name, py_object, + duplicate_of=duplicate_of, + duplicates=duplicates, + index=index, + tree=tree, + base_dir=base_dir) + + # TODO(deannarubin): use _tree to generate sidebar information. + + path = os.path.join(output_dir, parser.documentation_path(full_name)) + directory = os.path.dirname(path) + try: + if not os.path.exists(directory): + os.makedirs(directory) + with open(path, 'w') as f: + f.write(markdown) + except OSError as e: + print('Cannot write documentation for %s to %s: %s' % (full_name, + directory, e)) + raise + # TODO(deannarubin): write sidebar file? + + # Write a global index containing all full names with links. + with open(os.path.join(output_dir, 'full_index.md'), 'w') as f: + f.write(parser.generate_global_index('TensorFlow', 'tensorflow', + index, duplicate_of)) + + +def extract(): + """Extract docs from tf namespace and write them to disk.""" + visitor = doc_generator_visitor.DocGeneratorVisitor() + api_visitor = public_api.PublicAPIVisitor(visitor) + + # Access something in contrib so tf.contrib is properly loaded (it's hidden + # behind lazy loading) + # TODO(wicke): Enable contrib traversal once contrib is sealed. + # _ = tf.contrib.__name__ + + # Exclude some libaries in contrib from the documentation altogether. + # TODO(wicke): Shrink this list. + api_visitor.do_not_descend_map.update({ + 'contrib': [ + 'compiler', + 'factorization', + 'grid_rnn', + 'labeled_tensor', + 'ndlstm', + 'quantization', + 'session_bundle', + 'slim', + 'solvers', + 'specs', + 'tensor_forest', + 'tensorboard', + 'testing', + 'tfprof', + 'training', + ], + 'contrib.bayesflow': [ + 'entropy', 'monte_carlo', + 'special_math', 'stochastic_gradient_estimators', + 'stochastic_graph', 'stochastic_tensor', + 'stochastic_variables', 'variational_inference' + ], + 'contrib.distributions': ['bijector'], + 'contrib.graph_editor': [ + 'edit', + 'match', + 'reroute', + 'subgraph', + 'transform', + 'select', + 'util' + ], + 'contrib.layers': [ + 'feature_column', + 'summaries' + ], + 'contrib.learn': [ + 'datasets', + 'graph_actions', + 'io', + 'models', + 'monitors', + 'ops', + 'preprocessing', + 'utils', + ], + 'contrib.util': ['loader'], + }) + + traverse.traverse(tf, api_visitor) + + return visitor + + +def write(output_dir, base_dir, visitor): + """Write documentation for an index in a `DocGeneratorVisitor` to disk. + + This function will create `output_dir` if it doesn't exist, and write + the documentation contained in `visitor`. + + Args: + output_dir: The directory to write documentation to. Must not exist. + base_dir: The base dir of the library `visitor` has traversed. This is used + to compute relative paths for file references. + visitor: A `DocGeneratorVisitor` that has traversed a library located at + `base_dir`. + """ + duplicate_of, duplicates = visitor.find_duplicates() + write_docs(output_dir, os.path.abspath(base_dir), + duplicate_of, duplicates, visitor.index, visitor.tree) + + +if __name__ == '__main__': + argument_parser = argparse.ArgumentParser() + argument_parser.add_argument( + '--output_dir', + type=str, + default=None, + required=True, + help='Directory to write docs to. Must not exist.' + ) + + # This doc generator works on the TensorFlow codebase. Since this script lives + # at tensorflow/tools/docs, we can compute the base directory (three levels + # up), which is valid unless we're trying to apply this to a different code + # base, or are moving the script around. + script_dir = os.path.dirname(inspect.getfile(inspect.currentframe())) + default_base_dir = os.path.join(script_dir, '..', '..', '..') + + argument_parser.add_argument( + '--base_dir', + type=str, + default=default_base_dir, + help=('Base directory to to strip from file names referenced in docs. ' + 'Defaults to three directories up from the location of this file.') + ) + + flags, _ = argument_parser.parse_known_args() + + if os.path.exists(flags.output_dir): + raise RuntimeError('output_dir %s exists.\n' + 'Cowardly refusing to wipe it, please do that yourself.' + % flags.output_dir) + + write(flags.output_dir, flags.base_dir, extract()) diff --git a/tensorflow/tools/docs/generate_test.py b/tensorflow/tools/docs/generate_test.py new file mode 100644 index 00000000000..4594676109c --- /dev/null +++ b/tensorflow/tools/docs/generate_test.py @@ -0,0 +1,114 @@ +# Copyright 2015 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for doc generator traversal.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import sys +import tempfile + +from tensorflow.python.platform import googletest +from tensorflow.tools.docs import generate + + +def test_function(): + """Docstring for test_function.""" + pass + + +class TestClass(object): + """Docstring for TestClass itself.""" + + class ChildClass(object): + """Docstring for a child class.""" + + class GrandChildClass(object): + """Docstring for a child of a child class.""" + pass + + +class GenerateTest(googletest.TestCase): + + def test_extraction(self): + try: + generate.extract() + except RuntimeError: + print('*****************************************************************') + print('If this test fails, you have most likely introduced an unsealed') + print('module. Make sure to use remove_undocumented or similar utilities') + print('to avoid leaking symbols. See below for more information on the') + print('failure.') + print('*****************************************************************') + raise + + def test_write(self): + module = sys.modules[__name__] + + index = { + '': sys, # Can be any module, this test doesn't care about content. + 'TestModule': module, + 'test_function': test_function, + 'TestModule.test_function': test_function, + 'TestModule.TestClass': TestClass, + 'TestModule.TestClass.ChildClass': TestClass.ChildClass, + 'TestModule.TestClass.ChildClass.GrandChildClass': + TestClass.ChildClass.GrandChildClass, + } + + tree = { + '': ['TestModule', 'test_function'], + 'TestModule': ['test_function', 'TestClass'], + 'TestModule.TestClass': ['ChildClass'], + 'TestModule.TestClass.ChildClass': ['GrandChildClass'], + 'TestModule.TestClass.ChildClass.GrandChildClass': [] + } + + duplicate_of = { + 'TestModule.test_function': 'test_function' + } + + duplicates = { + 'test_function': ['test_function', 'TestModule.test_function'] + } + + output_dir = tempfile.mkdtemp() + base_dir = os.path.dirname(__file__) + + generate.write_docs(output_dir, base_dir, + duplicate_of, duplicates, + index, tree) + + # Make sure that the right files are written to disk. + self.assertTrue(os.path.exists(os.path.join(output_dir, 'index.md'))) + self.assertTrue(os.path.exists(os.path.join(output_dir, 'full_index.md'))) + self.assertTrue(os.path.exists(os.path.join(output_dir, 'TestModule.md'))) + self.assertTrue(os.path.exists(os.path.join( + output_dir, 'test_function.md'))) + self.assertTrue(os.path.exists(os.path.join( + output_dir, 'TestModule/TestClass.md'))) + self.assertTrue(os.path.exists(os.path.join( + output_dir, 'TestModule/TestClass/ChildClass.md'))) + self.assertTrue(os.path.exists(os.path.join( + output_dir, 'TestModule/TestClass/ChildClass/GrandChildClass.md'))) + # Make sure that duplicates are not written + self.assertFalse(os.path.exists(os.path.join( + output_dir, 'TestModule/test_function.md'))) + + +if __name__ == '__main__': + googletest.main() diff --git a/tensorflow/tools/docs/parser.py b/tensorflow/tools/docs/parser.py new file mode 100644 index 00000000000..1dde384fb59 --- /dev/null +++ b/tensorflow/tools/docs/parser.py @@ -0,0 +1,634 @@ +# Copyright 2015 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Turn Python docstrings into Markdown for TensorFlow documentation.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import functools +import inspect +import os +import re + + +# A regular expression capturing a python indentifier. +IDENTIFIER_RE = '[a-zA-Z_][a-zA-Z0-9_]*' + + +def documentation_path(full_name): + """Returns the file path for the documentation for the given API symbol. + + Given the fully qualified name of a library symbol, compute the path to which + to write the documentation for that symbol (relative to a base directory). + Documentation files are organized into directories that mirror the python + module/class structure. The path for the top-level module (whose full name is + '') is 'index.md'. + + Args: + full_name: Fully qualified name of a library symbol. + + Returns: + The file path to which to write the documentation for `full_name`. + """ + # The main page is special, since it has no name in here. + if not full_name: + dirs = ['index'] + else: + dirs = full_name.split('.') + + return os.path.join(*dirs) + '.md' + + +def _get_raw_docstring(py_object): + """Get the docs for a given python object. + + Args: + py_object: A python object to retrieve the docs for (class, function/method, + or module). + + Returns: + The docstring, or the empty string if no docstring was found. + """ + # For object instances, inspect.getdoc does give us the docstring of their + # type, which is not what we want. Only return the docstring if it is useful. + if (inspect.isclass(py_object) or inspect.ismethod(py_object) or + inspect.isfunction(py_object) or inspect.ismodule(py_object) or + isinstance(py_object, property)): + return inspect.getdoc(py_object) or '' + else: + return '' + + +def _get_brief_docstring(py_object): + """Gets the one line docstring of a python object.""" + return _get_raw_docstring(py_object).split('\n')[0] + + +def _reference_to_link(ref_full_name, relative_path_to_root, duplicate_of): + """Resolve a "@{symbol}" reference to a relative path, respecting duplicates. + + The input to this function should already be stripped of the '@' and '{}', and + its output is only the link, not the full Markdown. + + Args: + ref_full_name: The fully qualified name of the symbol to link to. + relative_path_to_root: The relative path from the location of the current + document to the root of the API documentation. + duplicate_of: A map from duplicate full names to master names. + + Returns: + A relative path that links from the documentation page of `from_full_name` + to the documentation page of `ref_full_name`. + """ + master_name = duplicate_of.get(ref_full_name, ref_full_name) + ref_path = documentation_path(master_name) + return os.path.join(relative_path_to_root, ref_path) + + +def _markdown_link(link_text, ref_full_name, relative_path_to_root, + duplicate_of): + """Resolve a "@{symbol}" reference to a Markdown link, respecting duplicates. + + The input to this function should already be stripped of the '@' and '{}'. + This function returns a Markdown link. It is assumed that this is a code + reference, so the link text will always be rendered as code (using backticks). + + `link_text` should refer to a library symbol. You can either refer to it with + or without the `tf.` prefix. + + Args: + link_text: The text of the Markdown link. + ref_full_name: The fully qualified name of the symbol to link to + (may optionally include 'tf.'). + relative_path_to_root: The relative path from the location of the current + document to the root of the API documentation. + duplicate_of: A map from duplicate full names to master names. + + Returns: + A markdown link from the documentation page of `from_full_name` + to the documentation page of `ref_full_name`. + """ + if ref_full_name.startswith('tf.'): + ref_full_name = ref_full_name[3:] + + return '[`%s`](%s)' % ( + link_text, + _reference_to_link(ref_full_name, relative_path_to_root, duplicate_of)) + + +def replace_references(string, relative_path_to_root, duplicate_of): + """Replace "@{symbol}" references with links to symbol's documentation page. + + This functions finds all occurrences of "@{symbol}" in `string` and replaces + them with markdown links to the documentation page for "symbol". + + `relative_path_to_root` is the relative path from the document that contains + the "@{symbol}" reference to the root of the API documentation that is linked + to. If the containing page is part of the same API docset, + `relative_path_to_root` can be set to + `os.path.dirname(documentation_path(name))`, where `name` is the python name + of the object whose documentation page the reference lives on. + + Args: + string: A string in which "@{symbol}" references should be replaced. + relative_path_to_root: The relative path from the contianing document to the + root of the API documentation that is being linked to. + duplicate_of: A map from duplicate names to preferred names of API symbols. + + Returns: + `string`, with "@{symbol}" references replaced by Markdown links. + """ + full_name_re = '%s(.%s)*' % (IDENTIFIER_RE, IDENTIFIER_RE) + symbol_reference_re = re.compile(r'@\{(' + full_name_re + r')\}') + match = symbol_reference_re.search(string) + while match: + symbol_name = match.group(1) + link_text = _markdown_link(symbol_name, symbol_name, + relative_path_to_root, duplicate_of) + + # Remove only the '@symbol' part of the match, and replace with the link. + string = string[:match.start()] + link_text + string[match.end():] + match = symbol_reference_re.search(string, + pos=match.start() + len(link_text)) + return string + + +def _md_docstring(py_object, relative_path_to_root, duplicate_of): + """Get the docstring from an object and make it into nice Markdown. + + For links within the same set of docs, the `relative_path_to_root` for a + docstring on the page for `full_name` can be set to + + ```python + relative_path_to_root = os.path.relpath( + os.path.dirname(documentation_path(full_name)) or '.', '.') + ``` + + Args: + py_object: A python object to retrieve the docs for (class, function/method, + or module). + relative_path_to_root: The relative path from the location of the current + document to the root of the API documentation. This is used to compute + links for "@symbol" references. + duplicate_of: A map from duplicate symbol names to master names. Used to + resolve "@symbol" references. + + Returns: + The docstring, or the empty string if no docstring was found. + """ + # TODO(wicke): If this is a partial, use the .func docstring and add a note. + raw_docstring = _get_raw_docstring(py_object) + raw_lines = raw_docstring.split('\n') + + # Define regular expressions used during parsing below. + symbol_list_item_re = re.compile(r'^ (%s): ' % IDENTIFIER_RE) + section_re = re.compile(r'^(\w+):\s*$') + + # Translate docstring line by line. + in_special_section = False + lines = [] + + def is_section_start(i): + # Previous line is empty, line i is "Word:", and next line is indented. + return (i > 0 and not raw_lines[i-1].strip() and + re.match(section_re, raw_lines[i]) and + len(raw_lines) > i+1 and raw_lines[i+1].startswith(' ')) + + for i, line in enumerate(raw_lines): + if not in_special_section and is_section_start(i): + in_special_section = True + lines.append('#### ' + section_re.sub(r'\1:', line)) + lines.append('') + continue + + # If the next line starts a new section, this one ends. Add an extra line. + if in_special_section and is_section_start(i+1): + in_special_section = False + lines.append('') + + if in_special_section: + # Translate symbols in 'Args:', 'Parameters:', 'Raises:', etc. sections. + lines.append(symbol_list_item_re.sub(r'* `\1`: ', line)) + else: + lines.append(line) + + docstring = '\n'.join(lines) + + # TODO(deannarubin): Improve formatting for devsite + # TODO(deannarubin): Interpret @compatibility and other formatting notes. + + return replace_references(docstring, relative_path_to_root, duplicate_of) + + +def _get_arg_spec(func): + """Extracts signature information from a function or functools.partial object. + + For functions, uses `inspect.getargspec`. For `functools.partial` objects, + corrects the signature of the underlying function to take into account the + removed arguments. + + Args: + func: A function whose signature to extract. + + Returns: + An `ArgSpec` namedtuple `(args, varargs, keywords, defaults)`, as returned + by `inspect.getargspec`. + """ + # getargspec does not work for functools.partial objects directly. + if isinstance(func, functools.partial): + argspec = inspect.getargspec(func.func) + # Remove the args from the original function that have been used up. + first_default_arg = ( + len(argspec.args or []) - len(argspec.defaults or [])) + partial_args = len(func.args) + argspec_args = [] + + if argspec.args: + argspec_args = list(argspec.args[partial_args:]) + + argspec_defaults = list(argspec.defaults or ()) + if argspec.defaults and partial_args > first_default_arg: + argspec_defaults = list(argspec.defaults[partial_args-first_default_arg:]) + + first_default_arg = max(0, first_default_arg - partial_args) + for kwarg in func.keywords: + if kwarg in argspec.args: + i = argspec_args.index(kwarg) + argspec_args.pop(i) + if i >= first_default_arg: + argspec_defaults.pop(i-first_default_arg) + else: + first_default_arg -= 1 + return inspect.ArgSpec(args=argspec_args, + varargs=argspec.varargs, + keywords=argspec.keywords, + defaults=tuple(argspec_defaults)) + else: # Regular function or method, getargspec will work fine. + return inspect.getargspec(func) + + +def _generate_signature(func): + """Given a function, returns a string representing its args. + + This function produces a string representing the arguments to a python + function, including surrounding parentheses. It uses inspect.getargspec, which + does not generalize well to Python 3.x, which is more flexible in how *args + and **kwargs are handled. This is not a problem in TF, since we have to remain + compatible to Python 2.7 anyway. + + This function uses `__name__` for callables if it is available. This can lead + to poor results for functools.partial and other callable objects. + + The returned string is Python code, so if it is included in a Markdown + document, it should be typeset as code (using backticks), or escaped. + + Args: + func: A function of method to extract the signature for (anything + `inspect.getargspec` will accept). + + Returns: + A string representing the signature of `func` as python code. + """ + + # This produces poor signatures for decorated functions. + # TODO(wicke): We need to use something like the decorator module to fix it. + + args_list = [] + + argspec = _get_arg_spec(func) + first_arg_with_default = ( + len(argspec.args or []) - len(argspec.defaults or [])) + + # Python documentation skips `self` when printing method signatures. + first_arg = 1 if inspect.ismethod(func) and 'self' in argspec.args[:1] else 0 + + # Add all args without defaults. + for arg in argspec.args[first_arg:first_arg_with_default]: + args_list.append(arg) + + # Add all args with defaults. + if argspec.defaults: + for arg, default in zip( + argspec.args[first_arg_with_default:], argspec.defaults): + # Some callables don't have __name__, fall back to including their repr. + # TODO(wicke): This could be improved at least for common cases. + if callable(default) and hasattr(default, '__name__'): + args_list.append('%s=%s' % (arg, default.__name__)) + else: + args_list.append('%s=%r' % (arg, default)) + + # Add *args and *kwargs. + if argspec.varargs: + args_list.append('*' + argspec.varargs) + if argspec.keywords: + args_list.append('**' + argspec.keywords) + + return '(%s)' % ', '.join(args_list) + + +def _generate_markdown_for_function(full_name, duplicate_names, + function, duplicate_of): + """Generate Markdown docs for a function or method. + + This function creates a documentation page for a function. It uses the + function name (incl. signature) as the title, followed by a list of duplicate + names (if there are any), and the Markdown formatted docstring of the + function. + + Args: + full_name: The preferred name of the function. Used in the title. Must not + be present in `duplicate_of` (master names never are). + duplicate_names: A sorted list of alternative names (incl. `full_name`). + function: The python object referenced by `full_name`. + duplicate_of: A map of duplicate full names to master names. Used to resolve + @{symbol} references in the docstring. + + Returns: + A string that can be written to a documentation file for this function. + """ + # TODO(wicke): Make sure this works for partials. + relative_path = os.path.relpath( + os.path.dirname(documentation_path(full_name)) or '.', '.') + docstring = _md_docstring(function, relative_path, duplicate_of) + signature = _generate_signature(function) + + if duplicate_names: + aliases = '\n'.join(['### `%s`' % (name + signature) + for name in duplicate_names]) + aliases += '\n\n' + else: + aliases = '' + + return '#`%s%s`\n\n%s%s' % (full_name, signature, aliases, docstring) + + +def _generate_markdown_for_class(full_name, duplicate_names, py_class, + duplicate_of, index, tree): + """Generate Markdown docs for a class. + + This function creates a documentation page for a class. It uses the + class name as the title, followed by a list of duplicate + names (if there are any), the Markdown formatted docstring of the + class, a list of links to all child class docs, a list of all properties + including their docstrings, a list of all methods incl. their docstrings, and + a list of all class member names (public fields). + + Args: + full_name: The preferred name of the class. Used in the title. Must not + be present in `duplicate_of` (master names never are). + duplicate_names: A sorted list of alternative names (incl. `full_name`). + py_class: The python object referenced by `full_name`. + duplicate_of: A map of duplicate full names to master names. Used to resolve + @{symbol} references in the docstrings. + index: A map from full names to python object references. + tree: A map from full names to the names of all documentable child objects. + + Returns: + A string that can be written to a documentation file for this class. + """ + relative_path = os.path.relpath( + os.path.dirname(documentation_path(full_name)) or '.', '.') + docstring = _md_docstring(py_class, relative_path, duplicate_of) + if duplicate_names: + aliases = '\n'.join(['### `class %s`' % name for name in duplicate_names]) + aliases += '\n\n' + else: + aliases = '' + + docs = '# `%s`\n\n%s%s\n\n' % (full_name, aliases, docstring) + + field_names = [] + properties = [] + methods = [] + class_links = [] + for member in tree[full_name]: + child_name = '.'.join([full_name, member]) + child = index[child_name] + + if isinstance(child, property): + properties.append((member, child)) + elif inspect.isclass(child): + class_links.append(_markdown_link('class ' + member, child_name, + relative_path, duplicate_of)) + elif inspect.ismethod(child) or inspect.isfunction(child): + methods.append((member, child)) + else: + # TODO(wicke): We may want to also remember the object itself. + field_names.append(member) + + if class_links: + docs += '## Child Classes\n' + docs += '\n\n'.join(sorted(class_links)) + docs += '\n\n' + + if properties: + docs += '## Properties\n\n' + for property_name, prop in sorted(properties, key=lambda x: x[0]): + docs += '### `%s`\n\n%s\n\n' % ( + property_name, _md_docstring(prop, relative_path, duplicate_of)) + docs += '\n\n' + + if methods: + docs += '## Methods\n\n' + for method_name, method in sorted(methods, key=lambda x: x[0]): + method_signature = method_name + _generate_signature(method) + docs += '### `%s`\n\n%s\n\n' % (method_signature, + _md_docstring(method, relative_path, + duplicate_of)) + docs += '\n\n' + + if field_names: + docs += '## Class Members\n\n' + # TODO(wicke): Document the value of the members, at least for basic types. + docs += '\n\n'.join(sorted(field_names)) + docs += '\n\n' + + return docs + + +def _generate_markdown_for_module(full_name, duplicate_names, module, + duplicate_of, index, tree): + """Generate Markdown docs for a module. + + This function creates a documentation page for a module. It uses the + module name as the title, followed by a list of duplicate + names (if there are any), the Markdown formatted docstring of the + class, and a list of links to all members of this module. + + Args: + full_name: The preferred name of the module. Used in the title. Must not + be present in `duplicate_of` (master names never are). + duplicate_names: A sorted list of alternative names (incl. `full_name`). + module: The python object referenced by `full_name`. + duplicate_of: A map of duplicate full names to master names. Used to resolve + @{symbol} references in the docstrings. + index: A map from full names to python object references. + tree: A map from full names to the names of all documentable child objects. + + Returns: + A string that can be written to a documentation file for this module. + """ + relative_path = os.path.relpath( + os.path.dirname(documentation_path(full_name)) or '.', '.') + docstring = _md_docstring(module, relative_path, duplicate_of) + if duplicate_names: + aliases = '\n'.join(['### Module `%s`' % name for name in duplicate_names]) + aliases += '\n\n' + else: + aliases = '' + + member_names = tree.get(full_name, []) + + # Make links to all members. + member_links = [] + for name in member_names: + member_full_name = full_name + '.' + name if full_name else name + member = index[member_full_name] + + if inspect.isclass(member): + link_text = 'class ' + name + elif inspect.isfunction(member): + link_text = name + _generate_signature(member) + else: + link_text = name + + member_links.append(_markdown_link(link_text, member_full_name, + relative_path, duplicate_of)) + + # TODO(deannarubin): Make this list into a table and add the brief docstring. + # (use _get_brief_docstring) + + return '# Module `%s`\n\n%s%s\n\n## Members\n\n%s' % ( + full_name, aliases, docstring, '\n\n'.join(member_links)) + + +def generate_markdown(full_name, py_object, + duplicate_of, duplicates, + index, tree, base_dir): + """Generate Markdown docs for a given object that's part of the TF API. + + This function uses _md_docstring to obtain the docs pertaining to + `object`. + + This function resolves '@symbol' references in the docstrings into links to + the appropriate location. It also adds a list of alternative names for the + symbol automatically. + + It assumes that the docs for each object live in a file given by + `documentation_path`, and that relative links to files within the + documentation are resolvable. + + The output is Markdown that can be written to file and published. + + Args: + full_name: The fully qualified name (excl. "tf.") of the symbol to be + documented. + py_object: The Python object to be documented. Its documentation is sourced + from `py_object`'s docstring. + duplicate_of: A `dict` mapping fully qualified names to "master" names. This + is used to resolve "@{symbol}" references to the "master" name. + duplicates: A `dict` mapping fully qualified names to a set of all + aliases of this name. This is used to automatically generate a list of all + aliases for each name. + index: A `dict` mapping fully qualified names to the corresponding Python + objects. Used to produce docs for child objects, and to check the validity + of "@{symbol}" references. + tree: A `dict` mapping a fully qualified name to the names of all its + members. Used to populate the members section of a class or module page. + base_dir: A base path that is stripped from file locations written to the + docs. + + Returns: + A string containing the Markdown docs for `py_object`. + + Raises: + RuntimeError: If an object is encountered for which we don't know how + to make docs. + """ + + # Which other aliases exist for the object referenced by full_name? + master_name = duplicate_of.get(full_name, full_name) + duplicate_names = duplicates.get(master_name, [full_name]) + + # TODO(wicke): Once other pieces are ready, enable this also for partials. + if (inspect.ismethod(py_object) or inspect.isfunction(py_object) or + # Some methods in classes from extensions come in as routines. + inspect.isroutine(py_object)): + markdown = _generate_markdown_for_function(master_name, duplicate_names, + py_object, duplicate_of) + elif inspect.isclass(py_object): + markdown = _generate_markdown_for_class(master_name, duplicate_names, + py_object, duplicate_of, + index, tree) + elif inspect.ismodule(py_object): + markdown = _generate_markdown_for_module(master_name, duplicate_names, + py_object, duplicate_of, + index, tree) + else: + raise RuntimeError('Cannot make docs for object %s: %r' % (full_name, + py_object)) + + # Every page gets a note on the bottom about where this object is defined + # TODO(wicke): If py_object is decorated, get the decorated object instead. + # TODO(wicke): Only use decorators that support this in TF. + + try: + path = os.path.relpath(inspect.getfile(py_object), base_dir) + + # TODO(wicke): If this is a generated file, point to the source instead. + + # Never include links outside this code base. + if not path.startswith('..'): + # TODO(wicke): Make this a link to github. + markdown += '\n\ndefined in %s\n\n' % path + except TypeError: # getfile throws TypeError if py_object is a builtin. + markdown += '\n\nthis is an alias for a Python built-in.' + + return markdown + + +def generate_global_index(library_name, root_name, index, duplicate_of): + """Given a dict of full names to python objects, generate an index page. + + The index page generated contains a list of links for all symbols in `index` + that have their own documentation page. + + Args: + library_name: The name for the documented library to use in the title. + root_name: The name to use for the root module. + index: A dict mapping full names to python objects. + duplicate_of: A map of duplicate names to preferred names. + + Returns: + A string containing an index page as Markdown. + """ + symbol_links = [] + for full_name, py_object in index.iteritems(): + index_name = full_name or root_name + if (inspect.ismodule(py_object) or inspect.isfunction(py_object) or + inspect.isclass(py_object)): + symbol_links.append((index_name, + _markdown_link(index_name, full_name, + '.', duplicate_of))) + + lines = ['# All symbols in %s' % library_name, ''] + for _, link in sorted(symbol_links, key=lambda x: x[0]): + lines.append('* %s' % link) + + # TODO(deannarubin): Make this list into a table and add the brief docstring. + # (use _get_brief_docstring) + + return '\n'.join(lines) diff --git a/tensorflow/tools/docs/parser_test.py b/tensorflow/tools/docs/parser_test.py new file mode 100644 index 00000000000..521e2d4ed3b --- /dev/null +++ b/tensorflow/tools/docs/parser_test.py @@ -0,0 +1,349 @@ +# Copyright 2015 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for documentation parser.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import functools +import inspect +import os +import sys + +from tensorflow.python.platform import googletest +from tensorflow.tools.docs import parser + + +def test_function_for_markdown_reference(unused_arg): + """Docstring with reference to @{test_function}.""" + pass + + +def test_function(unused_arg, unused_kwarg='default'): + """Docstring for test function.""" + pass + + +def test_function_with_args_kwargs(unused_arg, *unused_args, **unused_kwargs): + """Docstring for second test function.""" + pass + + +def test_function_with_fancy_docstring(arg): + """Function with a fancy docstring. + + Args: + arg: An argument. + + Returns: + arg: the input, and + arg: the input, again. + """ + return arg, arg + + +class TestClass(object): + """Docstring for TestClass itself.""" + + def a_method(self, arg='default'): + """Docstring for a method.""" + pass + + class ChildClass(object): + """Docstring for a child class.""" + pass + + @property + def a_property(self): + """Docstring for a property.""" + pass + + CLASS_MEMBER = 'a class member' + + +class ParserTest(googletest.TestCase): + + def test_documentation_path(self): + self.assertEqual('test.md', parser.documentation_path('test')) + self.assertEqual('test/module.md', parser.documentation_path('test.module')) + + def test_documentation_path_empty(self): + self.assertEqual('index.md', parser.documentation_path('')) + + def test_replace_references(self): + string = 'A @{reference}, another @{tf.reference}, and a @{third}.' + duplicate_of = {'third': 'fourth'} + result = parser.replace_references(string, '../..', duplicate_of) + self.assertEqual( + 'A [`reference`](../../reference.md), another ' + '[`tf.reference`](../../reference.md), ' + 'and a [`third`](../../fourth.md).', + result) + + def test_generate_markdown_for_class(self): + + index = { + 'TestClass': TestClass, + 'TestClass.a_method': TestClass.a_method, + 'TestClass.a_property': TestClass.a_property, + 'TestClass.ChildClass': TestClass.ChildClass, + 'TestClass.CLASS_MEMBER': TestClass.CLASS_MEMBER + } + + tree = { + 'TestClass': ['a_method', 'a_property', 'ChildClass', 'CLASS_MEMBER'] + } + + docs = parser.generate_markdown(full_name='TestClass', py_object=TestClass, + duplicate_of={}, duplicates={}, + index=index, tree=tree, base_dir='/') + + # Make sure all required docstrings are present. + self.assertTrue(inspect.getdoc(TestClass) in docs) + self.assertTrue(inspect.getdoc(TestClass.a_method) in docs) + self.assertTrue(inspect.getdoc(TestClass.a_property) in docs) + + # Make sure that the signature is extracted properly and omits self. + self.assertTrue('a_method(arg=\'default\')' in docs) + + # Make sure there is a link to the child class and it points the right way. + self.assertTrue('[`class ChildClass`](./TestClass/ChildClass.md)' in docs) + + # Make sure CLASS_MEMBER is mentioned. + self.assertTrue('CLASS_MEMBER' in docs) + + # Make sure this file is contained as the definition location. + self.assertTrue(os.path.relpath(__file__, '/') in docs) + + def test_generate_markdown_for_module(self): + module = sys.modules[__name__] + + index = { + 'TestModule': module, + 'TestModule.test_function': test_function, + 'TestModule.test_function_with_args_kwargs': + test_function_with_args_kwargs, + 'TestModule.TestClass': TestClass, + } + + tree = { + 'TestModule': ['TestClass', 'test_function', + 'test_function_with_args_kwargs'] + } + + docs = parser.generate_markdown(full_name='TestModule', py_object=module, + duplicate_of={}, duplicates={}, + index=index, tree=tree, base_dir='/') + + # Make sure all required docstrings are present. + self.assertTrue(inspect.getdoc(module) in docs) + + # Make sure that links to the members are there (not asserting on exact link + # text for functions). + self.assertTrue('./TestModule/test_function.md' in docs) + self.assertTrue('./TestModule/test_function_with_args_kwargs.md' in docs) + + # Make sure there is a link to the child class and it points the right way. + self.assertTrue('[`class TestClass`](./TestModule/TestClass.md)' in docs) + + # Make sure this file is contained as the definition location. + self.assertTrue(os.path.relpath(__file__, '/') in docs) + + def test_generate_markdown_for_function(self): + index = { + 'test_function': test_function + } + + tree = { + '': ['test_function'] + } + + docs = parser.generate_markdown(full_name='test_function', + py_object=test_function, + duplicate_of={}, duplicates={}, + index=index, tree=tree, base_dir='/') + + # Make sure docstring shows up. + self.assertTrue(inspect.getdoc(test_function) in docs) + + # Make sure the extracted signature is good. + self.assertTrue( + 'test_function(unused_arg, unused_kwarg=\'default\')' in docs) + + # Make sure this file is contained as the definition location. + self.assertTrue(os.path.relpath(__file__, '/') in docs) + + def test_generate_markdown_for_function_with_kwargs(self): + index = { + 'test_function_with_args_kwargs': test_function_with_args_kwargs + } + + tree = { + '': ['test_function_with_args_kwargs'] + } + + docs = parser.generate_markdown(full_name='test_function_with_args_kwargs', + py_object=test_function_with_args_kwargs, + duplicate_of={}, duplicates={}, + index=index, tree=tree, base_dir='/') + + # Make sure docstring shows up. + self.assertTrue(inspect.getdoc(test_function_with_args_kwargs) in docs) + + # Make sure the extracted signature is good. + self.assertTrue( + 'test_function_with_args_kwargs(unused_arg,' + ' *unused_args, **unused_kwargs)' in docs) + + def test_references_replaced_in_generated_markdown(self): + index = { + 'test_function_for_markdown_reference': + test_function_for_markdown_reference + } + + tree = { + '': ['test_function_for_markdown_reference'] + } + + docs = parser.generate_markdown( + full_name='test_function_for_markdown_reference', + py_object=test_function_for_markdown_reference, + duplicate_of={}, duplicates={}, + index=index, tree=tree, base_dir='/') + + # Make sure docstring shows up and is properly processed. + expected_docs = parser.replace_references( + inspect.getdoc(test_function_for_markdown_reference), + relative_path_to_root='.', duplicate_of={}) + + self.assertTrue(expected_docs in docs) + + def test_docstring_special_section(self): + index = { + 'test_function': test_function_with_fancy_docstring + } + + tree = { + '': 'test_function' + } + + docs = parser.generate_markdown( + full_name='test_function', + py_object=test_function_with_fancy_docstring, + duplicate_of={}, duplicates={}, + index=index, tree=tree, base_dir='/') + + expected = '\n'.join([ + 'Function with a fancy docstring.', + '', + '#### Args:', + '', + '* `arg`: An argument.', + '', + '', + '#### Returns:', + '', + '* `arg`: the input, and', + '* `arg`: the input, again.', + '']) + self.assertTrue(expected in docs) + + def test_generate_index(self): + module = sys.modules[__name__] + + index = { + 'TestModule': module, + 'test_function': test_function, + 'TestModule.test_function': test_function, + 'TestModule.TestClass': TestClass, + 'TestModule.TestClass.a_method': TestClass.a_method, + 'TestModule.TestClass.a_property': TestClass.a_property, + 'TestModule.TestClass.ChildClass': TestClass.ChildClass, + } + + duplicate_of = { + 'TestModule.test_function': 'test_function' + } + + docs = parser.generate_global_index('TestLibrary', 'test', + index=index, + duplicate_of=duplicate_of) + + # Make sure duplicates and non-top-level symbols are in the index, but + # methods and properties are not. + self.assertTrue('a_method' not in docs) + self.assertTrue('a_property' not in docs) + self.assertTrue('TestModule.TestClass' in docs) + self.assertTrue('TestModule.TestClass.ChildClass' in docs) + self.assertTrue('TestModule.test_function' in docs) + # Leading backtick to make sure it's included top-level. + # This depends on formatting, but should be stable. + self.assertTrue('`test_function' in docs) + + def test_argspec_for_functoos_partial(self): + + # pylint: disable=unused-argument + def test_function_for_partial1(arg1, arg2, kwarg1=1, kwarg2=2): + pass + + def test_function_for_partial2(arg1, arg2, *my_args, **my_kwargs): + pass + # pylint: enable=unused-argument + + # pylint: disable=protected-access + # Make sure everything works for regular functions. + expected = inspect.ArgSpec(['arg1', 'arg2', 'kwarg1', 'kwarg2'], None, None, + (1, 2)) + self.assertEqual(expected, parser._get_arg_spec(test_function_for_partial1)) + + # Make sure doing nothing works. + expected = inspect.ArgSpec(['arg1', 'arg2', 'kwarg1', 'kwarg2'], None, None, + (1, 2)) + partial = functools.partial(test_function_for_partial1) + self.assertEqual(expected, parser._get_arg_spec(partial)) + + # Make sure setting args from the front works. + expected = inspect.ArgSpec(['arg2', 'kwarg1', 'kwarg2'], None, None, (1, 2)) + partial = functools.partial(test_function_for_partial1, 1) + self.assertEqual(expected, parser._get_arg_spec(partial)) + + expected = inspect.ArgSpec(['kwarg2',], None, None, (2,)) + partial = functools.partial(test_function_for_partial1, 1, 2, 3) + self.assertEqual(expected, parser._get_arg_spec(partial)) + + # Make sure setting kwargs works. + expected = inspect.ArgSpec(['arg1', 'arg2', 'kwarg2'], None, None, (2,)) + partial = functools.partial(test_function_for_partial1, kwarg1=0) + self.assertEqual(expected, parser._get_arg_spec(partial)) + + expected = inspect.ArgSpec(['arg1', 'arg2', 'kwarg1'], None, None, (1,)) + partial = functools.partial(test_function_for_partial1, kwarg2=0) + self.assertEqual(expected, parser._get_arg_spec(partial)) + + expected = inspect.ArgSpec(['arg1'], None, None, ()) + partial = functools.partial(test_function_for_partial1, + arg2=0, kwarg1=0, kwarg2=0) + self.assertEqual(expected, parser._get_arg_spec(partial)) + + # Make sure *args, *kwargs is accounted for. + expected = inspect.ArgSpec([], 'my_args', 'my_kwargs', ()) + partial = functools.partial(test_function_for_partial2, 0, 1) + self.assertEqual(expected, parser._get_arg_spec(partial)) + + # pylint: enable=protected-access + +if __name__ == '__main__': + googletest.main()