From 196ce75c9953c9e7f6a7dad01b98548481ce22b0 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Sat, 11 Feb 2017 10:02:02 -0800 Subject: [PATCH] More doc generator changes. Change: 147249420 --- tensorflow/tools/docs/BUILD | 11 -- tensorflow/tools/docs/generate.py | 117 ++++++++++++++++++++-- tensorflow/tools/docs/generate_test.py | 6 +- tensorflow/tools/docs/parser.py | 78 ++++++++++----- tensorflow/tools/docs/parser_test.py | 50 ++++++--- tensorflow/tools/docs/substitute_links.py | 99 ------------------ 6 files changed, 201 insertions(+), 160 deletions(-) delete mode 100644 tensorflow/tools/docs/substitute_links.py diff --git a/tensorflow/tools/docs/BUILD b/tensorflow/tools/docs/BUILD index 022d85e40f2..b7b08a2c20f 100644 --- a/tensorflow/tools/docs/BUILD +++ b/tensorflow/tools/docs/BUILD @@ -108,17 +108,6 @@ py_test( ], ) -py_binary( - name = "substitute_links", - srcs = ["substitute_links.py"], - srcs_version = "PY2AND3", - deps = [ - ":generate", - ":parser", - ":py_guide_parser", - ], -) - filegroup( name = "doxy_config", srcs = ["tf-doxy_for_md-config"], diff --git a/tensorflow/tools/docs/generate.py b/tensorflow/tools/docs/generate.py index 6e313a8eef8..b34bd397e6e 100644 --- a/tensorflow/tools/docs/generate.py +++ b/tensorflow/tools/docs/generate.py @@ -33,7 +33,7 @@ from tensorflow.tools.docs import py_guide_parser def write_docs(output_dir, base_dir, duplicate_of, duplicates, index, tree, - reverse_index, guide_index): + reverse_index, doc_index, guide_index): """Write previously extracted docs to disk. Write a docs page for each symbol in `index` to a tree of docs at @@ -59,6 +59,7 @@ def write_docs(output_dir, base_dir, duplicate_of, duplicates, index, tree, tree: A `dict` mapping a fully qualified name to the names of all its members. Used to populate the members section of a class or module page. reverse_index: A `dict` mapping object ids to fully qualified names. + doc_index: A `dict` mapping a doc key to a DocInfo. guide_index: A `dict` mapping symbol name strings to GuideRef. """ # Make output_dir. @@ -94,6 +95,7 @@ def write_docs(output_dir, base_dir, duplicate_of, duplicates, index, tree, index=index, tree=tree, reverse_index=reverse_index, + doc_index=doc_index, guide_index=guide_index, base_dir=base_dir) @@ -185,6 +187,46 @@ def extract(): return visitor +class GetMarkdownTitle(py_guide_parser.PyGuideParser): + """Extract the title from a .md file.""" + + def __init__(self): + self.title = None + py_guide_parser.PyGuideParser.__init__(self) + + def process_title(self, _, title): + if self.title is None: # only use the first title + self.title = title + + +class DocInfo(object): + """A simple struct for holding a doc's url and title.""" + + def __init__(self, url, title): + self.url = url + self.title = title + + +def build_doc_index(src_dir): + """Build an index from a keyword designating a doc to DocInfo objects.""" + doc_index = {} + for dirpath, _, filenames in os.walk(src_dir): + suffix = os.path.relpath(path=dirpath, start=src_dir) + for base_name in filenames: + if not base_name.endswith('.md'): continue + title_parser = GetMarkdownTitle() + title_parser.process(os.path.join(dirpath, base_name)) + key_parts = os.path.join(suffix, base_name[:-3]).split('/') + if key_parts[-1] == 'index': + key_parts = key_parts[:-1] + doc_info = DocInfo(os.path.join(suffix, base_name), title_parser.title) + doc_index[key_parts[-1]] = doc_info + if len(key_parts) > 1: + doc_index['/'.join(key_parts[-2:])] = doc_info + + return doc_index + + class GuideRef(object): def __init__(self, base_name, title, section_title, section_tag): @@ -238,7 +280,7 @@ def build_guide_index(guide_src_dir): return index_generator.index -def write(output_dir, base_dir, guide_index, visitor): +def write(output_dir, base_dir, doc_index, guide_index, visitor): """Write documentation for an index in a `DocGeneratorVisitor` to disk. This function will create `output_dir` if it doesn't exist, and write @@ -248,13 +290,75 @@ def write(output_dir, base_dir, guide_index, visitor): output_dir: The directory to write documentation to. Must not exist. base_dir: The base dir of the library `visitor` has traversed. This is used to compute relative paths for file references. + doc_index: A `dict` mapping a doc key to a DocInfo. guide_index: A `dict` mapping symbol name strings to GuideRef. visitor: A `DocGeneratorVisitor` that has traversed a library located at `base_dir`. """ write_docs(output_dir, os.path.abspath(base_dir), visitor.duplicate_of, visitor.duplicates, - visitor.index, visitor.tree, visitor.reverse_index, guide_index) + visitor.index, visitor.tree, visitor.reverse_index, + doc_index, guide_index) + + +class UpdateTags(py_guide_parser.PyGuideParser): + """Rewrites a Python guide so that each section has an explicit tag.""" + + def process_section(self, line_number, section_title, tag): + self.replace_line(line_number, '## %s {#%s}' % (section_title, tag)) + + +def other_docs(src_dir, output_dir, visitor, doc_index): + """Convert all the files in `src_dir` and write results to `output_dir`.""" + header = '\n' + + # Iterate through all the source files and process them. + tag_updater = UpdateTags() + for dirpath, _, filenames in os.walk(src_dir): + # How to get from `dirpath` to api_docs/python/ + relative_path_to_root = os.path.relpath( + path=os.path.join(src_dir, 'api_docs/python'), start=dirpath) + + # Make the directory under output_dir. + new_dir = os.path.join(output_dir, + os.path.relpath(path=dirpath, start=src_dir)) + try: + if not os.path.exists(new_dir): + os.makedirs(new_dir) + except OSError as e: + print('Creating output dir "%s" failed: %s' % (new_dir, e)) + raise + + for base_name in filenames: + full_in_path = os.path.join(dirpath, base_name) + suffix = os.path.relpath(path=full_in_path, start=src_dir) + full_out_path = os.path.join(output_dir, suffix) + if not base_name.endswith('.md'): + print('Copying non-md file %s...' % suffix) + open(full_out_path, 'w').write(open(full_in_path).read()) + continue + if dirpath.endswith('/api_guides/python'): + print('Processing Python guide %s...' % base_name) + md_string = tag_updater.process(full_in_path) + else: + print('Processing doc %s...' % suffix) + md_string = open(full_in_path).read() + + output = parser.replace_references( + md_string, relative_path_to_root, visitor.duplicate_of, doc_index) + open(full_out_path, 'w').write(header + output) + + print('Done.') + + +def _main(src_dir, output_dir, base_dir): + doc_index = build_doc_index(src_dir) + visitor = extract() + write(os.path.join(output_dir, 'api_docs/python'), base_dir, + doc_index, + build_guide_index(os.path.join(src_dir, 'api_guides/python')), + visitor) + other_docs(src_dir, output_dir, visitor, doc_index) if __name__ == '__main__': @@ -268,7 +372,7 @@ if __name__ == '__main__': ) argument_parser.add_argument( - '--guide_src_dir', + '--src_dir', type=str, default=None, required=True, @@ -288,7 +392,7 @@ if __name__ == '__main__': type=str, default=default_base_dir, help=('Base directory to to strip from file names referenced in docs. ' - 'Defaults to three directories up from the location of this file.') + 'Defaults to two directories up from the location of this file.') ) flags, _ = argument_parser.parse_known_args() @@ -298,5 +402,4 @@ if __name__ == '__main__': 'Cowardly refusing to wipe it, please do that yourself.' % flags.output_dir) - write(flags.output_dir, flags.base_dir, - build_guide_index(flags.guide_src_dir), extract()) + _main(flags.src_dir, flags.output_dir, flags.base_dir) diff --git a/tensorflow/tools/docs/generate_test.py b/tensorflow/tools/docs/generate_test.py index 64d215ae779..a423a9d9aa9 100644 --- a/tensorflow/tools/docs/generate_test.py +++ b/tensorflow/tools/docs/generate_test.py @@ -89,9 +89,9 @@ class GenerateTest(googletest.TestCase): output_dir = tempfile.mkdtemp() base_dir = os.path.dirname(__file__) - generate.write_docs(output_dir, base_dir, - duplicate_of, duplicates, - index, tree, reverse_index={}, guide_index={}) + generate.write_docs(output_dir, base_dir, duplicate_of, duplicates, + index, tree, reverse_index={}, doc_index={}, + guide_index={}) # Make sure that the right files are written to disk. self.assertTrue(os.path.exists(os.path.join(output_dir, 'index.md'))) diff --git a/tensorflow/tools/docs/parser.py b/tensorflow/tools/docs/parser.py index 21505cf61d5..00f510ab057 100644 --- a/tensorflow/tools/docs/parser.py +++ b/tensorflow/tools/docs/parser.py @@ -31,8 +31,7 @@ import six # A regular expression capturing a python indentifier. IDENTIFIER_RE = '[a-zA-Z_][a-zA-Z0-9_]*' # A regular expression for capturing a @{symbol} reference. -_FULL_NAME_RE = '%s(.%s)*' % (IDENTIFIER_RE, IDENTIFIER_RE) -SYMBOL_REFERENCE_RE = re.compile(r'@\{(' + _FULL_NAME_RE + r')\}') +SYMBOL_REFERENCE_RE = re.compile(r'@\{([^}]+)\}') def documentation_path(full_name): @@ -125,7 +124,7 @@ def _markdown_link(link_text, ref_full_name, relative_path_to_root, _reference_to_link(ref_full_name, relative_path_to_root, duplicate_of)) -def _one_ref(string, relative_path_to_root, duplicate_of): +def _one_ref(string, relative_path_to_root, duplicate_of, doc_index): """Return a link for a single "@{symbol}" reference.""" # Look for link text after $. dollar = string.rfind('$') @@ -137,15 +136,32 @@ def _one_ref(string, relative_path_to_root, duplicate_of): # Handle different types of references. if string.startswith('$'): # Doc reference + if link_text == string: link_text = None + string = string[1:] # remove leading $ + + # If string has a #, split that part into `hash_tag` + hash_pos = string.find('#') + if hash_pos > -1: + hash_tag = string[hash_pos:] + string = string[:hash_pos] + else: + hash_tag = '' + + if string in doc_index: + if link_text is None: link_text = doc_index[string].title + return '[%s](%s%s)' % ( + link_text, os.path.join(relative_path_to_root, doc_index[string].url), + hash_tag) print('ERROR: Handle doc reference "@{%s}"' % string) - return 'TODO' + return 'TODO:%s' % string + elif string.startswith('tf'): # Python symbol # TODO(wicke): Figure out how to handle methods here. return _markdown_link( link_text, string, relative_path_to_root, duplicate_of) elif string.startswith('tensorflow::'): # C++ symbol if string == 'tensorflow::ClientSession': - ret = 'GRAPH_URL' + ret = 'CLIENT_SESSION_URL' elif string == 'tensorflow::Graph': ret = 'GRAPH_URL' elif string == 'tensorflow::Scope': @@ -157,13 +173,13 @@ def _one_ref(string, relative_path_to_root, duplicate_of): else: print('ERROR: Handle C++ reference "@{%s}"' % string) return 'TODO_C++:%s' % string - return os.path.join(relative_path_to_root, ret) + return '[`%s`](%s)' % (link_text, os.path.join(relative_path_to_root, ret)) # Error! print('ERROR: Did not understand "@{%s}"' % string) return 'ERROR:%s' % string -def replace_references(string, relative_path_to_root, duplicate_of): +def replace_references(string, relative_path_to_root, duplicate_of, doc_index): """Replace "@{symbol}" references with links to symbol's documentation page. This functions finds all occurrences of "@{symbol}" in `string` and replaces @@ -181,6 +197,8 @@ def replace_references(string, relative_path_to_root, duplicate_of): relative_path_to_root: The relative path from the containing document to the root of the API documentation that is being linked to. duplicate_of: A map from duplicate names to preferred names of API symbols. + doc_index: A `dict` mapping symbol name strings to objects with `url` + and `title` fields. Used to resolve @{$doc} references in docstrings. Returns: `string`, with "@{symbol}" references replaced by Markdown links. @@ -188,7 +206,8 @@ def replace_references(string, relative_path_to_root, duplicate_of): return re.sub(SYMBOL_REFERENCE_RE, lambda match: _one_ref(match.group(1), # pylint: disable=g-long-lambda relative_path_to_root, - duplicate_of), + duplicate_of, + doc_index), string) @@ -213,7 +232,7 @@ def _handle_compatibility(doc): return match_compatibility.subn(r'', doc)[0], compatibility_notes -def _md_docstring(py_object, relative_path_to_root, duplicate_of): +def _md_docstring(py_object, relative_path_to_root, duplicate_of, doc_index): """Get the docstring from an object and make it into nice Markdown. For links within the same set of docs, the `relative_path_to_root` for a @@ -232,6 +251,8 @@ def _md_docstring(py_object, relative_path_to_root, duplicate_of): links for "@symbol" references. duplicate_of: A map from duplicate symbol names to master names. Used to resolve "@symbol" references. + doc_index: A `dict` mapping symbol name strings to objects with `url` + and `title` fields. Used to resolve @{$doc} references in docstrings. Returns: The docstring, or the empty string if no docstring was found. @@ -239,7 +260,7 @@ def _md_docstring(py_object, relative_path_to_root, duplicate_of): # TODO(wicke): If this is a partial, use the .func docstring and add a note. raw_docstring = _get_raw_docstring(py_object) raw_docstring = replace_references(raw_docstring, relative_path_to_root, - duplicate_of) + duplicate_of, doc_index) raw_docstring, compatibility = _handle_compatibility(raw_docstring) raw_lines = raw_docstring.split('\n') @@ -438,7 +459,7 @@ def _get_guides_markdown(duplicate_names, guide_index, relative_path): def _generate_markdown_for_function(full_name, duplicate_names, function, duplicate_of, reverse_index, - guide_index): + doc_index, guide_index): """Generate Markdown docs for a function or method. This function creates a documentation page for a function. It uses the @@ -454,6 +475,8 @@ def _generate_markdown_for_function(full_name, duplicate_names, duplicate_of: A map of duplicate full names to master names. Used to resolve @{symbol} references in the docstring. reverse_index: A map from object ids in the index to full names. + doc_index: A `dict` mapping symbol name strings to objects with `url` + and `title` fields. Used to resolve @{$doc} references in docstrings. guide_index: A `dict` mapping symbol name strings to objects with a `make_md_link()` method. @@ -463,7 +486,7 @@ def _generate_markdown_for_function(full_name, duplicate_names, # TODO(wicke): Make sure this works for partials. relative_path = os.path.relpath( path='.', start=os.path.dirname(documentation_path(full_name)) or '.') - docstring = _md_docstring(function, relative_path, duplicate_of) + docstring = _md_docstring(function, relative_path, duplicate_of, doc_index) signature = _generate_signature(function, reverse_index) guides = _get_guides_markdown(duplicate_names, guide_index, relative_path) @@ -480,7 +503,7 @@ def _generate_markdown_for_function(full_name, duplicate_names, def _generate_markdown_for_class(full_name, duplicate_names, py_class, duplicate_of, index, tree, - reverse_index, guide_index): + reverse_index, doc_index, guide_index): """Generate Markdown docs for a class. This function creates a documentation page for a class. It uses the @@ -500,6 +523,8 @@ def _generate_markdown_for_class(full_name, duplicate_names, py_class, index: A map from full names to python object references. tree: A map from full names to the names of all documentable child objects. reverse_index: A map from object ids in the index to full names. + doc_index: A `dict` mapping symbol name strings to objects with `url` + and `title` fields. Used to resolve @{$doc} references in docstrings. guide_index: A `dict` mapping symbol name strings to objects with a `make_md_link()` method. @@ -508,7 +533,7 @@ def _generate_markdown_for_class(full_name, duplicate_names, py_class, """ relative_path = os.path.relpath( path='.', start=os.path.dirname(documentation_path(full_name)) or '.') - docstring = _md_docstring(py_class, relative_path, duplicate_of) + docstring = _md_docstring(py_class, relative_path, duplicate_of, doc_index) guides = _get_guides_markdown(duplicate_names, guide_index, relative_path) if duplicate_names: aliases = '\n'.join(['### `class %s`' % name for name in duplicate_names]) @@ -546,7 +571,8 @@ def _generate_markdown_for_class(full_name, duplicate_names, py_class, docs += '## Properties\n\n' for property_name, prop in sorted(properties, key=lambda x: x[0]): docs += '### `%s`\n\n%s\n\n' % ( - property_name, _md_docstring(prop, relative_path, duplicate_of)) + property_name, + _md_docstring(prop, relative_path, duplicate_of, doc_index)) docs += '\n\n' if methods: @@ -556,7 +582,7 @@ def _generate_markdown_for_class(full_name, duplicate_names, py_class, reverse_index) docs += '### `%s`\n\n%s\n\n' % (method_signature, _md_docstring(method, relative_path, - duplicate_of)) + duplicate_of, doc_index)) docs += '\n\n' if field_names: @@ -569,7 +595,7 @@ def _generate_markdown_for_class(full_name, duplicate_names, py_class, def _generate_markdown_for_module(full_name, duplicate_names, module, - duplicate_of, index, tree): + duplicate_of, index, tree, doc_index): """Generate Markdown docs for a module. This function creates a documentation page for a module. It uses the @@ -586,13 +612,15 @@ def _generate_markdown_for_module(full_name, duplicate_names, module, @{symbol} references in the docstrings. index: A map from full names to python object references. tree: A map from full names to the names of all documentable child objects. + doc_index: A `dict` mapping symbol name strings to objects with `url` + and `title` fields. Used to resolve @{$doc} references in docstrings. Returns: A string that can be written to a documentation file for this module. """ relative_path = os.path.relpath( path='.', start=os.path.dirname(documentation_path(full_name)) or '.') - docstring = _md_docstring(module, relative_path, duplicate_of) + docstring = _md_docstring(module, relative_path, duplicate_of, doc_index) if duplicate_names: aliases = '\n'.join(['### Module `%s`' % name for name in duplicate_names]) aliases += '\n\n' @@ -637,14 +665,14 @@ _CODE_URL_PREFIX = ( 'https://www.tensorflow.org/code/tensorflow/') -def generate_markdown(full_name, py_object, duplicate_of, duplicates, - index, tree, reverse_index, guide_index, base_dir): +def generate_markdown(full_name, py_object, duplicate_of, duplicates, index, + tree, reverse_index, doc_index, guide_index, base_dir): """Generate Markdown docs for a given object that's part of the TF API. This function uses _md_docstring to obtain the docs pertaining to `object`. - This function resolves '@symbol' references in the docstrings into links to + This function resolves '@{symbol}' references in the docstrings into links to the appropriate location. It also adds a list of alternative names for the symbol automatically. @@ -670,6 +698,8 @@ def generate_markdown(full_name, py_object, duplicate_of, duplicates, tree: A `dict` mapping a fully qualified name to the names of all its members. Used to populate the members section of a class or module page. reverse_index: A `dict` mapping objects in the index to full names. + doc_index: A `dict` mapping symbol name strings to objects with `url` + and `title` fields. Used to resolve @{$doc} references in docstrings. guide_index: A `dict` mapping symbol name strings to objects with a `make_md_link()` method. base_dir: A base path that is stripped from file locations written to the @@ -693,15 +723,15 @@ def generate_markdown(full_name, py_object, duplicate_of, duplicates, inspect.isroutine(py_object)): markdown = _generate_markdown_for_function( master_name, duplicate_names, py_object, duplicate_of, - reverse_index, guide_index) + reverse_index, doc_index, guide_index) elif inspect.isclass(py_object): markdown = _generate_markdown_for_class( master_name, duplicate_names, py_object, duplicate_of, index, tree, - reverse_index, guide_index) + reverse_index, doc_index, guide_index) elif inspect.ismodule(py_object): markdown = _generate_markdown_for_module(master_name, duplicate_names, py_object, duplicate_of, - index, tree) + index, tree, doc_index) else: raise RuntimeError('Cannot make docs for object %s: %r' % (full_name, py_object)) diff --git a/tensorflow/tools/docs/parser_test.py b/tensorflow/tools/docs/parser_test.py index a598a895c25..ff76e5c385b 100644 --- a/tensorflow/tools/docs/parser_test.py +++ b/tensorflow/tools/docs/parser_test.py @@ -94,13 +94,32 @@ class ParserTest(googletest.TestCase): def test_replace_references(self): string = 'A @{tf.reference}, another @{tf.reference}, and a @{tf.third}.' duplicate_of = {'tf.third': 'tf.fourth'} - result = parser.replace_references(string, '../..', duplicate_of) + result = parser.replace_references( + string, '../..', duplicate_of, doc_index={}) self.assertEqual( 'A [`tf.reference`](../../tf/reference.md), another ' '[`tf.reference`](../../tf/reference.md), ' 'and a [`tf.third`](../../tf/fourth.md).', result) + def test_doc_replace_references(self): + string = '@{$doc1} @{$doc1#abc} @{$doc1$link} @{$doc1#def$zelda} @{$do/c2}' + + class DocInfo(object): + pass + doc1 = DocInfo() + doc1.title = 'Title1' + doc1.url = 'URL1' + doc2 = DocInfo() + doc2.title = 'Two words' + doc2.url = 'somewhere/else' + doc_index = {'doc1': doc1, 'do/c2': doc2} + result = parser.replace_references(string, '..', {}, doc_index=doc_index) + self.assertEqual( + '[Title1](../URL1) [Title1](../URL1#abc) [link](../URL1) ' + '[zelda](../URL1#def) [Two words](../somewhere/else)', + result) + def test_generate_markdown_for_class(self): index = { @@ -115,10 +134,10 @@ class ParserTest(googletest.TestCase): 'TestClass': ['a_method', 'a_property', 'ChildClass', 'CLASS_MEMBER'] } - docs = parser.generate_markdown(full_name='TestClass', py_object=TestClass, - duplicate_of={}, duplicates={}, - index=index, tree=tree, reverse_index={}, - guide_index={}, base_dir='/') + docs = parser.generate_markdown( + full_name='TestClass', py_object=TestClass, duplicate_of={}, + duplicates={}, index=index, tree=tree, reverse_index={}, doc_index={}, + guide_index={}, base_dir='/') # Make sure all required docstrings are present. self.assertTrue(inspect.getdoc(TestClass) in docs) @@ -156,7 +175,7 @@ class ParserTest(googletest.TestCase): docs = parser.generate_markdown(full_name='TestModule', py_object=module, duplicate_of={}, duplicates={}, index=index, tree=tree, reverse_index={}, - guide_index={}, base_dir='/') + doc_index={}, guide_index={}, base_dir='/') # Make sure all required docstrings are present. self.assertTrue(inspect.getdoc(module) in docs) @@ -185,7 +204,7 @@ class ParserTest(googletest.TestCase): py_object=test_function, duplicate_of={}, duplicates={}, index=index, tree=tree, reverse_index={}, - guide_index={}, base_dir='/') + doc_index={}, guide_index={}, base_dir='/') # Make sure docstring shows up. self.assertTrue(inspect.getdoc(test_function) in docs) @@ -209,7 +228,7 @@ class ParserTest(googletest.TestCase): py_object=test_function_with_args_kwargs, duplicate_of={}, duplicates={}, index=index, tree=tree, reverse_index={}, - guide_index={}, base_dir='/') + doc_index={}, guide_index={}, base_dir='/') # Make sure docstring shows up. self.assertTrue(inspect.getdoc(test_function_with_args_kwargs) in docs) @@ -231,14 +250,14 @@ class ParserTest(googletest.TestCase): docs = parser.generate_markdown( full_name='test_function_for_markdown_reference', - py_object=test_function_for_markdown_reference, - duplicate_of={}, duplicates={}, - index=index, tree=tree, reverse_index={}, guide_index={}, base_dir='/') + py_object=test_function_for_markdown_reference, duplicate_of={}, + duplicates={}, index=index, tree=tree, reverse_index={}, doc_index={}, + guide_index={}, base_dir='/') # Make sure docstring shows up and is properly processed. expected_docs = parser.replace_references( inspect.getdoc(test_function_for_markdown_reference), - relative_path_to_root='.', duplicate_of={}) + relative_path_to_root='.', duplicate_of={}, doc_index={}) self.assertTrue(expected_docs in docs) @@ -252,10 +271,9 @@ class ParserTest(googletest.TestCase): } docs = parser.generate_markdown( - full_name='test_function', - py_object=test_function_with_fancy_docstring, - duplicate_of={}, duplicates={}, - index=index, tree=tree, reverse_index={}, guide_index={}, base_dir='/') + full_name='test_function', py_object=test_function_with_fancy_docstring, + duplicate_of={}, duplicates={}, index=index, tree=tree, + reverse_index={}, doc_index={}, guide_index={}, base_dir='/') expected = '\n'.join([ 'Function with a fancy docstring.', '', diff --git a/tensorflow/tools/docs/substitute_links.py b/tensorflow/tools/docs/substitute_links.py deleted file mode 100644 index 848aa6d186c..00000000000 --- a/tensorflow/tools/docs/substitute_links.py +++ /dev/null @@ -1,99 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Convert @{symbol} to MarkDown links in the Python API guides.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -import os - -from tensorflow.tools.docs import generate -from tensorflow.tools.docs import parser -from tensorflow.tools.docs import py_guide_parser - - -class UpdateTags(py_guide_parser.PyGuideParser): - """Rewrites a Python guide so that each section has an explicit tag.""" - - def process_section(self, line_number, section_title, tag): - self.replace_line(line_number, '## %s {#%s}' % (section_title, tag)) - - -def _main(input_dir, output_dir): - """Convert all the files in `input_dir` and write results to `output_dir`.""" - visitor = generate.extract() - - header = '\n' - - # Iterate through all the source files and process them. - tag_updater = UpdateTags() - for dirpath, _, filenames in os.walk(input_dir): - # How to get from `dirpath` to api_docs/python/ - relative_path_to_root = os.path.relpath( - path=os.path.join(input_dir, 'api_docs/python'), start=dirpath) - - # Make the directory under output_dir. - new_dir = os.path.join(output_dir, - os.path.relpath(path=dirpath, start=input_dir)) - try: - if not os.path.exists(new_dir): - os.makedirs(new_dir) - except OSError as e: - print('Creating output dir "%s" failed: %s' % (new_dir, e)) - raise - - for base_name in filenames: - full_in_path = os.path.join(dirpath, base_name) - suffix = os.path.relpath(path=full_in_path, start=input_dir) - full_out_path = os.path.join(output_dir, suffix) - if not base_name.endswith('.md'): - print('Copying non-md file %s...' % suffix) - open(full_out_path, 'w').write(open(full_in_path).read()) - continue - if dirpath.endswith('/api_guides/python'): - print('Processing Python guide %s...' % base_name) - md_string = tag_updater.process(full_in_path) - else: - print('Processing doc %s...' % suffix) - md_string = open(full_in_path).read() - - output = parser.replace_references( - md_string, relative_path_to_root, visitor.duplicate_of) - open(full_out_path, 'w').write(header + output) - - print('Done.') - - -if __name__ == '__main__': - argument_parser = argparse.ArgumentParser() - argument_parser.add_argument( - '--input_dir', - type=str, - default=None, - required=True, - help='Directory to copy docs from.' - ) - argument_parser.add_argument( - '--output_dir', - type=str, - default=None, - required=True, - help='Directory to write docs to.' - ) - flags, _ = argument_parser.parse_known_args() - _main(flags.input_dir, flags.output_dir)