experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

More doc generator changes.

Browse Source 
Change: 147249420
This commit is contained in:
A. Unique TensorFlower 2017-02-11 10:02:02 -08:00 committed by TensorFlower Gardener
parent c4423aec4a
commit 196ce75c99
6 changed files with 201 additions and 160 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
tensorflow/tools/docs/BUILD
View File

@ -108,17 +108,6 @@ py_test(
],
)
py_binary(
name = "substitute_links",
srcs = ["substitute_links.py"],
srcs_version = "PY2AND3",
deps = [
":generate",
":parser",
":py_guide_parser",
],
)
filegroup(
name = "doxy_config",
srcs = ["tf-doxy_for_md-config"],

117
tensorflow/tools/docs/generate.py
View File

@ -33,7 +33,7 @@ from tensorflow.tools.docs import py_guide_parser
def write_docs(output_dir, base_dir, duplicate_of, duplicates, index, tree,
reverse_index, guide_index):
reverse_index, doc_index, guide_index):
"""Write previously extracted docs to disk.
Write a docs page for each symbol in `index` to a tree of docs at
@ -59,6 +59,7 @@ def write_docs(output_dir, base_dir, duplicate_of, duplicates, index, tree,
tree: A `dict` mapping a fully qualified name to the names of all its
members. Used to populate the members section of a class or module page.
reverse_index: A `dict` mapping object ids to fully qualified names.
doc_index: A `dict` mapping a doc key to a DocInfo.
guide_index: A `dict` mapping symbol name strings to GuideRef.
"""
# Make output_dir.
@ -94,6 +95,7 @@ def write_docs(output_dir, base_dir, duplicate_of, duplicates, index, tree,
index=index,
tree=tree,
reverse_index=reverse_index,
doc_index=doc_index,
guide_index=guide_index,
base_dir=base_dir)
@ -185,6 +187,46 @@ def extract():
return visitor
class GetMarkdownTitle(py_guide_parser.PyGuideParser):
"""Extract the title from a .md file."""
def __init__(self):
self.title = None
py_guide_parser.PyGuideParser.__init__(self)
def process_title(self, _, title):
if self.title is None: # only use the first title
self.title = title
class DocInfo(object):
"""A simple struct for holding a doc's url and title."""
def __init__(self, url, title):
self.url = url
self.title = title
def build_doc_index(src_dir):
"""Build an index from a keyword designating a doc to DocInfo objects."""
doc_index = {}
for dirpath, _, filenames in os.walk(src_dir):
suffix = os.path.relpath(path=dirpath, start=src_dir)
for base_name in filenames:
if not base_name.endswith('.md'): continue
title_parser = GetMarkdownTitle()
title_parser.process(os.path.join(dirpath, base_name))
key_parts = os.path.join(suffix, base_name[:-3]).split('/')
if key_parts[-1] == 'index':
key_parts = key_parts[:-1]
doc_info = DocInfo(os.path.join(suffix, base_name), title_parser.title)
doc_index[key_parts[-1]] = doc_info
if len(key_parts) > 1:
doc_index['/'.join(key_parts[-2:])] = doc_info
return doc_index
class GuideRef(object):
def __init__(self, base_name, title, section_title, section_tag):
@ -238,7 +280,7 @@ def build_guide_index(guide_src_dir):
return index_generator.index
def write(output_dir, base_dir, guide_index, visitor):
def write(output_dir, base_dir, doc_index, guide_index, visitor):
"""Write documentation for an index in a `DocGeneratorVisitor` to disk.
This function will create `output_dir` if it doesn't exist, and write
@ -248,13 +290,75 @@ def write(output_dir, base_dir, guide_index, visitor):
output_dir: The directory to write documentation to. Must not exist.
base_dir: The base dir of the library `visitor` has traversed. This is used
to compute relative paths for file references.
doc_index: A `dict` mapping a doc key to a DocInfo.
guide_index: A `dict` mapping symbol name strings to GuideRef.
visitor: A `DocGeneratorVisitor` that has traversed a library located at
`base_dir`.
"""
write_docs(output_dir, os.path.abspath(base_dir),
visitor.duplicate_of, visitor.duplicates,
visitor.index, visitor.tree, visitor.reverse_index, guide_index)
visitor.index, visitor.tree, visitor.reverse_index,
doc_index, guide_index)
class UpdateTags(py_guide_parser.PyGuideParser):
"""Rewrites a Python guide so that each section has an explicit tag."""
def process_section(self, line_number, section_title, tag):
self.replace_line(line_number, '## %s {#%s}' % (section_title, tag))
def other_docs(src_dir, output_dir, visitor, doc_index):
"""Convert all the files in `src_dir` and write results to `output_dir`."""
header = '<!-- DO NOT EDIT! Automatically generated file. -->\n'
# Iterate through all the source files and process them.
tag_updater = UpdateTags()
for dirpath, _, filenames in os.walk(src_dir):
# How to get from `dirpath` to api_docs/python/
relative_path_to_root = os.path.relpath(
path=os.path.join(src_dir, 'api_docs/python'), start=dirpath)
# Make the directory under output_dir.
new_dir = os.path.join(output_dir,
os.path.relpath(path=dirpath, start=src_dir))
try:
if not os.path.exists(new_dir):
os.makedirs(new_dir)
except OSError as e:
print('Creating output dir "%s" failed: %s' % (new_dir, e))
raise
for base_name in filenames:
full_in_path = os.path.join(dirpath, base_name)
suffix = os.path.relpath(path=full_in_path, start=src_dir)
full_out_path = os.path.join(output_dir, suffix)
if not base_name.endswith('.md'):
print('Copying non-md file %s...' % suffix)
open(full_out_path, 'w').write(open(full_in_path).read())
continue
if dirpath.endswith('/api_guides/python'):
print('Processing Python guide %s...' % base_name)
md_string = tag_updater.process(full_in_path)
else:
print('Processing doc %s...' % suffix)
md_string = open(full_in_path).read()
output = parser.replace_references(
md_string, relative_path_to_root, visitor.duplicate_of, doc_index)
open(full_out_path, 'w').write(header + output)
print('Done.')
def _main(src_dir, output_dir, base_dir):
doc_index = build_doc_index(src_dir)
visitor = extract()
write(os.path.join(output_dir, 'api_docs/python'), base_dir,
doc_index,
build_guide_index(os.path.join(src_dir, 'api_guides/python')),
visitor)
other_docs(src_dir, output_dir, visitor, doc_index)
if __name__ == '__main__':
@ -268,7 +372,7 @@ if __name__ == '__main__':
)
argument_parser.add_argument(
'--guide_src_dir',
'--src_dir',
type=str,
default=None,
required=True,
@ -288,7 +392,7 @@ if __name__ == '__main__':
type=str,
default=default_base_dir,
help=('Base directory to to strip from file names referenced in docs. '
'Defaults to three directories up from the location of this file.')
'Defaults to two directories up from the location of this file.')
)
flags, _ = argument_parser.parse_known_args()
@ -298,5 +402,4 @@ if __name__ == '__main__':
'Cowardly refusing to wipe it, please do that yourself.'
% flags.output_dir)
write(flags.output_dir, flags.base_dir,
build_guide_index(flags.guide_src_dir), extract())
_main(flags.src_dir, flags.output_dir, flags.base_dir)

6
tensorflow/tools/docs/generate_test.py
View File

@ -89,9 +89,9 @@ class GenerateTest(googletest.TestCase):
output_dir = tempfile.mkdtemp()
base_dir = os.path.dirname(__file__)
generate.write_docs(output_dir, base_dir,
duplicate_of, duplicates,
index, tree, reverse_index={}, guide_index={})
generate.write_docs(output_dir, base_dir, duplicate_of, duplicates,
index, tree, reverse_index={}, doc_index={},
guide_index={})
# Make sure that the right files are written to disk.
self.assertTrue(os.path.exists(os.path.join(output_dir, 'index.md')))

78
tensorflow/tools/docs/parser.py
View File

@ -31,8 +31,7 @@ import six
# A regular expression capturing a python indentifier.
IDENTIFIER_RE = '[a-zA-Z_][a-zA-Z0-9_]*'
# A regular expression for capturing a @{symbol} reference.
_FULL_NAME_RE = '%s(.%s)*' % (IDENTIFIER_RE, IDENTIFIER_RE)
SYMBOL_REFERENCE_RE = re.compile(r'@\{(' + _FULL_NAME_RE + r')\}')
SYMBOL_REFERENCE_RE = re.compile(r'@\{([^}]+)\}')
def documentation_path(full_name):
@ -125,7 +124,7 @@ def _markdown_link(link_text, ref_full_name, relative_path_to_root,
_reference_to_link(ref_full_name, relative_path_to_root, duplicate_of))
def _one_ref(string, relative_path_to_root, duplicate_of):
def _one_ref(string, relative_path_to_root, duplicate_of, doc_index):
"""Return a link for a single "@{symbol}" reference."""
# Look for link text after $.
dollar = string.rfind('$')
@ -137,15 +136,32 @@ def _one_ref(string, relative_path_to_root, duplicate_of):
# Handle different types of references.
if string.startswith('$'): # Doc reference
if link_text == string: link_text = None
string = string[1:] # remove leading $
# If string has a #, split that part into `hash_tag`
hash_pos = string.find('#')
if hash_pos > -1:
hash_tag = string[hash_pos:]
string = string[:hash_pos]
else:
hash_tag = ''
if string in doc_index:
if link_text is None: link_text = doc_index[string].title
return '[%s](%s%s)' % (
link_text, os.path.join(relative_path_to_root, doc_index[string].url),
hash_tag)
print('ERROR: Handle doc reference "@{%s}"' % string)
return 'TODO'
return 'TODO:%s' % string
elif string.startswith('tf'): # Python symbol
# TODO(wicke): Figure out how to handle methods here.
return _markdown_link(
link_text, string, relative_path_to_root, duplicate_of)
elif string.startswith('tensorflow::'): # C++ symbol
if string == 'tensorflow::ClientSession':
ret = 'GRAPH_URL'
ret = 'CLIENT_SESSION_URL'
elif string == 'tensorflow::Graph':
ret = 'GRAPH_URL'
elif string == 'tensorflow::Scope':
@ -157,13 +173,13 @@ def _one_ref(string, relative_path_to_root, duplicate_of):
else:
print('ERROR: Handle C++ reference "@{%s}"' % string)
return 'TODO_C++:%s' % string
return os.path.join(relative_path_to_root, ret)
return '[`%s`](%s)' % (link_text, os.path.join(relative_path_to_root, ret))
# Error!
print('ERROR: Did not understand "@{%s}"' % string)
return 'ERROR:%s' % string
def replace_references(string, relative_path_to_root, duplicate_of):
def replace_references(string, relative_path_to_root, duplicate_of, doc_index):
"""Replace "@{symbol}" references with links to symbol's documentation page.
This functions finds all occurrences of "@{symbol}" in `string` and replaces
@ -181,6 +197,8 @@ def replace_references(string, relative_path_to_root, duplicate_of):
relative_path_to_root: The relative path from the containing document to the
root of the API documentation that is being linked to.
duplicate_of: A map from duplicate names to preferred names of API symbols.
doc_index: A `dict` mapping symbol name strings to objects with `url`
and `title` fields. Used to resolve @{$doc} references in docstrings.
Returns:
`string`, with "@{symbol}" references replaced by Markdown links.
@ -188,7 +206,8 @@ def replace_references(string, relative_path_to_root, duplicate_of):
return re.sub(SYMBOL_REFERENCE_RE,
lambda match: _one_ref(match.group(1), # pylint: disable=g-long-lambda
relative_path_to_root,
duplicate_of),
duplicate_of,
doc_index),
string)
@ -213,7 +232,7 @@ def _handle_compatibility(doc):
return match_compatibility.subn(r'', doc)[0], compatibility_notes
def _md_docstring(py_object, relative_path_to_root, duplicate_of):
def _md_docstring(py_object, relative_path_to_root, duplicate_of, doc_index):
"""Get the docstring from an object and make it into nice Markdown.
For links within the same set of docs, the `relative_path_to_root` for a
@ -232,6 +251,8 @@ def _md_docstring(py_object, relative_path_to_root, duplicate_of):
links for "@symbol" references.
duplicate_of: A map from duplicate symbol names to master names. Used to
resolve "@symbol" references.
doc_index: A `dict` mapping symbol name strings to objects with `url`
and `title` fields. Used to resolve @{$doc} references in docstrings.
Returns:
The docstring, or the empty string if no docstring was found.
@ -239,7 +260,7 @@ def _md_docstring(py_object, relative_path_to_root, duplicate_of):
# TODO(wicke): If this is a partial, use the .func docstring and add a note.
raw_docstring = _get_raw_docstring(py_object)
raw_docstring = replace_references(raw_docstring, relative_path_to_root,
duplicate_of)
duplicate_of, doc_index)
raw_docstring, compatibility = _handle_compatibility(raw_docstring)
raw_lines = raw_docstring.split('\n')
@ -438,7 +459,7 @@ def _get_guides_markdown(duplicate_names, guide_index, relative_path):
def _generate_markdown_for_function(full_name, duplicate_names,
function, duplicate_of, reverse_index,
guide_index):
doc_index, guide_index):
"""Generate Markdown docs for a function or method.
This function creates a documentation page for a function. It uses the
@ -454,6 +475,8 @@ def _generate_markdown_for_function(full_name, duplicate_names,
duplicate_of: A map of duplicate full names to master names. Used to resolve
@{symbol} references in the docstring.
reverse_index: A map from object ids in the index to full names.
doc_index: A `dict` mapping symbol name strings to objects with `url`
and `title` fields. Used to resolve @{$doc} references in docstrings.
guide_index: A `dict` mapping symbol name strings to objects with a
`make_md_link()` method.
@ -463,7 +486,7 @@ def _generate_markdown_for_function(full_name, duplicate_names,
# TODO(wicke): Make sure this works for partials.
relative_path = os.path.relpath(
path='.', start=os.path.dirname(documentation_path(full_name)) or '.')
docstring = _md_docstring(function, relative_path, duplicate_of)
docstring = _md_docstring(function, relative_path, duplicate_of, doc_index)
signature = _generate_signature(function, reverse_index)
guides = _get_guides_markdown(duplicate_names, guide_index, relative_path)
@ -480,7 +503,7 @@ def _generate_markdown_for_function(full_name, duplicate_names,
def _generate_markdown_for_class(full_name, duplicate_names, py_class,
duplicate_of, index, tree,
reverse_index, guide_index):
reverse_index, doc_index, guide_index):
"""Generate Markdown docs for a class.
This function creates a documentation page for a class. It uses the
@ -500,6 +523,8 @@ def _generate_markdown_for_class(full_name, duplicate_names, py_class,
index: A map from full names to python object references.
tree: A map from full names to the names of all documentable child objects.
reverse_index: A map from object ids in the index to full names.
doc_index: A `dict` mapping symbol name strings to objects with `url`
and `title` fields. Used to resolve @{$doc} references in docstrings.
guide_index: A `dict` mapping symbol name strings to objects with a
`make_md_link()` method.
@ -508,7 +533,7 @@ def _generate_markdown_for_class(full_name, duplicate_names, py_class,
"""
relative_path = os.path.relpath(
path='.', start=os.path.dirname(documentation_path(full_name)) or '.')
docstring = _md_docstring(py_class, relative_path, duplicate_of)
docstring = _md_docstring(py_class, relative_path, duplicate_of, doc_index)
guides = _get_guides_markdown(duplicate_names, guide_index, relative_path)
if duplicate_names:
aliases = '\n'.join(['### `class %s`' % name for name in duplicate_names])
@ -546,7 +571,8 @@ def _generate_markdown_for_class(full_name, duplicate_names, py_class,
docs += '## Properties\n\n'
for property_name, prop in sorted(properties, key=lambda x: x[0]):
docs += '### `%s`\n\n%s\n\n' % (
property_name, _md_docstring(prop, relative_path, duplicate_of))
property_name,
_md_docstring(prop, relative_path, duplicate_of, doc_index))
docs += '\n\n'
if methods:
@ -556,7 +582,7 @@ def _generate_markdown_for_class(full_name, duplicate_names, py_class,
reverse_index)
docs += '### `%s`\n\n%s\n\n' % (method_signature,
_md_docstring(method, relative_path,
duplicate_of))
duplicate_of, doc_index))
docs += '\n\n'
if field_names:
@ -569,7 +595,7 @@ def _generate_markdown_for_class(full_name, duplicate_names, py_class,
def _generate_markdown_for_module(full_name, duplicate_names, module,
duplicate_of, index, tree):
duplicate_of, index, tree, doc_index):
"""Generate Markdown docs for a module.
This function creates a documentation page for a module. It uses the
@ -586,13 +612,15 @@ def _generate_markdown_for_module(full_name, duplicate_names, module,
@{symbol} references in the docstrings.
index: A map from full names to python object references.
tree: A map from full names to the names of all documentable child objects.
doc_index: A `dict` mapping symbol name strings to objects with `url`
and `title` fields. Used to resolve @{$doc} references in docstrings.
Returns:
A string that can be written to a documentation file for this module.
"""
relative_path = os.path.relpath(
path='.', start=os.path.dirname(documentation_path(full_name)) or '.')
docstring = _md_docstring(module, relative_path, duplicate_of)
docstring = _md_docstring(module, relative_path, duplicate_of, doc_index)
if duplicate_names:
aliases = '\n'.join(['### Module `%s`' % name for name in duplicate_names])
aliases += '\n\n'
@ -637,14 +665,14 @@ _CODE_URL_PREFIX = (
'https://www.tensorflow.org/code/tensorflow/')
def generate_markdown(full_name, py_object, duplicate_of, duplicates,
index, tree, reverse_index, guide_index, base_dir):
def generate_markdown(full_name, py_object, duplicate_of, duplicates, index,
tree, reverse_index, doc_index, guide_index, base_dir):
"""Generate Markdown docs for a given object that's part of the TF API.
This function uses _md_docstring to obtain the docs pertaining to
`object`.
This function resolves '@symbol' references in the docstrings into links to
This function resolves '@{symbol}' references in the docstrings into links to
the appropriate location. It also adds a list of alternative names for the
symbol automatically.
@ -670,6 +698,8 @@ def generate_markdown(full_name, py_object, duplicate_of, duplicates,
tree: A `dict` mapping a fully qualified name to the names of all its
members. Used to populate the members section of a class or module page.
reverse_index: A `dict` mapping objects in the index to full names.
doc_index: A `dict` mapping symbol name strings to objects with `url`
and `title` fields. Used to resolve @{$doc} references in docstrings.
guide_index: A `dict` mapping symbol name strings to objects with a
`make_md_link()` method.
base_dir: A base path that is stripped from file locations written to the
@ -693,15 +723,15 @@ def generate_markdown(full_name, py_object, duplicate_of, duplicates,
inspect.isroutine(py_object)):
markdown = _generate_markdown_for_function(
master_name, duplicate_names, py_object, duplicate_of,
reverse_index, guide_index)
reverse_index, doc_index, guide_index)
elif inspect.isclass(py_object):
markdown = _generate_markdown_for_class(
master_name, duplicate_names, py_object, duplicate_of, index, tree,
reverse_index, guide_index)
reverse_index, doc_index, guide_index)
elif inspect.ismodule(py_object):
markdown = _generate_markdown_for_module(master_name, duplicate_names,
py_object, duplicate_of,
index, tree)
index, tree, doc_index)
else:
raise RuntimeError('Cannot make docs for object %s: %r' % (full_name,
py_object))

50
tensorflow/tools/docs/parser_test.py
View File

@ -94,13 +94,32 @@ class ParserTest(googletest.TestCase):
def test_replace_references(self):
string = 'A @{tf.reference}, another @{tf.reference}, and a @{tf.third}.'
duplicate_of = {'tf.third': 'tf.fourth'}
result = parser.replace_references(string, '../..', duplicate_of)
result = parser.replace_references(
string, '../..', duplicate_of, doc_index={})
self.assertEqual(
'A [`tf.reference`](../../tf/reference.md), another '
'[`tf.reference`](../../tf/reference.md), '
'and a [`tf.third`](../../tf/fourth.md).',
result)
def test_doc_replace_references(self):
string = '@{$doc1} @{$doc1#abc} @{$doc1$link} @{$doc1#def$zelda} @{$do/c2}'
class DocInfo(object):
pass
doc1 = DocInfo()
doc1.title = 'Title1'
doc1.url = 'URL1'
doc2 = DocInfo()
doc2.title = 'Two words'
doc2.url = 'somewhere/else'
doc_index = {'doc1': doc1, 'do/c2': doc2}
result = parser.replace_references(string, '..', {}, doc_index=doc_index)
self.assertEqual(
'[Title1](../URL1) [Title1](../URL1#abc) [link](../URL1) '
'[zelda](../URL1#def) [Two words](../somewhere/else)',
result)
def test_generate_markdown_for_class(self):
index = {
@ -115,10 +134,10 @@ class ParserTest(googletest.TestCase):
'TestClass': ['a_method', 'a_property', 'ChildClass', 'CLASS_MEMBER']
}
docs = parser.generate_markdown(full_name='TestClass', py_object=TestClass,
duplicate_of={}, duplicates={},
index=index, tree=tree, reverse_index={},
guide_index={}, base_dir='/')
docs = parser.generate_markdown(
full_name='TestClass', py_object=TestClass, duplicate_of={},
duplicates={}, index=index, tree=tree, reverse_index={}, doc_index={},
guide_index={}, base_dir='/')
# Make sure all required docstrings are present.
self.assertTrue(inspect.getdoc(TestClass) in docs)
@ -156,7 +175,7 @@ class ParserTest(googletest.TestCase):
docs = parser.generate_markdown(full_name='TestModule', py_object=module,
duplicate_of={}, duplicates={},
index=index, tree=tree, reverse_index={},
guide_index={}, base_dir='/')
doc_index={}, guide_index={}, base_dir='/')
# Make sure all required docstrings are present.
self.assertTrue(inspect.getdoc(module) in docs)
@ -185,7 +204,7 @@ class ParserTest(googletest.TestCase):
py_object=test_function,
duplicate_of={}, duplicates={},
index=index, tree=tree, reverse_index={},
guide_index={}, base_dir='/')
doc_index={}, guide_index={}, base_dir='/')
# Make sure docstring shows up.
self.assertTrue(inspect.getdoc(test_function) in docs)
@ -209,7 +228,7 @@ class ParserTest(googletest.TestCase):
py_object=test_function_with_args_kwargs,
duplicate_of={}, duplicates={},
index=index, tree=tree, reverse_index={},
guide_index={}, base_dir='/')
doc_index={}, guide_index={}, base_dir='/')
# Make sure docstring shows up.
self.assertTrue(inspect.getdoc(test_function_with_args_kwargs) in docs)
@ -231,14 +250,14 @@ class ParserTest(googletest.TestCase):
docs = parser.generate_markdown(
full_name='test_function_for_markdown_reference',
py_object=test_function_for_markdown_reference,
duplicate_of={}, duplicates={},
index=index, tree=tree, reverse_index={}, guide_index={}, base_dir='/')
py_object=test_function_for_markdown_reference, duplicate_of={},
duplicates={}, index=index, tree=tree, reverse_index={}, doc_index={},
guide_index={}, base_dir='/')
# Make sure docstring shows up and is properly processed.
expected_docs = parser.replace_references(
inspect.getdoc(test_function_for_markdown_reference),
relative_path_to_root='.', duplicate_of={})
relative_path_to_root='.', duplicate_of={}, doc_index={})
self.assertTrue(expected_docs in docs)
@ -252,10 +271,9 @@ class ParserTest(googletest.TestCase):
}
docs = parser.generate_markdown(
full_name='test_function',
py_object=test_function_with_fancy_docstring,
duplicate_of={}, duplicates={},
index=index, tree=tree, reverse_index={}, guide_index={}, base_dir='/')
full_name='test_function', py_object=test_function_with_fancy_docstring,
duplicate_of={}, duplicates={}, index=index, tree=tree,
reverse_index={}, doc_index={}, guide_index={}, base_dir='/')
expected = '\n'.join([
'Function with a fancy docstring.',
'',

99
tensorflow/tools/docs/substitute_links.py
View File

@ -1,99 +0,0 @@
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Convert @{symbol} to MarkDown links in the Python API guides."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import argparse
import os
from tensorflow.tools.docs import generate
from tensorflow.tools.docs import parser
from tensorflow.tools.docs import py_guide_parser
class UpdateTags(py_guide_parser.PyGuideParser):
"""Rewrites a Python guide so that each section has an explicit tag."""
def process_section(self, line_number, section_title, tag):
self.replace_line(line_number, '## %s {#%s}' % (section_title, tag))
def _main(input_dir, output_dir):
"""Convert all the files in `input_dir` and write results to `output_dir`."""
visitor = generate.extract()
header = '<!-- DO NOT EDIT! Automatically generated file. -->\n'
# Iterate through all the source files and process them.
tag_updater = UpdateTags()
for dirpath, _, filenames in os.walk(input_dir):
# How to get from `dirpath` to api_docs/python/
relative_path_to_root = os.path.relpath(
path=os.path.join(input_dir, 'api_docs/python'), start=dirpath)
# Make the directory under output_dir.
new_dir = os.path.join(output_dir,
os.path.relpath(path=dirpath, start=input_dir))
try:
if not os.path.exists(new_dir):
os.makedirs(new_dir)
except OSError as e:
print('Creating output dir "%s" failed: %s' % (new_dir, e))
raise
for base_name in filenames:
full_in_path = os.path.join(dirpath, base_name)
suffix = os.path.relpath(path=full_in_path, start=input_dir)
full_out_path = os.path.join(output_dir, suffix)
if not base_name.endswith('.md'):
print('Copying non-md file %s...' % suffix)
open(full_out_path, 'w').write(open(full_in_path).read())
continue
if dirpath.endswith('/api_guides/python'):
print('Processing Python guide %s...' % base_name)
md_string = tag_updater.process(full_in_path)
else:
print('Processing doc %s...' % suffix)
md_string = open(full_in_path).read()
output = parser.replace_references(
md_string, relative_path_to_root, visitor.duplicate_of)
open(full_out_path, 'w').write(header + output)
print('Done.')
if __name__ == '__main__':
argument_parser = argparse.ArgumentParser()
argument_parser.add_argument(
'--input_dir',
type=str,
default=None,
required=True,
help='Directory to copy docs from.'
)
argument_parser.add_argument(
'--output_dir',
type=str,
default=None,
required=True,
help='Directory to write docs to.'
)
flags, _ = argument_parser.parse_known_args()
_main(flags.input_dir, flags.output_dir)