Install codegen in bazel workspace to allow docs tests to run.

PiperOrigin-RevId: 159477794
This commit is contained in:
Mark Daoust 2017-06-19 14:23:24 -07:00 committed by TensorFlower Gardener
parent 3b41352a31
commit ba7b03f26f
7 changed files with 114 additions and 29 deletions

View File

@ -37,6 +37,7 @@ py_library(
srcs = ["parser.py"],
srcs_version = "PY2AND3",
visibility = ["//visibility:public"],
deps = ["@com_github_andreif_codegen"],
)
py_test(
@ -44,7 +45,6 @@ py_test(
size = "small",
srcs = ["parser_test.py"],
srcs_version = "PY2AND3",
tags = ["manual"],
deps = [
":parser",
"//tensorflow/python:platform_test",
@ -78,13 +78,10 @@ py_test(
size = "small",
srcs = ["generate_lib_test.py"],
srcs_version = "PY2AND3",
tags = ["manual"],
deps = [
":generate_lib",
":parser",
"//tensorflow:tensorflow_py",
"//tensorflow/python:platform_test",
"//tensorflow/python/debug:debug_py",
],
)
@ -105,7 +102,6 @@ py_test(
srcs = ["build_docs_test.py"],
data = ["//tensorflow:docs_src"],
srcs_version = "PY2AND3",
tags = ["manual"],
deps = [
":generate_lib",
"//tensorflow:tensorflow_py",

View File

@ -19,6 +19,8 @@ from __future__ import division
from __future__ import print_function
import os
import sys
import textwrap
import tensorflow as tf
from tensorflow.python import debug as tf_debug
@ -29,19 +31,40 @@ from tensorflow.tools.docs import generate_lib
class Flags(object):
resource_root = resource_loader.get_root_dir_with_all_resources()
src_dir = os.path.join(resource_root, 'third_party/tensorflow/docs_src')
base_dir = os.path.join(resource_root, 'third_party/tensorflow/')
src_dir = os.path.join(resource_root, 'tensorflow/docs_src')
base_dir = os.path.join(resource_root, 'tensorflow/')
output_dir = googletest.GetTempDir()
class BuildDocsTest(googletest.TestCase):
def testBuildDocs(self):
if sys.version_info >= (3, 0):
print('Warning: Doc generation is not supported from python3.')
return
doc_generator = generate_lib.DocGenerator()
doc_generator.set_py_modules([('tf', tf), ('tfdbg', tf_debug)])
status = doc_generator.build(Flags())
try:
status = doc_generator.build(Flags())
except RuntimeError as e:
if not e.args[0].startswith('Modules nested too deep'):
raise
msg = textwrap.dedent("""\
%s
****************************************************************
If this test fails here, you have most likely introduced an
unsealed module. Make sure to use `remove_undocumented` or similar
utilities to avoid leaking symbols. See above for more information
on the exact point of failure.
****************************************************************
""" % e.args[0])
raise RuntimeError(msg)
if status:
self.fail('Found %s Errors!' % status)

View File

@ -20,6 +20,7 @@ from __future__ import print_function
import argparse
import os
import sys
import six
@ -415,6 +416,8 @@ class DocGenerator(object):
"""Main entry point for generating docs."""
def __init__(self):
if sys.version_info >= (3, 0):
print('Warning: Doc generation is not supported from python3.')
self.argument_parser = argparse.ArgumentParser()
self._py_modules = None
self._private_map = _get_default_private_map()

View File

@ -21,9 +21,6 @@ from __future__ import print_function
import os
import sys
import tensorflow as tf
from tensorflow.python import debug as tf_debug
from tensorflow.python.platform import googletest
from tensorflow.tools.docs import generate_lib
from tensorflow.tools.docs import parser
@ -54,22 +51,6 @@ class DummyVisitor(object):
class GenerateTest(googletest.TestCase):
def test_extraction(self):
py_modules = [('tf', tf), ('tfdbg', tf_debug)]
try:
generate_lib.extract(py_modules,
generate_lib._get_default_private_map(),
generate_lib._get_default_do_not_descend_map())
except RuntimeError:
print('*****************************************************************')
print('If this test fails, you have most likely introduced an unsealed')
print('module. Make sure to use remove_undocumented or similar utilities')
print('to avoid leaking symbols. See below for more information on the')
print('failure.')
print('*****************************************************************')
raise
def test_write(self):
module = sys.modules[__name__]

View File

@ -491,13 +491,13 @@ Returns:
class TestParseFunctionDetails(googletest.TestCase):
def testParseFunctionDetails(self):
def test_parse_function_details(self):
docstring, function_details = parser._parse_function_details(RELU_DOC)
self.assertEqual(len(function_details), 2)
args = function_details[0]
self.assertEqual(args.keyword, 'Args')
self.assertEmpty(args.header)
self.assertEqual(len(args.header), 0)
self.assertEqual(len(args.items), 2)
self.assertEqual(args.items[0][0], 'features')
self.assertEqual(args.items[1][0], 'name')
@ -515,5 +515,60 @@ class TestParseFunctionDetails(googletest.TestCase):
docstring + ''.join(str(detail) for detail in function_details))
class TestGenerateSignature(googletest.TestCase):
def test_known_object(self):
if sys.version_info >= (3, 0):
print('Warning: Doc generation is not supported from python3.')
return
known_object = object()
reverse_index = {id(known_object): 'location.of.object.in.api'}
def example_fun(arg=known_object): # pylint: disable=unused-argument
pass
sig = parser._generate_signature(example_fun, reverse_index)
self.assertEqual(sig, ['arg=location.of.object.in.api'])
def test_literals(self):
if sys.version_info >= (3, 0):
print('Warning: Doc generation is not supported from python3.')
return
def example_fun(a=5, b=5.0, c=None, d=True, e='hello', f=(1, (2, 3))): # pylint: disable=g-bad-name, unused-argument
pass
sig = parser._generate_signature(example_fun, reverse_index={})
self.assertEqual(
sig, ['a=5', 'b=5.0', 'c=None', 'd=True', "e='hello'", 'f=(1, (2, 3))'])
def test_dotted_name(self):
if sys.version_info >= (3, 0):
print('Warning: Doc generation is not supported from python3.')
return
# pylint: disable=g-bad-name
class a(object):
class b(object):
class c(object):
class d(object):
def __init__(self, *args):
pass
# pylint: enable=g-bad-name
e = {'f': 1}
def example_fun(arg1=a.b.c.d, arg2=a.b.c.d(1, 2), arg3=e['f']): # pylint: disable=unused-argument
pass
sig = parser._generate_signature(example_fun, reverse_index={})
self.assertEqual(sig, ['arg1=a.b.c.d', 'arg2=a.b.c.d(1, 2)', "arg3=e['f']"])
if __name__ == '__main__':
googletest.main()

View File

@ -297,6 +297,17 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
build_file = str(Label("//third_party:backports_weakref.BUILD")),
)
native.new_http_archive(
name = "com_github_andreif_codegen",
urls = [
"http://mirror.bazel.build/github.com/andreif/codegen/archive/1.0.tar.gz",
"https://github.com/andreif/codegen/archive/1.0.tar.gz",
],
sha256 = "2dadd04a2802de27e0fe5a19b76538f6da9d39ff244036afa00c1bba754de5ee",
strip_prefix = "codegen-1.0",
build_file = str(Label("//third_party:codegen.BUILD")),
)
filegroup_external(
name = "org_python_license",
licenses = ["notice"], # Python 2.0

16
third_party/codegen.BUILD vendored Normal file
View File

@ -0,0 +1,16 @@
# -*- mode: python; -*-
#
# Description:
# Extension to ast that allow ast -> python code generation.
package(default_visibility = ["//visibility:public"])
licenses(["notice"]) # New BSD
exports_files(["LICENSE"])
py_library(
name = "com_github_andreif_codegen",
srcs = glob(["codegen.py"]),
srcs_version = "PY2AND3",
)