Upgrade to gast 0.3.2.

PiperOrigin-RevId: 289172767
Change-Id: I98b02508aebdd15da78353db03f4f242257b0d72
This commit is contained in:
Dan Moldovan 2020-01-10 15:04:08 -08:00 committed by TensorFlower Gardener
parent fced7a8078
commit a396eb4a3a
23 changed files with 153 additions and 90 deletions

View File

@ -38,8 +38,10 @@ class AssertTransformer(converter.Base):
if node.msg is None:
return templates.replace(
template, test=node.test, msg=gast.Str('Assertion error'))
elif isinstance(node.msg, gast.Str):
template,
test=node.test,
msg=gast.Constant('Assertion error', kind=None))
elif isinstance(node.msg, gast.Constant):
return templates.replace(template, test=node.test, msg=node.msg)
else:
raise NotImplementedError('can only convert string messages for now.')

View File

@ -72,7 +72,11 @@ class _ArgTemplateBuilder(object):
def add_stararg(self, a):
self._consume_args()
self._argspec.append(
gast.Call(gast.Name('tuple', gast.Load(), None), [a], ()))
gast.Call(
gast.Name(
'tuple', ctx=gast.Load(), annotation=None, type_comment=None),
args=[a],
keywords=()))
def finalize(self):
self._consume_args()
@ -161,7 +165,10 @@ class CallTreeTransformer(converter.Base):
"""Ties together all keyword and **kwarg arguments in a single dict."""
if node.keywords:
return gast.Call(
gast.Name('dict', gast.Load(), None), args=(), keywords=node.keywords)
gast.Name(
'dict', ctx=gast.Load(), annotation=None, type_comment=None),
args=(),
keywords=node.keywords)
else:
return parser.parse_expression('None')

View File

@ -163,7 +163,7 @@ class ControlFlowTransformer(converter.Base):
opts_dict = loop_directives[directives.set_loop_options]
str_keys, values = zip(*opts_dict.items())
keys = [gast.Str(s) for s in str_keys]
keys = [gast.Constant(s, kind=None) for s in str_keys]
values = list(values) # ast and gast don't play well with tuples.
return gast.Dict(keys, values)
@ -176,7 +176,7 @@ class ControlFlowTransformer(converter.Base):
assignments += templates.replace(
template,
var=s,
symbol_name=gast.Str(s.ssf()))
symbol_name=gast.Constant(s.ssf(), kind=None))
return assignments
def visit_If(self, node):
@ -297,9 +297,9 @@ class ControlFlowTransformer(converter.Base):
composites, state_getter_name, state_setter_name)
basic_symbol_names = tuple(
gast.Str(str(symbol)) for symbol in returned_from_cond)
gast.Constant(str(symbol), kind=None) for symbol in returned_from_cond)
composite_symbol_names = tuple(
gast.Str(str(symbol)) for symbol in composites)
gast.Constant(str(symbol), kind=None) for symbol in composites)
cond_expr = self._create_cond_expr(cond_results, cond_var_name, body_name,
orelse_name, state_getter_name,
@ -395,9 +395,9 @@ class ControlFlowTransformer(converter.Base):
composite_loop_vars, state_getter_name, state_setter_name)
basic_symbol_names = tuple(
gast.Str(str(symbol)) for symbol in basic_loop_vars)
gast.Constant(str(symbol), kind=None) for symbol in basic_loop_vars)
composite_symbol_names = tuple(
gast.Str(str(symbol)) for symbol in composite_loop_vars)
gast.Constant(str(symbol), kind=None) for symbol in composite_loop_vars)
opts = self._create_loop_options(node)
@ -518,9 +518,9 @@ class ControlFlowTransformer(converter.Base):
undefined_assigns = self._create_undefined_assigns(possibly_undefs)
basic_symbol_names = tuple(
gast.Str(str(symbol)) for symbol in basic_loop_vars)
gast.Constant(str(symbol), kind=None) for symbol in basic_loop_vars)
composite_symbol_names = tuple(
gast.Str(str(symbol)) for symbol in composite_loop_vars)
gast.Constant(str(symbol), kind=None) for symbol in composite_loop_vars)
opts = self._create_loop_options(node)

View File

@ -165,7 +165,7 @@ class ControlFlowTransformer(converter.Base):
opts_dict = loop_directives[directives.set_loop_options]
str_keys, values = zip(*opts_dict.items())
keys = [gast.Str(s) for s in str_keys]
keys = [gast.Constant(s, kind=None) for s in str_keys]
values = list(values) # ast and gast don't play well with tuples.
return gast.Dict(keys, values)
@ -178,7 +178,7 @@ class ControlFlowTransformer(converter.Base):
assignments += templates.replace(
template,
var=s,
symbol_name=gast.Str(s.ssf()))
symbol_name=gast.Constant(s.ssf(), kind=None))
return assignments
def visit_If(self, node):
@ -299,9 +299,9 @@ class ControlFlowTransformer(converter.Base):
composites, state_getter_name, state_setter_name)
basic_symbol_names = tuple(
gast.Str(str(symbol)) for symbol in returned_from_cond)
gast.Constant(str(symbol), kind=None) for symbol in returned_from_cond)
composite_symbol_names = tuple(
gast.Str(str(symbol)) for symbol in composites)
gast.Constant(str(symbol), kind=None) for symbol in composites)
cond_expr = self._create_cond_expr(cond_results, cond_var_name, body_name,
orelse_name, state_getter_name,
@ -397,9 +397,9 @@ class ControlFlowTransformer(converter.Base):
composite_loop_vars, state_getter_name, state_setter_name)
basic_symbol_names = tuple(
gast.Str(str(symbol)) for symbol in basic_loop_vars)
gast.Constant(str(symbol), kind=None) for symbol in basic_loop_vars)
composite_symbol_names = tuple(
gast.Str(str(symbol)) for symbol in composite_loop_vars)
gast.Constant(str(symbol), kind=None) for symbol in composite_loop_vars)
opts = self._create_loop_options(node)
@ -520,9 +520,9 @@ class ControlFlowTransformer(converter.Base):
undefined_assigns = self._create_undefined_assigns(possibly_undefs)
basic_symbol_names = tuple(
gast.Str(str(symbol)) for symbol in basic_loop_vars)
gast.Constant(str(symbol), kind=None) for symbol in basic_loop_vars)
composite_symbol_names = tuple(
gast.Str(str(symbol)) for symbol in composite_loop_vars)
gast.Constant(str(symbol), kind=None) for symbol in composite_loop_vars)
opts = self._create_loop_options(node)

View File

@ -41,7 +41,7 @@ class DirectivesTest(converter_testing.TestCase):
def_, = anno.getanno(node.body[0].targets[0],
anno.Static.DEFINITIONS)
d = def_.directives[directives.set_element_type]
self.assertEqual(d['dtype'].s, 'a')
self.assertEqual(d['dtype'].value, 'a')
self.assertEqual(d['shape'].id, 'string_var')
def test_argument_target(self):
@ -54,8 +54,8 @@ class DirectivesTest(converter_testing.TestCase):
def_, = anno.getanno(node.args.args[0], anno.Static.DEFINITIONS)
d = def_.directives[directives.set_element_type]
self.assertEqual(d['dtype'].n, 1)
self.assertEqual(d['shape'].n, 2)
self.assertEqual(d['dtype'].value, 1)
self.assertEqual(d['shape'].value, 2)
def test_loop_target(self):
@ -69,7 +69,7 @@ class DirectivesTest(converter_testing.TestCase):
d = anno.getanno(node.body[1], anno.Basic.DIRECTIVES)
d = d[directives.set_loop_options]
self.assertEqual(d['parallel_iterations'].n, 10)
self.assertEqual(d['parallel_iterations'].value, 10)
self.assertEqual(d['back_prop'].id, 'a')
self.assertNotIn('swap_memory', d)

View File

@ -81,7 +81,7 @@ class FunctionBodyTransformer(converter.Base):
template,
options=self._function_scope_options().to_ast(),
function_context=function_context_name,
function_context_name=gast.Str(function_context_name),
function_context_name=gast.Constant(function_context_name, kind=None),
body=node.body)
self.state[_Function].exit()
@ -102,7 +102,7 @@ class FunctionBodyTransformer(converter.Base):
if node.body:
first_statement = node.body[0]
if (isinstance(first_statement, gast.Expr) and
isinstance(first_statement.value, gast.Str)):
isinstance(first_statement.value, gast.Constant)):
docstring_node = first_statement
node.body = node.body[1:]
@ -113,8 +113,8 @@ class FunctionBodyTransformer(converter.Base):
"""
wrapped_body = templates.replace(
template,
function_name=gast.Str(node.name),
context_name=gast.Str(function_context_name),
function_name=gast.Constant(node.name, kind=None),
context_name=gast.Constant(function_context_name, kind=None),
options=self._function_scope_options().to_ast(),
function_context=function_context_name,
body=node.body)

View File

@ -349,7 +349,7 @@ class ReturnStatementsTransformer(converter.Base):
docstring = None
if converted_body:
if (isinstance(converted_body[0], gast.Expr) and
isinstance(converted_body[0].value, gast.Str)):
isinstance(converted_body[0].value, gast.Constant)):
docstring = converted_body[0]
converted_body = converted_body[1:]

View File

@ -599,7 +599,9 @@ def convert_class_to_ast(c, program_ctx):
renames[qual_names.QN(base.__name__)] = qual_names.QN(alias)
# Generate the definition of the converted class.
bases = [gast.Name(n, gast.Load(), None) for n in base_names]
bases = [
gast.Name(n, ctx=gast.Load(), annotation=None, type_comment=None)
for n in base_names]
class_def = gast.ClassDef(
class_name,
bases=bases,
@ -706,7 +708,11 @@ def convert_func_to_ast(f, program_ctx, do_rename=True):
if isinstance(node, gast.Lambda):
node = gast.Assign(
targets=[gast.Name(new_name, gast.Store(), None)], value=node)
targets=[
gast.Name(
new_name, ctx=gast.Store(), annotation=None, type_comment=None)
],
value=node)
elif do_rename:
node.name = new_name
else:

View File

@ -86,7 +86,11 @@ class SymbolRenamer(gast.NodeTransformer):
def _process_name_node(self, node):
qn = anno.getanno(node, anno.Basic.QN)
if qn in self.name_map:
new_node = gast.Name(str(self.name_map[qn]), node.ctx, None)
new_node = gast.Name(
str(self.name_map[qn]),
ctx=node.ctx,
annotation=None,
type_comment=None)
# All annotations get carried over.
for k in anno.keys(node):
anno.copyanno(node, new_node, k)
@ -133,7 +137,7 @@ def keywords_to_dict(keywords):
keys = []
values = []
for kw in keywords:
keys.append(gast.Str(kw.arg))
keys.append(gast.Constant(kw.arg, kind=None))
values.append(kw.value)
return gast.Dict(keys=keys, values=values)

View File

@ -31,6 +31,7 @@ import collections
import gast
import six
from tensorflow.python.autograph.pyct import gast_util
from tensorflow.python.autograph.pyct import templates
from tensorflow.python.autograph.pyct import transformer
@ -118,19 +119,18 @@ class AnfTransformer(transformer.Base):
# These could be pulled out, but are generally considered to already be in
# A-normal form. Thus they are left in by default, but could be pulled
# out if the configuration calls for it.
try:
# TODO(b/140808434): Fix this.
# gast pre-0.3
if gast_util.GAST2:
literal_node_types = (
gast.Num, gast.Str, gast.Bytes, gast.NameConstant,
gast.Name # Name is here to cover True, False, and None in Python 2
)
except AttributeError:
# gast 0.3+
elif gast_util.GAST3:
literal_node_types = (
gast.Constant,
gast.Name # Name is here to cover True, False, and None in Python 2
)
else:
assert False
self._overrides = [
(ASTEdgePattern(ANY, ANY, literal_node_types), LEAVE),
@ -523,14 +523,9 @@ def _is_trivial(node):
)
if isinstance(node, trivial_node_types) and not _is_py2_name_constant(node):
return True
try:
# gast pre-0.3
if isinstance(node, gast.Ellipsis):
return True
except AttributeError:
# gast 0.3+
if isinstance(node, gast.Constant) and node.value == Ellipsis:
return True
if gast_util.is_ellipsis(node):
return True
return False

View File

@ -524,7 +524,7 @@ class AnfConfiguredTest(AnfTestBase):
# Checking that the nodes for `True`, `False`, and `None` can be manipulated
# by a configuration. This is non-trivial, because in Python 2 those are
# represented as `Name`, which is the same node type as variable references.
specials = (gast.Name, gast.NameConstant)
specials = (gast.Name, gast.Constant)
config = [(anf.ASTEdgePattern(gast.Call, anf.ANY, specials), anf.REPLACE)]
def test_function(f):

View File

@ -51,21 +51,30 @@ class LoaderTest(test.TestCase):
node = gast.FunctionDef(
name='f',
args=gast.arguments(
args=[gast.Name('a', gast.Param(), None)],
args=[
gast.Name(
'a', ctx=gast.Param(), annotation=None, type_comment=None)
],
posonlyargs=[],
vararg=None,
kwonlyargs=[],
kw_defaults=[],
kwarg=None,
defaults=[],
kw_defaults=[]),
defaults=[]),
body=[
gast.Return(
gast.BinOp(
op=gast.Add(),
left=gast.Name('a', gast.Load(), None),
right=gast.Num(1)))
left=gast.Name(
'a',
ctx=gast.Load(),
annotation=None,
type_comment=None),
right=gast.Constant(1, kind=None)))
],
decorator_list=[],
returns=None)
returns=None,
type_comment=None)
module, source, _ = loader.load_ast(node)

View File

@ -136,16 +136,29 @@ string""")
def test_unparse(self):
node = gast.If(
test=gast.Num(1),
test=gast.Constant(1, kind=None),
body=[
gast.Assign(
targets=[gast.Name('a', gast.Store(), None)],
value=gast.Name('b', gast.Load(), None))
targets=[
gast.Name(
'a',
ctx=gast.Store(),
annotation=None,
type_comment=None)
],
value=gast.Name(
'b', ctx=gast.Load(), annotation=None, type_comment=None))
],
orelse=[
gast.Assign(
targets=[gast.Name('a', gast.Store(), None)],
value=gast.Str('c'))
targets=[
gast.Name(
'a',
ctx=gast.Store(),
annotation=None,
type_comment=None)
],
value=gast.Constant('c', kind=None))
])
source = parser.unparse(node, indentation=' ')

View File

@ -33,6 +33,10 @@ from tensorflow.python.autograph.pyct import anno
from tensorflow.python.autograph.pyct import parser
class CallerMustSetThis(object):
pass
class Symbol(collections.namedtuple('Symbol', ['name'])):
"""Represents a Python symbol."""
@ -188,20 +192,25 @@ class QN(object):
return ssf_string + ssfs[-1]
def ast(self):
"""AST representation."""
# The caller must adjust the context appropriately.
if self.has_subscript():
return gast.Subscript(self.parent.ast(), gast.Index(self.qn[-1].ast()),
None)
return gast.Subscript(
value=self.parent.ast(),
slice=gast.Index(self.qn[-1].ast()),
ctx=CallerMustSetThis)
if self.has_attr():
return gast.Attribute(self.parent.ast(), self.qn[-1], None)
return gast.Attribute(
value=self.parent.ast(), attr=self.qn[-1], ctx=CallerMustSetThis)
base = self.qn[0]
if isinstance(base, str):
return gast.Name(base, None, None)
return gast.Name(
base, ctx=CallerMustSetThis, annotation=None, type_comment=None)
elif isinstance(base, StringLiteral):
return gast.Str(base.value)
return gast.Constant(base.value, kind=None)
elif isinstance(base, NumberLiteral):
return gast.Num(base.value)
return gast.Constant(base.value, kind=None)
else:
assert False, ('the constructor should prevent types other than '
'str, StringLiteral and NumberLiteral')
@ -233,10 +242,8 @@ class QnResolver(gast.NodeTransformer):
# TODO(mdan): Support range and multi-dimensional indices.
# Continuing silently because some demos use these.
return node
if isinstance(s.value, gast.Num):
subscript = QN(NumberLiteral(s.value.n))
elif isinstance(s.value, gast.Str):
subscript = QN(StringLiteral(s.value.s))
if isinstance(s.value, gast.Constant):
subscript = QN(NumberLiteral(s.value.value))
else:
# The index may be an expression, case in which a name doesn't make sense.
if anno.hasanno(node.slice.value, anno.Basic.QN):

View File

@ -150,7 +150,7 @@ class QNTest(test.TestCase):
d = {QN('a'): 'a', QN('b'): 'b'}
self.assertEqual(d[QN('a')], 'a')
self.assertEqual(d[QN('b')], 'b')
self.assertTrue(QN('c') not in d)
self.assertNotIn(QN('c'), d)
def test_literals(self):
a = QN('a')
@ -161,7 +161,7 @@ class QNTest(test.TestCase):
self.assertNotEqual(hash(a_sub_str_b), hash(a_sub_b))
a_sub_three = QN(a, subscript=QN(qual_names.NumberLiteral(3)))
self.assertEqual(a_sub_three.ast().slice.value.n, 3)
self.assertEqual(a_sub_three.ast().slice.value.value, 3)
def test_support_set(self):
a = QN('a')

View File

@ -221,7 +221,7 @@ def _convert_to_ast(n):
# unknown. ctx must be filled in according to the template being used.
# See ReplaceTransformer.visit_Name.
if isinstance(n, str):
return gast.Name(id=n, ctx=None, annotation=None)
return gast.Name(id=n, ctx=None, annotation=None, type_comment=None)
if isinstance(n, qual_names.QN):
return n.ast()
if isinstance(n, list):

View File

@ -110,12 +110,28 @@ class TemplatesTest(test.TestCase, parameterized.TestCase):
return a
"""
class ShouldBeReplaced(object):
pass
node = templates.replace(
template,
block=[
gast.Assign([
gast.Name('a', None, None)
], gast.BinOp(gast.Name('a', None, None), gast.Add(), gast.Num(1))),
gast.Assign(
[
gast.Name(
'a',
ctx=ShouldBeReplaced,
annotation=None,
type_comment=None)
],
gast.BinOp(
gast.Name(
'a',
ctx=ShouldBeReplaced,
annotation=None,
type_comment=None), gast.Add(),
gast.Constant(1, kind=None)),
),
] * 2)[0]
result, _, _ = loader.load_ast(node)
self.assertEqual(3, result.test_fn(1))

View File

@ -208,8 +208,9 @@ class TransformerTest(test.TestCase):
class TestTransformer(transformer.Base):
# Extract all string constants from the block.
def visit_Str(self, node):
self.set_local('string', self.get_local('string', default='') + node.s)
def visit_Constant(self, node):
self.set_local(
'string', self.get_local('string', default='') + str(node.value))
return self.generic_visit(node)
def _annotate_result(self, node):
@ -236,7 +237,7 @@ class TransformerTest(test.TestCase):
return 'b'
else:
_ = 'c'
while True:
while 4:
raise '1'
return 'nor this'
@ -247,9 +248,9 @@ class TransformerTest(test.TestCase):
while_node = for_node.body[1].orelse[1]
self.assertFalse(anno.hasanno(for_node, 'string'))
self.assertEqual('abc', anno.getanno(for_node, 'test'))
self.assertEqual('3a2bc', anno.getanno(for_node, 'test'))
self.assertFalse(anno.hasanno(while_node, 'string'))
self.assertEqual('1', anno.getanno(while_node, 'test'))
self.assertEqual('41', anno.getanno(while_node, 'test'))
def test_local_scope_info_stack_checks_integrity(self):
@ -289,7 +290,10 @@ class TransformerTest(test.TestCase):
def _process_body_item(self, node):
if isinstance(node, gast.Assign) and (node.value.id == 'y'):
if_node = gast.If(gast.Name('x', gast.Load(), None), [node], [])
if_node = gast.If(
gast.Name(
'x', ctx=gast.Load(), annotation=None, type_comment=None),
[node], [])
return if_node, if_node.body
return node, None

View File

@ -477,7 +477,7 @@ install_tensorflow_pip() {
# Install the gast package in the virtualenv. Installing it in user system
# packages does not appear to port it over when creating a virtualenv.
${PIP_BIN_PATH} install --upgrade "gast==0.2.2" || \
${PIP_BIN_PATH} install --upgrade "gast==0.3.2" || \
die "Error: gast install, upgrade FAILED"
}

View File

@ -130,7 +130,7 @@ function install_pip_deps {
# LINT.IfChange(ubuntu_pip_installations)
# TODO(aselle): Change all these to be --user instead of sudo.
${SUDO_CMD} ${PIP_CMD} install keras_preprocessing==1.1.0 --no-deps
${SUDO_CMD} ${PIP_CMD} install gast==0.2.2
${SUDO_CMD} ${PIP_CMD} install gast==0.3.2
${SUDO_CMD} ${PIP_CMD} install h5py==2.8.0
${SUDO_CMD} ${PIP_CMD} install six==1.12.0
${SUDO_CMD} ${PIP_CMD} install grpcio
@ -163,7 +163,7 @@ function install_ubuntu_16_pip_deps {
"${PIP_CMD}" install keras_preprocessing==1.1.0 --no-deps --user
"${PIP_CMD}" install numpy==1.14.5 --user
"${PIP_CMD}" install --user --upgrade "future>=0.17.1"
"${PIP_CMD}" install gast==0.2.2 --user
"${PIP_CMD}" install gast==0.3.2 --user
"${PIP_CMD}" install h5py==2.8.0 --user
"${PIP_CMD}" install six==1.12.0 --user
"${PIP_CMD}" install grpcio --user
@ -208,7 +208,7 @@ function install_macos_pip_deps {
${SUDO_CMD} ${PIP_CMD} install six==1.12.0
${SUDO_CMD} ${PIP_CMD} install scikit-learn==0.20.3
${SUDO_CMD} ${PIP_CMD} install numpy==1.14.5
${SUDO_CMD} ${PIP_CMD} install gast==0.2.2
${SUDO_CMD} ${PIP_CMD} install gast==0.3.2
${SUDO_CMD} ${PIP_CMD} install h5py==2.8.0
${SUDO_CMD} ${PIP_CMD} install --upgrade grpcio
${SUDO_CMD} ${PIP_CMD} install --upgrade "tb-nightly>=2.1.*"

View File

@ -42,7 +42,7 @@ IF "%PYTHON_DIRECTORY%"=="Python37" (
%PIP_EXE% install absl-py==0.5.0
%PIP_EXE% install colorama==0.3.9
%PIP_EXE% install cycler==0.10.0
%PIP_EXE% install gast==0.2.0
%PIP_EXE% install gast==0.3.2
%PIP_EXE% install jedi==0.11.1
%PIP_EXE% install oauth2client==4.1.2
%PIP_EXE% install portpicker==1.2.0

View File

@ -54,7 +54,7 @@ REQUIRED_PACKAGES = [
'astor >= 0.6.0',
'backports.weakref >= 1.0rc1;python_version<"3.4"',
'enum34 >= 1.1.6;python_version<"3.4"',
'gast == 0.2.2',
'gast == 0.3.2',
'google_pasta >= 0.1.8',
'h5py >= 2.10.0, < 2.11.0',
'keras_preprocessing >= 1.1.0',

View File

@ -359,12 +359,12 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""):
tf_http_archive(
name = "gast_archive",
build_file = clean_dep("//third_party:gast.BUILD"),
sha256 = "fe939df4583692f0512161ec1c880e0a10e71e6a232da045ab8edd3756fbadf0",
strip_prefix = "gast-0.2.2",
sha256 = "5c7617f1f6c8b8b426819642b16b9016727ddaecd16af9a07753e537eba8a3a5",
strip_prefix = "gast-0.3.2",
system_build_file = clean_dep("//third_party/systemlibs:gast.BUILD"),
urls = [
"https://storage.googleapis.com/mirror.tensorflow.org/pypi.python.org/packages/4e/35/11749bf99b2d4e3cceb4d55ca22590b0d7c2c62b9de38ac4a4a7f4687421/gast-0.2.2.tar.gz",
"https://files.pythonhosted.org/packages/4e/35/11749bf99b2d4e3cceb4d55ca22590b0d7c2c62b9de38ac4a4a7f4687421/gast-0.2.2.tar.gz",
"https://storage.googleapis.com/mirror.tensorflow.org/files.pythonhosted.org/packages/1f/04/4e36c33f8eb5c5b6c622a1f4859352a6acca7ab387257d4b3c191d23ec1d/gast-0.3.2.tar.gz",
"https://files.pythonhosted.org/packages/1f/04/4e36c33f8eb5c5b6c622a1f4859352a6acca7ab387257d4b3c191d23ec1d/gast-0.3.2.tar.gz",
],
)