From 8e3adf77b2a148fb2c6fea8fea2d0217bca3339f Mon Sep 17 00:00:00 2001 From: Dan Moldovan Date: Mon, 20 Jan 2020 05:39:12 -0800 Subject: [PATCH] Remove line continuations before applying dedent_block. Fixes #35765. PiperOrigin-RevId: 290602586 Change-Id: I262db91fc869e0d69bc52b35f5a74da8d3157cf4 --- tensorflow/python/autograph/pyct/parser.py | 7 ++++++ .../python/autograph/pyct/parser_test.py | 24 +++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/tensorflow/python/autograph/pyct/parser.py b/tensorflow/python/autograph/pyct/parser.py index 9efcb101030..88b0e163929 100644 --- a/tensorflow/python/autograph/pyct/parser.py +++ b/tensorflow/python/autograph/pyct/parser.py @@ -43,9 +43,16 @@ STANDARD_PREAMBLE_LEN = 2 _LEADING_WHITESPACE = re.compile(r'\s*') +def _unfold_continuations(code_string): + """Removes any backslash line continuations from the code.""" + return code_string.replace('\\\n', '') + + def dedent_block(code_string): """Dedents a code so that its first line starts at row zero.""" + code_string = _unfold_continuations(code_string) + token_gen = tokenize.generate_tokens(six.StringIO(code_string).readline) block_indentation = None diff --git a/tensorflow/python/autograph/pyct/parser_test.py b/tensorflow/python/autograph/pyct/parser_test.py index dd8192a031b..4cce4323a2d 100644 --- a/tensorflow/python/autograph/pyct/parser_test.py +++ b/tensorflow/python/autograph/pyct/parser_test.py @@ -129,6 +129,30 @@ string""") f = self._eval_code(parser.dedent_block(code), 'f') self.assertEqual(f(), (1, 2, 3)) + def test_dedent_block_continuation(self): + + code = r""" + def f(): + a = \ + 1 + return a + """ + + f = self._eval_code(parser.dedent_block(code), 'f') + self.assertEqual(f(), 1) + + def test_dedent_block_continuation_in_string(self): + + code = r""" + def f(): + a = "a \ + b" + return a + """ + + f = self._eval_code(parser.dedent_block(code), 'f') + self.assertEqual(f(), 'a b') + def test_parse_expression(self): node = parser.parse_expression('a.b') self.assertEqual('a', node.value.id)