Update to make find_rocm_config.py to execute properly in RBE (remote build environment)

third_party/gpus/compress_find_rocm_config.py

@@ -0,0 +1,37 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Compresses the contents of 'find_rocm_config.py'.
+
+The compressed file is what is actually being used. It works around remote
+config not being able to upload files yet.
+"""
+import base64
+import zlib
+
+
+def main():
+  with open('find_rocm_config.py', 'rb') as f:
+    data = f.read()
+
+  compressed = zlib.compress(data)
+  b64encoded = base64.b64encode(compressed)
+
+  with open('find_rocm_config.py.gz.base64', 'wb') as f:
+    f.write(b64encoded)
+
+
+if __name__ == '__main__':
+  main()
+

third_party/gpus/find_rocm_config.py.gz.base64

@@ -0,0 +1 @@
+eJzNWm1v2zYQ/q5fQSgoKrWOkvbTkCEfvDRDvbVJYGcbhiYwaIm2ucqiRlJxjaL/fXckJUuynDh1gi5AUVu6e3i8e+5Fog/ImchXks/mmrw9fntMrueMXLNMCflrKpakX+i5kCoi/TQlQxRTZMgUk3csibwD74B84DGIs4QUWcIk0aDfz2kM/7k7PfInk4qLjLyNjkmAAr675Yc/A8JKFGRBVyQTmhSKAQRXZMpTRtiXmOWa8IzEYpGnnGYxI0uu52YZBwJmkL8dhJhoCtIU5HP4Nq3LEaqNwfg31zo/OTpaLpcRNcZGQs6OUiuojj4Mzs4vRueHYLBR+SNLmVJEsn8LLmGrkxWhOdgT0wlYmdIlEZLQmWRwTwu0dym55tmsR5SY6iWVDFASrrTkk0I3nFVaB3uuC4C7aEb8/ogMRj75pT8ajHqA8dfg+v3lH9fkr/5w2L+4HpyPyOWQnF1evBtcDy4v4NuvpH/xN/l9cPGuRxi4CpZhX3KJ9oORHN1oQkdGjDUMmAprkMpZzKc8hn1ls4LOGJmJOyYz2A7JmVxwhcFUYF4CKClfcE21ubKxKVzm9En/PN/3ryTPkIaXZwtYfiKpXKExZM4orp9AiGItJGfGRnJn2QeUEmAgOtbscqU0W0Seh4RXseTAM8WoBC4o44pt8EhM1UTpQcTRa1p5cHGBFEiYRldlxsVclkYYoNzaj/qxyKZ8VkjjQNRTOhGFjoxVOUWiixIcGeJigzSbS1HM5kgSlt1xKbIFyzS5o5IbUgZg/8fxVf/6fRh5gykkF9xLedJakju39Ox2rB9KA405TEoTasl0IU3YCVwCB8UiYU3/afqZ2X2VMVjVLIakwVuVXZ12R3W8VIjPNhjW9zaeZUxsIEy2z6lMDtGeBGKoIe89VUzqPJhKsSATqpxTXWFY21bZGxHw1dpEcA9UJa8SNG6CtDwSuT6SIl74KFJg+aNgi4a4T2mR4n7SgnnIVs+DnBMSwifKT0KVn6AuuE/AJM+LUwppemYidI5ODs5NBYRIhSceAeMVSHmwCBnPmB671cZoyRh3FhgxG6q6lXUlIwyc0jRNa0pg6ruStNbRZcRd1BbEKVnmoCZuD5ZrA5LTreaBOJ8Sv/KwjwEUKnJcQEO6Adcyn2ratyB/sE0eP0WS0dQsvSEUrj21ca/uL2w9QnHNxi6Jx1mxmDAZLOg/QvYIOAz/A7V4Xnf/m2P4e2WEyGv89spIwmcjWl/C0rvED9CCHsnogpWRGbpQQB3PwVgo5gAE6iZW4MGzenWKbFgwb1KsQHCfi0jkrET2pQ+9IYOEgZp+6hd6eviTH6KrElRQlhAEOirYCb6ULDIfA//ArkleKPI6uElehz55YQztmaVCowYBNuIWhNhcABhzMZpB3cqDN6G76ZwFBScwcqGHHqQcUrWeB0b6Zbm8//WbSTzb326ylxFsFuAD4zPy2koTM1Fg68MKA14wM8XXb9AxayqGCmsVDoXPGmxzH+sdGMdmDL0J+UHTl2EZOzAmsey2VbWDZCe4HxQ2t5oMUoETMV5yt4yRa/7+I3gVtohnU3HkBA8TdudXDkdLSxX2BeYIFdQBwzIWWzxrNm9yvGxUxgzjZ4Q2PaJyWQM5bFjvNgYb+HRrbphpzXCvoYXlcloRxN3B6QeGjFMyNVwMwgiv5EHYklsv0tSMFAw3OvAj5xebe6eWXE3dT8e3TsbkZLfMGyeTuzzoknl7W6566IcVqmM16SgRyIeOy5hmXQzpKFtlybWcA8WvZk2/Q98n1sePL2Sg9s1rVUi7YIP8c57LItN8wXZIgZpwy4LH5QHgHPEsTouEHeFn+FcCRvOnzYn3gysytDZ/b2ocwDMV1JJagYZ5xc5RX/AZAqaGsq5hyYYVx3+eD0cwzffIci7ANDtMWCxACmoi44/93+AB4BX2F+gsjTuDi8thaQI0EMn4nZ34y55kuW9b/WFM07iA3g7WMQn1XHHzsAWznDAzkMVZUiilQCbMBWHAJoXCnqFwqMop7B4yGIY5yHp87FCNROxqdnWPQWw39uY38/SxEOgEv5GUb2pd2aAiPTd4vE6rrawtk2s7rTtnjnpabSzbSK4Fx8K5Q2I5wf26iwWpEst9faa8+ji4BPQn7zaPIBpYcHV+sS/X2ig1upVl/fEoMNqevff36CRb+NDdSxosW9O+E+PJ+klj0fY4NYHHoN0mKiO5H+0dSsV79/3wmYjv4H8k82HO++VDf7Qv9Tdgvo/7bZi9yb+NFlsnqRrdGsNUB8pTzlO1ZdsJIKEn75YARnKvYcqB1PlfB/1f839nhg37F+9KhrW41UR0lKgFoEGJDncDJbYF4sH231ypzYLpVO9GAhTcuwgCSJ0D8PUZSyCg/8gK2PKZ0Xx8AWyjoObj618LxWjuWf66+LC1+q1Z1mD6JsZT1r71om3Sa0ljcNZOvHeye1Pf4tTZb688A/Et8A/u/tfD/tn58An6fxuoNgEcrANUHcwIZl+pzSk8iFLH2NIVrv7Ws+d4ryzYwo6tidBgXiMXOpGeMh0aS7ffryh8qt7x9YqT3S8jKpz6SxZ75blaQrXAj8yMygh3Xv3x+9rCBsz39YU2zNW+jWE7PbpTok29xhuRTqQnS4n20i4lNl63t49HKEl4jKdTeD4opvbUyhiSMTzRdBvB1+jRA2dXHadk3ja6d+XkvYx/OapOHtdnjYbsVY000BXjN1ewVYspPGyEwHxbf/1kXwRrIdLPXBtp/7Z8vdw45Co1oiJPqGbBLucZ4RatXV4Eb9N96D3XNr0HXxXco3j/I9Y9ivdO5ffoPTTY3OPYhzpAWO8mBsIlzIJCVTdk1HJlOYmvmD+zVa885MqIElKzJNhMrgiSd6GCsKrg5ncBgf9CnZAXCk/+gjWSsd/9XqfGeDznYVZfrVRkf08Q4a9jWODfZOfD4eXwBOh7k91kfsl2pWUAgGGlBsmg8cjQ8yABx2M85RuPyekp8cdj3ON47J+Yuofb9f4Dez+DwA==

third_party/gpus/rocm_configure.bzl

@@ -383,7 +383,26 @@ def _find_libs(repository_ctx, rocm_config, bash_bin):
 
 def _exec_find_rocm_config(repository_ctx, script_path):
     python_bin = get_python_bin(repository_ctx)
-    return execute(repository_ctx, [python_bin, script_path])
+    # If used with remote execution then repository_ctx.execute() can't
+    # access files from the source tree. A trick is to read the contents
+    # of the file in Starlark and embed them as part of the command. In
+    # this case the trick is not sufficient as the find_cuda_config.py
+    # script has more than 8192 characters. 8192 is the command length
+    # limit of cmd.exe on Windows. Thus we additionally need to compress
+    # the contents locally and decompress them as part of the execute().
+    compressed_contents = repository_ctx.read(script_path)
+    decompress_and_execute_cmd = (
+        "from zlib import decompress;" +
+        "from base64 import b64decode;" +
+        "from os import system;" +
+        "script = decompress(b64decode('%s'));" % compressed_contents +
+        "f = open('script.py', 'wb');" +
+        "f.write(script);" +
+        "f.close();" +
+        "system('\"%s\" script.py');" % (python_bin)
+    )
+
+    return execute(repository_ctx, [python_bin, "-c", decompress_and_execute_cmd])
 
 def find_rocm_config(repository_ctx, script_path):
     """Returns ROCm config dictionary from running find_rocm_config.py"""
@@ -552,7 +571,7 @@ def _create_local_rocm_repository(repository_ctx):
         "rocm:rocm_config.h",
     ]}
 
-    find_rocm_config_script = repository_ctx.path(Label("@org_tensorflow//third_party/gpus:find_rocm_config.py"))
+    find_rocm_config_script = repository_ctx.path(Label("@org_tensorflow//third_party/gpus:find_rocm_config.py.gz.base64"))
 
     bash_bin = get_bash_bin(repository_ctx)
     rocm_config = _get_rocm_config(repository_ctx, bash_bin, find_rocm_config_script)