From b7796f3c856965f68699b2b527fc2872e2aa71ad Mon Sep 17 00:00:00 2001 From: Jakob Buchgraber Date: Tue, 18 Feb 2020 11:50:58 -0800 Subject: [PATCH] cuda_configure: make find_cuda_config() compatible with remote execution repository_ctx.execute() does not support uploading of files from the source tree. I initially tried constructing a command that simply embeds the file's contents. However that did not work on Windows because the file is larger than 8192 characters. So my best idea was to compress it locally and embed the compressed contents in the command and to uncompress it remotely. This works but comes with the drawback that we need to compress it first. This can't be done as part of the repository_rule either because within one repository_rule every execute() runs either locally or remotely. I thus decided to check in the compressed version in the source tree. It's very much a temporary measure as I'll add the ability to upload files to a future version of Bazel. PiperOrigin-RevId: 295787408 Change-Id: I1545dd86cdec7e4b20cba43d6a134ad6d1a08109 --- tensorflow/opensource_only.files | 1 + third_party/gpus/compress_find_cuda_config.py | 37 +++++++++++++++++++ third_party/gpus/cuda_configure.bzl | 31 +++++++++++++--- .../gpus/find_cuda_config.py.gz.base64 | 1 + third_party/nccl/nccl_configure.bzl | 2 +- third_party/tensorrt/tensorrt_configure.bzl | 2 +- 6 files changed, 67 insertions(+), 7 deletions(-) create mode 100644 third_party/gpus/compress_find_cuda_config.py create mode 100644 third_party/gpus/find_cuda_config.py.gz.base64 diff --git a/tensorflow/opensource_only.files b/tensorflow/opensource_only.files index 4cec73276da..c282a6021ee 100644 --- a/tensorflow/opensource_only.files +++ b/tensorflow/opensource_only.files @@ -95,6 +95,7 @@ tensorflow/third_party/gpus/cuda/build_defs.bzl.tpl tensorflow/third_party/gpus/cuda/cuda_config.h.tpl tensorflow/third_party/gpus/cuda_configure.bzl tensorflow/third_party/gpus/find_cuda_config.py +tensorflow/third_party/gpus/find_cuda_config.py.gz.base64 tensorflow/third_party/gpus/rocm/BUILD tensorflow/third_party/gpus/rocm/BUILD.tpl tensorflow/third_party/gpus/rocm/build_defs.bzl.tpl diff --git a/third_party/gpus/compress_find_cuda_config.py b/third_party/gpus/compress_find_cuda_config.py new file mode 100644 index 00000000000..98be39d9245 --- /dev/null +++ b/third_party/gpus/compress_find_cuda_config.py @@ -0,0 +1,37 @@ +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Compresses the contents of find_cuda_config.py.oss. + +The compressed file is what is actually being used. It works around remote +config not being able to upload files yet. +""" +import base64 +import zlib + + +def main(): + with open('find_cuda_config.py.oss', 'rb') as f: + data = f.read() + + compressed = zlib.compress(data) + b64encoded = base64.b64encode(compressed) + + with open('find_cuda_config.py.gz.base64.oss', 'wb') as f: + f.write(b64encoded) + + +if __name__ == '__main__': + main() + diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl index 1f132e96f2c..5dcdfdbad73 100644 --- a/third_party/gpus/cuda_configure.bzl +++ b/third_party/gpus/cuda_configure.bzl @@ -579,14 +579,35 @@ def _cudart_static_linkopt(cpu_value): """Returns additional platform-specific linkopts for cudart.""" return "" if cpu_value == "Darwin" else "\"-lrt\"," +def _exec_find_cuda_config(repository_ctx, script_path, cuda_libraries): + python_bin = get_python_bin(repository_ctx) + + # If used with remote execution then repository_ctx.execute() can't + # access files from the source tree. A trick is to read the contents + # of the file in Starlark and embed them as part of the command. In + # this case the trick is not sufficient as the find_cuda_config.py + # script has more than 8192 characters. 8192 is the command length + # limit of cmd.exe on Windows. Thus we additionally need to compress + # the contents locally and decompress them as part of the execute(). + compressed_contents = repository_ctx.read(script_path) + decompress_and_execute_cmd = ( + "from zlib import decompress;" + + "from base64 import b64decode;" + + "from os import system;" + + "script = decompress(b64decode('%s'));" % compressed_contents + + "f = open('script.py', 'wb');" + + "f.write(script);" + + "f.close();" + + "system('%s script.py %s');" % (python_bin, " ".join(cuda_libraries)) + ) + + return execute(repository_ctx, [python_bin, "-c", decompress_and_execute_cmd]) + # TODO(csigg): Only call once instead of from here, tensorrt_configure.bzl, # and nccl_configure.bzl. def find_cuda_config(repository_ctx, script_path, cuda_libraries): """Returns CUDA config dictionary from running find_cuda_config.py""" - exec_result = raw_exec(repository_ctx, [ - get_python_bin(repository_ctx), - script_path, - ] + cuda_libraries) + exec_result = _exec_find_cuda_config(repository_ctx, script_path, cuda_libraries) if exec_result.return_code: auto_configure_fail("Failed to run find_cuda_config.py: %s" % err_out(exec_result)) @@ -858,7 +879,7 @@ def _create_local_cuda_repository(repository_ctx): "cuda:cuda_config.h", ]} tpl_paths["cuda:BUILD"] = _tpl_path(repository_ctx, "cuda:BUILD.windows" if is_windows(repository_ctx) else "cuda:BUILD") - find_cuda_config_script = repository_ctx.path(Label("@org_tensorflow//third_party/gpus:find_cuda_config.py")) + find_cuda_config_script = repository_ctx.path(Label("@org_tensorflow//third_party/gpus:find_cuda_config.py.gz.base64")) cuda_config = _get_cuda_config(repository_ctx, find_cuda_config_script) diff --git a/third_party/gpus/find_cuda_config.py.gz.base64 b/third_party/gpus/find_cuda_config.py.gz.base64 new file mode 100644 index 00000000000..418acdfd5ac --- /dev/null +++ b/third_party/gpus/find_cuda_config.py.gz.base64 @@ -0,0 +1 @@ +eJzNXHtz47iR/5+fAseJy6RHpu291FZOF++V157JKvHZU7Zm9lK2o0AkLHFNkQofkpVUvvt1NwASBCm/JlsZVa1HAtFAd+PXLxDYd+w0W27yeDYv2XeHR//FxnPBxiItsvxjkq3ZSVXOs7wI2EmSsCvsVrArUYh8JaLAeee8Y+dxCN1FxKo0Ejkrgf5kyUP4Rz0ZsC8iL+IsZd8Fh8zDDq565Pr/DSNssoot+IalWcmqQsAQccHu40Qw8RiKZcnilIXZYpnEPA0FW8flnKZRgwAb7M9qiGxacujNof8Sft2b/RgviWH8zMtyOTw4WK/XASdmgyyfHSSyY3FwPjr9cHH9YR8YJpLPaSKKguXib1Wcg6jTDeNL4CfkU+Ay4WuW5YzPcgHPygz5XedxGaezASuy+3LNcwGjRHFR5vG0KlvK0tyBzGYHUBdPmXtyzUbXLvvx5Hp0PYAxfh6Nf7r8PGY/n1xdnVyMRx+u2eUVO728OBuNR5cX8OsjO7n4M/vT6OJswASoCqYRj8sc+QcmY1QjLR27FqLFwH0mGSqWIozv4xDkSmcVnwk2y1YiT0EcthT5Ii5wMQtgL4JRkngRl7yklo5QOM3xv/TjuK77KY9TgOHp57MTmH6a83yDzLC54Dh/BEsUllkeC+KRrST6AFIZMIiKJSk3RSkWgeMg4IswjwFnheA5YKEgVWwbHoFZtEcZwIqj1srCgcYFQiASJaoqJRXHuWaCBlpK/pE+zNL7eFblpECkK8ooq8qAuFrycl5IPNHoRIxUNQ5r0QBget0QgvM8q2ZzJtJVnGfpQqSls+J5jGgFUx7dg6mxFU/iyGIgVkoaSOGkVjS7xJzIc1r4XJRVTiBg0ATqCrNIKG0mAGO0Pak8XAagvY+BeRi/4ZIj27MKuQOmrqvlMssR+Q0Zmg0tgxenYVJF0BRWP56fXPsD+HJ2cTFgF6en5wNSjHRaV+P2kpb8AQeqeZpyQLqJkIYfMGqUevxxgnNOPp2Mf7p2DBUyrULkHPzRgu8XYslBdUA8S7IpTRIwY/Ykyx4kmiR4Cgc51aCSSCJ3Ned5tI8qjACExGhRTU027/NsgewB9yQCYSNwYClb/KKG0Y3WUoFu2OV1Y9ORuOdVUmI/cLXR0HEYGGtaPQ7BKx5URX6QZCFPDsIq4gNqkNqVWCQudpNIoobtL3cDoP8ZljZbF0NW88H61DYA/+Ow5nM6vL39lGeznC/YR1TF7e3Fl9HZ6IT94dNnCEuLZYUulI2zLHmIy9tbHP72ds9xPoJCpzx8AL8a0VIAd/E0TuJygw53IUwMJUVGUYUnYJEpdFxJFTod9WbSWfawDqtEoWTTNrQAVEewctSS7td67h1FaVuuPqvDIvRtsMVOoihGY+RJB6xAvf+VHxiCLIo+GjpfPlxdQ/DQ64JN48vL8z+NxrSaQCPNrqbBHy2qfhqwUGOei4v2RLJpdHE9Pjk/10RozzVz+MNijppMGukCJj+dXekhtCOgIcYQxy+vrsbGMHVTa2rni/al/QAIwdNNhU4pwKTBEB53MaTuPgabXXRx6JYF4IE7NQyUg4ZIvFiWG+xdpYafzhgPKcPh6Ub3Vf4LogDgH+fjKnBptwGma3AxdMr7ye/Vsx8mapAhA6aCv7efSR80AUANWRAE7YfqS/MUAq3jQLoATpnFmf6WFfobejz9fZnwEpnRv8Fvq2/gxpZ5FkL2UbdsCgfSguUmgbAyxJwHFXw825fP98F97fNyv8yWTplvhrCgZJzFHLxBwtQg63kczh2VHo6o7QOGobo7plJIUATFkq9TTYdRaCIeRViV2pPLoRqGRLqVH8cJEw551Cm5P5rQ+0A8gMp9nHsJj6EbeFk2iQtQKnhWj57IgFlrKpCZg+dDfoTpMPRzDcK1dKkvIFXO1yRe8DB7CekZz2GemnLBS0x+NIQ8HpYVTyY1hHXWoVtoAkDJ6VyED6hHQf6T/K9OdxZCqExHUcto76g4gEWFSGSjBrU9DUtVVg3Wh2ksmmVM2ZcapM1nPTZ+OmNZnbFHAeHQIHmNJ9WfI2Z/joJ22zivhEkQfGcTWL8/Qtx6hiL4zycpuh97DmJKr0Q+K4Y1ta23ISU1WmmdlAnCax7UxG0Vt0kxLQ4xX9L5jIxGkOCWPEkoCdUDXckMc8h+VsDC3nLsBl4Ssu2kOEtFIJEJfyFuW0sO7vMCukhhlX1I3UELVEVQgS1BRB4aWVedaOPA1njHNgCxhlt6vtMDv+NOU6s38dIZjedlgTWv1zFAbbm8nCSCF+XrTFfp8pjdgN/zVj7lqSvKRy0WoGQsPTdw/TtDqD7CrnQWqSEh+6HRhhZkJkodpbQomHgOWMoXQvsbBQx0Biq0wPRADsUWZdOnZqUWSBgghwlWY/A8zoJsKfTIbu5CdE4h94Zc89ityvv937l+kMMQSKD8KJNQowUM6KvnvpNzsp2Cvfduo/e+y3aI0QFN5RMZ4I+6a9tSKqC2YAZl2tI7Mhbfrb1xCMsuipinExAyqsLSu4/zooT8VkD2HdnKAOuhHB2z4WmcqoIcXCpRUQovCZVC1Hw3WREgWfBLFqfePYwuV/MeFSVJ8SfpVdLfmWuVRBNZCkyoPvD6uOqk2e36oTZTLFusmNmwiejRZDQXrAQFbs/Vza6P6ZV7UID4B3Uj0KlM6tjIRYIQY9ZEPvFuWiMDJPaX7p1P0bzEmkGuOpYZACgA8x47/oF5wZ7vypXDCILDi5Ls2MSanEGaQQtOKrUxkaVmU/DC3kEksCb0XF6Ecez6ElIq6/mcxvjwjLrUyY85YBMRuihEngMeRZ4GACwTYtdrI9M3sFlQee5hkevJAXzfBIMqLSdYOio80FfT56h1NtslT2YLsO7u4cqJRPUHB0KbgM+RtX6+BzIYRs7aSaoabNWBC1Shcv8A5PFasdSta1t3YD14Wxm72oGO4DFag7X0RU8Mt3njWvX5/k6BLsckAvBanVzVZLJt9UlT9w601W/ReoWVW+638+72RS4SWWrXm1i4F2JshcjdD8sbKRbdmllXbfx0GpRsduvePrmP/VlaNQ/FY5nz4uD086fx6ODJAWUfelb7OV0Y/Qskb238PCs69P7+t62frR/9suKDR5OsJfxeWzQwrQnVlhPaw/Nwv0HKOajF0L+RZ+WhSAdNXzRXVwaQG/c21SDb3Sl2EZ7krVF6+hLXfqQZwL+TXsac8LWDtqnvDLdlFmtaKadZlUTkWWhbEqvvHbkTB9+MnbfNEKw0ze7pn8ZYPUMZXU0ZgtXWQ3UnUr1Kx6aAPQnB9qGUh6NIju9RgB7L9QD/eK2QLyOeObFvpSr4DLXJ40K8HTItRSg7aBFrM5MJ1LZaU5udTE3kFjZpCMxMFwGiaHaRN01+0RMBDAYBbyYHFDzQ4nQUsupqm9bdKfaCaIMUgE5MeDV2qel9a3Bwtk/Wae3PEzn1zfDojiK0gEDfy9aTfEBkKDL3GW6emx41tWc4si1Qt93oE+hQE4nXGcxguxC2CAOGke4VyOLSiso5Lzslp0SbVk5Tbfz77LZTnxrSkufyVcf4rZs+Hf8wsCbd7i8U9ROLqnqYQAaISb1jHqXl0qFga1VMYJLJRkvlKpPpdzRvwIbKZ16MjpahPAF2K+VqM/6MA3larJaGKH2UOZ+FzM7Kw6RIt6VKV9BocIe7A9tLetd89aBrG5Xvt3L8WmO4bWNm7u5OFOxE5HD1pAcH7Ojw8HBQc7FDv2W77zgqxTY0W6u14XsrclwkDubuc9p/0Yq8YYyuLn2nUwNZ4rB3+CoAtw0E49NsJYJaC6ZLrsXui80kd166A6tAMfCQrsKwDw1GLDoFWiaraP26O0uk/Qt80XUbvb8N4M+AffH0d/rju7VX1HX11lJeV/D7ihco5Huq757tnDqd2154d+vo7fs5bdBCAymIUgtQBf4IxKNwO4kJhXLZwdVUagvM0HC9Wk+4j56KYhqnsgAYNOwMrPrRXkxfMr9atGGyfTrsewAoisQqDo06q4BSyS7bsFqBijuK+T4VtKWskS1yYNetW/aCo8NgGrrKmJdlbFrzdv463tQlYrBmvx7oNQYB/d0eJ1kbF27CyddpMJK9yVIvq6+Va7x+6+2v9O/LTer/Oz/RU8tYoxRXRyUqa5blI5dHYGrtBUQ+vjy79MIins38Ib2yQpOaZkBqbtbhpjzoLNASqTnq+kzHcs1smuX4In7eTlQsZYDuAkgd/cGLiG3VEPVBPYAvM/sudzeHd+w/jvseHN2pzR9KT8za0B2lEAWLGIROS/lawtTqECvEVcFoz2WL7+7OR+ulPME/NOpNg3OHbftr9VFbEyg59LNBYUcyv01sqG0bsYl3i7pZMc2hsYamqVvzdFasGRUt7DmJLHP2LfLnZLKN2BLKXBotlo0bpPinlSBNE168JEWy4uNQ7z123s9YW3bg0Y5c2f0lGZaZY5m7lM+8PjF6YiilWAR277nNeY7J/5788fLKpeSsaRtdYNu2fMXs+ulkfPpTEy51nqarUNNJMvb2JIzWgy/jt6RiX52H9Sdh6FbVIZmqUE55wX/JmpN/WZpsArVpTQJsS9nMCvvwzjHz4k651gZSe+BmI6fr6xSrdZZcsCgT8uiYzIzI/zWPt/s8xrz2tJYR0GI3+xPv8HVwLuQ5tXqCOL2HSmrFITPEUxlTAQBVr4bRNgbsl6pQe3V07gVf7NkYel3oNyBkLZ/xnpgOQaKZ8KKtEA9San/LYtoAQ/lfk1jgcG5nLfvjCHV52qdu8aZE+bQ77ThS2y1GafrrFI6NU3upS7MdGp4sM3wZ/ZRuTP8kV3X+4cuHc+2vnvJWX1Ewpunr3dRXOqmtdSIs2Mu9zqvKwzSV1WEzw7bUp+khQ3Dze9Du9SZcI+FXwToNw+TbRDWdc6xBLX9pTNOvVvT9NdCMuvlGwEzL9GtgGQcGjZrj9yLZ7IB5r/Fz0OrzBhgT3VehuKQzsHn5jSL5S3MutwG02fhUytnq2cL8O/ZXxcRfMXxzECgVOYd6lmXTX6CwxdPhbE3XLqBjyaC2oy1cuh2gso6qiNOZGg6ee4sqLuMlbv/HC0iRpiLJ1ir2NxqhQxGtdFBlbDxJvEbP2iDqnSGc5Is6WybfR8rrFwv+IDrZHtOXLuhgR/CsmddnTN5q7xerUXov8m8ozVZnX4xctk4r9anv79kiW4nIPLZH+WW+qC/ZKLmU4oN5J5d8i5bq0b4RZSEAtBd4oaN8lZ9cxSg1WK49S6+7tDuBS7ObBp2+b3CdNe1b3Sda8gTPqU1EuvLgP7VLqk43Hd/c2W+F7KtA5EzopFvfTYLmhbQem86J1UePWmeTmuYb3ftOr9nANQ45KO5aZ/PEjIdSwheJweXmYetwL3hGGqVfElzn7iWOQd8djd47F3RipsxglF15IAnPrvzu+8n3v21OtuyyNZ3VWOZiFWdVkWxkGUi7ld3beQ5TN6PYGZ00Bi+Q4d0f6KA0RVPtDuRln3Us7z3iRSJ0DNbq+61zxE+tV/d06F+8g5u/HLC7Pf89CXa7ft9IdfA/vwHD6VvdJ0+N3rRfMxhnw56Drd8c+QE/mMR/l4bchHcDCHUX+ZJzIC/cai0VUDjO5QVIdfVAIVq5jM6Grt7m7j+CoaiIJBfLhIfCc29v6dSaCXDsoGTovLXsIjmKQ7pCldMd4Kduciq4SBGaW2PH7Ibns1UAsR7cvTwTiw307mlTBPj95mh4d+d03r9Zpwhd65IVSkaCtQ5QWevnti71PRtQXnT8sjMnvStrnVUwHsNq6ZUUj8BbIdfxTvp2deL1H/+U60q7q3RCs9agcTxzm4ydO2Nu6+yU8oN0ULVaRuBave4L62aCvm2nztrI4W7aO/B35j6OZrZhRBtkWUES6Nkn3s2RjGjqs98z7+hwwI7qrO8d+7GzraW2nNa8UFda5HZTffwMN56M1zv6aoTFa6OE/h2pXkSaV/hqTNoKb/L8np1wkwt7v6pvNRRQ0nQLUqByr6FiBzC3G2d60GLvcPSbYnMPcavcPTtcBoP2RkctHdWOPcJRPbddtk687BHNqnd7JDMvSz4jmLnH0fBmFb21VDql6pOsTre2S9cb+nsk7ElWe6S073M+I6ldB7f57cldSWq07oeBvtJCo8alWNRBC7TyAJxF8lqOSzkm3T5otdJbJqPoI9fzcIdKsqLwykyZ1Y00GesWHGo6GqMu5og5sRngOaRKGAdqu3FRc10zQVf5PXenGMo9feY1I/n99RW6ILV3j4FP/i8AAvzfWwivKHNPqNobH0KkKPH4g+NgsKcMZDKhq4aTCUoymbg4khTK+X+SX8ik \ No newline at end of file diff --git a/third_party/nccl/nccl_configure.bzl b/third_party/nccl/nccl_configure.bzl index 363a65f1f43..eba838cd98e 100644 --- a/third_party/nccl/nccl_configure.bzl +++ b/third_party/nccl/nccl_configure.bzl @@ -75,7 +75,7 @@ def _nccl_configure_impl(repository_ctx): # function to be restarted with all previous state being lost. This # can easily lead to a O(n^2) runtime in the number of labels. # See https://github.com/tensorflow/tensorflow/commit/62bd3534525a036f07d9851b3199d68212904778 - find_cuda_config_path = repository_ctx.path(Label("@org_tensorflow//third_party/gpus:find_cuda_config.py")) + find_cuda_config_path = repository_ctx.path(Label("@org_tensorflow//third_party/gpus:find_cuda_config.py.gz.base64")) nccl_version = get_host_environ(repository_ctx, _TF_NCCL_VERSION, "") if nccl_version: diff --git a/third_party/tensorrt/tensorrt_configure.bzl b/third_party/tensorrt/tensorrt_configure.bzl index b3375dc224f..3466ed3b3bb 100644 --- a/third_party/tensorrt/tensorrt_configure.bzl +++ b/third_party/tensorrt/tensorrt_configure.bzl @@ -114,7 +114,7 @@ def _tensorrt_configure_impl(repository_ctx): # function to be restarted with all previous state being lost. This # can easily lead to a O(n^2) runtime in the number of labels. # See https://github.com/tensorflow/tensorflow/commit/62bd3534525a036f07d9851b3199d68212904778 - find_cuda_config_path = repository_ctx.path(Label("@org_tensorflow//third_party/gpus:find_cuda_config.py")) + find_cuda_config_path = repository_ctx.path(Label("@org_tensorflow//third_party/gpus:find_cuda_config.py.gz.base64")) tpl_paths = { "build_defs.bzl": _tpl_path(repository_ctx, "build_defs.bzl"), "BUILD": _tpl_path(repository_ctx, "BUILD"),