From 6deb2d610d7671ace9f5a5ddc8c61c30dca72665 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 5 Feb 2020 02:16:58 -0800 Subject: [PATCH] python_configure: move common functionality to common.bzl These functions will also be used in cuda_configure.bzl et al. PiperOrigin-RevId: 293326676 Change-Id: I2d2751691948851b756a3ff3cbc064c05f96c84f --- tensorflow/opensource_only.files | 2 + third_party/py/python_configure.bzl | 93 +++++------- third_party/remote_config/BUILD | 0 third_party/remote_config/common.bzl | 211 +++++++++++++++++++++++++++ 4 files changed, 250 insertions(+), 56 deletions(-) create mode 100644 third_party/remote_config/BUILD create mode 100644 third_party/remote_config/common.bzl diff --git a/tensorflow/opensource_only.files b/tensorflow/opensource_only.files index 80cf81ceb79..50796aff50f 100644 --- a/tensorflow/opensource_only.files +++ b/tensorflow/opensource_only.files @@ -146,6 +146,8 @@ tensorflow/third_party/py/numpy/BUILD tensorflow/third_party/py/python_configure.bzl tensorflow/third_party/pybind11.BUILD tensorflow/third_party/python_runtime/BUILD +tensorflow/third_party/remote_config/BUILD +tensorflow/third_party/remote_config/common.bzl tensorflow/third_party/repo.bzl tensorflow/third_party/rules_closure.patch tensorflow/third_party/six.BUILD diff --git a/third_party/py/python_configure.bzl b/third_party/py/python_configure.bzl index 98079c33775..2995564c1d1 100644 --- a/third_party/py/python_configure.bzl +++ b/third_party/py/python_configure.bzl @@ -6,10 +6,21 @@ * `PYTHON_LIB_PATH`: Location of python libraries. """ -_BAZEL_SH = "BAZEL_SH" -_PYTHON_BIN_PATH = "PYTHON_BIN_PATH" -_PYTHON_LIB_PATH = "PYTHON_LIB_PATH" -_TF_PYTHON_CONFIG_REPO = "TF_PYTHON_CONFIG_REPO" +load( + "//third_party/remote_config:common.bzl", + "BAZEL_SH", + "PYTHON_BIN_PATH", + "PYTHON_LIB_PATH", + "TF_PYTHON_CONFIG_REPO", + "auto_config_fail", + "execute", + "get_bash_bin", + "get_host_environ", + "get_python_bin", + "is_windows", + "raw_exec", + "read_dir", +) def _which(repository_ctx, program_name): """Returns the full path to a program on the execution platform.""" @@ -33,7 +44,7 @@ def _get_environ(repository_ctx, name, default_value = None): cmd = "echo -n \"$%s\"" % name result = _execute( repository_ctx, - [_get_bash_bin(repository_ctx), "-c", cmd], + [get_bash_bin(repository_ctx), "-c", cmd], empty_stdout_fine = True, ) if len(result.stdout) == 0: @@ -154,7 +165,7 @@ def _symlink_genrule_for_dir( if src_dir != None: src_dir = _norm_path(src_dir) dest_dir = _norm_path(dest_dir) - files = "\n".join(sorted(_read_dir(repository_ctx, src_dir).splitlines())) + files = "\n".join(read_dir(repository_ctx, src_dir)) # Create a list with the src_dir stripped to use for outputs. dest_files = files.replace(src_dir, "").splitlines() @@ -179,39 +190,9 @@ def _symlink_genrule_for_dir( ) return genrule -def _get_python_bin(repository_ctx): - """Gets the python bin path.""" - python_bin = _get_host_environ(repository_ctx, _PYTHON_BIN_PATH) - if python_bin != None: - return python_bin - python_bin_path = _which(repository_ctx, "python") - if python_bin_path == None: - _fail("Cannot find python in PATH, please make sure " + - "python is installed and add its directory in PATH, or --define " + - "%s='/something/else'.\nPATH=%s" % ( - _PYTHON_BIN_PATH, - _get_environ("PATH", ""), - )) - return python_bin_path - -def _get_bash_bin(repository_ctx): - """Gets the bash bin path.""" - bash_bin = _get_host_environ(repository_ctx, _BAZEL_SH) - if bash_bin != None: - return bash_bin - bash_bin_path = _which(repository_ctx, "bash") - if bash_bin_path == None: - _fail("Cannot find bash in PATH, please make sure " + - "bash is installed and add its directory in PATH, or --define " + - "%s='/path/to/bash'.\nPATH=%s" % ( - _BAZEL_SH, - _get_environ("PATH", ""), - )) - return bash_bin_path - def _get_python_lib(repository_ctx, python_bin): """Gets the python lib path.""" - python_lib = _get_host_environ(repository_ctx, _PYTHON_LIB_PATH) + python_lib = get_host_environ(repository_ctx, PYTHON_LIB_PATH) if python_lib != None: return python_lib @@ -248,29 +229,29 @@ def _get_python_lib(repository_ctx, python_bin): cmd += "from os import system;" cmd += "system(\"%s script.py\");" % python_bin - result = _execute(repository_ctx, [python_bin, "-c", cmd]) + result = execute(repository_ctx, [python_bin, "-c", cmd]) return result.stdout.strip() def _check_python_lib(repository_ctx, python_lib): """Checks the python lib path.""" cmd = 'test -d "%s" -a -x "%s"' % (python_lib, python_lib) - result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd]) + result = raw_exec(repository_ctx, [get_bash_bin(repository_ctx), "-c", cmd]) if result.return_code == 1: - _fail("Invalid python library path: %s" % python_lib) + auto_config_fail("Invalid python library path: %s" % python_lib) def _check_python_bin(repository_ctx, python_bin): """Checks the python bin path.""" cmd = '[[ -x "%s" ]] && [[ ! -d "%s" ]]' % (python_bin, python_bin) - result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd]) + result = raw_exec(repository_ctx, [get_bash_bin(repository_ctx), "-c", cmd]) if result.return_code == 1: - _fail("--define %s='%s' is not executable. Is it the python binary?" % ( - _PYTHON_BIN_PATH, + auto_config_fail("--define %s='%s' is not executable. Is it the python binary?" % ( + PYTHON_BIN_PATH, python_bin, )) def _get_python_include(repository_ctx, python_bin): """Gets the python include path.""" - result = _execute( + result = execute( repository_ctx, [ python_bin, @@ -281,14 +262,14 @@ def _get_python_include(repository_ctx, python_bin): ], error_msg = "Problem getting python include path.", error_details = ("Is the Python binary path set up right? " + - "(See ./configure or " + _PYTHON_BIN_PATH + ".) " + + "(See ./configure or " + PYTHON_BIN_PATH + ".) " + "Is distutils installed?"), ) return result.stdout.splitlines()[0] def _get_python_import_lib_name(repository_ctx, python_bin): """Get Python import library name (pythonXY.lib) on Windows.""" - result = _execute( + result = execute( repository_ctx, [ python_bin, @@ -299,13 +280,13 @@ def _get_python_import_lib_name(repository_ctx, python_bin): ], error_msg = "Problem getting python import library.", error_details = ("Is the Python binary path set up right? " + - "(See ./configure or " + _PYTHON_BIN_PATH + ".) "), + "(See ./configure or " + PYTHON_BIN_PATH + ".) "), ) return result.stdout.splitlines()[0] def _get_numpy_include(repository_ctx, python_bin): """Gets the numpy include path.""" - return _execute( + return execute( repository_ctx, [ python_bin, @@ -326,7 +307,7 @@ def _create_local_python_repository(repository_ctx): # can easily lead to a O(n^2) runtime in the number of labels. build_tpl = repository_ctx.path(Label("//third_party/py:BUILD.tpl")) - python_bin = _get_python_bin(repository_ctx) + python_bin = get_python_bin(repository_ctx) _check_python_bin(repository_ctx, python_bin) python_lib = _get_python_lib(repository_ctx, python_bin) _check_python_lib(repository_ctx, python_lib) @@ -342,7 +323,7 @@ def _create_local_python_repository(repository_ctx): # To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib # See https://docs.python.org/3/extending/windows.html - if _is_windows(repository_ctx): + if is_windows(repository_ctx): python_include = _norm_path(python_include) python_import_lib_name = _get_python_import_lib_name(repository_ctx, python_bin) python_import_lib_src = python_include.rsplit("/", 1)[0] + "/libs/" + python_import_lib_name @@ -375,10 +356,10 @@ def _create_remote_python_repository(repository_ctx, remote_config_repo): def _python_autoconf_impl(repository_ctx): """Implementation of the python_autoconf repository rule.""" - if _get_host_environ(repository_ctx, _TF_PYTHON_CONFIG_REPO) != None: + if get_host_environ(repository_ctx, TF_PYTHON_CONFIG_REPO) != None: _create_remote_python_repository( repository_ctx, - _get_host_environ(repository_ctx, _TF_PYTHON_CONFIG_REPO), + get_host_environ(repository_ctx, TF_PYTHON_CONFIG_REPO), ) else: _create_local_python_repository(repository_ctx) @@ -386,10 +367,10 @@ def _python_autoconf_impl(repository_ctx): python_configure = repository_rule( implementation = _python_autoconf_impl, environ = [ - _BAZEL_SH, - _PYTHON_BIN_PATH, - _PYTHON_LIB_PATH, - _TF_PYTHON_CONFIG_REPO, + BAZEL_SH, + PYTHON_BIN_PATH, + PYTHON_LIB_PATH, + TF_PYTHON_CONFIG_REPO, ], ) """Detects and configures the local Python. diff --git a/third_party/remote_config/BUILD b/third_party/remote_config/BUILD new file mode 100644 index 00000000000..e69de29bb2d diff --git a/third_party/remote_config/common.bzl b/third_party/remote_config/common.bzl new file mode 100644 index 00000000000..328173ed396 --- /dev/null +++ b/third_party/remote_config/common.bzl @@ -0,0 +1,211 @@ +"""Functions common across configure rules.""" + +BAZEL_SH = "BAZEL_SH" +PYTHON_BIN_PATH = "PYTHON_BIN_PATH" +PYTHON_LIB_PATH = "PYTHON_LIB_PATH" +TF_PYTHON_CONFIG_REPO = "TF_PYTHON_CONFIG_REPO" + +def auto_config_fail(msg): + """Output failure message when auto configuration fails.""" + red = "\033[0;31m" + no_color = "\033[0m" + fail("%sConfiguration Error:%s %s\n" % (red, no_color, msg)) + +def which(repository_ctx, program_name): + """Returns the full path to a program on the execution platform. + + Args: + repository_ctx: the repository_ctx + program_name: name of the program on the PATH + + Returns: + The full path to a program on the execution platform. + """ + if is_windows(repository_ctx): + if not program_name.endswith(".exe"): + program_name = program_name + ".exe" + result = execute(repository_ctx, ["where.exe", program_name]) + else: + result = execute(repository_ctx, ["which", program_name]) + return result.stdout.rstrip() + +def get_python_bin(repository_ctx): + """Gets the python bin path. + + Args: + repository_ctx: the repository_ctx + + Returns: + The python bin path. + """ + python_bin = get_host_environ(repository_ctx, PYTHON_BIN_PATH) + if python_bin != None: + return python_bin + python_bin_path = which(repository_ctx, "python") + if python_bin_path == None: + auto_config_fail("Cannot find python in PATH, please make sure " + + "python is installed and add its directory in PATH, or --define " + + "%s='/something/else'.\nPATH=%s" % ( + PYTHON_BIN_PATH, + get_environ("PATH", ""), + )) + return python_bin_path + +def get_bash_bin(repository_ctx): + """Gets the bash bin path. + + Args: + repository_ctx: the repository_ctx + + Returns: + The bash bin path. + """ + bash_bin = get_host_environ(repository_ctx, BAZEL_SH) + if bash_bin != None: + return bash_bin + bash_bin_path = which(repository_ctx, "bash") + if bash_bin_path == None: + auto_config_fail("Cannot find bash in PATH, please make sure " + + "bash is installed and add its directory in PATH, or --define " + + "%s='/path/to/bash'.\nPATH=%s" % ( + BAZEL_SH, + get_environ("PATH", ""), + )) + return bash_bin_path + +def read_dir(repository_ctx, src_dir): + """Returns a sorted list with all files in a directory. + + Finds all files inside a directory, traversing subfolders and following + symlinks. + + Args: + repository_ctx: the repository_ctx + src_dir: the directory to traverse + + Returns: + A sorted list with all files in a directory. + """ + if is_windows(repository_ctx): + src_dir = src_dir.replace("/", "\\") + find_result = execute( + repository_ctx, + ["cmd.exe", "/c", "dir", src_dir, "/b", "/s", "/a-d"], + empty_stdout_fine = True, + ) + + # src_files will be used in genrule.outs where the paths must + # use forward slashes. + result = find_result.stdout.replace("\\", "/") + else: + find_result = execute( + repository_ctx, + ["find", src_dir, "-follow", "-type", "f"], + empty_stdout_fine = True, + ) + result = find_result.stdout + return sorted(result.splitlines()) + +def get_environ(repository_ctx, name, default_value = None): + """Returns the value of an environment variable on the execution platform. + + Args: + repository_ctx: the repository_ctx + name: the name of environment variable + default_value: the value to return if not set + + Returns: + The value of the environment variable 'name' on the execution platform + or 'default_value' if it's not set. + """ + if is_windows(repository_ctx): + result = execute( + repository_ctx, + ["cmd.exe", "/c", "echo", "%" + name + "%"], + empty_stdout_fine = True, + ) + else: + cmd = "echo -n \"$%s\"" % name + result = execute( + repository_ctx, + [get_bash_bin(repository_ctx), "-c", cmd], + empty_stdout_fine = True, + ) + if len(result.stdout) == 0: + return default_value + return result.stdout + +def get_host_environ(repository_ctx, name): + """Returns the value of an environment variable on the host platform. + + The host platform is the machine that Bazel runs on. + + Args: + repository_ctx: the repository_ctx + name: the name of environment variable + + Returns: + The value of the environment variable 'name' on the host platform. + """ + return repository_ctx.os.environ.get(name) + +def is_windows(repository_ctx): + """Returns true if the execution platform is Windows. + + Args: + repository_ctx: the repository_ctx + + Returns: + If the execution platform is Windows. + """ + os_name = "" + if hasattr(repository_ctx.attr, "exec_properties") and "OSFamily" in repository_ctx.attr.exec_properties: + os_name = repository_ctx.attr.exec_properties["OSFamily"] + else: + os_name = repository_ctx.os.name + + return os_name.lower().find("windows") != -1 + +def execute( + repository_ctx, + cmdline, + error_msg = None, + error_details = None, + empty_stdout_fine = False): + """Executes an arbitrary shell command. + + Args: + repository_ctx: the repository_ctx object + cmdline: list of strings, the command to execute + error_msg: string, a summary of the error if the command fails + error_details: string, details about the error or steps to fix it + empty_stdout_fine: bool, if True, an empty stdout result is fine, + otherwise it's an error + Returns: + The result of repository_ctx.execute(cmdline) + """ + result = raw_exec(repository_ctx, cmdline) + if result.stderr or not (empty_stdout_fine or result.stdout): + fail( + "\n".join([ + error_msg.strip() if error_msg else "Repository command failed", + result.stderr.strip(), + error_details if error_details else "", + ]), + ) + return result + +def raw_exec(repository_ctx, cmdline): + """Executes a command via repository_ctx.execute() and returns the result. + + This method is useful for debugging purposes. For example, to print all + commands executed as well as their return code. + + Args: + repository_ctx: the repository_ctx + cmdline: the list of args + + Returns: + The 'exec_result' of repository_ctx.execute(). + """ + return repository_ctx.execute(cmdline)