python_configure: move common functionality to common.bzl

These functions will also be used in cuda_configure.bzl et al.

PiperOrigin-RevId: 293326676
Change-Id: I2d2751691948851b756a3ff3cbc064c05f96c84f

tensorflow/opensource_only.files

@@ -146,6 +146,8 @@ tensorflow/third_party/py/numpy/BUILD
 tensorflow/third_party/py/python_configure.bzl
 tensorflow/third_party/pybind11.BUILD
 tensorflow/third_party/python_runtime/BUILD
+tensorflow/third_party/remote_config/BUILD
+tensorflow/third_party/remote_config/common.bzl
 tensorflow/third_party/repo.bzl
 tensorflow/third_party/rules_closure.patch
 tensorflow/third_party/six.BUILD

third_party/py/python_configure.bzl

@@ -6,10 +6,21 @@
   * `PYTHON_LIB_PATH`: Location of python libraries.
 """
 
-_BAZEL_SH = "BAZEL_SH"
+load(
-_PYTHON_BIN_PATH = "PYTHON_BIN_PATH"
+    "//third_party/remote_config:common.bzl",
-_PYTHON_LIB_PATH = "PYTHON_LIB_PATH"
+    "BAZEL_SH",
-_TF_PYTHON_CONFIG_REPO = "TF_PYTHON_CONFIG_REPO"
+    "PYTHON_BIN_PATH",
+    "PYTHON_LIB_PATH",
+    "TF_PYTHON_CONFIG_REPO",
+    "auto_config_fail",
+    "execute",
+    "get_bash_bin",
+    "get_host_environ",
+    "get_python_bin",
+    "is_windows",
+    "raw_exec",
+    "read_dir",
+)
 
 def _which(repository_ctx, program_name):
     """Returns the full path to a program on the execution platform."""
@@ -33,7 +44,7 @@ def _get_environ(repository_ctx, name, default_value = None):
         cmd = "echo -n \"$%s\"" % name
         result = _execute(
             repository_ctx,
-            [_get_bash_bin(repository_ctx), "-c", cmd],
+            [get_bash_bin(repository_ctx), "-c", cmd],
             empty_stdout_fine = True,
         )
     if len(result.stdout) == 0:
@@ -154,7 +165,7 @@ def _symlink_genrule_for_dir(
     if src_dir != None:
         src_dir = _norm_path(src_dir)
         dest_dir = _norm_path(dest_dir)
-        files = "\n".join(sorted(_read_dir(repository_ctx, src_dir).splitlines()))
+        files = "\n".join(read_dir(repository_ctx, src_dir))
 
         # Create a list with the src_dir stripped to use for outputs.
         dest_files = files.replace(src_dir, "").splitlines()
@@ -179,39 +190,9 @@ def _symlink_genrule_for_dir(
     )
     return genrule
 
-def _get_python_bin(repository_ctx):
-    """Gets the python bin path."""
-    python_bin = _get_host_environ(repository_ctx, _PYTHON_BIN_PATH)
-    if python_bin != None:
-        return python_bin
-    python_bin_path = _which(repository_ctx, "python")
-    if python_bin_path == None:
-        _fail("Cannot find python in PATH, please make sure " +
-              "python is installed and add its directory in PATH, or --define " +
-              "%s='/something/else'.\nPATH=%s" % (
-                  _PYTHON_BIN_PATH,
-                  _get_environ("PATH", ""),
-              ))
-    return python_bin_path
-
-def _get_bash_bin(repository_ctx):
-    """Gets the bash bin path."""
-    bash_bin = _get_host_environ(repository_ctx, _BAZEL_SH)
-    if bash_bin != None:
-        return bash_bin
-    bash_bin_path = _which(repository_ctx, "bash")
-    if bash_bin_path == None:
-        _fail("Cannot find bash in PATH, please make sure " +
-              "bash is installed and add its directory in PATH, or --define " +
-              "%s='/path/to/bash'.\nPATH=%s" % (
-                  _BAZEL_SH,
-                  _get_environ("PATH", ""),
-              ))
-    return bash_bin_path
-
 def _get_python_lib(repository_ctx, python_bin):
     """Gets the python lib path."""
-    python_lib = _get_host_environ(repository_ctx, _PYTHON_LIB_PATH)
+    python_lib = get_host_environ(repository_ctx, PYTHON_LIB_PATH)
     if python_lib != None:
         return python_lib
 
@@ -248,29 +229,29 @@ def _get_python_lib(repository_ctx, python_bin):
     cmd += "from os import system;"
     cmd += "system(\"%s script.py\");" % python_bin
 
-    result = _execute(repository_ctx, [python_bin, "-c", cmd])
+    result = execute(repository_ctx, [python_bin, "-c", cmd])
     return result.stdout.strip()
 
 def _check_python_lib(repository_ctx, python_lib):
     """Checks the python lib path."""
     cmd = 'test -d "%s" -a -x "%s"' % (python_lib, python_lib)
-    result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd])
+    result = raw_exec(repository_ctx, [get_bash_bin(repository_ctx), "-c", cmd])
     if result.return_code == 1:
-        _fail("Invalid python library path: %s" % python_lib)
+        auto_config_fail("Invalid python library path: %s" % python_lib)
 
 def _check_python_bin(repository_ctx, python_bin):
     """Checks the python bin path."""
     cmd = '[[ -x "%s" ]] && [[ ! -d "%s" ]]' % (python_bin, python_bin)
-    result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd])
+    result = raw_exec(repository_ctx, [get_bash_bin(repository_ctx), "-c", cmd])
     if result.return_code == 1:
-        _fail("--define %s='%s' is not executable. Is it the python binary?" % (
+        auto_config_fail("--define %s='%s' is not executable. Is it the python binary?" % (
-            _PYTHON_BIN_PATH,
+            PYTHON_BIN_PATH,
             python_bin,
         ))
 
 def _get_python_include(repository_ctx, python_bin):
     """Gets the python include path."""
-    result = _execute(
+    result = execute(
         repository_ctx,
         [
             python_bin,
@@ -281,14 +262,14 @@ def _get_python_include(repository_ctx, python_bin):
         ],
         error_msg = "Problem getting python include path.",
         error_details = ("Is the Python binary path set up right? " +
-                         "(See ./configure or " + _PYTHON_BIN_PATH + ".) " +
+                         "(See ./configure or " + PYTHON_BIN_PATH + ".) " +
                          "Is distutils installed?"),
     )
     return result.stdout.splitlines()[0]
 
 def _get_python_import_lib_name(repository_ctx, python_bin):
     """Get Python import library name (pythonXY.lib) on Windows."""
-    result = _execute(
+    result = execute(
         repository_ctx,
         [
             python_bin,
@@ -299,13 +280,13 @@ def _get_python_import_lib_name(repository_ctx, python_bin):
         ],
         error_msg = "Problem getting python import library.",
         error_details = ("Is the Python binary path set up right? " +
-                         "(See ./configure or " + _PYTHON_BIN_PATH + ".) "),
+                         "(See ./configure or " + PYTHON_BIN_PATH + ".) "),
     )
     return result.stdout.splitlines()[0]
 
 def _get_numpy_include(repository_ctx, python_bin):
     """Gets the numpy include path."""
-    return _execute(
+    return execute(
         repository_ctx,
         [
             python_bin,
@@ -326,7 +307,7 @@ def _create_local_python_repository(repository_ctx):
     # can easily lead to a O(n^2) runtime in the number of labels.
     build_tpl = repository_ctx.path(Label("//third_party/py:BUILD.tpl"))
 
-    python_bin = _get_python_bin(repository_ctx)
+    python_bin = get_python_bin(repository_ctx)
     _check_python_bin(repository_ctx, python_bin)
     python_lib = _get_python_lib(repository_ctx, python_bin)
     _check_python_lib(repository_ctx, python_lib)
@@ -342,7 +323,7 @@ def _create_local_python_repository(repository_ctx):
 
     # To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib
     # See https://docs.python.org/3/extending/windows.html
-    if _is_windows(repository_ctx):
+    if is_windows(repository_ctx):
         python_include = _norm_path(python_include)
         python_import_lib_name = _get_python_import_lib_name(repository_ctx, python_bin)
         python_import_lib_src = python_include.rsplit("/", 1)[0] + "/libs/" + python_import_lib_name
@@ -375,10 +356,10 @@ def _create_remote_python_repository(repository_ctx, remote_config_repo):
 
 def _python_autoconf_impl(repository_ctx):
     """Implementation of the python_autoconf repository rule."""
-    if _get_host_environ(repository_ctx, _TF_PYTHON_CONFIG_REPO) != None:
+    if get_host_environ(repository_ctx, TF_PYTHON_CONFIG_REPO) != None:
         _create_remote_python_repository(
             repository_ctx,
-            _get_host_environ(repository_ctx, _TF_PYTHON_CONFIG_REPO),
+            get_host_environ(repository_ctx, TF_PYTHON_CONFIG_REPO),
         )
     else:
         _create_local_python_repository(repository_ctx)
@@ -386,10 +367,10 @@ def _python_autoconf_impl(repository_ctx):
 python_configure = repository_rule(
     implementation = _python_autoconf_impl,
     environ = [
-        _BAZEL_SH,
+        BAZEL_SH,
-        _PYTHON_BIN_PATH,
+        PYTHON_BIN_PATH,
-        _PYTHON_LIB_PATH,
+        PYTHON_LIB_PATH,
-        _TF_PYTHON_CONFIG_REPO,
+        TF_PYTHON_CONFIG_REPO,
     ],
 )
 """Detects and configures the local Python.

third_party/remote_config/common.bzl

@@ -0,0 +1,211 @@
+"""Functions common across configure rules."""
+
+BAZEL_SH = "BAZEL_SH"
+PYTHON_BIN_PATH = "PYTHON_BIN_PATH"
+PYTHON_LIB_PATH = "PYTHON_LIB_PATH"
+TF_PYTHON_CONFIG_REPO = "TF_PYTHON_CONFIG_REPO"
+
+def auto_config_fail(msg):
+    """Output failure message when auto configuration fails."""
+    red = "\033[0;31m"
+    no_color = "\033[0m"
+    fail("%sConfiguration Error:%s %s\n" % (red, no_color, msg))
+
+def which(repository_ctx, program_name):
+    """Returns the full path to a program on the execution platform.
+
+    Args:
+      repository_ctx: the repository_ctx
+      program_name: name of the program on the PATH
+
+    Returns:
+      The full path to a program on the execution platform.
+    """
+    if is_windows(repository_ctx):
+        if not program_name.endswith(".exe"):
+            program_name = program_name + ".exe"
+        result = execute(repository_ctx, ["where.exe", program_name])
+    else:
+        result = execute(repository_ctx, ["which", program_name])
+    return result.stdout.rstrip()
+
+def get_python_bin(repository_ctx):
+    """Gets the python bin path.
+
+    Args:
+      repository_ctx: the repository_ctx
+
+    Returns:
+      The python bin path.
+    """
+    python_bin = get_host_environ(repository_ctx, PYTHON_BIN_PATH)
+    if python_bin != None:
+        return python_bin
+    python_bin_path = which(repository_ctx, "python")
+    if python_bin_path == None:
+        auto_config_fail("Cannot find python in PATH, please make sure " +
+                         "python is installed and add its directory in PATH, or --define " +
+                         "%s='/something/else'.\nPATH=%s" % (
+                             PYTHON_BIN_PATH,
+                             get_environ("PATH", ""),
+                         ))
+    return python_bin_path
+
+def get_bash_bin(repository_ctx):
+    """Gets the bash bin path.
+
+    Args:
+      repository_ctx: the repository_ctx
+
+    Returns:
+      The bash bin path.
+    """
+    bash_bin = get_host_environ(repository_ctx, BAZEL_SH)
+    if bash_bin != None:
+        return bash_bin
+    bash_bin_path = which(repository_ctx, "bash")
+    if bash_bin_path == None:
+        auto_config_fail("Cannot find bash in PATH, please make sure " +
+                         "bash is installed and add its directory in PATH, or --define " +
+                         "%s='/path/to/bash'.\nPATH=%s" % (
+                             BAZEL_SH,
+                             get_environ("PATH", ""),
+                         ))
+    return bash_bin_path
+
+def read_dir(repository_ctx, src_dir):
+    """Returns a sorted list with all files in a directory.
+
+    Finds all files inside a directory, traversing subfolders and following
+    symlinks.
+
+    Args:
+      repository_ctx: the repository_ctx
+      src_dir: the directory to traverse
+
+    Returns:
+      A sorted list with all files in a directory.
+    """
+    if is_windows(repository_ctx):
+        src_dir = src_dir.replace("/", "\\")
+        find_result = execute(
+            repository_ctx,
+            ["cmd.exe", "/c", "dir", src_dir, "/b", "/s", "/a-d"],
+            empty_stdout_fine = True,
+        )
+
+        # src_files will be used in genrule.outs where the paths must
+        # use forward slashes.
+        result = find_result.stdout.replace("\\", "/")
+    else:
+        find_result = execute(
+            repository_ctx,
+            ["find", src_dir, "-follow", "-type", "f"],
+            empty_stdout_fine = True,
+        )
+        result = find_result.stdout
+    return sorted(result.splitlines())
+
+def get_environ(repository_ctx, name, default_value = None):
+    """Returns the value of an environment variable on the execution platform.
+
+    Args:
+      repository_ctx: the repository_ctx
+      name: the name of environment variable
+      default_value: the value to return if not set
+
+    Returns:
+      The value of the environment variable 'name' on the execution platform
+      or 'default_value' if it's not set.
+    """
+    if is_windows(repository_ctx):
+        result = execute(
+            repository_ctx,
+            ["cmd.exe", "/c", "echo", "%" + name + "%"],
+            empty_stdout_fine = True,
+        )
+    else:
+        cmd = "echo -n \"$%s\"" % name
+        result = execute(
+            repository_ctx,
+            [get_bash_bin(repository_ctx), "-c", cmd],
+            empty_stdout_fine = True,
+        )
+    if len(result.stdout) == 0:
+        return default_value
+    return result.stdout
+
+def get_host_environ(repository_ctx, name):
+    """Returns the value of an environment variable on the host platform.
+
+    The host platform is the machine that Bazel runs on.
+
+    Args:
+      repository_ctx: the repository_ctx
+      name: the name of environment variable
+
+    Returns:
+      The value of the environment variable 'name' on the host platform.
+    """
+    return repository_ctx.os.environ.get(name)
+
+def is_windows(repository_ctx):
+    """Returns true if the execution platform is Windows.
+
+    Args:
+      repository_ctx: the repository_ctx
+
+    Returns:
+      If the execution platform is Windows.
+    """
+    os_name = ""
+    if hasattr(repository_ctx.attr, "exec_properties") and "OSFamily" in repository_ctx.attr.exec_properties:
+        os_name = repository_ctx.attr.exec_properties["OSFamily"]
+    else:
+        os_name = repository_ctx.os.name
+
+    return os_name.lower().find("windows") != -1
+
+def execute(
+        repository_ctx,
+        cmdline,
+        error_msg = None,
+        error_details = None,
+        empty_stdout_fine = False):
+    """Executes an arbitrary shell command.
+
+    Args:
+      repository_ctx: the repository_ctx object
+      cmdline: list of strings, the command to execute
+      error_msg: string, a summary of the error if the command fails
+      error_details: string, details about the error or steps to fix it
+      empty_stdout_fine: bool, if True, an empty stdout result is fine,
+        otherwise it's an error
+    Returns:
+      The result of repository_ctx.execute(cmdline)
+    """
+    result = raw_exec(repository_ctx, cmdline)
+    if result.stderr or not (empty_stdout_fine or result.stdout):
+        fail(
+            "\n".join([
+                error_msg.strip() if error_msg else "Repository command failed",
+                result.stderr.strip(),
+                error_details if error_details else "",
+            ]),
+        )
+    return result
+
+def raw_exec(repository_ctx, cmdline):
+    """Executes a command via repository_ctx.execute() and returns the result.
+
+    This method is useful for debugging purposes. For example, to print all
+    commands executed as well as their return code.
+
+    Args:
+      repository_ctx: the repository_ctx
+      cmdline: the list of args
+
+    Returns:
+      The 'exec_result' of repository_ctx.execute().
+    """
+    return repository_ctx.execute(cmdline)