experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

python_configure: move common functionality to common.bzl

Browse Source 
These functions will also be used in cuda_configure.bzl et al.

PiperOrigin-RevId: 293326676
Change-Id: I2d2751691948851b756a3ff3cbc064c05f96c84f
This commit is contained in:
A. Unique TensorFlower 2020-02-05 02:16:58 -08:00 committed by TensorFlower Gardener
parent 6e5d3265bc
commit 6deb2d610d
4 changed files with 250 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
tensorflow/opensource_only.files
View File

@ -146,6 +146,8 @@ tensorflow/third_party/py/numpy/BUILD
tensorflow/third_party/py/python_configure.bzl
tensorflow/third_party/pybind11.BUILD
tensorflow/third_party/python_runtime/BUILD
tensorflow/third_party/remote_config/BUILD
tensorflow/third_party/remote_config/common.bzl
tensorflow/third_party/repo.bzl
tensorflow/third_party/rules_closure.patch
tensorflow/third_party/six.BUILD

93
third_party/py/python_configure.bzl vendored
View File

@ -6,10 +6,21 @@
* `PYTHON_LIB_PATH`: Location of python libraries.
"""
_BAZEL_SH = "BAZEL_SH"
_PYTHON_BIN_PATH = "PYTHON_BIN_PATH"
_PYTHON_LIB_PATH = "PYTHON_LIB_PATH"
_TF_PYTHON_CONFIG_REPO = "TF_PYTHON_CONFIG_REPO"
load(
"//third_party/remote_config:common.bzl",
"BAZEL_SH",
"PYTHON_BIN_PATH",
"PYTHON_LIB_PATH",
"TF_PYTHON_CONFIG_REPO",
"auto_config_fail",
"execute",
"get_bash_bin",
"get_host_environ",
"get_python_bin",
"is_windows",
"raw_exec",
"read_dir",
)
def _which(repository_ctx, program_name):
"""Returns the full path to a program on the execution platform."""
@ -33,7 +44,7 @@ def _get_environ(repository_ctx, name, default_value = None):
cmd = "echo -n \"$%s\"" % name
result = _execute(
repository_ctx,
[_get_bash_bin(repository_ctx), "-c", cmd],
[get_bash_bin(repository_ctx), "-c", cmd],
empty_stdout_fine = True,
)
if len(result.stdout) == 0:
@ -154,7 +165,7 @@ def _symlink_genrule_for_dir(
if src_dir != None:
src_dir = _norm_path(src_dir)
dest_dir = _norm_path(dest_dir)
files = "\n".join(sorted(_read_dir(repository_ctx, src_dir).splitlines()))
files = "\n".join(read_dir(repository_ctx, src_dir))
# Create a list with the src_dir stripped to use for outputs.
dest_files = files.replace(src_dir, "").splitlines()
@ -179,39 +190,9 @@ def _symlink_genrule_for_dir(
)
return genrule
def _get_python_bin(repository_ctx):
"""Gets the python bin path."""
python_bin = _get_host_environ(repository_ctx, _PYTHON_BIN_PATH)
if python_bin != None:
return python_bin
python_bin_path = _which(repository_ctx, "python")
if python_bin_path == None:
_fail("Cannot find python in PATH, please make sure " +
"python is installed and add its directory in PATH, or --define " +
"%s='/something/else'.\nPATH=%s" % (
_PYTHON_BIN_PATH,
_get_environ("PATH", ""),
))
return python_bin_path
def _get_bash_bin(repository_ctx):
"""Gets the bash bin path."""
bash_bin = _get_host_environ(repository_ctx, _BAZEL_SH)
if bash_bin != None:
return bash_bin
bash_bin_path = _which(repository_ctx, "bash")
if bash_bin_path == None:
_fail("Cannot find bash in PATH, please make sure " +
"bash is installed and add its directory in PATH, or --define " +
"%s='/path/to/bash'.\nPATH=%s" % (
_BAZEL_SH,
_get_environ("PATH", ""),
))
return bash_bin_path
def _get_python_lib(repository_ctx, python_bin):
"""Gets the python lib path."""
python_lib = _get_host_environ(repository_ctx, _PYTHON_LIB_PATH)
python_lib = get_host_environ(repository_ctx, PYTHON_LIB_PATH)
if python_lib != None:
return python_lib
@ -248,29 +229,29 @@ def _get_python_lib(repository_ctx, python_bin):
cmd += "from os import system;"
cmd += "system(\"%s script.py\");" % python_bin
result = _execute(repository_ctx, [python_bin, "-c", cmd])
result = execute(repository_ctx, [python_bin, "-c", cmd])
return result.stdout.strip()
def _check_python_lib(repository_ctx, python_lib):
"""Checks the python lib path."""
cmd = 'test -d "%s" -a -x "%s"' % (python_lib, python_lib)
result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd])
result = raw_exec(repository_ctx, [get_bash_bin(repository_ctx), "-c", cmd])
if result.return_code == 1:
_fail("Invalid python library path: %s" % python_lib)
auto_config_fail("Invalid python library path: %s" % python_lib)
def _check_python_bin(repository_ctx, python_bin):
"""Checks the python bin path."""
cmd = '[[ -x "%s" ]] && [[ ! -d "%s" ]]' % (python_bin, python_bin)
result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd])
result = raw_exec(repository_ctx, [get_bash_bin(repository_ctx), "-c", cmd])
if result.return_code == 1:
_fail("--define %s='%s' is not executable. Is it the python binary?" % (
_PYTHON_BIN_PATH,
auto_config_fail("--define %s='%s' is not executable. Is it the python binary?" % (
PYTHON_BIN_PATH,
python_bin,
))
def _get_python_include(repository_ctx, python_bin):
"""Gets the python include path."""
result = _execute(
result = execute(
repository_ctx,
[
python_bin,
@ -281,14 +262,14 @@ def _get_python_include(repository_ctx, python_bin):
],
error_msg = "Problem getting python include path.",
error_details = ("Is the Python binary path set up right? " +
"(See ./configure or " + _PYTHON_BIN_PATH + ".) " +
"(See ./configure or " + PYTHON_BIN_PATH + ".) " +
"Is distutils installed?"),
)
return result.stdout.splitlines()[0]
def _get_python_import_lib_name(repository_ctx, python_bin):
"""Get Python import library name (pythonXY.lib) on Windows."""
result = _execute(
result = execute(
repository_ctx,
[
python_bin,
@ -299,13 +280,13 @@ def _get_python_import_lib_name(repository_ctx, python_bin):
],
error_msg = "Problem getting python import library.",
error_details = ("Is the Python binary path set up right? " +
"(See ./configure or " + _PYTHON_BIN_PATH + ".) "),
"(See ./configure or " + PYTHON_BIN_PATH + ".) "),
)
return result.stdout.splitlines()[0]
def _get_numpy_include(repository_ctx, python_bin):
"""Gets the numpy include path."""
return _execute(
return execute(
repository_ctx,
[
python_bin,
@ -326,7 +307,7 @@ def _create_local_python_repository(repository_ctx):
# can easily lead to a O(n^2) runtime in the number of labels.
build_tpl = repository_ctx.path(Label("//third_party/py:BUILD.tpl"))
python_bin = _get_python_bin(repository_ctx)
python_bin = get_python_bin(repository_ctx)
_check_python_bin(repository_ctx, python_bin)
python_lib = _get_python_lib(repository_ctx, python_bin)
_check_python_lib(repository_ctx, python_lib)
@ -342,7 +323,7 @@ def _create_local_python_repository(repository_ctx):
# To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib
# See https://docs.python.org/3/extending/windows.html
if _is_windows(repository_ctx):
if is_windows(repository_ctx):
python_include = _norm_path(python_include)
python_import_lib_name = _get_python_import_lib_name(repository_ctx, python_bin)
python_import_lib_src = python_include.rsplit("/", 1)[0] + "/libs/" + python_import_lib_name
@ -375,10 +356,10 @@ def _create_remote_python_repository(repository_ctx, remote_config_repo):
def _python_autoconf_impl(repository_ctx):
"""Implementation of the python_autoconf repository rule."""
if _get_host_environ(repository_ctx, _TF_PYTHON_CONFIG_REPO) != None:
if get_host_environ(repository_ctx, TF_PYTHON_CONFIG_REPO) != None:
_create_remote_python_repository(
repository_ctx,
_get_host_environ(repository_ctx, _TF_PYTHON_CONFIG_REPO),
get_host_environ(repository_ctx, TF_PYTHON_CONFIG_REPO),
)
else:
_create_local_python_repository(repository_ctx)
@ -386,10 +367,10 @@ def _python_autoconf_impl(repository_ctx):
python_configure = repository_rule(
implementation = _python_autoconf_impl,
environ = [
_BAZEL_SH,
_PYTHON_BIN_PATH,
_PYTHON_LIB_PATH,
_TF_PYTHON_CONFIG_REPO,
BAZEL_SH,
PYTHON_BIN_PATH,
PYTHON_LIB_PATH,
TF_PYTHON_CONFIG_REPO,
],
)
"""Detects and configures the local Python.

0
third_party/remote_config/BUILD vendored Normal file
View File

211
third_party/remote_config/common.bzl vendored Normal file
View File

@ -0,0 +1,211 @@
"""Functions common across configure rules."""
BAZEL_SH = "BAZEL_SH"
PYTHON_BIN_PATH = "PYTHON_BIN_PATH"
PYTHON_LIB_PATH = "PYTHON_LIB_PATH"
TF_PYTHON_CONFIG_REPO = "TF_PYTHON_CONFIG_REPO"
def auto_config_fail(msg):
"""Output failure message when auto configuration fails."""
red = "\033[0;31m"
no_color = "\033[0m"
fail("%sConfiguration Error:%s %s\n" % (red, no_color, msg))
def which(repository_ctx, program_name):
"""Returns the full path to a program on the execution platform.
Args:
repository_ctx: the repository_ctx
program_name: name of the program on the PATH
Returns:
The full path to a program on the execution platform.
"""
if is_windows(repository_ctx):
if not program_name.endswith(".exe"):
program_name = program_name + ".exe"
result = execute(repository_ctx, ["where.exe", program_name])
else:
result = execute(repository_ctx, ["which", program_name])
return result.stdout.rstrip()
def get_python_bin(repository_ctx):
"""Gets the python bin path.
Args:
repository_ctx: the repository_ctx
Returns:
The python bin path.
"""
python_bin = get_host_environ(repository_ctx, PYTHON_BIN_PATH)
if python_bin != None:
return python_bin
python_bin_path = which(repository_ctx, "python")
if python_bin_path == None:
auto_config_fail("Cannot find python in PATH, please make sure " +
"python is installed and add its directory in PATH, or --define " +
"%s='/something/else'.\nPATH=%s" % (
PYTHON_BIN_PATH,
get_environ("PATH", ""),
))
return python_bin_path
def get_bash_bin(repository_ctx):
"""Gets the bash bin path.
Args:
repository_ctx: the repository_ctx
Returns:
The bash bin path.
"""
bash_bin = get_host_environ(repository_ctx, BAZEL_SH)
if bash_bin != None:
return bash_bin
bash_bin_path = which(repository_ctx, "bash")
if bash_bin_path == None:
auto_config_fail("Cannot find bash in PATH, please make sure " +
"bash is installed and add its directory in PATH, or --define " +
"%s='/path/to/bash'.\nPATH=%s" % (
BAZEL_SH,
get_environ("PATH", ""),
))
return bash_bin_path
def read_dir(repository_ctx, src_dir):
"""Returns a sorted list with all files in a directory.
Finds all files inside a directory, traversing subfolders and following
symlinks.
Args:
repository_ctx: the repository_ctx
src_dir: the directory to traverse
Returns:
A sorted list with all files in a directory.
"""
if is_windows(repository_ctx):
src_dir = src_dir.replace("/", "\\")
find_result = execute(
repository_ctx,
["cmd.exe", "/c", "dir", src_dir, "/b", "/s", "/a-d"],
empty_stdout_fine = True,
)
# src_files will be used in genrule.outs where the paths must
# use forward slashes.
result = find_result.stdout.replace("\\", "/")
else:
find_result = execute(
repository_ctx,
["find", src_dir, "-follow", "-type", "f"],
empty_stdout_fine = True,
)
result = find_result.stdout
return sorted(result.splitlines())
def get_environ(repository_ctx, name, default_value = None):
"""Returns the value of an environment variable on the execution platform.
Args:
repository_ctx: the repository_ctx
name: the name of environment variable
default_value: the value to return if not set
Returns:
The value of the environment variable 'name' on the execution platform
or 'default_value' if it's not set.
"""
if is_windows(repository_ctx):
result = execute(
repository_ctx,
["cmd.exe", "/c", "echo", "%" + name + "%"],
empty_stdout_fine = True,
)
else:
cmd = "echo -n \"$%s\"" % name
result = execute(
repository_ctx,
[get_bash_bin(repository_ctx), "-c", cmd],
empty_stdout_fine = True,
)
if len(result.stdout) == 0:
return default_value
return result.stdout
def get_host_environ(repository_ctx, name):
"""Returns the value of an environment variable on the host platform.
The host platform is the machine that Bazel runs on.
Args:
repository_ctx: the repository_ctx
name: the name of environment variable
Returns:
The value of the environment variable 'name' on the host platform.
"""
return repository_ctx.os.environ.get(name)
def is_windows(repository_ctx):
"""Returns true if the execution platform is Windows.
Args:
repository_ctx: the repository_ctx
Returns:
If the execution platform is Windows.
"""
os_name = ""
if hasattr(repository_ctx.attr, "exec_properties") and "OSFamily" in repository_ctx.attr.exec_properties:
os_name = repository_ctx.attr.exec_properties["OSFamily"]
else:
os_name = repository_ctx.os.name
return os_name.lower().find("windows") != -1
def execute(
repository_ctx,
cmdline,
error_msg = None,
error_details = None,
empty_stdout_fine = False):
"""Executes an arbitrary shell command.
Args:
repository_ctx: the repository_ctx object
cmdline: list of strings, the command to execute
error_msg: string, a summary of the error if the command fails
error_details: string, details about the error or steps to fix it
empty_stdout_fine: bool, if True, an empty stdout result is fine,
otherwise it's an error
Returns:
The result of repository_ctx.execute(cmdline)
"""
result = raw_exec(repository_ctx, cmdline)
if result.stderr or not (empty_stdout_fine or result.stdout):
fail(
"\n".join([
error_msg.strip() if error_msg else "Repository command failed",
result.stderr.strip(),
error_details if error_details else "",
]),
)
return result
def raw_exec(repository_ctx, cmdline):
"""Executes a command via repository_ctx.execute() and returns the result.
This method is useful for debugging purposes. For example, to print all
commands executed as well as their return code.
Args:
repository_ctx: the repository_ctx
cmdline: the list of args
Returns:
The 'exec_result' of repository_ctx.execute().
"""
return repository_ctx.execute(cmdline)