0f631f8fb1
- @local_execution_config_platform: local platform to allow selecting locally executed tools on - @local_execution_config_python: python configured for execution on the local machine during otherwise remote builds Mark rules that are required to run locally to require our local platform. This allows python paths to differ between the remote docker image and local machine. For example, the local machine might have python 3.7 installed in /usr/bin/python3, while the remote docker should use a python installed in /usr/local/bin/python3.8. PiperOrigin-RevId: 307585019 Change-Id: I29313121beb967b77ae123e7d1b614c688cb40ca
295 lines
9.8 KiB
Python
295 lines
9.8 KiB
Python
"""Repository rule for Python autoconfiguration.
|
|
|
|
`python_configure` depends on the following environment variables:
|
|
|
|
* `PYTHON_BIN_PATH`: location of python binary.
|
|
* `PYTHON_LIB_PATH`: Location of python libraries.
|
|
"""
|
|
|
|
load(
|
|
"//third_party/remote_config:common.bzl",
|
|
"BAZEL_SH",
|
|
"PYTHON_BIN_PATH",
|
|
"PYTHON_LIB_PATH",
|
|
"TF_PYTHON_CONFIG_REPO",
|
|
"auto_config_fail",
|
|
"config_repo_label",
|
|
"execute",
|
|
"get_bash_bin",
|
|
"get_host_environ",
|
|
"get_python_bin",
|
|
"is_windows",
|
|
"raw_exec",
|
|
"read_dir",
|
|
)
|
|
|
|
def _genrule(src_dir, genrule_name, command, outs):
|
|
"""Returns a string with a genrule.
|
|
|
|
Genrule executes the given command and produces the given outputs.
|
|
"""
|
|
return (
|
|
"genrule(\n" +
|
|
' name = "' +
|
|
genrule_name + '",\n' +
|
|
" outs = [\n" +
|
|
outs +
|
|
"\n ],\n" +
|
|
' cmd = """\n' +
|
|
command +
|
|
'\n """,\n' +
|
|
")\n"
|
|
)
|
|
|
|
def _norm_path(path):
|
|
"""Returns a path with '/' and remove the trailing slash."""
|
|
path = path.replace("\\", "/")
|
|
if path[-1] == "/":
|
|
path = path[:-1]
|
|
return path
|
|
|
|
def _symlink_genrule_for_dir(
|
|
repository_ctx,
|
|
src_dir,
|
|
dest_dir,
|
|
genrule_name,
|
|
src_files = [],
|
|
dest_files = []):
|
|
"""Returns a genrule to symlink(or copy if on Windows) a set of files.
|
|
|
|
If src_dir is passed, files will be read from the given directory; otherwise
|
|
we assume files are in src_files and dest_files
|
|
"""
|
|
if src_dir != None:
|
|
src_dir = _norm_path(src_dir)
|
|
dest_dir = _norm_path(dest_dir)
|
|
files = "\n".join(read_dir(repository_ctx, src_dir))
|
|
|
|
# Create a list with the src_dir stripped to use for outputs.
|
|
dest_files = files.replace(src_dir, "").splitlines()
|
|
src_files = files.splitlines()
|
|
command = []
|
|
outs = []
|
|
for i in range(len(dest_files)):
|
|
if dest_files[i] != "":
|
|
# If we have only one file to link we do not want to use the dest_dir, as
|
|
# $(@D) will include the full path to the file.
|
|
dest = "$(@D)/" + dest_dir + dest_files[i] if len(dest_files) != 1 else "$(@D)/" + dest_files[i]
|
|
|
|
# Copy the headers to create a sandboxable setup.
|
|
cmd = "cp -f"
|
|
command.append(cmd + ' "%s" "%s"' % (src_files[i], dest))
|
|
outs.append(' "' + dest_dir + dest_files[i] + '",')
|
|
genrule = _genrule(
|
|
src_dir,
|
|
genrule_name,
|
|
" && ".join(command),
|
|
"\n".join(outs),
|
|
)
|
|
return genrule
|
|
|
|
def _get_python_lib(repository_ctx, python_bin):
|
|
"""Gets the python lib path."""
|
|
python_lib = get_host_environ(repository_ctx, PYTHON_LIB_PATH)
|
|
if python_lib != None:
|
|
return python_lib
|
|
|
|
# The interesting program to execute.
|
|
print_lib = [
|
|
"from __future__ import print_function",
|
|
"import site",
|
|
"import os",
|
|
"python_paths = []",
|
|
"if os.getenv('PYTHONPATH') is not None:",
|
|
" python_paths = os.getenv('PYTHONPATH').split(':')",
|
|
"try:",
|
|
" library_paths = site.getsitepackages()",
|
|
"except AttributeError:",
|
|
" from distutils.sysconfig import get_python_lib",
|
|
" library_paths = [get_python_lib()]",
|
|
"all_paths = set(python_paths + library_paths)",
|
|
"paths = []",
|
|
"for path in all_paths:",
|
|
" if os.path.isdir(path):",
|
|
" paths.append(path)",
|
|
"if len(paths) >=1:",
|
|
" print(paths[0])",
|
|
]
|
|
|
|
# The below script writes the above program to a file
|
|
# and executes it. This is to work around the limitation
|
|
# of not being able to upload files as part of execute.
|
|
cmd = "from os import linesep;"
|
|
cmd += "f = open('script.py', 'w');"
|
|
for line in print_lib:
|
|
cmd += "f.write(\"%s\" + linesep);" % line
|
|
cmd += "f.close();"
|
|
cmd += "from os import system;"
|
|
cmd += "system(\"%s script.py\");" % python_bin
|
|
|
|
result = execute(repository_ctx, [python_bin, "-c", cmd])
|
|
return result.stdout.strip()
|
|
|
|
def _check_python_lib(repository_ctx, python_lib):
|
|
"""Checks the python lib path."""
|
|
cmd = 'test -d "%s" -a -x "%s"' % (python_lib, python_lib)
|
|
result = raw_exec(repository_ctx, [get_bash_bin(repository_ctx), "-c", cmd])
|
|
if result.return_code == 1:
|
|
auto_config_fail("Invalid python library path: %s" % python_lib)
|
|
|
|
def _check_python_bin(repository_ctx, python_bin):
|
|
"""Checks the python bin path."""
|
|
cmd = '[[ -x "%s" ]] && [[ ! -d "%s" ]]' % (python_bin, python_bin)
|
|
result = raw_exec(repository_ctx, [get_bash_bin(repository_ctx), "-c", cmd])
|
|
if result.return_code == 1:
|
|
auto_config_fail("--define %s='%s' is not executable. Is it the python binary?" % (
|
|
PYTHON_BIN_PATH,
|
|
python_bin,
|
|
))
|
|
|
|
def _get_python_include(repository_ctx, python_bin):
|
|
"""Gets the python include path."""
|
|
result = execute(
|
|
repository_ctx,
|
|
[
|
|
python_bin,
|
|
"-c",
|
|
"from __future__ import print_function;" +
|
|
"from distutils import sysconfig;" +
|
|
"print(sysconfig.get_python_inc())",
|
|
],
|
|
error_msg = "Problem getting python include path.",
|
|
error_details = ("Is the Python binary path set up right? " +
|
|
"(See ./configure or " + PYTHON_BIN_PATH + ".) " +
|
|
"Is distutils installed?"),
|
|
)
|
|
return result.stdout.splitlines()[0]
|
|
|
|
def _get_python_import_lib_name(repository_ctx, python_bin):
|
|
"""Get Python import library name (pythonXY.lib) on Windows."""
|
|
result = execute(
|
|
repository_ctx,
|
|
[
|
|
python_bin,
|
|
"-c",
|
|
"import sys;" +
|
|
'print("python" + str(sys.version_info[0]) + ' +
|
|
' str(sys.version_info[1]) + ".lib")',
|
|
],
|
|
error_msg = "Problem getting python import library.",
|
|
error_details = ("Is the Python binary path set up right? " +
|
|
"(See ./configure or " + PYTHON_BIN_PATH + ".) "),
|
|
)
|
|
return result.stdout.splitlines()[0]
|
|
|
|
def _get_numpy_include(repository_ctx, python_bin):
|
|
"""Gets the numpy include path."""
|
|
return execute(
|
|
repository_ctx,
|
|
[
|
|
python_bin,
|
|
"-c",
|
|
"from __future__ import print_function;" +
|
|
"import numpy;" +
|
|
" print(numpy.get_include());",
|
|
],
|
|
error_msg = "Problem getting numpy include path.",
|
|
error_details = "Is numpy installed?",
|
|
).stdout.splitlines()[0]
|
|
|
|
def _create_local_python_repository(repository_ctx):
|
|
"""Creates the repository containing files set up to build with Python."""
|
|
|
|
# Resolve all labels before doing any real work. Resolving causes the
|
|
# function to be restarted with all previous state being lost. This
|
|
# can easily lead to a O(n^2) runtime in the number of labels.
|
|
build_tpl = repository_ctx.path(Label("//third_party/py:BUILD.tpl"))
|
|
|
|
python_bin = get_python_bin(repository_ctx)
|
|
_check_python_bin(repository_ctx, python_bin)
|
|
python_lib = _get_python_lib(repository_ctx, python_bin)
|
|
_check_python_lib(repository_ctx, python_lib)
|
|
python_include = _get_python_include(repository_ctx, python_bin)
|
|
numpy_include = _get_numpy_include(repository_ctx, python_bin) + "/numpy"
|
|
python_include_rule = _symlink_genrule_for_dir(
|
|
repository_ctx,
|
|
python_include,
|
|
"python_include",
|
|
"python_include",
|
|
)
|
|
python_import_lib_genrule = ""
|
|
|
|
# To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib
|
|
# See https://docs.python.org/3/extending/windows.html
|
|
if is_windows(repository_ctx):
|
|
python_include = _norm_path(python_include)
|
|
python_import_lib_name = _get_python_import_lib_name(repository_ctx, python_bin)
|
|
python_import_lib_src = python_include.rsplit("/", 1)[0] + "/libs/" + python_import_lib_name
|
|
python_import_lib_genrule = _symlink_genrule_for_dir(
|
|
repository_ctx,
|
|
None,
|
|
"",
|
|
"python_import_lib",
|
|
[python_import_lib_src],
|
|
[python_import_lib_name],
|
|
)
|
|
numpy_include_rule = _symlink_genrule_for_dir(
|
|
repository_ctx,
|
|
numpy_include,
|
|
"numpy_include/numpy",
|
|
"numpy_include",
|
|
)
|
|
|
|
repository_ctx.template("BUILD", build_tpl, {
|
|
"%{PYTHON_BIN_PATH}": python_bin,
|
|
"%{PYTHON_INCLUDE_GENRULE}": python_include_rule,
|
|
"%{PYTHON_IMPORT_LIB_GENRULE}": python_import_lib_genrule,
|
|
"%{NUMPY_INCLUDE_GENRULE}": numpy_include_rule,
|
|
})
|
|
|
|
def _create_remote_python_repository(repository_ctx, remote_config_repo):
|
|
"""Creates pointers to a remotely configured repo set up to build with Python.
|
|
"""
|
|
repository_ctx.template("BUILD", config_repo_label(remote_config_repo, ":BUILD"), {})
|
|
|
|
def _python_autoconf_impl(repository_ctx):
|
|
"""Implementation of the python_autoconf repository rule."""
|
|
if get_host_environ(repository_ctx, TF_PYTHON_CONFIG_REPO) != None:
|
|
_create_remote_python_repository(
|
|
repository_ctx,
|
|
get_host_environ(repository_ctx, TF_PYTHON_CONFIG_REPO),
|
|
)
|
|
else:
|
|
_create_local_python_repository(repository_ctx)
|
|
|
|
_ENVIRONS = [
|
|
BAZEL_SH,
|
|
PYTHON_BIN_PATH,
|
|
PYTHON_LIB_PATH,
|
|
]
|
|
|
|
remote_python_configure = repository_rule(
|
|
implementation = _create_local_python_repository,
|
|
environ = _ENVIRONS,
|
|
remotable = True,
|
|
attrs = {
|
|
"environ": attr.string_dict(),
|
|
},
|
|
)
|
|
|
|
python_configure = repository_rule(
|
|
implementation = _python_autoconf_impl,
|
|
environ = _ENVIRONS + [TF_PYTHON_CONFIG_REPO],
|
|
)
|
|
"""Detects and configures the local Python.
|
|
|
|
Add the following to your WORKSPACE FILE:
|
|
|
|
```python
|
|
python_configure(name = "local_config_python")
|
|
```
|
|
|
|
Args:
|
|
name: A unique name for this workspace rule.
|
|
"""
|