Add RBE configuration that allows building against multiple python versions.
This allows release builds to use RBE. PiperOrigin-RevId: 309014910 Change-Id: Iec428478a61ec444647b9c4bac8bd8372fb305fb
This commit is contained in:
A. Unique TensorFlower
TensorFlower Gardener
parent
90223fd671
commit
bb68770fe5
16
.bazelrc
16
.bazelrc
@ -380,6 +380,22 @@ build:rbe_linux_cuda_nvcc --repo_env=TF_NCCL_CONFIG_REPO="@ubuntu16.04-py3-gcc7_
|
||||
build:rbe_linux_cuda_nvcc --define=using_cuda_nvcc=true
|
||||
test:rbe_linux_cuda_nvcc --config=rbe_linux_cuda_base
|
||||
|
||||
build:rbe_linux_cuda_nvcc_base --config=rbe_linux_cuda_base
|
||||
build:rbe_linux_cuda_nvcc_base --crosstool_top="@ubuntu16.04-gcc7_manylinux2010-cuda10.1-cudnn7-tensorrt6.0_config_cuda//crosstool:toolchain"
|
||||
build:rbe_linux_cuda_nvcc_base --extra_toolchains="@ubuntu16.04-gcc7_manylinux2010-cuda10.1-cudnn7-tensorrt6.0_config_cuda//crosstool:toolchain-linux-x86_64"
|
||||
build:rbe_linux_cuda_nvcc_base --extra_execution_platforms="@ubuntu16.04-gcc7_manylinux2010-cuda10.1-cudnn7-tensorrt6.0_config_platform//:platform"
|
||||
build:rbe_linux_cuda_nvcc_base --host_platform="@ubuntu16.04-gcc7_manylinux2010-cuda10.1-cudnn7-tensorrt6.0_config_platform//:platform"
|
||||
build:rbe_linux_cuda_nvcc_base --platforms="@ubuntu16.04-gcc7_manylinux2010-cuda10.1-cudnn7-tensorrt6.0_config_platform//:platform"
|
||||
build:rbe_linux_cuda_nvcc_base --repo_env=TF_CUDA_CONFIG_REPO="@ubuntu16.04-gcc7_manylinux2010-cuda10.1-cudnn7-tensorrt6.0_config_cuda"
|
||||
build:rbe_linux_cuda_nvcc_base --repo_env=TF_TENSORRT_CONFIG_REPO="@ubuntu16.04-gcc7_manylinux2010-cuda10.1-cudnn7-tensorrt6.0_config_tensorrt"
|
||||
build:rbe_linux_cuda_nvcc_base --repo_env=TF_NCCL_CONFIG_REPO="@ubuntu16.04-gcc7_manylinux2010-cuda10.1-cudnn7-tensorrt6.0_config_nccl"
|
||||
build:rbe_linux_cuda_nvcc_base --define=using_cuda_nvcc=true
|
||||
build:rbe_linux_cuda_nvcc_py27 --config=rbe_linux_cuda_nvcc_base --repo_env=TF_PYTHON_CONFIG_REPO="@ubuntu16.04-gcc7_manylinux2010-cuda10.1-cudnn7-tensorrt6.0_config_python2.7"
|
||||
build:rbe_linux_cuda_nvcc_py35 --config=rbe_linux_cuda_nvcc_base --repo_env=TF_PYTHON_CONFIG_REPO="@ubuntu16.04-gcc7_manylinux2010-cuda10.1-cudnn7-tensorrt6.0_config_python3.5"
|
||||
build:rbe_linux_cuda_nvcc_py36 --config=rbe_linux_cuda_nvcc_base --repo_env=TF_PYTHON_CONFIG_REPO="@ubuntu16.04-gcc7_manylinux2010-cuda10.1-cudnn7-tensorrt6.0_config_python3.6"
|
||||
build:rbe_linux_cuda_nvcc_py37 --config=rbe_linux_cuda_nvcc_base --repo_env=TF_PYTHON_CONFIG_REPO="@ubuntu16.04-gcc7_manylinux2010-cuda10.1-cudnn7-tensorrt6.0_config_python3.7"
|
||||
build:rbe_linux_cuda_nvcc_py38 --config=rbe_linux_cuda_nvcc_base --repo_env=TF_PYTHON_CONFIG_REPO="@ubuntu16.04-gcc7_manylinux2010-cuda10.1-cudnn7-tensorrt6.0_config_python3.8"
|
||||
|
||||
build:rbe_linux_cuda_clang --config=rbe_linux_cuda_base
|
||||
build:rbe_linux_cuda_clang --crosstool_top="@ubuntu16.04-py3-clang_manylinux2010-cuda10.1-cudnn7-tensorrt6.0_config_cuda//crosstool:toolchain"
|
||||
build:rbe_linux_cuda_clang --extra_toolchains="@ubuntu16.04-py3-clang_manylinux2010-cuda10.1-cudnn7-tensorrt6.0_config_cuda//crosstool:toolchain-linux-x86_64"
|
||||
|
||||
@ -9,6 +9,7 @@ container_digests = {
|
||||
"cuda10.1-cudnn7-centos6": "sha256:454b899657e87893ee5e68dc0f87df59b6a0a7418ae09cafcc3dd65ac71feca9",
|
||||
"cuda10.0-cudnn7-ubuntu16.04-manylinux2010": "sha256:5812d9d0ef0a3276fc5faaf4cd01f3d6e03d635893a6e2d2e04f6f01d626c432",
|
||||
"cuda10.1-cudnn7-ubuntu16.04-manylinux2010": "sha256:cc7f760195d7bbe283b45ae740409751d0b74d8ffbdc2f7a3cb62c71a71fbe25",
|
||||
"cuda10.1-cudnn7-ubuntu16.04-manylinux2010-multipython": "sha256:c460570b88eab3da92f06fdf30098d89be4de0f3b010ee3d39086f4d000dd3b8",
|
||||
"rocm-ubuntu16.04": "sha256:e645447dd6127325f3e97b8bf23424f637a8579d963b34fcc6772cf7cfaa0ebe",
|
||||
"windows-1803": "sha256:f109576c7c0c8a1783ff22b666e8923b52dbbe7933f69a1c7a7275202c304a12",
|
||||
}
|
||||
|
||||
29
third_party/toolchains/remote_config/configs.bzl
vendored
29
third_party/toolchains/remote_config/configs.bzl
vendored
@ -10,7 +10,7 @@ def initialize_rbe_configs():
|
||||
tensorflow_rbe_config(
|
||||
name = "ubuntu16.04-manylinux2010-py3",
|
||||
os = "ubuntu16.04-manylinux2010",
|
||||
python_version = "3",
|
||||
python_versions = ["3"],
|
||||
compiler = "",
|
||||
)
|
||||
|
||||
@ -21,11 +21,27 @@ def initialize_rbe_configs():
|
||||
cuda_version = "10.0",
|
||||
cudnn_version = "7",
|
||||
os = "ubuntu16.04-manylinux2010",
|
||||
python_version = "3",
|
||||
python_versions = ["3"],
|
||||
tensorrt_install_path = "/usr",
|
||||
tensorrt_version = "5.1",
|
||||
)
|
||||
|
||||
tensorflow_rbe_config(
|
||||
name = "ubuntu16.04-gcc7_manylinux2010-cuda10.1-cudnn7-tensorrt6.0",
|
||||
compiler = "/dt7/usr/bin/gcc",
|
||||
compiler_prefix = "/usr/bin",
|
||||
cuda_version = "10.1",
|
||||
cudnn_version = "7",
|
||||
os = "ubuntu16.04-manylinux2010-multipython",
|
||||
python_versions = ["2.7", "3.5", "3.6", "3.7", "3.8"],
|
||||
tensorrt_install_path = "/usr",
|
||||
tensorrt_version = "6.0",
|
||||
python_install_path = "/usr/local",
|
||||
)
|
||||
|
||||
# TODO(klimek): Delete this once all users are migrated to a python-version
|
||||
# independent configuration. In the future, use
|
||||
# "ubuntu16.04-gcc7_manylinux2010-cuda10.1-cudnn7-tensorrt6.0" instead.
|
||||
tensorflow_rbe_config(
|
||||
name = "ubuntu16.04-py3-gcc7_manylinux2010-cuda10.1-cudnn7-tensorrt6.0",
|
||||
compiler = "/dt7/usr/bin/gcc",
|
||||
@ -33,7 +49,7 @@ def initialize_rbe_configs():
|
||||
cuda_version = "10.1",
|
||||
cudnn_version = "7",
|
||||
os = "ubuntu16.04-manylinux2010",
|
||||
python_version = "3",
|
||||
python_versions = ["3"],
|
||||
tensorrt_install_path = "/usr",
|
||||
tensorrt_version = "6.0",
|
||||
)
|
||||
@ -43,18 +59,19 @@ def initialize_rbe_configs():
|
||||
compiler = "/clang_r42cab985fd95ba4f3f290e7bb26b93805edb447d/bin/clang",
|
||||
cuda_version = "10.1",
|
||||
cudnn_version = "7",
|
||||
os = "ubuntu16.04-manylinux2010",
|
||||
python_version = "3",
|
||||
os = "ubuntu16.04-manylinux2010-multipython",
|
||||
python_versions = ["2.7", "3.5", "3.6", "3.7", "3.8"],
|
||||
tensorrt_install_path = "/usr",
|
||||
tensorrt_version = "6.0",
|
||||
sysroot = "/dt7",
|
||||
python_install_path = "/usr/local",
|
||||
)
|
||||
|
||||
tensorflow_rbe_config(
|
||||
name = "ubuntu16.04-py3_opt-gcc5-rocm",
|
||||
compiler = "gcc",
|
||||
os = "ubuntu16.04",
|
||||
python_version = "3",
|
||||
python_versions = ["3"],
|
||||
rocm_version = "2.5", # Any version will do.
|
||||
)
|
||||
|
||||
|
||||
@ -24,6 +24,13 @@ containers = {
|
||||
"digest": container_digests["cuda10.1-cudnn7-ubuntu16.04-manylinux2010"],
|
||||
},
|
||||
|
||||
# Built with //tensorflow/tools/ci_build/Dockerfile.rbe.cuda10.1-cudnn7-ubuntu16.04-manylinux2010-multipython.
|
||||
"cuda10.1-cudnn7-ubuntu16.04-manylinux2010-multipython": {
|
||||
"registry": "gcr.io",
|
||||
"repository": "tensorflow-testing/nosla-cuda10.1-cudnn7-ubuntu16.04-manylinux2010-multipython",
|
||||
"digest": container_digests["cuda10.1-cudnn7-ubuntu16.04-manylinux2010-multipython"],
|
||||
},
|
||||
|
||||
# Built with //tensorflow/tools/ci_build/Dockerfile.rbe.rocm-ubuntu16.04
|
||||
"rocm-ubuntu16.04": {
|
||||
"registry": "gcr.io",
|
||||
|
||||
@ -12,7 +12,7 @@ def _container_image_uri(container_name):
|
||||
container = containers[container_name]
|
||||
return "docker://%s/%s@%s" % (container["registry"], container["repository"], container["digest"])
|
||||
|
||||
def _tensorflow_rbe_config(name, compiler, python_version, os, rocm_version = None, cuda_version = None, cudnn_version = None, tensorrt_version = None, tensorrt_install_path = None, cudnn_install_path = None, compiler_prefix = None, sysroot = None):
|
||||
def _tensorflow_rbe_config(name, compiler, python_versions, os, rocm_version = None, cuda_version = None, cudnn_version = None, tensorrt_version = None, tensorrt_install_path = None, cudnn_install_path = None, compiler_prefix = None, sysroot = None, python_install_path = "/usr"):
|
||||
if cuda_version != None and rocm_version != None:
|
||||
fail("Specifying both cuda_version and rocm_version is not supported.")
|
||||
|
||||
@ -26,7 +26,6 @@ def _tensorflow_rbe_config(name, compiler, python_version, os, rocm_version = No
|
||||
"BAZEL_TARGET_SYSTEM": "x86_64-unknown-linux-gnu",
|
||||
"CC_TOOLCHAIN_NAME": "linux_gnu_x86",
|
||||
"CC": compiler,
|
||||
"PYTHON_BIN_PATH": "/usr/bin/python%s" % python_version,
|
||||
"CLEAR_CACHE": "1",
|
||||
"HOST_CXX_COMPILER": compiler,
|
||||
"HOST_C_COMPILER": compiler,
|
||||
@ -121,27 +120,35 @@ def _tensorflow_rbe_config(name, compiler, python_version, os, rocm_version = No
|
||||
environ = env,
|
||||
exec_properties = exec_properties,
|
||||
)
|
||||
elif python_version != None:
|
||||
elif python_versions != None:
|
||||
container_image = _container_image_uri(os)
|
||||
exec_properties = {
|
||||
"container-image": container_image,
|
||||
"Pool": "default",
|
||||
}
|
||||
|
||||
else:
|
||||
fail("Neither cuda_version, rocm_version nor python_version specified.")
|
||||
|
||||
for python_version in python_versions:
|
||||
env.update({
|
||||
"PYTHON_BIN_PATH": "%s/bin/python%s" % (python_install_path, python_version),
|
||||
})
|
||||
remote_platform_configure(
|
||||
name = "%s_config_platform" % name,
|
||||
platform = "linux",
|
||||
platform_exec_properties = exec_properties,
|
||||
)
|
||||
|
||||
# For backwards compatibility do not add the python version to the name
|
||||
# if we only create a single python configuration.
|
||||
version = python_version if len(python_versions) > 1 else ""
|
||||
remote_python_configure(
|
||||
name = "%s_config_python" % name,
|
||||
name = "%s_config_python%s" % (name, version),
|
||||
environ = env,
|
||||
exec_properties = exec_properties,
|
||||
platform_constraint = "@%s_config_platform//:platform_constraint" % name,
|
||||
)
|
||||
else:
|
||||
fail("Neither cuda_version, rocm_version nor python_version specified.")
|
||||
|
||||
def _tensorflow_rbe_win_config(name, python_bin_path, container_name = "windows-1803"):
|
||||
container_image = _container_image_uri(container_name)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user