diff --git a/tensorflow/tools/ci_build/linux/mkl/Dockerfile.devel-mkl b/tensorflow/tools/ci_build/linux/mkl/Dockerfile.devel-mkl new file mode 100755 index 00000000000..151cc5c7e40 --- /dev/null +++ b/tensorflow/tools/ci_build/linux/mkl/Dockerfile.devel-mkl @@ -0,0 +1,47 @@ +ARG ROOT_CONTAINER_TAG=devel +ARG ROOT_CONTAINER=tensorflow/tensorflow + +FROM ${ROOT_CONTAINER}:${ROOT_CONTAINER_TAG} + +LABEL maintainer="Clayne Robison " + +# These parameters can be overridden +ARG PYTHON="python" +ARG WHL_DIR="/tmp/pip" +ARG PIP="pip" +ARG TARGET_PLATFORM="haswell" + +# Download and build TensorFlow from the latest sources found in the root container +# make sure that if they pass in a tag, that it is loaded or we'll get an error +WORKDIR / + +COPY tensorflow/ /tensorflow/ + +WORKDIR /tensorflow + +RUN yes "" | ${PYTHON} configure.py + +ENV CI_BUILD_PYTHON ${PYTHON} + +# This script detects the version of gcc in the container, sets the appropriate +# compiler flags based on parameters +ADD set-build-env.py . +RUN ${PYTHON} set-build-env.py -p ${TARGET_PLATFORM} -f /root/.mkl.bazelrc --disable-v2 + +# Pull the compiler flags we just wrote into root user's .bazelrc file +RUN echo "import /root/.mkl.bazelrc" >>/root/.bazelrc + +RUN bazel --bazelrc=/root/.bazelrc build -c opt \ + tensorflow/tools/pip_package:build_pip_package && \ + bazel-bin/tensorflow/tools/pip_package/build_pip_package "${WHL_DIR}" && \ + ${PIP} --no-cache-dir install --upgrade "${WHL_DIR}"/tensorflow-*.whl && \ + rm -rf /root/.cache + # Clean up Bazel cache when done. + +# TensorBoard +EXPOSE 6006 +# IPython +EXPOSE 8888 + +WORKDIR /root + diff --git a/tensorflow/tools/ci_build/linux/mkl/build-dev-container.sh b/tensorflow/tools/ci_build/linux/mkl/build-dev-container.sh index b497326d98d..381de90e07a 100755 --- a/tensorflow/tools/ci_build/linux/mkl/build-dev-container.sh +++ b/tensorflow/tools/ci_build/linux/mkl/build-dev-container.sh @@ -16,6 +16,10 @@ # Build a whl and container with Intel(R) MKL support # Usage: build-dev-container.sh +DEBUG=1 +DOCKER_BINARY="docker" +TMP_DIR=$(pwd) + # Helper function to traverse directories up until given file is found. function upsearch () { test / == "$PWD" && return || \ @@ -23,81 +27,248 @@ function upsearch () { cd .. && upsearch "$1" } +function debug() +{ + if [[ ${DEBUG} == 1 ]] ; then + echo $1 + fi +} + +function die() +{ + echo $1 + exit 1 +} + # Set up WORKSPACE. WORKSPACE="${WORKSPACE:-$(upsearch WORKSPACE)}" -TF_DOCKER_BUILD_DEVEL_BRANCH=${TF_DOCKER_BUILD_DEVEL_BRANCH:-master} -TF_DOCKER_BUILD_IMAGE_NAME=${TF_DOCKER_BUILD_IMAGE_NAME:-intel-mkl/tensorflow} +ROOT_CONTAINER=${ROOT_CONTAINER:-tensorflow/tensorflow} +TF_ROOT_CONTAINER_TAG=${ROOT_CONTAINER_TAG:-devel} +TF_BUILD_VERSION=${TF_DOCKER_BUILD_DEVEL_BRANCH:-master} +TF_REPO=${TF_REPO:-https://github.com/tensorflow/tensorflow} +FINAL_IMAGE_NAME=${TF_DOCKER_BUILD_IMAGE_NAME:-intel-mkl/tensorflow} TF_DOCKER_BUILD_VERSION=${TF_DOCKER_BUILD_VERSION:-nightly} +BUILD_AVX_CONTAINERS=${BUILD_AVX_CONTAINERS:-no} +BUILD_AVX2_CONTAINERS=${BUILD_AVX2_CONTAINERS:-no} +BUILD_SKX_CONTAINERS=${BUILD_SKX_CONTAINERS:-no} +BUILD_CLX_CONTAINERS=${BUILD_CLX_CONTAINERS:-no} +CONTAINER_PORT=${TF_DOCKER_BUILD_PORT:-8888} +BUILD_TF_V2_CONTAINERS=${BUILD_TF_V2_CONTAINERS:-no} +ENABLE_SECURE_BUILD=${ENABLE_SECURE_BUILD:-no} -echo "TF_DOCKER_BUILD_DEVEL_BRANCH=${TF_DOCKER_BUILD_DEVEL_BRANCH}" -echo "TF_DOCKER_BUILD_IMAGE_NAME=${TF_DOCKER_BUILD_IMAGE_NAME}" -echo "TF_DOCKER_BUILD_VERSION=${TF_DOCKER_BUILD_VERSION}" +debug "ROOT_CONTAINER=${ROOT_CONTAINER}" +debug "TF_ROOT_CONTAINER_TAG=${TF_ROOT_CONTAINER_TAG}" +debug "TF_BUILD_VERSION=${TF_BUILD_VERSION}" +debug "FINAL_IMAGE_NAME=${FINAL_IMAGE_NAME}" +debug "TF_DOCKER_BUILD_VERSION=${TF_DOCKER_BUILD_VERSION}" +debug "BUILD_AVX_CONTAINERS=${BUILD_AVX_CONTAINERS}" +debug "BUILD_AVX2_CONTAINERS=${BUILD_AVX2_CONTAINERS}" +debug "BUILD_SKX_CONTAINERS=${BUILD_SKX_CONTAINERS}" +debug "BUILD_CLX_CONTAINERS=${BUILD_CLX_CONTAINERS}" +debug "BUILD_TF_V2_CONTAINERS=${BUILD_TF_V2_CONTAINERS}" +debug "ENABLE_SECURE_BUILD=${ENABLE_SECURE_BUILD}" +debug "TMP_DIR=${TMP_DIR}" -# Build containers for AVX -# Include the instructions for sandybridge and later, but tune for ivybridge -TF_BAZEL_BUILD_OPTIONS="--config=mkl --copt=-march=sandybridge --copt=-mtune=ivybridge --copt=-O3 --cxxopt=-D_GLIBCXX_USE_CXX11_ABI=0" +function build_container() +{ + if [[ $# -lt 2 ]]; then + die "Usage: build_container ." + fi + TEMP_IMAGE_NAME=${1} + debug "TEMP_IMAGE_NAME=${TEMP_IMAGE_NAME}" + shift + TF_DOCKER_BUILD_ARGS=("${@}") -# build the python 2 container and whl -TF_DOCKER_BUILD_TYPE="MKL" \ - TF_DOCKER_BUILD_IS_DEVEL="YES" \ - TF_DOCKER_BUILD_DEVEL_BRANCH="${TF_DOCKER_BUILD_DEVEL_BRANCH}" \ - TF_DOCKER_BUILD_IMAGE_NAME="${TF_DOCKER_BUILD_IMAGE_NAME}" \ - TF_DOCKER_BUILD_VERSION="${TF_DOCKER_BUILD_VERSION}" \ - TF_BAZEL_BUILD_OPTIONS="${TF_BAZEL_BUILD_OPTIONS}" \ - ${WORKSPACE}/tensorflow/tools/docker/parameterized_docker_build.sh + # Add the proxy info build args + TF_DOCKER_BUILD_ARGS+=("--build-arg http_proxy=${http_proxy}") + TF_DOCKER_BUILD_ARGS+=("--build-arg https_proxy=${https_proxy}") + TF_DOCKER_BUILD_ARGS+=("--build-arg socks_proxy=${socks_proxy}") + TF_DOCKER_BUILD_ARGS+=("--build-arg no_proxy=${no_proxy}") + TF_DOCKER_BUILD_ARGS+=("--build-arg HTTP_PROXY=${http_proxy}") + TF_DOCKER_BUILD_ARGS+=("--build-arg SOCKS_PROXY=${socks_proxy}") + TF_DOCKER_BUILD_ARGS+=("--build-arg NO_PROXY=${no_proxy}") -# build the python 3 container and whl -TF_DOCKER_BUILD_TYPE="MKL" \ - TF_DOCKER_BUILD_IS_DEVEL="YES" \ - TF_DOCKER_BUILD_DEVEL_BRANCH="${TF_DOCKER_BUILD_DEVEL_BRANCH}" \ - TF_DOCKER_BUILD_IMAGE_NAME="${TF_DOCKER_BUILD_IMAGE_NAME}" \ - TF_DOCKER_BUILD_VERSION="${TF_DOCKER_BUILD_VERSION}" \ - TF_DOCKER_BUILD_PYTHON_VERSION="PYTHON3" \ - TF_BAZEL_BUILD_OPTIONS="${TF_BAZEL_BUILD_OPTIONS}" \ - ${WORKSPACE}/tensorflow/tools/docker/parameterized_docker_build.sh + #Add --config=v2 build arg for TF v2 + if [[ ${BUILD_TF_V2_CONTAINERS} == "no" ]]; then + TF_DOCKER_BUILD_ARGS+=("--build-arg CONFIG_V2_DISABLE=--disable-v2") + fi -# build the python3.6 container and whl -TF_DOCKER_BUILD_TYPE="MKL" \ - TF_DOCKER_BUILD_IS_DEVEL="YES" \ - TF_DOCKER_BUILD_DEVEL_BRANCH="${TF_DOCKER_BUILD_DEVEL_BRANCH}" \ - TF_DOCKER_BUILD_IMAGE_NAME="${TF_DOCKER_BUILD_IMAGE_NAME}" \ - TF_DOCKER_BUILD_VERSION="${TF_DOCKER_BUILD_VERSION}" \ - TF_DOCKER_BUILD_PYTHON_VERSION="PYTHON3.6" \ - TF_BAZEL_BUILD_OPTIONS="${TF_BAZEL_BUILD_OPTIONS}" \ - ${WORKSPACE}/tensorflow/tools/docker/parameterized_docker_build.sh + #Add build arg for Secure Build + if [[ ${ENABLE_SECURE_BUILD} == "yes" ]]; then + TF_DOCKER_BUILD_ARGS+=("--build-arg ENABLE_SECURE_BUILD=--secure-build") + fi + # Perform docker build + debug "Building docker image with image name and tag: ${TEMP_IMAGE_NAME}" + CMD="${DOCKER_BINARY} build ${TF_DOCKER_BUILD_ARGS[@]} --no-cache --pull -t ${TEMP_IMAGE_NAME} -f Dockerfile.devel-mkl ." + debug "CMD=${CMD}" + ${CMD} -# Build containers for AVX2 -# Include the instructions for haswell and later, but tune for broadwell -TF_BAZEL_BUILD_OPTIONS="--config=mkl --copt=-march=haswell --copt=-mtune=broadwell --copt=-O3 --cxxopt=-D_GLIBCXX_USE_CXX11_ABI=0" + if [[ $? == "0" ]]; then + debug "${DOCKER_BINARY} build of ${TEMP_IMAGE_NAME} succeeded" + else + die "FAIL: ${DOCKER_BINARY} build of ${TEMP_IMAGE_NAME} failed" + fi +} -# build the python 2 container and whl -TF_DOCKER_BUILD_TYPE="MKL" \ - TF_DOCKER_BUILD_IS_DEVEL="YES" \ - TF_DOCKER_BUILD_DEVEL_BRANCH="${TF_DOCKER_BUILD_DEVEL_BRANCH}" \ - TF_DOCKER_BUILD_IMAGE_NAME="${TF_DOCKER_BUILD_IMAGE_NAME}" \ - TF_DOCKER_BUILD_VERSION="${TF_DOCKER_BUILD_VERSION}-avx2" \ - TF_BAZEL_BUILD_OPTIONS="${TF_BAZEL_BUILD_OPTIONS}" \ - ${WORKSPACE}/tensorflow/tools/docker/parameterized_docker_build.sh +function test_container() +{ + if [[ "$#" != "1" ]]; then + die "Usage: ${FUNCNAME} " + fi -# build the python 3 container and whl -TF_DOCKER_BUILD_TYPE="MKL" \ - TF_DOCKER_BUILD_IS_DEVEL="YES" \ - TF_DOCKER_BUILD_DEVEL_BRANCH="${TF_DOCKER_BUILD_DEVEL_BRANCH}" \ - TF_DOCKER_BUILD_IMAGE_NAME="${TF_DOCKER_BUILD_IMAGE_NAME}" \ - TF_DOCKER_BUILD_VERSION="${TF_DOCKER_BUILD_VERSION}-avx2" \ - TF_DOCKER_BUILD_PYTHON_VERSION="PYTHON3" \ - TF_BAZEL_BUILD_OPTIONS="${TF_BAZEL_BUILD_OPTIONS}" \ - ${WORKSPACE}/tensorflow/tools/docker/parameterized_docker_build.sh + TEMP_IMAGE_NAME=${1} -# build the python3.6 container and whl -TF_DOCKER_BUILD_TYPE="MKL" \ - TF_DOCKER_BUILD_IS_DEVEL="YES" \ - TF_DOCKER_BUILD_DEVEL_BRANCH="${TF_DOCKER_BUILD_DEVEL_BRANCH}" \ - TF_DOCKER_BUILD_IMAGE_NAME="${TF_DOCKER_BUILD_IMAGE_NAME}" \ - TF_DOCKER_BUILD_VERSION="${TF_DOCKER_BUILD_VERSION}-avx2" \ - TF_DOCKER_BUILD_PYTHON_VERSION="PYTHON3.6" \ - TF_BAZEL_BUILD_OPTIONS="${TF_BAZEL_BUILD_OPTIONS}" \ - ${WORKSPACE}/tensorflow/tools/docker/parameterized_docker_build.sh + # Make sure that there is no other containers of the same image running + if "${DOCKER_BINARY}" ps | grep -q "${TEMP_IMAGE_NAME}"; then + die "ERROR: It appears that there are docker containers of the image "\ + "${TEMP_IMAGE_NAME} running. Please stop them before proceeding" + fi + + # Start a docker container from the newly-built docker image + DOCKER_RUN_LOG="${TMP_DIR}/docker_run.log" + debug " Log file is at: ${DOCKER_RUN_LOG}" + + debug "Running docker container from image ${TEMP_IMAGE_NAME}..." + RUN_CMD="${DOCKER_BINARY} run --rm -d -p ${CONTAINER_PORT}:${CONTAINER_PORT} ${TEMP_IMAGE_NAME} tail -f /dev/null 2>&1 > ${DOCKER_RUN_LOG}" + debug "RUN_CMD=${RUN_CMD}" + ${RUN_CMD} + + # Get the container ID + CONTAINER_ID="" + while [[ -z ${CONTAINER_ID} ]]; do + sleep 1 + debug "Polling for container ID..." + CONTAINER_ID=$("${DOCKER_BINARY}" ps | grep "${TEMP_IMAGE_NAME}" | awk '{print $1}') + done + + debug "ID of the running docker container: ${CONTAINER_ID}" + + debug "Performing basic sanity checks on the running container..." + TEST_CMD=$(${DOCKER_BINARY} exec ${CONTAINER_ID} bash -c "${PYTHON} -c 'import tensorflow as tf; print(tf.pywrap_tensorflow.IsMklEnabled())'") + debug "Running test command: ${TEST_CMD}" + if [ "${TEST_CMD}" = "True" ] ; then + echo "PASS: MKL enabled test in ${TEMP_IMAGE_NAME}" + else + die "FAIL: MKL enabled test in ${TEMP_IMAGE_NAME}" + fi + + # Stop the running docker container + sleep 1 + "${DOCKER_BINARY}" stop --time=0 ${CONTAINER_ID} +} + +function checkout_tensorflow() +{ + if [[ "$#" != "2" ]]; then + die "Usage: ${FUNCNAME} " + fi + + TF_REPO="${1}" + TF_BUILD_VERSION="${2}" + TENSORFLOW_DIR="tensorflow" + + debug "Checking out ${TF_REPO}:${TF_BUILD_VERSION} into ${TENSORFLOW_DIR}" + + # Clean any existing tensorflow sources + rm -rf "${TENSORFLOW_DIR}" + + # Let's make this simeple for now; we can be more fancy later + git clone ${TF_REPO} ${TENSORFLOW_DIR} + cd ${TENSORFLOW_DIR} + git checkout ${TF_BUILD_VERSION} + if [ $? -ne 0 ]; then + die "Unable to find ${TF_BUILD_VERSION} on ${TF_REPO}" + fi + cd .. +} + +function tag_container() +{ + # Apply the final image name and tag + TEMP_IMAGE_NAME="${1}" + FINAL_IMG="${2}" + + DOCKER_VER=$("${DOCKER_BINARY}" version | grep Version | head -1 | awk '{print $NF}') + if [[ -z "${DOCKER_VER}" ]]; then + die "ERROR: Failed to determine ${DOCKER_BINARY} version" + fi + DOCKER_MAJOR_VER=$(echo "${DOCKER_VER}" | cut -d. -f 1) + DOCKER_MINOR_VER=$(echo "${DOCKER_VER}" | cut -d. -f 2) + + FORCE_TAG="" + if [[ "${DOCKER_MAJOR_VER}" -le 1 ]] && \ + [[ "${DOCKER_MINOR_VER}" -le 9 ]]; then + FORCE_TAG="--force" + fi + + "${DOCKER_BINARY}" tag ${FORCE_TAG} "${TEMP_IMAGE_NAME}" "${FINAL_IMG}" || \ + die "Failed to tag intermediate docker image ${TEMP_IMAGE_NAME} as ${FINAL_IMG}" + + debug "Successfully tagged docker image: ${FINAL_IMG}" +} + +PYTHON_VERSIONS=("python" "python3") +PLATFORMS=() +if [[ ${BUILD_AVX_CONTAINERS} == "yes" ]]; then + PLATFORMS+=("sandybridge") +fi + +if [[ ${BUILD_AVX2_CONTAINERS} == "yes" ]]; then + PLATFORMS+=("haswell") +fi + +if [[ ${BUILD_SKX_CONTAINERS} == "yes" ]]; then + PLATFORMS+=("skylake") +fi + +if [[ ${BUILD_CLX_CONTAINERS} == "yes" ]]; then + PLATFORMS+=("icelake") +fi + +# Checking out sources needs to be done only once +checkout_tensorflow "${TF_REPO}" "${TF_BUILD_VERSION}" + +for PLATFORM in "${PLATFORMS[@]}" +do + for PYTHON in "${PYTHON_VERSIONS[@]}" + do + # Clear the build args array + TF_DOCKER_BUILD_ARGS=("--build-arg TARGET_PLATFORM=${PLATFORM}") + TF_DOCKER_BUILD_ARGS+=("--build-arg ROOT_CONTAINER=${ROOT_CONTAINER}") + FINAL_TAG="${TF_DOCKER_BUILD_VERSION}" + ROOT_CONTAINER_TAG="${TF_ROOT_CONTAINER_TAG}" + + if [[ ${PLATFORM} == "haswell" ]]; then + FINAL_TAG="${FINAL_TAG}-avx2" + fi + + if [[ ${PLATFORM} == "skylake" ]]; then + FINAL_TAG="${FINAL_TAG}-avx512" + fi + + if [[ ${PLATFORM} == "icelake" ]]; then + FINAL_TAG="${FINAL_TAG}-avx512-VNNI" + fi + + # Add -devel-mkl to the image tag + FINAL_TAG="${FINAL_TAG}-devel-mkl" + if [[ "${PYTHON}" == "python3" ]]; then + TF_DOCKER_BUILD_ARGS+=("--build-arg WHL_DIR=/tmp/pip3") + TF_DOCKER_BUILD_ARGS+=("--build-arg PIP=pip3") + FINAL_TAG="${FINAL_TAG}-py3" + ROOT_CONTAINER_TAG="${ROOT_CONTAINER_TAG}-py3" + fi + + TF_DOCKER_BUILD_ARGS+=("--build-arg PYTHON=${PYTHON}") + TF_DOCKER_BUILD_ARGS+=("--build-arg ROOT_CONTAINER_TAG=${ROOT_CONTAINER_TAG}") + + # Intermediate image name with tag + TEMP_IMAGE_NAME="${USER}/tensorflow:${FINAL_TAG}" + build_container "${TEMP_IMAGE_NAME}" "${TF_DOCKER_BUILD_ARGS[@]}" + test_container "${TEMP_IMAGE_NAME}" + tag_container "${TEMP_IMAGE_NAME}" "${FINAL_IMAGE_NAME}:${FINAL_TAG}" + done +done diff --git a/tensorflow/tools/ci_build/linux/mkl/set-build-env.py b/tensorflow/tools/ci_build/linux/mkl/set-build-env.py new file mode 100755 index 00000000000..ba19c6174b7 --- /dev/null +++ b/tensorflow/tools/ci_build/linux/mkl/set-build-env.py @@ -0,0 +1,225 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Configure build environment for certain Intel platforms.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import os +import subprocess + +NEHALEM_CPU_INSTRUCTIONS = [ + "MMX", "SSE", "SSE2", "SSE3", "SSSE3", "SSE4.1", "SSE4.2", "POPCNT" +] + +SANDYBRIDGE_CPU_INSTRUCTIONS = NEHALEM_CPU_INSTRUCTIONS[:] +SANDYBRIDGE_CPU_INSTRUCTIONS.extend(["AVX", "AES", "PCLMUL"]) + +HASWELL_CPU_INSTRUCTIONS = SANDYBRIDGE_CPU_INSTRUCTIONS[:] +HASWELL_CPU_INSTRUCTIONS.extend( + ["FSGSBASE", "RDRND", "FMA", "BMI", "BMI2", "F16C", "MOVBE", "AVX2"]) + +SKYLAKE_CPU_INSTRUCTIONS = HASWELL_CPU_INSTRUCTIONS[:] +SKYLAKE_CPU_INSTRUCTIONS.extend([ + "PKU", "RDSEED", "ADCX", "PREFETCHW", "CLFLUSHOPT", "XSAVEC", "XSAVES", + "AVX512F", "CLWB", "AVX512VL", "AVX512BW", "AVX512DQ", "AVX512CD" +]) + +ICELAKE_CPU_INSTRUCTIONS = SKYLAKE_CPU_INSTRUCTIONS[:] +ICELAKE_CPU_INSTRUCTIONS.extend([ + "AVX512VBMI", "AVX512IFMA", "SHA", "CLWB", "UMIP", "RDPID", "GFNI", + "AVX512VBMI2", "AVX512VPOPCNTDQ", "AVX512BITALG", "AVX512VNNI", + "VPCLMULQDQ", "VAES" +]) + +BASIC_BUILD_OPTS = ["--cxxopt=-D_GLIBCXX_USE_CXX11_ABI=0", "--copt=-O3"] + +SECURE_BUILD_OPTS = [ + "--copt=-Wformat", "--copt=-Wformat-security", "--copt=-fstack-protector", + "--copt=-fPIC", "--copt=-fpic", "--linkopt=-znoexecstack", + "--linkopt=-zrelro", "--linkopt=-znow", "--linkopt=-fstack-protector" +] + + +class BuildEnvSetter(object): + """Prepares the proper environment settings for various Intel platforms.""" + default_platform_ = "haswell" + PLATFORMS = { + "nehalem": { + "min_gcc_major_version": "4", + "min_gcc_minor_version": "8", + "flags": NEHALEM_CPU_INSTRUCTIONS + }, + "sandybridge": { + "min_gcc_major_version": "4", + "min_gcc_minor_version": "8", + "flags": SANDYBRIDGE_CPU_INSTRUCTIONS + }, + "haswell": { + "min_gcc_major_version": "4", + "min_gcc_minor_version": "8", + "flags": HASWELL_CPU_INSTRUCTIONS + }, + "skylake": { + "min_gcc_major_version": "6", + "min_gcc_minor_version": "0", + "flags": SKYLAKE_CPU_INSTRUCTIONS + }, + "icelake": { + "min_gcc_major_version": "8", + "min_gcc_minor_version": "0", + "flags": ICELAKE_CPU_INSTRUCTIONS + } + } + + def __init__(self): + self.args = None + self.bazel_flags_ = "build " + self.go() + + def gcc_version_ok(self, min_gcc_major_version, min_gcc_minor_version): + """Make sure the GCC version installed on the machine is acceptable.""" + # check to see if gcc is present + gcc_path = "" + gcc_path_cmd = "command -v gcc" + try: + print("gcc_path_cmd = {}".format(gcc_path_cmd)) + gcc_path = subprocess.check_output(gcc_path_cmd, shell=True, + stderr=subprocess.STDOUT).\ + strip() + print("gcc located here: {}".format(gcc_path)) + if not os.access(gcc_path, os.F_OK | os.X_OK): + raise ValueError( + "{} does not exist or is not executable.".format(gcc_path)) + + gcc_output = subprocess.check_output([gcc_path, "-dumpversion"], + stderr=subprocess.STDOUT) + # handle python2 vs 3 (bytes vs str type) + if isinstance(gcc_output, bytes): + gcc_output = gcc_output.decode("utf-8") + print("gcc version: {}".format(gcc_output)) + gcc_info = gcc_output.split(".") + if gcc_info[0] < min_gcc_major_version: + print("Your MAJOR version of GCC is too old: {}; " + "it must be at least {}.{}".format(gcc_info[0], + min_gcc_major_version, + min_gcc_minor_version)) + return False + + elif gcc_info[0] == min_gcc_major_version: + if gcc_info[1] < min_gcc_minor_version: + print("Your MINOR version of GCC is too old: {}; " + "it must be at least {}.{}".format(gcc_info[1], + min_gcc_major_version, + min_gcc_minor_version)) + return False + return True + else: + self._debug("gcc version OK: {}.{}".format(gcc_info[0], gcc_info[1])) + return True + except subprocess.CalledProcessException as e: + print("Problem getting gcc info: {}".format(e)) + return False + + def parse_args(self): + """Set up argument parser, and parse CLI args.""" + arg_parser = argparse.ArgumentParser( + description="Parse the arguments for the " + "TensorFlow build environment " + " setter") + arg_parser.add_argument( + "--disable-mkl", + dest="disable_mkl", + help="Turn off MKL. By default the compiler flag " + "--config=mkl is enabled.", + action="store_true") + arg_parser.add_argument( + "--disable-v2", + dest="disable_v2", + help="Don't build TensorFlow v2. By default the " + " compiler flag --config=v2 is enabled.", + action="store_true") + arg_parser.add_argument( + "-s", + "--secure-build", + dest="secure_build", + help="Enable secure build flags.", + action="store_true") + arg_parser.add_argument( + "-p", + "--platform", + choices=self.PLATFORMS.keys(), + help="The target platform.", + dest="target_platform", + default=self.default_platform_) + arg_parser.add_argument( + "-f", + "--bazelrc-file", + dest="bazelrc_file", + help="The full path to the bazelrc file into which " + "the build command will be written. The path " + "will be relative to the container " + " environment.", + required=True) + + self.args = arg_parser.parse_args() + + def validate_args(self): + if os.path.exists(self.args.bazelrc_file): + if os.path.isfile(self.args.bazelrc_file): + self._debug("The file {} exists and will be deleted.".format( + self.args.bazelrc_file)) + elif os.path.isdir(self.args.bazelrc_file): + raise ValueError("{} is not a valid file name".format( + self.args.bazelrc_file)) + return True + + def set_build_args(self): + """Generate Bazel build flags.""" + for flag in BASIC_BUILD_OPTS: + self.bazel_flags_ += "{} ".format(flag) + if self.args.secure_build: + for flag in SECURE_BUILD_OPTS: + self.bazel_flags_ += "{} ".format(flag) + for flag in self.PLATFORMS.get(self.args.target_platform)["flags"]: + self.bazel_flags_ += "--copt=-m{} ".format(flag.lower()) + if not self.args.disable_mkl: + self.bazel_flags_ += "--config=mkl " + if not self.args.disable_v2: + self.bazel_flags_ += "--config=v2 " + + def write_build_args(self): + self._debug("Writing build flags: {}".format(self.bazel_flags_)) + with open(self.args.bazelrc_file, "w") as f: + f.write(self.bazel_flags_) + + def _debug(self, msg): + print(msg) + + def go(self): + self.parse_args() + target_platform = self.PLATFORMS.get(self.args.target_platform) + if self.validate_args() and \ + self.gcc_version_ok(target_platform["min_gcc_major_version"], + target_platform["min_gcc_minor_version"]): + self.set_build_args() + self.write_build_args() + else: + print("Error.") + + +env_setter = BuildEnvSetter()