Merge pull request #27169 from Intel-tensorflow:new-mkl-ci
PiperOrigin-RevId: 244390263
This commit is contained in:
TensorFlower Gardener
commit
96ce8df98a
47
tensorflow/tools/ci_build/linux/mkl/Dockerfile.devel-mkl
Executable file
47
tensorflow/tools/ci_build/linux/mkl/Dockerfile.devel-mkl
Executable file
@ -0,0 +1,47 @@
|
||||
ARG ROOT_CONTAINER_TAG=devel
|
||||
ARG ROOT_CONTAINER=tensorflow/tensorflow
|
||||
|
||||
FROM ${ROOT_CONTAINER}:${ROOT_CONTAINER_TAG}
|
||||
|
||||
LABEL maintainer="Clayne Robison <clayne.b.robison@intel.com>"
|
||||
|
||||
# These parameters can be overridden
|
||||
ARG PYTHON="python"
|
||||
ARG WHL_DIR="/tmp/pip"
|
||||
ARG PIP="pip"
|
||||
ARG TARGET_PLATFORM="haswell"
|
||||
|
||||
# Download and build TensorFlow from the latest sources found in the root container
|
||||
# make sure that if they pass in a tag, that it is loaded or we'll get an error
|
||||
WORKDIR /
|
||||
|
||||
COPY tensorflow/ /tensorflow/
|
||||
|
||||
WORKDIR /tensorflow
|
||||
|
||||
RUN yes "" | ${PYTHON} configure.py
|
||||
|
||||
ENV CI_BUILD_PYTHON ${PYTHON}
|
||||
|
||||
# This script detects the version of gcc in the container, sets the appropriate
|
||||
# compiler flags based on parameters
|
||||
ADD set-build-env.py .
|
||||
RUN ${PYTHON} set-build-env.py -p ${TARGET_PLATFORM} -f /root/.mkl.bazelrc --disable-v2
|
||||
|
||||
# Pull the compiler flags we just wrote into root user's .bazelrc file
|
||||
RUN echo "import /root/.mkl.bazelrc" >>/root/.bazelrc
|
||||
|
||||
RUN bazel --bazelrc=/root/.bazelrc build -c opt \
|
||||
tensorflow/tools/pip_package:build_pip_package && \
|
||||
bazel-bin/tensorflow/tools/pip_package/build_pip_package "${WHL_DIR}" && \
|
||||
${PIP} --no-cache-dir install --upgrade "${WHL_DIR}"/tensorflow-*.whl && \
|
||||
rm -rf /root/.cache
|
||||
# Clean up Bazel cache when done.
|
||||
|
||||
# TensorBoard
|
||||
EXPOSE 6006
|
||||
# IPython
|
||||
EXPOSE 8888
|
||||
|
||||
WORKDIR /root
|
||||
|
||||
@ -16,6 +16,10 @@
|
||||
# Build a whl and container with Intel(R) MKL support
|
||||
# Usage: build-dev-container.sh
|
||||
|
||||
DEBUG=1
|
||||
DOCKER_BINARY="docker"
|
||||
TMP_DIR=$(pwd)
|
||||
|
||||
# Helper function to traverse directories up until given file is found.
|
||||
function upsearch () {
|
||||
test / == "$PWD" && return || \
|
||||
@ -23,81 +27,248 @@ function upsearch () {
|
||||
cd .. && upsearch "$1"
|
||||
}
|
||||
|
||||
function debug()
|
||||
{
|
||||
if [[ ${DEBUG} == 1 ]] ; then
|
||||
echo $1
|
||||
fi
|
||||
}
|
||||
|
||||
function die()
|
||||
{
|
||||
echo $1
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Set up WORKSPACE.
|
||||
WORKSPACE="${WORKSPACE:-$(upsearch WORKSPACE)}"
|
||||
|
||||
TF_DOCKER_BUILD_DEVEL_BRANCH=${TF_DOCKER_BUILD_DEVEL_BRANCH:-master}
|
||||
TF_DOCKER_BUILD_IMAGE_NAME=${TF_DOCKER_BUILD_IMAGE_NAME:-intel-mkl/tensorflow}
|
||||
ROOT_CONTAINER=${ROOT_CONTAINER:-tensorflow/tensorflow}
|
||||
TF_ROOT_CONTAINER_TAG=${ROOT_CONTAINER_TAG:-devel}
|
||||
TF_BUILD_VERSION=${TF_DOCKER_BUILD_DEVEL_BRANCH:-master}
|
||||
TF_REPO=${TF_REPO:-https://github.com/tensorflow/tensorflow}
|
||||
FINAL_IMAGE_NAME=${TF_DOCKER_BUILD_IMAGE_NAME:-intel-mkl/tensorflow}
|
||||
TF_DOCKER_BUILD_VERSION=${TF_DOCKER_BUILD_VERSION:-nightly}
|
||||
BUILD_AVX_CONTAINERS=${BUILD_AVX_CONTAINERS:-no}
|
||||
BUILD_AVX2_CONTAINERS=${BUILD_AVX2_CONTAINERS:-no}
|
||||
BUILD_SKX_CONTAINERS=${BUILD_SKX_CONTAINERS:-no}
|
||||
BUILD_CLX_CONTAINERS=${BUILD_CLX_CONTAINERS:-no}
|
||||
CONTAINER_PORT=${TF_DOCKER_BUILD_PORT:-8888}
|
||||
BUILD_TF_V2_CONTAINERS=${BUILD_TF_V2_CONTAINERS:-no}
|
||||
ENABLE_SECURE_BUILD=${ENABLE_SECURE_BUILD:-no}
|
||||
|
||||
echo "TF_DOCKER_BUILD_DEVEL_BRANCH=${TF_DOCKER_BUILD_DEVEL_BRANCH}"
|
||||
echo "TF_DOCKER_BUILD_IMAGE_NAME=${TF_DOCKER_BUILD_IMAGE_NAME}"
|
||||
echo "TF_DOCKER_BUILD_VERSION=${TF_DOCKER_BUILD_VERSION}"
|
||||
debug "ROOT_CONTAINER=${ROOT_CONTAINER}"
|
||||
debug "TF_ROOT_CONTAINER_TAG=${TF_ROOT_CONTAINER_TAG}"
|
||||
debug "TF_BUILD_VERSION=${TF_BUILD_VERSION}"
|
||||
debug "FINAL_IMAGE_NAME=${FINAL_IMAGE_NAME}"
|
||||
debug "TF_DOCKER_BUILD_VERSION=${TF_DOCKER_BUILD_VERSION}"
|
||||
debug "BUILD_AVX_CONTAINERS=${BUILD_AVX_CONTAINERS}"
|
||||
debug "BUILD_AVX2_CONTAINERS=${BUILD_AVX2_CONTAINERS}"
|
||||
debug "BUILD_SKX_CONTAINERS=${BUILD_SKX_CONTAINERS}"
|
||||
debug "BUILD_CLX_CONTAINERS=${BUILD_CLX_CONTAINERS}"
|
||||
debug "BUILD_TF_V2_CONTAINERS=${BUILD_TF_V2_CONTAINERS}"
|
||||
debug "ENABLE_SECURE_BUILD=${ENABLE_SECURE_BUILD}"
|
||||
debug "TMP_DIR=${TMP_DIR}"
|
||||
|
||||
# Build containers for AVX
|
||||
# Include the instructions for sandybridge and later, but tune for ivybridge
|
||||
TF_BAZEL_BUILD_OPTIONS="--config=mkl --copt=-march=sandybridge --copt=-mtune=ivybridge --copt=-O3 --cxxopt=-D_GLIBCXX_USE_CXX11_ABI=0"
|
||||
function build_container()
|
||||
{
|
||||
if [[ $# -lt 2 ]]; then
|
||||
die "Usage: build_container <TEMP_IMAGE_NAME> <TF_DOCKER_BUILD_ARGS>."
|
||||
fi
|
||||
TEMP_IMAGE_NAME=${1}
|
||||
debug "TEMP_IMAGE_NAME=${TEMP_IMAGE_NAME}"
|
||||
shift
|
||||
TF_DOCKER_BUILD_ARGS=("${@}")
|
||||
|
||||
# build the python 2 container and whl
|
||||
TF_DOCKER_BUILD_TYPE="MKL" \
|
||||
TF_DOCKER_BUILD_IS_DEVEL="YES" \
|
||||
TF_DOCKER_BUILD_DEVEL_BRANCH="${TF_DOCKER_BUILD_DEVEL_BRANCH}" \
|
||||
TF_DOCKER_BUILD_IMAGE_NAME="${TF_DOCKER_BUILD_IMAGE_NAME}" \
|
||||
TF_DOCKER_BUILD_VERSION="${TF_DOCKER_BUILD_VERSION}" \
|
||||
TF_BAZEL_BUILD_OPTIONS="${TF_BAZEL_BUILD_OPTIONS}" \
|
||||
${WORKSPACE}/tensorflow/tools/docker/parameterized_docker_build.sh
|
||||
# Add the proxy info build args
|
||||
TF_DOCKER_BUILD_ARGS+=("--build-arg http_proxy=${http_proxy}")
|
||||
TF_DOCKER_BUILD_ARGS+=("--build-arg https_proxy=${https_proxy}")
|
||||
TF_DOCKER_BUILD_ARGS+=("--build-arg socks_proxy=${socks_proxy}")
|
||||
TF_DOCKER_BUILD_ARGS+=("--build-arg no_proxy=${no_proxy}")
|
||||
TF_DOCKER_BUILD_ARGS+=("--build-arg HTTP_PROXY=${http_proxy}")
|
||||
TF_DOCKER_BUILD_ARGS+=("--build-arg SOCKS_PROXY=${socks_proxy}")
|
||||
TF_DOCKER_BUILD_ARGS+=("--build-arg NO_PROXY=${no_proxy}")
|
||||
|
||||
# build the python 3 container and whl
|
||||
TF_DOCKER_BUILD_TYPE="MKL" \
|
||||
TF_DOCKER_BUILD_IS_DEVEL="YES" \
|
||||
TF_DOCKER_BUILD_DEVEL_BRANCH="${TF_DOCKER_BUILD_DEVEL_BRANCH}" \
|
||||
TF_DOCKER_BUILD_IMAGE_NAME="${TF_DOCKER_BUILD_IMAGE_NAME}" \
|
||||
TF_DOCKER_BUILD_VERSION="${TF_DOCKER_BUILD_VERSION}" \
|
||||
TF_DOCKER_BUILD_PYTHON_VERSION="PYTHON3" \
|
||||
TF_BAZEL_BUILD_OPTIONS="${TF_BAZEL_BUILD_OPTIONS}" \
|
||||
${WORKSPACE}/tensorflow/tools/docker/parameterized_docker_build.sh
|
||||
#Add --config=v2 build arg for TF v2
|
||||
if [[ ${BUILD_TF_V2_CONTAINERS} == "no" ]]; then
|
||||
TF_DOCKER_BUILD_ARGS+=("--build-arg CONFIG_V2_DISABLE=--disable-v2")
|
||||
fi
|
||||
|
||||
# build the python3.6 container and whl
|
||||
TF_DOCKER_BUILD_TYPE="MKL" \
|
||||
TF_DOCKER_BUILD_IS_DEVEL="YES" \
|
||||
TF_DOCKER_BUILD_DEVEL_BRANCH="${TF_DOCKER_BUILD_DEVEL_BRANCH}" \
|
||||
TF_DOCKER_BUILD_IMAGE_NAME="${TF_DOCKER_BUILD_IMAGE_NAME}" \
|
||||
TF_DOCKER_BUILD_VERSION="${TF_DOCKER_BUILD_VERSION}" \
|
||||
TF_DOCKER_BUILD_PYTHON_VERSION="PYTHON3.6" \
|
||||
TF_BAZEL_BUILD_OPTIONS="${TF_BAZEL_BUILD_OPTIONS}" \
|
||||
${WORKSPACE}/tensorflow/tools/docker/parameterized_docker_build.sh
|
||||
#Add build arg for Secure Build
|
||||
if [[ ${ENABLE_SECURE_BUILD} == "yes" ]]; then
|
||||
TF_DOCKER_BUILD_ARGS+=("--build-arg ENABLE_SECURE_BUILD=--secure-build")
|
||||
fi
|
||||
|
||||
# Perform docker build
|
||||
debug "Building docker image with image name and tag: ${TEMP_IMAGE_NAME}"
|
||||
CMD="${DOCKER_BINARY} build ${TF_DOCKER_BUILD_ARGS[@]} --no-cache --pull -t ${TEMP_IMAGE_NAME} -f Dockerfile.devel-mkl ."
|
||||
debug "CMD=${CMD}"
|
||||
${CMD}
|
||||
|
||||
# Build containers for AVX2
|
||||
# Include the instructions for haswell and later, but tune for broadwell
|
||||
TF_BAZEL_BUILD_OPTIONS="--config=mkl --copt=-march=haswell --copt=-mtune=broadwell --copt=-O3 --cxxopt=-D_GLIBCXX_USE_CXX11_ABI=0"
|
||||
if [[ $? == "0" ]]; then
|
||||
debug "${DOCKER_BINARY} build of ${TEMP_IMAGE_NAME} succeeded"
|
||||
else
|
||||
die "FAIL: ${DOCKER_BINARY} build of ${TEMP_IMAGE_NAME} failed"
|
||||
fi
|
||||
}
|
||||
|
||||
# build the python 2 container and whl
|
||||
TF_DOCKER_BUILD_TYPE="MKL" \
|
||||
TF_DOCKER_BUILD_IS_DEVEL="YES" \
|
||||
TF_DOCKER_BUILD_DEVEL_BRANCH="${TF_DOCKER_BUILD_DEVEL_BRANCH}" \
|
||||
TF_DOCKER_BUILD_IMAGE_NAME="${TF_DOCKER_BUILD_IMAGE_NAME}" \
|
||||
TF_DOCKER_BUILD_VERSION="${TF_DOCKER_BUILD_VERSION}-avx2" \
|
||||
TF_BAZEL_BUILD_OPTIONS="${TF_BAZEL_BUILD_OPTIONS}" \
|
||||
${WORKSPACE}/tensorflow/tools/docker/parameterized_docker_build.sh
|
||||
function test_container()
|
||||
{
|
||||
if [[ "$#" != "1" ]]; then
|
||||
die "Usage: ${FUNCNAME} <TEMP_IMAGE_NAME>"
|
||||
fi
|
||||
|
||||
# build the python 3 container and whl
|
||||
TF_DOCKER_BUILD_TYPE="MKL" \
|
||||
TF_DOCKER_BUILD_IS_DEVEL="YES" \
|
||||
TF_DOCKER_BUILD_DEVEL_BRANCH="${TF_DOCKER_BUILD_DEVEL_BRANCH}" \
|
||||
TF_DOCKER_BUILD_IMAGE_NAME="${TF_DOCKER_BUILD_IMAGE_NAME}" \
|
||||
TF_DOCKER_BUILD_VERSION="${TF_DOCKER_BUILD_VERSION}-avx2" \
|
||||
TF_DOCKER_BUILD_PYTHON_VERSION="PYTHON3" \
|
||||
TF_BAZEL_BUILD_OPTIONS="${TF_BAZEL_BUILD_OPTIONS}" \
|
||||
${WORKSPACE}/tensorflow/tools/docker/parameterized_docker_build.sh
|
||||
TEMP_IMAGE_NAME=${1}
|
||||
|
||||
# build the python3.6 container and whl
|
||||
TF_DOCKER_BUILD_TYPE="MKL" \
|
||||
TF_DOCKER_BUILD_IS_DEVEL="YES" \
|
||||
TF_DOCKER_BUILD_DEVEL_BRANCH="${TF_DOCKER_BUILD_DEVEL_BRANCH}" \
|
||||
TF_DOCKER_BUILD_IMAGE_NAME="${TF_DOCKER_BUILD_IMAGE_NAME}" \
|
||||
TF_DOCKER_BUILD_VERSION="${TF_DOCKER_BUILD_VERSION}-avx2" \
|
||||
TF_DOCKER_BUILD_PYTHON_VERSION="PYTHON3.6" \
|
||||
TF_BAZEL_BUILD_OPTIONS="${TF_BAZEL_BUILD_OPTIONS}" \
|
||||
${WORKSPACE}/tensorflow/tools/docker/parameterized_docker_build.sh
|
||||
# Make sure that there is no other containers of the same image running
|
||||
if "${DOCKER_BINARY}" ps | grep -q "${TEMP_IMAGE_NAME}"; then
|
||||
die "ERROR: It appears that there are docker containers of the image "\
|
||||
"${TEMP_IMAGE_NAME} running. Please stop them before proceeding"
|
||||
fi
|
||||
|
||||
# Start a docker container from the newly-built docker image
|
||||
DOCKER_RUN_LOG="${TMP_DIR}/docker_run.log"
|
||||
debug " Log file is at: ${DOCKER_RUN_LOG}"
|
||||
|
||||
debug "Running docker container from image ${TEMP_IMAGE_NAME}..."
|
||||
RUN_CMD="${DOCKER_BINARY} run --rm -d -p ${CONTAINER_PORT}:${CONTAINER_PORT} ${TEMP_IMAGE_NAME} tail -f /dev/null 2>&1 > ${DOCKER_RUN_LOG}"
|
||||
debug "RUN_CMD=${RUN_CMD}"
|
||||
${RUN_CMD}
|
||||
|
||||
# Get the container ID
|
||||
CONTAINER_ID=""
|
||||
while [[ -z ${CONTAINER_ID} ]]; do
|
||||
sleep 1
|
||||
debug "Polling for container ID..."
|
||||
CONTAINER_ID=$("${DOCKER_BINARY}" ps | grep "${TEMP_IMAGE_NAME}" | awk '{print $1}')
|
||||
done
|
||||
|
||||
debug "ID of the running docker container: ${CONTAINER_ID}"
|
||||
|
||||
debug "Performing basic sanity checks on the running container..."
|
||||
TEST_CMD=$(${DOCKER_BINARY} exec ${CONTAINER_ID} bash -c "${PYTHON} -c 'import tensorflow as tf; print(tf.pywrap_tensorflow.IsMklEnabled())'")
|
||||
debug "Running test command: ${TEST_CMD}"
|
||||
if [ "${TEST_CMD}" = "True" ] ; then
|
||||
echo "PASS: MKL enabled test in ${TEMP_IMAGE_NAME}"
|
||||
else
|
||||
die "FAIL: MKL enabled test in ${TEMP_IMAGE_NAME}"
|
||||
fi
|
||||
|
||||
# Stop the running docker container
|
||||
sleep 1
|
||||
"${DOCKER_BINARY}" stop --time=0 ${CONTAINER_ID}
|
||||
}
|
||||
|
||||
function checkout_tensorflow()
|
||||
{
|
||||
if [[ "$#" != "2" ]]; then
|
||||
die "Usage: ${FUNCNAME} <REPO_URL> <BRANCH/TAG/COMMIT-ID>"
|
||||
fi
|
||||
|
||||
TF_REPO="${1}"
|
||||
TF_BUILD_VERSION="${2}"
|
||||
TENSORFLOW_DIR="tensorflow"
|
||||
|
||||
debug "Checking out ${TF_REPO}:${TF_BUILD_VERSION} into ${TENSORFLOW_DIR}"
|
||||
|
||||
# Clean any existing tensorflow sources
|
||||
rm -rf "${TENSORFLOW_DIR}"
|
||||
|
||||
# Let's make this simeple for now; we can be more fancy later
|
||||
git clone ${TF_REPO} ${TENSORFLOW_DIR}
|
||||
cd ${TENSORFLOW_DIR}
|
||||
git checkout ${TF_BUILD_VERSION}
|
||||
if [ $? -ne 0 ]; then
|
||||
die "Unable to find ${TF_BUILD_VERSION} on ${TF_REPO}"
|
||||
fi
|
||||
cd ..
|
||||
}
|
||||
|
||||
function tag_container()
|
||||
{
|
||||
# Apply the final image name and tag
|
||||
TEMP_IMAGE_NAME="${1}"
|
||||
FINAL_IMG="${2}"
|
||||
|
||||
DOCKER_VER=$("${DOCKER_BINARY}" version | grep Version | head -1 | awk '{print $NF}')
|
||||
if [[ -z "${DOCKER_VER}" ]]; then
|
||||
die "ERROR: Failed to determine ${DOCKER_BINARY} version"
|
||||
fi
|
||||
DOCKER_MAJOR_VER=$(echo "${DOCKER_VER}" | cut -d. -f 1)
|
||||
DOCKER_MINOR_VER=$(echo "${DOCKER_VER}" | cut -d. -f 2)
|
||||
|
||||
FORCE_TAG=""
|
||||
if [[ "${DOCKER_MAJOR_VER}" -le 1 ]] && \
|
||||
[[ "${DOCKER_MINOR_VER}" -le 9 ]]; then
|
||||
FORCE_TAG="--force"
|
||||
fi
|
||||
|
||||
"${DOCKER_BINARY}" tag ${FORCE_TAG} "${TEMP_IMAGE_NAME}" "${FINAL_IMG}" || \
|
||||
die "Failed to tag intermediate docker image ${TEMP_IMAGE_NAME} as ${FINAL_IMG}"
|
||||
|
||||
debug "Successfully tagged docker image: ${FINAL_IMG}"
|
||||
}
|
||||
|
||||
PYTHON_VERSIONS=("python" "python3")
|
||||
PLATFORMS=()
|
||||
if [[ ${BUILD_AVX_CONTAINERS} == "yes" ]]; then
|
||||
PLATFORMS+=("sandybridge")
|
||||
fi
|
||||
|
||||
if [[ ${BUILD_AVX2_CONTAINERS} == "yes" ]]; then
|
||||
PLATFORMS+=("haswell")
|
||||
fi
|
||||
|
||||
if [[ ${BUILD_SKX_CONTAINERS} == "yes" ]]; then
|
||||
PLATFORMS+=("skylake")
|
||||
fi
|
||||
|
||||
if [[ ${BUILD_CLX_CONTAINERS} == "yes" ]]; then
|
||||
PLATFORMS+=("icelake")
|
||||
fi
|
||||
|
||||
# Checking out sources needs to be done only once
|
||||
checkout_tensorflow "${TF_REPO}" "${TF_BUILD_VERSION}"
|
||||
|
||||
for PLATFORM in "${PLATFORMS[@]}"
|
||||
do
|
||||
for PYTHON in "${PYTHON_VERSIONS[@]}"
|
||||
do
|
||||
# Clear the build args array
|
||||
TF_DOCKER_BUILD_ARGS=("--build-arg TARGET_PLATFORM=${PLATFORM}")
|
||||
TF_DOCKER_BUILD_ARGS+=("--build-arg ROOT_CONTAINER=${ROOT_CONTAINER}")
|
||||
FINAL_TAG="${TF_DOCKER_BUILD_VERSION}"
|
||||
ROOT_CONTAINER_TAG="${TF_ROOT_CONTAINER_TAG}"
|
||||
|
||||
if [[ ${PLATFORM} == "haswell" ]]; then
|
||||
FINAL_TAG="${FINAL_TAG}-avx2"
|
||||
fi
|
||||
|
||||
if [[ ${PLATFORM} == "skylake" ]]; then
|
||||
FINAL_TAG="${FINAL_TAG}-avx512"
|
||||
fi
|
||||
|
||||
if [[ ${PLATFORM} == "icelake" ]]; then
|
||||
FINAL_TAG="${FINAL_TAG}-avx512-VNNI"
|
||||
fi
|
||||
|
||||
# Add -devel-mkl to the image tag
|
||||
FINAL_TAG="${FINAL_TAG}-devel-mkl"
|
||||
if [[ "${PYTHON}" == "python3" ]]; then
|
||||
TF_DOCKER_BUILD_ARGS+=("--build-arg WHL_DIR=/tmp/pip3")
|
||||
TF_DOCKER_BUILD_ARGS+=("--build-arg PIP=pip3")
|
||||
FINAL_TAG="${FINAL_TAG}-py3"
|
||||
ROOT_CONTAINER_TAG="${ROOT_CONTAINER_TAG}-py3"
|
||||
fi
|
||||
|
||||
TF_DOCKER_BUILD_ARGS+=("--build-arg PYTHON=${PYTHON}")
|
||||
TF_DOCKER_BUILD_ARGS+=("--build-arg ROOT_CONTAINER_TAG=${ROOT_CONTAINER_TAG}")
|
||||
|
||||
# Intermediate image name with tag
|
||||
TEMP_IMAGE_NAME="${USER}/tensorflow:${FINAL_TAG}"
|
||||
build_container "${TEMP_IMAGE_NAME}" "${TF_DOCKER_BUILD_ARGS[@]}"
|
||||
test_container "${TEMP_IMAGE_NAME}"
|
||||
tag_container "${TEMP_IMAGE_NAME}" "${FINAL_IMAGE_NAME}:${FINAL_TAG}"
|
||||
done
|
||||
done
|
||||
|
||||
|
||||
225
tensorflow/tools/ci_build/linux/mkl/set-build-env.py
Executable file
225
tensorflow/tools/ci_build/linux/mkl/set-build-env.py
Executable file
@ -0,0 +1,225 @@
|
||||
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""Configure build environment for certain Intel platforms."""
|
||||
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
NEHALEM_CPU_INSTRUCTIONS = [
|
||||
"MMX", "SSE", "SSE2", "SSE3", "SSSE3", "SSE4.1", "SSE4.2", "POPCNT"
|
||||
]
|
||||
|
||||
SANDYBRIDGE_CPU_INSTRUCTIONS = NEHALEM_CPU_INSTRUCTIONS[:]
|
||||
SANDYBRIDGE_CPU_INSTRUCTIONS.extend(["AVX", "AES", "PCLMUL"])
|
||||
|
||||
HASWELL_CPU_INSTRUCTIONS = SANDYBRIDGE_CPU_INSTRUCTIONS[:]
|
||||
HASWELL_CPU_INSTRUCTIONS.extend(
|
||||
["FSGSBASE", "RDRND", "FMA", "BMI", "BMI2", "F16C", "MOVBE", "AVX2"])
|
||||
|
||||
SKYLAKE_CPU_INSTRUCTIONS = HASWELL_CPU_INSTRUCTIONS[:]
|
||||
SKYLAKE_CPU_INSTRUCTIONS.extend([
|
||||
"PKU", "RDSEED", "ADCX", "PREFETCHW", "CLFLUSHOPT", "XSAVEC", "XSAVES",
|
||||
"AVX512F", "CLWB", "AVX512VL", "AVX512BW", "AVX512DQ", "AVX512CD"
|
||||
])
|
||||
|
||||
ICELAKE_CPU_INSTRUCTIONS = SKYLAKE_CPU_INSTRUCTIONS[:]
|
||||
ICELAKE_CPU_INSTRUCTIONS.extend([
|
||||
"AVX512VBMI", "AVX512IFMA", "SHA", "CLWB", "UMIP", "RDPID", "GFNI",
|
||||
"AVX512VBMI2", "AVX512VPOPCNTDQ", "AVX512BITALG", "AVX512VNNI",
|
||||
"VPCLMULQDQ", "VAES"
|
||||
])
|
||||
|
||||
BASIC_BUILD_OPTS = ["--cxxopt=-D_GLIBCXX_USE_CXX11_ABI=0", "--copt=-O3"]
|
||||
|
||||
SECURE_BUILD_OPTS = [
|
||||
"--copt=-Wformat", "--copt=-Wformat-security", "--copt=-fstack-protector",
|
||||
"--copt=-fPIC", "--copt=-fpic", "--linkopt=-znoexecstack",
|
||||
"--linkopt=-zrelro", "--linkopt=-znow", "--linkopt=-fstack-protector"
|
||||
]
|
||||
|
||||
|
||||
class BuildEnvSetter(object):
|
||||
"""Prepares the proper environment settings for various Intel platforms."""
|
||||
default_platform_ = "haswell"
|
||||
PLATFORMS = {
|
||||
"nehalem": {
|
||||
"min_gcc_major_version": "4",
|
||||
"min_gcc_minor_version": "8",
|
||||
"flags": NEHALEM_CPU_INSTRUCTIONS
|
||||
},
|
||||
"sandybridge": {
|
||||
"min_gcc_major_version": "4",
|
||||
"min_gcc_minor_version": "8",
|
||||
"flags": SANDYBRIDGE_CPU_INSTRUCTIONS
|
||||
},
|
||||
"haswell": {
|
||||
"min_gcc_major_version": "4",
|
||||
"min_gcc_minor_version": "8",
|
||||
"flags": HASWELL_CPU_INSTRUCTIONS
|
||||
},
|
||||
"skylake": {
|
||||
"min_gcc_major_version": "6",
|
||||
"min_gcc_minor_version": "0",
|
||||
"flags": SKYLAKE_CPU_INSTRUCTIONS
|
||||
},
|
||||
"icelake": {
|
||||
"min_gcc_major_version": "8",
|
||||
"min_gcc_minor_version": "0",
|
||||
"flags": ICELAKE_CPU_INSTRUCTIONS
|
||||
}
|
||||
}
|
||||
|
||||
def __init__(self):
|
||||
self.args = None
|
||||
self.bazel_flags_ = "build "
|
||||
self.go()
|
||||
|
||||
def gcc_version_ok(self, min_gcc_major_version, min_gcc_minor_version):
|
||||
"""Make sure the GCC version installed on the machine is acceptable."""
|
||||
# check to see if gcc is present
|
||||
gcc_path = ""
|
||||
gcc_path_cmd = "command -v gcc"
|
||||
try:
|
||||
print("gcc_path_cmd = {}".format(gcc_path_cmd))
|
||||
gcc_path = subprocess.check_output(gcc_path_cmd, shell=True,
|
||||
stderr=subprocess.STDOUT).\
|
||||
strip()
|
||||
print("gcc located here: {}".format(gcc_path))
|
||||
if not os.access(gcc_path, os.F_OK | os.X_OK):
|
||||
raise ValueError(
|
||||
"{} does not exist or is not executable.".format(gcc_path))
|
||||
|
||||
gcc_output = subprocess.check_output([gcc_path, "-dumpversion"],
|
||||
stderr=subprocess.STDOUT)
|
||||
# handle python2 vs 3 (bytes vs str type)
|
||||
if isinstance(gcc_output, bytes):
|
||||
gcc_output = gcc_output.decode("utf-8")
|
||||
print("gcc version: {}".format(gcc_output))
|
||||
gcc_info = gcc_output.split(".")
|
||||
if gcc_info[0] < min_gcc_major_version:
|
||||
print("Your MAJOR version of GCC is too old: {}; "
|
||||
"it must be at least {}.{}".format(gcc_info[0],
|
||||
min_gcc_major_version,
|
||||
min_gcc_minor_version))
|
||||
return False
|
||||
|
||||
elif gcc_info[0] == min_gcc_major_version:
|
||||
if gcc_info[1] < min_gcc_minor_version:
|
||||
print("Your MINOR version of GCC is too old: {}; "
|
||||
"it must be at least {}.{}".format(gcc_info[1],
|
||||
min_gcc_major_version,
|
||||
min_gcc_minor_version))
|
||||
return False
|
||||
return True
|
||||
else:
|
||||
self._debug("gcc version OK: {}.{}".format(gcc_info[0], gcc_info[1]))
|
||||
return True
|
||||
except subprocess.CalledProcessException as e:
|
||||
print("Problem getting gcc info: {}".format(e))
|
||||
return False
|
||||
|
||||
def parse_args(self):
|
||||
"""Set up argument parser, and parse CLI args."""
|
||||
arg_parser = argparse.ArgumentParser(
|
||||
description="Parse the arguments for the "
|
||||
"TensorFlow build environment "
|
||||
" setter")
|
||||
arg_parser.add_argument(
|
||||
"--disable-mkl",
|
||||
dest="disable_mkl",
|
||||
help="Turn off MKL. By default the compiler flag "
|
||||
"--config=mkl is enabled.",
|
||||
action="store_true")
|
||||
arg_parser.add_argument(
|
||||
"--disable-v2",
|
||||
dest="disable_v2",
|
||||
help="Don't build TensorFlow v2. By default the "
|
||||
" compiler flag --config=v2 is enabled.",
|
||||
action="store_true")
|
||||
arg_parser.add_argument(
|
||||
"-s",
|
||||
"--secure-build",
|
||||
dest="secure_build",
|
||||
help="Enable secure build flags.",
|
||||
action="store_true")
|
||||
arg_parser.add_argument(
|
||||
"-p",
|
||||
"--platform",
|
||||
choices=self.PLATFORMS.keys(),
|
||||
help="The target platform.",
|
||||
dest="target_platform",
|
||||
default=self.default_platform_)
|
||||
arg_parser.add_argument(
|
||||
"-f",
|
||||
"--bazelrc-file",
|
||||
dest="bazelrc_file",
|
||||
help="The full path to the bazelrc file into which "
|
||||
"the build command will be written. The path "
|
||||
"will be relative to the container "
|
||||
" environment.",
|
||||
required=True)
|
||||
|
||||
self.args = arg_parser.parse_args()
|
||||
|
||||
def validate_args(self):
|
||||
if os.path.exists(self.args.bazelrc_file):
|
||||
if os.path.isfile(self.args.bazelrc_file):
|
||||
self._debug("The file {} exists and will be deleted.".format(
|
||||
self.args.bazelrc_file))
|
||||
elif os.path.isdir(self.args.bazelrc_file):
|
||||
raise ValueError("{} is not a valid file name".format(
|
||||
self.args.bazelrc_file))
|
||||
return True
|
||||
|
||||
def set_build_args(self):
|
||||
"""Generate Bazel build flags."""
|
||||
for flag in BASIC_BUILD_OPTS:
|
||||
self.bazel_flags_ += "{} ".format(flag)
|
||||
if self.args.secure_build:
|
||||
for flag in SECURE_BUILD_OPTS:
|
||||
self.bazel_flags_ += "{} ".format(flag)
|
||||
for flag in self.PLATFORMS.get(self.args.target_platform)["flags"]:
|
||||
self.bazel_flags_ += "--copt=-m{} ".format(flag.lower())
|
||||
if not self.args.disable_mkl:
|
||||
self.bazel_flags_ += "--config=mkl "
|
||||
if not self.args.disable_v2:
|
||||
self.bazel_flags_ += "--config=v2 "
|
||||
|
||||
def write_build_args(self):
|
||||
self._debug("Writing build flags: {}".format(self.bazel_flags_))
|
||||
with open(self.args.bazelrc_file, "w") as f:
|
||||
f.write(self.bazel_flags_)
|
||||
|
||||
def _debug(self, msg):
|
||||
print(msg)
|
||||
|
||||
def go(self):
|
||||
self.parse_args()
|
||||
target_platform = self.PLATFORMS.get(self.args.target_platform)
|
||||
if self.validate_args() and \
|
||||
self.gcc_version_ok(target_platform["min_gcc_major_version"],
|
||||
target_platform["min_gcc_minor_version"]):
|
||||
self.set_build_args()
|
||||
self.write_build_args()
|
||||
else:
|
||||
print("Error.")
|
||||
|
||||
|
||||
env_setter = BuildEnvSetter()
|
||||
Loading…
x
Reference in New Issue
Block a user