From 08ed32dbb9e8f67eec9efce3807b5bdb3933eb2f Mon Sep 17 00:00:00 2001 From: Yun Peng Date: Thu, 8 Jun 2017 08:43:21 +0200 Subject: [PATCH] Windows: Make TensorFlow build without --cpu=x64_windows_msvc (#10466) * Windows: Make TensorFlow build without --cpu=x64_windows_msvc Since from Bazel 0.5.0, MSVC toolchain became the default toolchain on Windows. So --cpu=x64_windows_msvc is not required as long as we adjust the BUILD files in TensorFlow. --cpu=x64_windows_msvc is also supported for now, but is depracated. The configuration for cpu value x64_windows_msvc is a duplicate of x64_windows, which should be removed in the future. * Fix breakage on macOS --- tensorflow/BUILD | 7 ++ tensorflow/core/BUILD | 51 ++++++++------- .../core/platform/default/build_config.bzl | 34 ++++++---- tensorflow/java/BUILD | 1 + tensorflow/tensorflow.bzl | 23 ++++--- .../ci_build/windows/bazel/common_env.sh | 2 +- tensorflow/tools/pip_package/BUILD | 1 + tensorflow/tools/proto_text/BUILD | 1 + third_party/curl.BUILD | 65 ++++++++++++------- third_party/farmhash.BUILD | 10 ++- third_party/gif.BUILD | 14 +++- third_party/jpeg/jpeg.BUILD | 20 ++++-- third_party/nasm.BUILD | 15 ++++- third_party/snappy.BUILD | 1 + third_party/swig.BUILD | 10 ++- 15 files changed, 173 insertions(+), 82 deletions(-) diff --git a/tensorflow/BUILD b/tensorflow/BUILD index 32108e36046..d16f9ccd14b 100644 --- a/tensorflow/BUILD +++ b/tensorflow/BUILD @@ -71,6 +71,12 @@ config_setting( config_setting( name = "windows", + values = {"cpu": "x64_windows"}, + visibility = ["//visibility:public"], +) + +config_setting( + name = "windows_msvc", values = {"cpu": "x64_windows_msvc"}, visibility = ["//visibility:public"], ) @@ -458,6 +464,7 @@ cc_binary( "//tensorflow/c:exported_symbols.lds", ], "//tensorflow:windows": [], + "//tensorflow:windows_msvc": [], "//conditions:default": [ "-z defs", "-s", diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index 47e3b99f3b2..0a854d67b7a 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -413,6 +413,7 @@ tf_cuda_library( "util/work_sharder.h", ] + select({ "//tensorflow:windows": [], + "//tensorflow:windows_msvc": [], "//conditions:default": [ "util/memmapped_file_system.h", "util/memmapped_file_system_writer.h", @@ -1208,32 +1209,35 @@ tf_proto_library_cc( ], ) +LIB_INTERNAL_WINDOWS_DEPS = glob( + [ + "lib/**/*.h", + "lib/**/*.cc", + "platform/*.h", + "platform/*.cc", + "platform/profile_utils/**/*.h", + "platform/profile_utils/**/*.cc", + ], + exclude = [ + "**/*test*", + "lib/hash/crc32c_accelerate.cc", + "lib/gif/**/*", + "lib/jpeg/**/*", + "platform/gif.h", + "platform/jpeg.h", + "platform/**/env_time.cc", + "platform/**/cuda.h", + "platform/**/cuda_libdevice_path.cc", + "platform/**/stream_executor.h", + "platform/load_library.cc", + ], +) + cc_library( name = "lib_internal", srcs = select({ - "//tensorflow:windows": glob( - [ - "lib/**/*.h", - "lib/**/*.cc", - "platform/*.h", - "platform/*.cc", - "platform/profile_utils/**/*.h", - "platform/profile_utils/**/*.cc", - ], - exclude = [ - "**/*test*", - "lib/hash/crc32c_accelerate.cc", - "lib/gif/**/*", - "lib/jpeg/**/*", - "platform/gif.h", - "platform/jpeg.h", - "platform/**/env_time.cc", - "platform/**/cuda.h", - "platform/**/cuda_libdevice_path.cc", - "platform/**/stream_executor.h", - "platform/load_library.cc", - ], - ), + "//tensorflow:windows": LIB_INTERNAL_WINDOWS_DEPS, + "//tensorflow:windows_msvc": LIB_INTERNAL_WINDOWS_DEPS, "//conditions:default": glob( [ "lib/**/*.h", @@ -1437,6 +1441,7 @@ tf_cuda_library( ], ) + select({ "//tensorflow:windows": [], + "//tensorflow:windows_msvc": [], "//conditions:default": [ "util/memmapped_file_system.h", "util/memmapped_file_system.cc", diff --git a/tensorflow/core/platform/default/build_config.bzl b/tensorflow/core/platform/default/build_config.bzl index 342aefbae62..94f255663eb 100644 --- a/tensorflow/core/platform/default/build_config.bzl +++ b/tensorflow/core/platform/default/build_config.bzl @@ -98,12 +98,14 @@ def tf_proto_library(name, srcs = [], has_services = None, ) def tf_additional_lib_hdrs(exclude = []): + windows_hdrs = native.glob([ + "platform/default/*.h", + "platform/windows/*.h", + "platform/posix/error.h", + ], exclude = exclude) return select({ - "//tensorflow:windows" : native.glob([ - "platform/default/*.h", - "platform/windows/*.h", - "platform/posix/error.h", - ], exclude = exclude), + "//tensorflow:windows" : windows_hdrs, + "//tensorflow:windows_msvc" : windows_hdrs, "//conditions:default" : native.glob([ "platform/default/*.h", "platform/posix/*.h", @@ -111,12 +113,14 @@ def tf_additional_lib_hdrs(exclude = []): }) def tf_additional_lib_srcs(exclude = []): + windows_srcs = native.glob([ + "platform/default/*.cc", + "platform/windows/*.cc", + "platform/posix/error.cc", + ], exclude = exclude) return select({ - "//tensorflow:windows" : native.glob([ - "platform/default/*.cc", - "platform/windows/*.cc", - "platform/posix/error.cc", - ], exclude = exclude), + "//tensorflow:windows" : windows_srcs, + "//tensorflow:windows_msvc" : windows_srcs, "//conditions:default" : native.glob([ "platform/default/*.cc", "platform/posix/*.cc", @@ -148,11 +152,13 @@ def tf_env_time_hdrs(): ] def tf_env_time_srcs(): + win_env_time = native.glob([ + "platform/windows/env_time.cc", + "platform/env_time.cc", + ], exclude = []) return select({ - "//tensorflow:windows" : native.glob([ - "platform/windows/env_time.cc", - "platform/env_time.cc", - ], exclude = []), + "//tensorflow:windows" : win_env_time, + "//tensorflow:windows_msvc" : win_env_time, "//conditions:default" : native.glob([ "platform/posix/env_time.cc", "platform/env_time.cc", diff --git a/tensorflow/java/BUILD b/tensorflow/java/BUILD index c305f8e56f0..9abb63c9661 100644 --- a/tensorflow/java/BUILD +++ b/tensorflow/java/BUILD @@ -163,6 +163,7 @@ cc_binary( LINKER_EXPORTED_SYMBOLS, ], "//tensorflow:windows": [], + "//tensorflow:windows_msvc": [], "//conditions:default": [ "-z defs", "-s", diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl index b766d6c2469..dba8d6de631 100644 --- a/tensorflow/tensorflow.bzl +++ b/tensorflow/tensorflow.bzl @@ -112,6 +112,7 @@ def if_not_mobile(a): def if_not_windows(a): return select({ clean_dep("//tensorflow:windows"): [], + clean_dep("//tensorflow:windows_msvc"): [], "//conditions:default": a, }) @@ -120,6 +121,7 @@ def if_x86(a): return select({ clean_dep("//tensorflow:linux_x86_64"): a, clean_dep("//tensorflow:windows"): a, + clean_dep("//tensorflow:windows_msvc"): a, "//conditions:default": [], }) @@ -129,6 +131,15 @@ def if_darwin(a): "//conditions:default": [], }) +WIN_COPTS = [ + "/DLANG_CXX11", + "/D__VERSION__=\\\"MSVC\\\"", + "/DPLATFORM_WINDOWS", + "/DTF_COMPILE_LIBRARY", + "/DEIGEN_HAS_C99_MATH", + "/DTENSORFLOW_USE_EIGEN_THREADPOOL", +] + # LINT.IfChange def tf_copts(): return ([ @@ -144,14 +155,8 @@ def tf_copts(): "-O2", ], clean_dep("//tensorflow:darwin"): [], - clean_dep("//tensorflow:windows"): [ - "/DLANG_CXX11", - "/D__VERSION__=\\\"MSVC\\\"", - "/DPLATFORM_WINDOWS", - "/DTF_COMPILE_LIBRARY", - "/DEIGEN_HAS_C99_MATH", - "/DTENSORFLOW_USE_EIGEN_THREADPOOL", - ], + clean_dep("//tensorflow:windows"): WIN_COPTS, + clean_dep("//tensorflow:windows_msvc"): WIN_COPTS, clean_dep("//tensorflow:ios"): ["-std=c++11"], "//conditions:default": ["-pthread"] })) @@ -996,6 +1001,7 @@ def tf_py_wrap_cc(name, clean_dep("//tensorflow:tf_exported_symbols.lds") ], clean_dep("//tensorflow:windows"): [], + clean_dep("//tensorflow:windows_msvc"): [], "//conditions:default": [ "-Wl,--version-script", clean_dep("//tensorflow:tf_version_script.lds") @@ -1006,6 +1012,7 @@ def tf_py_wrap_cc(name, clean_dep("//tensorflow:tf_exported_symbols.lds") ], clean_dep("//tensorflow:windows"): [], + clean_dep("//tensorflow:windows_msvc"): [], "//conditions:default": [ clean_dep("//tensorflow:tf_version_script.lds") ] diff --git a/tensorflow/tools/ci_build/windows/bazel/common_env.sh b/tensorflow/tools/ci_build/windows/bazel/common_env.sh index 47274d8c723..8853dc53b17 100644 --- a/tensorflow/tools/ci_build/windows/bazel/common_env.sh +++ b/tensorflow/tools/ci_build/windows/bazel/common_env.sh @@ -55,4 +55,4 @@ export PATH="/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v8.0/extras/CUPT export PATH="/c/tools/cuda/bin:$PATH" # Set the common build options on Windows -export BUILD_OPTS='--cpu=x64_windows_msvc --host_cpu=x64_windows_msvc --copt=-w --host_copt=-w --verbose_failures --experimental_ui' +export BUILD_OPTS='--copt=-w --host_copt=-w --verbose_failures --experimental_ui' diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD index 09fe6c53cd9..ef0475d479e 100644 --- a/tensorflow/tools/pip_package/BUILD +++ b/tensorflow/tools/pip_package/BUILD @@ -132,6 +132,7 @@ sh_binary( srcs = ["build_pip_package.sh"], data = select({ "//tensorflow:windows": [":simple_console_for_windows"], + "//tensorflow:windows_msvc": [":simple_console_for_windows"], "//conditions:default": [ ":licenses", "MANIFEST.in", diff --git a/tensorflow/tools/proto_text/BUILD b/tensorflow/tools/proto_text/BUILD index 2d14538c8dc..3a60c8c9583 100644 --- a/tensorflow/tools/proto_text/BUILD +++ b/tensorflow/tools/proto_text/BUILD @@ -44,6 +44,7 @@ cc_library( hdrs = ["gen_proto_text_functions_lib.h"], linkopts = select({ "//tensorflow:windows": [], + "//tensorflow:windows_msvc": [], "//tensorflow:darwin": [ "-lm", "-lpthread", diff --git a/third_party/curl.BUILD b/third_party/curl.BUILD index 43f6599accd..882967df1c1 100644 --- a/third_party/curl.BUILD +++ b/third_party/curl.BUILD @@ -5,6 +5,26 @@ licenses(["notice"]) # MIT/X derivative license exports_files(["COPYING"]) +CURL_WIN_COPTS = [ + "/I%prefix%/curl/lib", + "/DHAVE_CONFIG_H", + "/DCURL_DISABLE_FTP", + "/DCURL_DISABLE_NTLM", + "/DHAVE_LIBZ", + "/DHAVE_ZLIB_H", + # Defining _USING_V110_SDK71_ is hackery to defeat curl's incorrect + # detection of what OS releases we can build on with VC 2012. This + # may not be needed (or may have to change) if the WINVER setting + # changes in //third_party/msvc/vc_12_0/CROSSTOOL. + "/D_USING_V110_SDK71_", +] + +CURL_WIN_SRCS = [ + "lib/asyn-thread.c", + "lib/inet_ntop.c", + "lib/system_win32.c", +] + cc_library( name = "curl", srcs = [ @@ -210,11 +230,8 @@ cc_library( "@%ws%//tensorflow:ios": [ "lib/vtls/darwinssl.c", ], - "@%ws%//tensorflow:windows": [ - "lib/asyn-thread.c", - "lib/inet_ntop.c", - "lib/system_win32.c", - ], + "@%ws%//tensorflow:windows": CURL_WIN_SRCS, + "@%ws%//tensorflow:windows_msvc": CURL_WIN_SRCS, "//conditions:default": [ "lib/vtls/openssl.c", ], @@ -231,19 +248,8 @@ cc_library( "include/curl/typecheck-gcc.h", ], copts = select({ - "@%ws%//tensorflow:windows": [ - "/I%prefix%/curl/lib", - "/DHAVE_CONFIG_H", - "/DCURL_DISABLE_FTP", - "/DCURL_DISABLE_NTLM", - "/DHAVE_LIBZ", - "/DHAVE_ZLIB_H", - # Defining _USING_V110_SDK71_ is hackery to defeat curl's incorrect - # detection of what OS releases we can build on with VC 2012. This - # may not be needed (or may have to change) if the WINVER setting - # changes in //third_party/msvc/vc_12_0/CROSSTOOL. - "/D_USING_V110_SDK71_", - ], + "@%ws%//tensorflow:windows": CURL_WIN_COPTS, + "@%ws%//tensorflow:windows_msvc": CURL_WIN_COPTS, "//conditions:default": [ "-I%prefix%/curl/lib", "-D_GNU_SOURCE", @@ -262,6 +268,10 @@ cc_library( # See curl.h for discussion of write size and Windows "/DCURL_MAX_WRITE_SIZE=16384", ], + "@%ws%//tensorflow:windows_msvc": [ + # See curl.h for discussion of write size and Windows + "/DCURL_MAX_WRITE_SIZE=16384", + ], "//conditions:default": [ "-DCURL_MAX_WRITE_SIZE=65536", ], @@ -279,7 +289,10 @@ cc_library( ], "@%ws%//tensorflow:ios": [], "@%ws%//tensorflow:windows": [ - "ws2_32.lib", + "-Wl,ws2_32.lib", + ], + "@%ws%//tensorflow:windows_msvc": [ + "-Wl,ws2_32.lib", ], "//conditions:default": [ "-lrt", @@ -291,12 +304,19 @@ cc_library( ] + select({ "@%ws%//tensorflow:ios": [], "@%ws%//tensorflow:windows": [], + "@%ws%//tensorflow:windows_msvc": [], "//conditions:default": [ "@boringssl//:ssl", ], }), ) +CURL_BIN_WIN_COPTS = [ + "/I%prefix%/curl/lib", + "/DHAVE_CONFIG_H", + "/DCURL_DISABLE_LIBCURL_OPTION", +] + cc_binary( name = "curl_bin", srcs = [ @@ -386,11 +406,8 @@ cc_binary( "src/tool_xattr.h", ], copts = select({ - "@%ws%//tensorflow:windows": [ - "/I%prefix%/curl/lib", - "/DHAVE_CONFIG_H", - "/DCURL_DISABLE_LIBCURL_OPTION", - ], + "@%ws%//tensorflow:windows": CURL_BIN_WIN_COPTS, + "@%ws%//tensorflow:windows_msvc": CURL_BIN_WIN_COPTS, "//conditions:default": [ "-I%prefix%/curl/lib", "-D_GNU_SOURCE", diff --git a/third_party/farmhash.BUILD b/third_party/farmhash.BUILD index 6a1d4da6e55..a51e1511c1f 100644 --- a/third_party/farmhash.BUILD +++ b/third_party/farmhash.BUILD @@ -3,12 +3,19 @@ licenses(["notice"]) # MIT exports_files(["COPYING"]) config_setting( - name = "windows", + name = "windows_msvc", values = { "cpu": "x64_windows_msvc", }, ) +config_setting( + name = "windows", + values = { + "cpu": "x64_windows", + }, +) + cc_library( name = "farmhash", srcs = ["src/farmhash.cc"], @@ -16,6 +23,7 @@ cc_library( # Disable __builtin_expect support on Windows copts = select({ ":windows": ["/DFARMHASH_OPTIONAL_BUILTIN_EXPECT"], + ":windows_msvc": ["/DFARMHASH_OPTIONAL_BUILTIN_EXPECT"], "//conditions:default": [], }), includes = ["src/."], diff --git a/third_party/gif.BUILD b/third_party/gif.BUILD index fec7449130c..ad6821af3cc 100644 --- a/third_party/gif.BUILD +++ b/third_party/gif.BUILD @@ -24,6 +24,7 @@ cc_library( visibility = ["//visibility:public"], deps = select({ ":windows": [":windows_polyfill"], + ":windows_msvc": [":windows_polyfill"], "//conditions:default": [], }), ) @@ -41,6 +42,15 @@ genrule( ) config_setting( - name = "windows", - values = {"cpu": "x64_windows_msvc"}, + name = "windows_msvc", + values = { + "cpu": "x64_windows_msvc", + }, +) + +config_setting( + name = "windows", + values = { + "cpu": "x64_windows", + }, ) diff --git a/third_party/jpeg/jpeg.BUILD b/third_party/jpeg/jpeg.BUILD index 78e03eadcfa..f6078052ece 100644 --- a/third_party/jpeg/jpeg.BUILD +++ b/third_party/jpeg/jpeg.BUILD @@ -9,17 +9,20 @@ load("@%ws%//third_party:common.bzl", "template_rule") libjpegturbo_nocopts = "-[W]error" +WIN_COPTS = [ + "/Ox", + "/w14711", # function 'function' selected for inline expansion + "/w14710", # 'function' : function not inlined +] + libjpegturbo_copts = select({ ":android": [ "-O2", "-fPIE", "-w", ], - ":windows": [ - "/Ox", - "/w14711", # function 'function' selected for inline expansion - "/w14710", # 'function' : function not inlined - ], + ":windows": WIN_COPTS, + ":windows_msvc": WIN_COPTS, "//conditions:default": [ "-O3", "-w", @@ -370,6 +373,7 @@ genrule( outs = ["jconfig.h"], cmd = select({ ":windows": "cp $(location jconfig_win.h) $@", + ":windows_msvc": "cp $(location jconfig_win.h) $@", ":k8": "cp $(location jconfig_nowin_simd.h) $@", ":armeabi-v7a": "cp $(location jconfig_nowin_simd.h) $@", ":arm64-v8a": "cp $(location jconfig_nowin_simd.h) $@", @@ -386,6 +390,7 @@ genrule( outs = ["jconfigint.h"], cmd = select({ ":windows": "cp $(location jconfigint_win.h) $@", + ":windows_msvc": "cp $(location jconfigint_win.h) $@", "//conditions:default": "cp $(location jconfigint_nowin.h) $@", }), ) @@ -482,5 +487,10 @@ config_setting( config_setting( name = "windows", + values = {"cpu": "x64_windows"}, +) + +config_setting( + name = "windows_msvc", values = {"cpu": "x64_windows_msvc"}, ) diff --git a/third_party/nasm.BUILD b/third_party/nasm.BUILD index b3cf17a97e3..341d58068be 100644 --- a/third_party/nasm.BUILD +++ b/third_party/nasm.BUILD @@ -101,6 +101,7 @@ cc_binary( ], copts = select({ ":windows": [], + ":windows_msvc": [], "//conditions:default": [ "-w", "-std=c99", @@ -108,12 +109,22 @@ cc_binary( }), defines = select({ ":windows": [], + ":windows_msvc": [], "//conditions:default": ["HAVE_SNPRINTF"], }), visibility = ["@jpeg//:__pkg__"], ) config_setting( - name = "windows", - values = {"cpu": "x64_windows_msvc"}, + name = "windows_msvc", + values = { + "cpu": "x64_windows_msvc", + }, +) + +config_setting( + name = "windows", + values = { + "cpu": "x64_windows", + }, ) diff --git a/third_party/snappy.BUILD b/third_party/snappy.BUILD index 37eebe291e8..120028dc52a 100644 --- a/third_party/snappy.BUILD +++ b/third_party/snappy.BUILD @@ -35,6 +35,7 @@ genrule( "-e 's/@ac_cv_have_stdint_h@/1/g' " + select({ "@%ws%//tensorflow:windows": "-e 's/@ac_cv_have_sys_uio_h@/0/g' ", + "@%ws%//tensorflow:windows_msvc": "-e 's/@ac_cv_have_sys_uio_h@/0/g' ", "//conditions:default": "-e 's/@ac_cv_have_sys_uio_h@/1/g' ", }) + "-e 's/@SNAPPY_MAJOR@/1/g' " + diff --git a/third_party/swig.BUILD b/third_party/swig.BUILD index bea5d6b5314..d698fa934ba 100644 --- a/third_party/swig.BUILD +++ b/third_party/swig.BUILD @@ -70,7 +70,8 @@ cc_binary( "Source/Swig/wrapfunc.c", ], copts = ["$(STACK_FRAME_UNLIMITED)"] + select({ - ":x64_windows_msvc": [], + ":windows": [], + ":windows_msvc": [], "//conditions:default": [ "-Wno-parentheses", "-Wno-unused-variable", @@ -331,6 +332,11 @@ genrule( ) config_setting( - name = "x64_windows_msvc", + name = "windows_msvc", values = {"cpu": "x64_windows_msvc"}, ) + +config_setting( + name = "windows", + values = {"cpu": "x64_windows"}, +)