From 427fad6af6482280814b96eb27f7173fff3554bb Mon Sep 17 00:00:00 2001
From: Derek Murray <derek.murray@gmail.com>
Date: Fri, 2 Sep 2016 09:54:20 -0700
Subject: [PATCH] Add more capabilities to the CMake build (#4163)
This commit includes changes that enable the following in the CMake build:
* Building protobuf from source.
* Building grpc from source.
* Building the TensorFlow distributed runtime.
* Building the TensorFlow Python bindings (excluding tf.contrib).
This commit also includes minor changes to remove the implicit
dependency between a CPU-only build of the runtime and stream executor,
and makes minor changes for compatibility with Protobuf 3.0.0 (regarding
compatibility between Protobuf's and TensorFlow's int64 representation).
---
tensorflow/contrib/cmake/CMakeLists.txt | 6 +
.../contrib/cmake/external/boringssl.cmake | 1 +
.../contrib/cmake/external/farmhash.cmake | 1 +
tensorflow/contrib/cmake/external/gif.cmake | 3 +
tensorflow/contrib/cmake/external/grpc.cmake | 27 ++
.../contrib/cmake/external/highwayhash.cmake | 2 +-
tensorflow/contrib/cmake/external/jpeg.cmake | 2 +-
.../contrib/cmake/external/jsoncpp.cmake | 1 +
tensorflow/contrib/cmake/external/png.cmake | 1 +
.../contrib/cmake/external/protobuf.cmake | 22 +
tensorflow/contrib/cmake/external/re2.cmake | 7 +-
.../contrib/cmake/patches/grpc/CMakeLists.txt | 315 ++++++++++++++
.../cmake/tf_core_distributed_runtime.cmake | 98 +++++
.../contrib/cmake/tf_core_framework.cmake | 9 +-
tensorflow/contrib/cmake/tf_core_ops.cmake | 1 +
tensorflow/contrib/cmake/tf_python.cmake | 393 ++++++++++++++++++
tensorflow/contrib/cmake/tf_tools.cmake | 1 +
.../core/distributed_runtime/graph_mgr.cc | 2 +-
.../distributed_runtime/master_session.cc | 2 +-
tensorflow/core/distributed_runtime/rpc/BUILD | 2 +-
.../rpc/grpc_remote_worker.cc | 4 +-
.../rpc/grpc_worker_service.cc | 7 +
.../core/distributed_runtime/tensor_coding.cc | 4 +-
tensorflow/tools/ci_build/Dockerfile.cmake | 1 -
24 files changed, 894 insertions(+), 18 deletions(-)
create mode 100644 tensorflow/contrib/cmake/external/grpc.cmake
create mode 100644 tensorflow/contrib/cmake/external/protobuf.cmake
create mode 100644 tensorflow/contrib/cmake/patches/grpc/CMakeLists.txt
create mode 100644 tensorflow/contrib/cmake/tf_core_distributed_runtime.cmake
create mode 100644 tensorflow/contrib/cmake/tf_python.cmake
diff --git a/tensorflow/contrib/cmake/CMakeLists.txt b/tensorflow/contrib/cmake/CMakeLists.txt
index 4fe5960d3b7..3556e143b8e 100644
--- a/tensorflow/contrib/cmake/CMakeLists.txt
+++ b/tensorflow/contrib/cmake/CMakeLists.txt
@@ -38,6 +38,8 @@ set (DOWNLOAD_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/downloads"
CACHE PATH "Location where external projects will be downloaded.")
mark_as_advanced(DOWNLOAD_LOCATION)
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+
# External dependencies
include(gif)
include(png)
@@ -48,6 +50,8 @@ include(jsoncpp)
include(boringssl)
include(farmhash)
include(highwayhash)
+include(protobuf)
+include(grpc)
# Let's get to work!
include(tf_core_framework.cmake)
@@ -57,10 +61,12 @@ include(tf_core_cpu.cmake)
include(tf_models.cmake)
include(tf_core_ops.cmake)
include(tf_core_direct_session.cmake)
+include(tf_core_distributed_runtime.cmake)
include(tf_core_kernels.cmake)
include(tf_cc_ops.cmake)
include(tf_tutorials.cmake)
include(tf_tools.cmake)
+include(tf_python.cmake)
if (tensorflow_BUILD_TESTS)
include(tests.cmake)
diff --git a/tensorflow/contrib/cmake/external/boringssl.cmake b/tensorflow/contrib/cmake/external/boringssl.cmake
index ca86e1f2fda..7d6245f0c3f 100644
--- a/tensorflow/contrib/cmake/external/boringssl.cmake
+++ b/tensorflow/contrib/cmake/external/boringssl.cmake
@@ -27,5 +27,6 @@ ExternalProject_Add(boringssl
CMAKE_CACHE_ARGS
-DCMAKE_BUILD_TYPE:STRING=Release
-DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF
+ -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
)
diff --git a/tensorflow/contrib/cmake/external/farmhash.cmake b/tensorflow/contrib/cmake/external/farmhash.cmake
index 3cf8d5a50a1..11397fe4830 100644
--- a/tensorflow/contrib/cmake/external/farmhash.cmake
+++ b/tensorflow/contrib/cmake/external/farmhash.cmake
@@ -27,6 +27,7 @@ ExternalProject_Add(farmhash
${farmhash_BUILD}/configure
--prefix=${farmhash_INSTALL}
--enable-shared=yes
+ CXXFLAGS=-fPIC
)
# put farmhash includes in the directory where they are expected
diff --git a/tensorflow/contrib/cmake/external/gif.cmake b/tensorflow/contrib/cmake/external/gif.cmake
index 9fdaeec13f6..32c63690676 100644
--- a/tensorflow/contrib/cmake/external/gif.cmake
+++ b/tensorflow/contrib/cmake/external/gif.cmake
@@ -10,6 +10,8 @@ set(gif_HEADERS
"${gif_INSTALL}/include/gif_lib.h"
)
+set(ENV{CFLAGS} "$ENV{CFLAGS} -fPIC")
+
ExternalProject_Add(gif
PREFIX gif
URL ${gif_URL}
@@ -20,6 +22,7 @@ ExternalProject_Add(gif
INSTALL_COMMAND $(MAKE) install
CONFIGURE_COMMAND
${CMAKE_CURRENT_BINARY_DIR}/gif/src/gif/configure
+ --with-pic
--prefix=${gif_INSTALL}
--enable-shared=yes
)
diff --git a/tensorflow/contrib/cmake/external/grpc.cmake b/tensorflow/contrib/cmake/external/grpc.cmake
new file mode 100644
index 00000000000..30ddcad6d7e
--- /dev/null
+++ b/tensorflow/contrib/cmake/external/grpc.cmake
@@ -0,0 +1,27 @@
+include (ExternalProject)
+
+set(GRPC_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/include)
+set(GRPC_URL https://github.com/grpc/grpc.git)
+set(GRPC_BUILD ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc)
+set(GRPC_TAG 3bc78cd0b5bd784a235c01612d634b1ec5f8fb97)
+set(GRPC_LIBRARIES
+ ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/libgrpc++_unsecure.a
+ ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/libgrpc_unsecure.a
+ ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/libgpr.a)
+
+ExternalProject_Add(grpc
+ PREFIX grpc
+ DEPENDS protobuf
+ GIT_REPOSITORY ${GRPC_URL}
+ GIT_TAG ${GRPC_TAG}
+ DOWNLOAD_DIR "${DOWNLOAD_LOCATION}"
+ BUILD_IN_SOURCE 1
+ PATCH_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_SOURCE_DIR}/patches/grpc/CMakeLists.txt ${GRPC_BUILD}
+ INSTALL_COMMAND ""
+ CMAKE_CACHE_ARGS
+ -DCMAKE_BUILD_TYPE:STRING=Release
+ -DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF
+ -DPROTOBUF_INCLUDE_DIRS:STRING=${PROTOBUF_INCLUDE_DIRS}
+ -DPROTOBUF_LIBRARIES:STRING=${PROTOBUF_LIBRARIES}
+)
+
diff --git a/tensorflow/contrib/cmake/external/highwayhash.cmake b/tensorflow/contrib/cmake/external/highwayhash.cmake
index 2213534bf86..fec44c28983 100644
--- a/tensorflow/contrib/cmake/external/highwayhash.cmake
+++ b/tensorflow/contrib/cmake/external/highwayhash.cmake
@@ -19,7 +19,7 @@ set(highwayhash_HEADERS
ExternalProject_Add(highwayhash
PREFIX highwayhash
GIT_REPOSITORY ${highwayhash_URL}
- GIT_TAG ${highwayhash_HASH}
+ GIT_TAG ${highwayhash_TAG}
DOWNLOAD_DIR "${DOWNLOAD_LOCATION}"
BUILD_IN_SOURCE 1
BUILD_COMMAND $(MAKE)
diff --git a/tensorflow/contrib/cmake/external/jpeg.cmake b/tensorflow/contrib/cmake/external/jpeg.cmake
index 4b6b6489508..edef25a35b9 100644
--- a/tensorflow/contrib/cmake/external/jpeg.cmake
+++ b/tensorflow/contrib/cmake/external/jpeg.cmake
@@ -44,7 +44,6 @@ if (WIN32)
)
else()
-
ExternalProject_Add(jpeg
PREFIX jpeg
URL ${jpeg_URL}
@@ -57,6 +56,7 @@ else()
${jpeg_BUILD}/configure
--prefix=${jpeg_INSTALL}
--enable-shared=yes
+ CFLAGS=-fPIC
)
endif()
diff --git a/tensorflow/contrib/cmake/external/jsoncpp.cmake b/tensorflow/contrib/cmake/external/jsoncpp.cmake
index b392f070884..22d8139b45a 100644
--- a/tensorflow/contrib/cmake/external/jsoncpp.cmake
+++ b/tensorflow/contrib/cmake/external/jsoncpp.cmake
@@ -25,5 +25,6 @@ ExternalProject_Add(jsoncpp
CMAKE_CACHE_ARGS
-DCMAKE_BUILD_TYPE:STRING=Release
-DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF
+ -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
)
diff --git a/tensorflow/contrib/cmake/external/png.cmake b/tensorflow/contrib/cmake/external/png.cmake
index ca3633430d7..56d6ae7050c 100644
--- a/tensorflow/contrib/cmake/external/png.cmake
+++ b/tensorflow/contrib/cmake/external/png.cmake
@@ -22,6 +22,7 @@ ExternalProject_Add(png
-DCMAKE_BUILD_TYPE:STRING=Release
-DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF
-DCMAKE_INSTALL_PREFIX:STRING=${png_INSTALL}
+ -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
)
## put png includes in the directory where they are expected
diff --git a/tensorflow/contrib/cmake/external/protobuf.cmake b/tensorflow/contrib/cmake/external/protobuf.cmake
new file mode 100644
index 00000000000..dc74e9a338e
--- /dev/null
+++ b/tensorflow/contrib/cmake/external/protobuf.cmake
@@ -0,0 +1,22 @@
+include (ExternalProject)
+
+set(PROTOBUF_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf/src)
+set(PROTOBUF_URL https://github.com/google/protobuf/releases/download/v3.0.0/protobuf-cpp-3.0.0.zip)
+set(PROTOBUF_HASH SHA256=e886ea7d08267fc3d866ac42d6dd7461ae11c491836adef6f34c04cad0be3078)
+set(PROTOBUF_LIBRARIES ${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf/libprotobuf.a)
+set(PROTOBUF_PROTOC_EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf/protoc)
+
+ExternalProject_Add(protobuf
+ PREFIX protobuf
+ URL ${PROTOBUF_URL}
+ DOWNLOAD_DIR "${DOWNLOAD_LOCATION}"
+ BUILD_IN_SOURCE 1
+ SOURCE_DIR ${CMAKE_BINARY_DIR}/protobuf/src/protobuf
+ CONFIGURE_COMMAND ${CMAKE_COMMAND} cmake/ -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON
+ INSTALL_COMMAND ""
+ CMAKE_CACHE_ARGS
+ -DCMAKE_BUILD_TYPE:STRING=Release
+ -DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF
+ -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
+)
+
diff --git a/tensorflow/contrib/cmake/external/re2.cmake b/tensorflow/contrib/cmake/external/re2.cmake
index 2a193c92db8..1da2ff6be23 100644
--- a/tensorflow/contrib/cmake/external/re2.cmake
+++ b/tensorflow/contrib/cmake/external/re2.cmake
@@ -14,6 +14,8 @@ set(re2_INCLUDES ${re2_BUILD})
# For the rest, we'll just add the build dir as an include dir.
set(re2_HEADERS
"${re2_BUILD}/re2/re2.h"
+ "${re2_BUILD}/re2/stringpiece.h"
+ "${re2_BUILD}/re2/variadic_function.h"
)
ExternalProject_Add(re2
@@ -26,11 +28,12 @@ ExternalProject_Add(re2
CMAKE_CACHE_ARGS
-DCMAKE_BUILD_TYPE:STRING=Release
-DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF
+ -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
)
## put re2 includes in the directory where they are expected
add_custom_target(re2_create_destination_dir
- COMMAND ${CMAKE_COMMAND} -E make_directory ${re2_INCLUDE_DIR}
+ COMMAND ${CMAKE_COMMAND} -E make_directory ${re2_INCLUDE_DIR}/re2
DEPENDS re2)
add_custom_target(re2_copy_headers_to_destination
@@ -38,7 +41,7 @@ add_custom_target(re2_copy_headers_to_destination
foreach(header_file ${re2_HEADERS})
add_custom_command(TARGET re2_copy_headers_to_destination PRE_BUILD
- COMMAND ${CMAKE_COMMAND} -E copy ${header_file} ${re2_INCLUDE_DIR})
+ COMMAND ${CMAKE_COMMAND} -E copy ${header_file} ${re2_INCLUDE_DIR}/re2)
endforeach()
ADD_LIBRARY(re2_lib STATIC IMPORTED
diff --git a/tensorflow/contrib/cmake/patches/grpc/CMakeLists.txt b/tensorflow/contrib/cmake/patches/grpc/CMakeLists.txt
new file mode 100644
index 00000000000..6cc2e4e375d
--- /dev/null
+++ b/tensorflow/contrib/cmake/patches/grpc/CMakeLists.txt
@@ -0,0 +1,315 @@
+# GRPC global cmake file, modified for the TensorFlow build system.
+# This currently builds C and C++ code.
+
+# This file is based on the CMakeLists.txt available from here:
+# https://github.com/grpc/grpc/blob/3bc78cd0b5bd784a235c01612d634b1ec5f8fb97/CMakeLists.txt
+# with modifications to remove dependencies on SSL, and to reuse
+# previously compiled libprotobuf.
+#
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+cmake_minimum_required(VERSION 2.8)
+
+set(PACKAGE_NAME "grpc")
+set(PACKAGE_VERSION "1.0.0-pre2-tensorflow")
+set(PACKAGE_STRING "${PACKAGE_NAME} ${PACKAGE_VERSION}")
+set(PACKAGE_TARNAME "${PACKAGE_NAME}-${PACKAGE_VERSION}")
+set(PACKAGE_BUGREPORT "https://github.com/grpc/grpc/issues/")
+project(${PACKAGE_NAME} C CXX)
+
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c11")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
+
+add_library(gpr
+ src/core/lib/profiling/basic_timers.c
+ src/core/lib/profiling/stap_timers.c
+ src/core/lib/support/alloc.c
+ src/core/lib/support/avl.c
+ src/core/lib/support/backoff.c
+ src/core/lib/support/cmdline.c
+ src/core/lib/support/cpu_iphone.c
+ src/core/lib/support/cpu_linux.c
+ src/core/lib/support/cpu_posix.c
+ src/core/lib/support/cpu_windows.c
+ src/core/lib/support/env_linux.c
+ src/core/lib/support/env_posix.c
+ src/core/lib/support/env_windows.c
+ src/core/lib/support/histogram.c
+ src/core/lib/support/host_port.c
+ src/core/lib/support/log.c
+ src/core/lib/support/log_android.c
+ src/core/lib/support/log_linux.c
+ src/core/lib/support/log_posix.c
+ src/core/lib/support/log_windows.c
+ src/core/lib/support/murmur_hash.c
+ src/core/lib/support/slice.c
+ src/core/lib/support/slice_buffer.c
+ src/core/lib/support/stack_lockfree.c
+ src/core/lib/support/string.c
+ src/core/lib/support/string_posix.c
+ src/core/lib/support/string_util_windows.c
+ src/core/lib/support/string_windows.c
+ src/core/lib/support/subprocess_posix.c
+ src/core/lib/support/subprocess_windows.c
+ src/core/lib/support/sync.c
+ src/core/lib/support/sync_posix.c
+ src/core/lib/support/sync_windows.c
+ src/core/lib/support/thd.c
+ src/core/lib/support/thd_posix.c
+ src/core/lib/support/thd_windows.c
+ src/core/lib/support/time.c
+ src/core/lib/support/time_posix.c
+ src/core/lib/support/time_precise.c
+ src/core/lib/support/time_windows.c
+ src/core/lib/support/tls_pthread.c
+ src/core/lib/support/tmpfile_msys.c
+ src/core/lib/support/tmpfile_posix.c
+ src/core/lib/support/tmpfile_windows.c
+ src/core/lib/support/wrap_memcpy.c
+)
+
+target_include_directories(gpr
+ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
+ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
+ PRIVATE ${PROTOBUF_INCLUDE_DIRS}
+)
+
+add_library(grpc_unsecure
+ src/core/lib/surface/init.c
+ src/core/lib/surface/init_unsecure.c
+ src/core/lib/channel/channel_args.c
+ src/core/lib/channel/channel_stack.c
+ src/core/lib/channel/channel_stack_builder.c
+ src/core/lib/channel/compress_filter.c
+ src/core/lib/channel/connected_channel.c
+ src/core/lib/channel/http_client_filter.c
+ src/core/lib/channel/http_server_filter.c
+ src/core/lib/compression/compression.c
+ src/core/lib/compression/message_compress.c
+ src/core/lib/debug/trace.c
+ src/core/lib/http/format_request.c
+ src/core/lib/http/httpcli.c
+ src/core/lib/http/parser.c
+ src/core/lib/iomgr/closure.c
+ src/core/lib/iomgr/endpoint.c
+ src/core/lib/iomgr/endpoint_pair_posix.c
+ src/core/lib/iomgr/endpoint_pair_windows.c
+ src/core/lib/iomgr/error.c
+ src/core/lib/iomgr/ev_epoll_linux.c
+ src/core/lib/iomgr/ev_poll_and_epoll_posix.c
+ src/core/lib/iomgr/ev_poll_posix.c
+ src/core/lib/iomgr/ev_posix.c
+ src/core/lib/iomgr/exec_ctx.c
+ src/core/lib/iomgr/executor.c
+ src/core/lib/iomgr/iocp_windows.c
+ src/core/lib/iomgr/iomgr.c
+ src/core/lib/iomgr/iomgr_posix.c
+ src/core/lib/iomgr/iomgr_windows.c
+ src/core/lib/iomgr/load_file.c
+ src/core/lib/iomgr/network_status_tracker.c
+ src/core/lib/iomgr/polling_entity.c
+ src/core/lib/iomgr/pollset_set_windows.c
+ src/core/lib/iomgr/pollset_windows.c
+ src/core/lib/iomgr/resolve_address_posix.c
+ src/core/lib/iomgr/resolve_address_windows.c
+ src/core/lib/iomgr/sockaddr_utils.c
+ src/core/lib/iomgr/socket_utils_common_posix.c
+ src/core/lib/iomgr/socket_utils_linux.c
+ src/core/lib/iomgr/socket_utils_posix.c
+ src/core/lib/iomgr/socket_windows.c
+ src/core/lib/iomgr/tcp_client_posix.c
+ src/core/lib/iomgr/tcp_client_windows.c
+ src/core/lib/iomgr/tcp_posix.c
+ src/core/lib/iomgr/tcp_server_posix.c
+ src/core/lib/iomgr/tcp_server_windows.c
+ src/core/lib/iomgr/tcp_windows.c
+ src/core/lib/iomgr/time_averaged_stats.c
+ src/core/lib/iomgr/timer.c
+ src/core/lib/iomgr/timer_heap.c
+ src/core/lib/iomgr/udp_server.c
+ src/core/lib/iomgr/unix_sockets_posix.c
+ src/core/lib/iomgr/unix_sockets_posix_noop.c
+ src/core/lib/iomgr/wakeup_fd_eventfd.c
+ src/core/lib/iomgr/wakeup_fd_nospecial.c
+ src/core/lib/iomgr/wakeup_fd_pipe.c
+ src/core/lib/iomgr/wakeup_fd_posix.c
+ src/core/lib/iomgr/workqueue_posix.c
+ src/core/lib/iomgr/workqueue_windows.c
+ src/core/lib/json/json.c
+ src/core/lib/json/json_reader.c
+ src/core/lib/json/json_string.c
+ src/core/lib/json/json_writer.c
+ src/core/lib/surface/alarm.c
+ src/core/lib/surface/api_trace.c
+ src/core/lib/surface/byte_buffer.c
+ src/core/lib/surface/byte_buffer_reader.c
+ src/core/lib/surface/call.c
+ src/core/lib/surface/call_details.c
+ src/core/lib/surface/call_log_batch.c
+ src/core/lib/surface/channel.c
+ src/core/lib/surface/channel_init.c
+ src/core/lib/surface/channel_ping.c
+ src/core/lib/surface/channel_stack_type.c
+ src/core/lib/surface/completion_queue.c
+ src/core/lib/surface/event_string.c
+ src/core/lib/surface/lame_client.c
+ src/core/lib/surface/metadata_array.c
+ src/core/lib/surface/server.c
+ src/core/lib/surface/validate_metadata.c
+ src/core/lib/surface/version.c
+ src/core/lib/transport/byte_stream.c
+ src/core/lib/transport/connectivity_state.c
+ src/core/lib/transport/metadata.c
+ src/core/lib/transport/metadata_batch.c
+ src/core/lib/transport/static_metadata.c
+ src/core/lib/transport/transport.c
+ src/core/lib/transport/transport_op_string.c
+ src/core/ext/transport/chttp2/server/insecure/server_chttp2.c
+ src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.c
+ src/core/ext/transport/chttp2/transport/bin_decoder.c
+ src/core/ext/transport/chttp2/transport/bin_encoder.c
+ src/core/ext/transport/chttp2/transport/chttp2_plugin.c
+ src/core/ext/transport/chttp2/transport/chttp2_transport.c
+ src/core/ext/transport/chttp2/transport/frame_data.c
+ src/core/ext/transport/chttp2/transport/frame_goaway.c
+ src/core/ext/transport/chttp2/transport/frame_ping.c
+ src/core/ext/transport/chttp2/transport/frame_rst_stream.c
+ src/core/ext/transport/chttp2/transport/frame_settings.c
+ src/core/ext/transport/chttp2/transport/frame_window_update.c
+ src/core/ext/transport/chttp2/transport/hpack_encoder.c
+ src/core/ext/transport/chttp2/transport/hpack_parser.c
+ src/core/ext/transport/chttp2/transport/hpack_table.c
+ src/core/ext/transport/chttp2/transport/huffsyms.c
+ src/core/ext/transport/chttp2/transport/incoming_metadata.c
+ src/core/ext/transport/chttp2/transport/parsing.c
+ src/core/ext/transport/chttp2/transport/status_conversion.c
+ src/core/ext/transport/chttp2/transport/stream_lists.c
+ src/core/ext/transport/chttp2/transport/stream_map.c
+ src/core/ext/transport/chttp2/transport/timeout_encoding.c
+ src/core/ext/transport/chttp2/transport/varint.c
+ src/core/ext/transport/chttp2/transport/writing.c
+ src/core/ext/transport/chttp2/alpn/alpn.c
+ src/core/ext/transport/chttp2/client/insecure/channel_create.c
+ src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c
+ src/core/ext/client_config/channel_connectivity.c
+ src/core/ext/client_config/client_channel.c
+ src/core/ext/client_config/client_channel_factory.c
+ src/core/ext/client_config/client_config.c
+ src/core/ext/client_config/client_config_plugin.c
+ src/core/ext/client_config/connector.c
+ src/core/ext/client_config/default_initial_connect_string.c
+ src/core/ext/client_config/initial_connect_string.c
+ src/core/ext/client_config/lb_policy.c
+ src/core/ext/client_config/lb_policy_factory.c
+ src/core/ext/client_config/lb_policy_registry.c
+ src/core/ext/client_config/parse_address.c
+ src/core/ext/client_config/resolver.c
+ src/core/ext/client_config/resolver_factory.c
+ src/core/ext/client_config/resolver_registry.c
+ src/core/ext/client_config/subchannel.c
+ src/core/ext/client_config/subchannel_call_holder.c
+ src/core/ext/client_config/subchannel_index.c
+ src/core/ext/client_config/uri_parser.c
+ src/core/ext/resolver/dns/native/dns_resolver.c
+ src/core/ext/resolver/sockaddr/sockaddr_resolver.c
+ src/core/ext/load_reporting/load_reporting.c
+ src/core/ext/load_reporting/load_reporting_filter.c
+ src/core/ext/lb_policy/grpclb/load_balancer_api.c
+ src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c
+ third_party/nanopb/pb_common.c
+ third_party/nanopb/pb_decode.c
+ third_party/nanopb/pb_encode.c
+ src/core/ext/lb_policy/pick_first/pick_first.c
+ src/core/ext/lb_policy/round_robin/round_robin.c
+ src/core/ext/census/context.c
+ src/core/ext/census/gen/census.pb.c
+ src/core/ext/census/grpc_context.c
+ src/core/ext/census/grpc_filter.c
+ src/core/ext/census/grpc_plugin.c
+ src/core/ext/census/initialize.c
+ src/core/ext/census/mlog.c
+ src/core/ext/census/operation.c
+ src/core/ext/census/placeholders.c
+ src/core/ext/census/tracing.c
+ src/core/plugin_registry/grpc_unsecure_plugin_registry.c
+)
+
+target_include_directories(grpc_unsecure
+ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
+ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
+ PRIVATE ${PROTOBUF_ROOT_DIR}/src
+)
+
+target_link_libraries(grpc_unsecure
+ gpr
+)
+
+add_library(grpc++_unsecure
+ src/cpp/common/insecure_create_auth_context.cc
+ src/cpp/client/channel.cc
+ src/cpp/client/client_context.cc
+ src/cpp/client/create_channel.cc
+ src/cpp/client/create_channel_internal.cc
+ src/cpp/client/create_channel_posix.cc
+ src/cpp/client/credentials.cc
+ src/cpp/client/generic_stub.cc
+ src/cpp/client/insecure_credentials.cc
+ src/cpp/common/channel_arguments.cc
+ src/cpp/common/completion_queue.cc
+ src/cpp/common/core_codegen.cc
+ src/cpp/common/rpc_method.cc
+ src/cpp/server/async_generic_service.cc
+ src/cpp/server/create_default_thread_pool.cc
+ src/cpp/server/dynamic_thread_pool.cc
+ src/cpp/server/insecure_server_credentials.cc
+ src/cpp/server/server.cc
+ src/cpp/server/server_builder.cc
+ src/cpp/server/server_context.cc
+ src/cpp/server/server_credentials.cc
+ src/cpp/server/server_posix.cc
+ src/cpp/util/byte_buffer.cc
+ src/cpp/util/slice.cc
+ src/cpp/util/status.cc
+ src/cpp/util/string_ref.cc
+ src/cpp/util/time.cc
+ src/cpp/codegen/codegen_init.cc
+)
+
+target_include_directories(grpc++_unsecure
+ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
+ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
+ PRIVATE ${PROTOBUF_INCLUDE_DIRS}
+)
+
+target_link_libraries(grpc++_unsecure
+ ${PROTOBUF_LIBRARIES}
+ gpr
+ grpc_unsecure
+)
diff --git a/tensorflow/contrib/cmake/tf_core_distributed_runtime.cmake b/tensorflow/contrib/cmake/tf_core_distributed_runtime.cmake
new file mode 100644
index 00000000000..d52b18a85e2
--- /dev/null
+++ b/tensorflow/contrib/cmake/tf_core_distributed_runtime.cmake
@@ -0,0 +1,98 @@
+########################################################
+# tf_core_distributed_runtime library
+########################################################
+file(GLOB_RECURSE tf_core_distributed_runtime_srcs
+ "${tensorflow_source_dir}/tensorflow/core/distributed_runtime/*.h"
+ "${tensorflow_source_dir}/tensorflow/core/distributed_runtime/*.cc"
+)
+
+file(GLOB_RECURSE tf_core_distributed_runtime_exclude_srcs
+ "${tensorflow_source_dir}/tensorflow/core/distributed_runtime/*test*.h"
+ "${tensorflow_source_dir}/tensorflow/core/distributed_runtime/*test*.cc"
+ "${tensorflow_source_dir}/tensorflow/core/distributed_runtime/rpc/grpc_tensorflow_server.cc"
+)
+
+list(REMOVE_ITEM tf_core_distributed_runtime_srcs ${tf_core_distributed_runtime_exclude_srcs})
+
+add_library(tf_core_distributed_runtime OBJECT ${tf_core_distributed_runtime_srcs})
+
+add_dependencies(tf_core_distributed_runtime
+ tf_core_cpu grpc
+ re2_copy_headers_to_destination
+)
+
+target_include_directories(tf_core_distributed_runtime PRIVATE
+ ${tensorflow_source_dir}
+ ${eigen_INCLUDE_DIRS}
+ ${GRPC_INCLUDE_DIRS}
+ ${re2_INCLUDE_DIR}
+)
+
+target_compile_options(tf_core_distributed_runtime PRIVATE
+ -fno-exceptions
+ -DEIGEN_AVOID_STL_ARRAY
+)
+
+# C++11
+target_compile_features(tf_core_distributed_runtime PRIVATE
+ cxx_rvalue_references
+)
+
+########################################################
+# grpc_tensorflow_server executable
+########################################################
+set(grpc_tensorflow_server_srcs
+ "${tensorflow_source_dir}/tensorflow/core/distributed_runtime/rpc/grpc_tensorflow_server.cc"
+)
+
+add_executable(grpc_tensorflow_server
+ ${grpc_tensorflow_server_srcs}
+ $<TARGET_OBJECTS:tf_core_lib>
+ $<TARGET_OBJECTS:tf_core_cpu>
+ $<TARGET_OBJECTS:tf_core_framework>
+ $<TARGET_OBJECTS:tf_core_kernels>
+ $<TARGET_OBJECTS:tf_cc_framework>
+ $<TARGET_OBJECTS:tf_cc_ops>
+ $<TARGET_OBJECTS:tf_core_ops>
+ $<TARGET_OBJECTS:tf_core_direct_session>
+ $<TARGET_OBJECTS:tf_core_distributed_runtime>
+)
+
+add_dependencies(tf_core_distributed_runtime
+ grpc
+ re2_copy_headers_to_destination
+)
+
+target_include_directories(grpc_tensorflow_server PUBLIC
+ ${tensorflow_source_dir}
+ ${eigen_INCLUDE_DIRS}
+ ${GRPC_INCLUDE_DIRS}
+)
+
+find_package(ZLIB REQUIRED)
+
+target_link_libraries(grpc_tensorflow_server PUBLIC
+ ${CMAKE_THREAD_LIBS_INIT}
+ ${PROTOBUF_LIBRARIES}
+ ${GRPC_LIBRARIES}
+ tf_protos_cc
+ re2_lib
+ ${boringssl_STATIC_LIBRARIES}
+ ${farmhash_STATIC_LIBRARIES}
+ ${gif_STATIC_LIBRARIES}
+ ${jpeg_STATIC_LIBRARIES}
+ ${jsoncpp_STATIC_LIBRARIES}
+ ${png_STATIC_LIBRARIES}
+ ${ZLIB_LIBRARIES}
+ ${CMAKE_DL_LIBS}
+)
+
+target_compile_options(grpc_tensorflow_server PRIVATE
+ -fno-exceptions
+ -DEIGEN_AVOID_STL_ARRAY
+)
+
+# C++11
+target_compile_features(grpc_tensorflow_server PRIVATE
+ cxx_rvalue_references
+)
diff --git a/tensorflow/contrib/cmake/tf_core_framework.cmake b/tensorflow/contrib/cmake/tf_core_framework.cmake
index 0dbe3c194d9..3965f843635 100644
--- a/tensorflow/contrib/cmake/tf_core_framework.cmake
+++ b/tensorflow/contrib/cmake/tf_core_framework.cmake
@@ -24,8 +24,8 @@ function(RELATIVE_PROTOBUF_GENERATE_CPP SRCS HDRS ROOT_DIR)
OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${REL_DIR}/${FIL_WE}.pb.cc"
"${CMAKE_CURRENT_BINARY_DIR}/${REL_DIR}/${FIL_WE}.pb.h"
COMMAND ${PROTOBUF_PROTOC_EXECUTABLE}
- ARGS --cpp_out ${CMAKE_CURRENT_BINARY_DIR} -I ${ROOT_DIR} ${ABS_FIL}
- DEPENDS ${ABS_FIL} ${PROTOBUF_PROTOC_EXECUTABLE}
+ ARGS --cpp_out ${CMAKE_CURRENT_BINARY_DIR} -I ${ROOT_DIR} ${ABS_FIL} -I ${PROTOBUF_INCLUDE_DIRS}
+ DEPENDS ${ABS_FIL} protobuf
COMMENT "Running C++ protocol buffer compiler on ${FIL}"
VERBATIM )
endforeach()
@@ -71,13 +71,10 @@ endfunction()
# tf_protos_cc library
########################################################
-# Build proto library
-include(FindProtobuf)
-find_package(Protobuf REQUIRED)
include_directories(${PROTOBUF_INCLUDE_DIRS})
include_directories(${CMAKE_CURRENT_BINARY_DIR})
file(GLOB_RECURSE tf_protos_cc_srcs RELATIVE ${tensorflow_source_dir}
- "${tensorflow_source_dir}/tensorflow/*.proto"
+ "${tensorflow_source_dir}/tensorflow/core/*.proto"
)
RELATIVE_PROTOBUF_GENERATE_CPP(PROTO_SRCS PROTO_HDRS
${tensorflow_source_dir} ${tf_protos_cc_srcs}
diff --git a/tensorflow/contrib/cmake/tf_core_ops.cmake b/tensorflow/contrib/cmake/tf_core_ops.cmake
index c6124b689cc..2aa03facafb 100644
--- a/tensorflow/contrib/cmake/tf_core_ops.cmake
+++ b/tensorflow/contrib/cmake/tf_core_ops.cmake
@@ -16,6 +16,7 @@ set(tf_op_lib_names
"attention_ops"
"candidate_sampling_ops"
"control_flow_ops"
+ "ctc_ops"
"data_flow_ops"
"image_ops"
"io_ops"
diff --git a/tensorflow/contrib/cmake/tf_python.cmake b/tensorflow/contrib/cmake/tf_python.cmake
new file mode 100644
index 00000000000..1efb59e201b
--- /dev/null
+++ b/tensorflow/contrib/cmake/tf_python.cmake
@@ -0,0 +1,393 @@
+# CMake rules for generating the TensorFlow Python bindings.
+#
+# Known limitations:
+# * Generates output in a hard-coded path ${CMAKE_CURRENT_BINARY_DIR}/tf_python.
+# * No support for dynamic library loading.
+# * No support for tf.contrib. (TODO(mrry): Add rules for building op libraries.)
+# * No support for Python 3. (TODO(mrry): Add override for FindPythonInterp.)
+#
+# The _pywrap_tensorflow target builds everything.
+
+########################################################
+# Resolve installed dependencies
+########################################################
+
+# 1. Resolve the installed version of SWIG.
+FIND_PACKAGE(SWIG REQUIRED)
+INCLUDE(${SWIG_USE_FILE})
+
+# 2. Resolve the installed version of Python (for Python.h and python).
+# TODO(mrry): Parameterize the build script to enable Python 3 building.
+include(FindPythonInterp)
+if(NOT PYTHON_INCLUDE_DIR)
+ set(PYTHON_NOT_FOUND false)
+ exec_program("${PYTHON_EXECUTABLE}"
+ ARGS "-c 'import distutils.sysconfig; print distutils.sysconfig.get_python_inc()'"
+ OUTPUT_VARIABLE PYTHON_INCLUDE_DIR
+ RETURN_VALUE PYTHON_NOT_FOUND)
+ message(${PYTHON_INCLUDE_DIR})
+ if(${PYTHON_NOT_FOUND})
+ message(FATAL_ERROR
+ "Cannot get Python include directory. Is distutils installed?")
+ endif(${PYTHON_NOT_FOUND})
+endif(NOT PYTHON_INCLUDE_DIR)
+FIND_PACKAGE(PythonLibs)
+
+# 3. Resolve the installed version of NumPy (for numpy/arrayobject.h).
+if(NOT NUMPY_INCLUDE_DIR)
+ set(NUMPY_NOT_FOUND false)
+ exec_program("${PYTHON_EXECUTABLE}"
+ ARGS "-c 'import numpy; print numpy.get_include()'"
+ OUTPUT_VARIABLE NUMPY_INCLUDE_DIR
+ RETURN_VALUE NUMPY_NOT_FOUND)
+ if(${NUMPY_NOT_FOUND})
+ message(FATAL_ERROR
+ "Cannot get NumPy include directory: Is NumPy installed?")
+ endif(${NUMPY_NOT_FOUND})
+endif(NOT NUMPY_INCLUDE_DIR)
+
+# 4. Resolve the installed version of zlib (for libz.so).
+find_package(ZLIB REQUIRED)
+
+
+########################################################
+# Build the Python directory structure.
+########################################################
+
+# TODO(mrry): Configure this to build in a directory other than tf_python/
+# TODO(mrry): Assemble the Python files into a PIP package.
+
+# tf_python_srcs contains all static .py files
+file(GLOB_RECURSE tf_python_srcs RELATIVE ${tensorflow_source_dir}
+ "${tensorflow_source_dir}/tensorflow/python/*.py"
+)
+list(APPEND tf_python_srcs "tensorflow/__init__.py")
+
+# tf_python_copy_scripts_to_destination copies all Python files
+# (including static source and generated protobuf wrappers, but *not*
+# generated TensorFlow op wrappers) into tf_python/.
+add_custom_target(tf_python_copy_scripts_to_destination)
+
+# Copy static files to tf_python/.
+foreach(script ${tf_python_srcs})
+ get_filename_component(REL_DIR ${script} DIRECTORY)
+ add_custom_command(TARGET tf_python_copy_scripts_to_destination PRE_BUILD
+ COMMAND ${CMAKE_COMMAND} -E copy ${tensorflow_source_dir}/${script} ${CMAKE_CURRENT_BINARY_DIR}/tf_python/${script})
+endforeach()
+
+# Generates the Python protobuf wrappers.
+# ROOT_DIR must be absolute; subsequent arguments are interpreted as
+# paths of .proto files, and must be relative to ROOT_DIR.
+function(RELATIVE_PROTOBUF_GENERATE_PYTHON ROOT_DIR)
+ if(NOT ARGN)
+ message(SEND_ERROR "Error: RELATIVE_PROTOBUF_GENERATE_PYTHON() called without any proto files")
+ return()
+ endif()
+ foreach(FIL ${ARGN})
+ set(ABS_FIL ${ROOT_DIR}/${FIL})
+ get_filename_component(FIL_WE ${FIL} NAME_WE)
+ get_filename_component(FIL_DIR ${ABS_FIL} PATH)
+ file(RELATIVE_PATH REL_DIR ${ROOT_DIR} ${FIL_DIR})
+ add_custom_command(
+ TARGET tf_python_copy_scripts_to_destination PRE_LINK
+ COMMAND ${PROTOBUF_PROTOC_EXECUTABLE}
+ ARGS --python_out ${CMAKE_CURRENT_BINARY_DIR}/tf_python/ -I ${ROOT_DIR} -I ${PROTOBUF_INCLUDE_DIRS} ${ABS_FIL}
+ DEPENDS ${ABS_FIL} ${PROTOBUF_PROTOC_EXECUTABLE} protobuf
+ COMMENT "Running Pyton protocol buffer compiler on ${FIL}"
+ VERBATIM )
+ endforeach()
+endfunction()
+
+file(GLOB_RECURSE tf_protos_python_srcs RELATIVE ${tensorflow_source_dir}
+ "${tensorflow_source_dir}/tensorflow/core/*.proto"
+ "${tensorflow_source_dir}/tensorflow/python/*.proto"
+)
+RELATIVE_PROTOBUF_GENERATE_PYTHON(
+ ${tensorflow_source_dir} ${tf_protos_python_srcs}
+)
+
+# tf_python_touchup_modules adds empty __init__.py files to all
+# directories containing Python code, so that Python will recognize
+# them as modules.
+add_custom_target(tf_python_touchup_modules
+ DEPENDS tf_python_copy_scripts_to_destination)
+
+function(add_python_module MODULE_NAME)
+ add_custom_command(TARGET tf_python_touchup_modules PRE_BUILD
+ COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_CURRENT_BINARY_DIR}/tf_python/${MODULE_NAME}")
+ add_custom_command(TARGET tf_python_touchup_modules PRE_BUILD
+ COMMAND ${CMAKE_COMMAND} -E touch "${CMAKE_CURRENT_BINARY_DIR}/tf_python/${MODULE_NAME}/__init__.py")
+endfunction()
+
+add_python_module("tensorflow")
+add_python_module("tensorflow/core")
+add_python_module("tensorflow/core/example")
+add_python_module("tensorflow/core/framework")
+add_python_module("tensorflow/core/lib")
+add_python_module("tensorflow/core/lib/core")
+add_python_module("tensorflow/core/protobuf")
+add_python_module("tensorflow/core/util")
+add_python_module("tensorflow/python")
+add_python_module("tensorflow/python/client")
+add_python_module("tensorflow/python/framework")
+add_python_module("tensorflow/python/ops")
+add_python_module("tensorflow/python/kernel_tests")
+add_python_module("tensorflow/python/lib")
+add_python_module("tensorflow/python/lib/core")
+add_python_module("tensorflow/python/lib/core/io")
+add_python_module("tensorflow/python/platform")
+add_python_module("tensorflow/python/platform/default")
+add_python_module("tensorflow/python/platform/summary")
+add_python_module("tensorflow/python/platform/summary/impl")
+add_python_module("tensorflow/python/tools")
+add_python_module("tensorflow/python/training")
+add_python_module("tensorflow/python/util")
+add_python_module("tensorflow/python/util/protobuf")
+add_python_module("tensorflow/contrib")
+add_python_module("tensorflow/contrib/bayesflow")
+add_python_module("tensorflow/contrib/bayesflow/python")
+add_python_module("tensorflow/contrib/bayesflow/python/ops")
+add_python_module("tensorflow/contrib/bayesflow/python/ops/bernoulli")
+add_python_module("tensorflow/contrib/framework")
+add_python_module("tensorflow/contrib/framework/python")
+add_python_module("tensorflow/contrib/framework/python/framework")
+add_python_module("tensorflow/contrib/layers")
+add_python_module("tensorflow/contrib/layers/python")
+add_python_module("tensorflow/contrib/layers/python/layers")
+add_python_module("tensorflow/contrib/layers/python/ops")
+
+
+
+########################################################
+# tf_python_op_gen_main library
+########################################################
+set(tf_python_op_gen_main_srcs
+ "${tensorflow_source_dir}/tensorflow/python/framework/python_op_gen.cc"
+ "${tensorflow_source_dir}/tensorflow/python/framework/python_op_gen_main.cc"
+ "${tensorflow_source_dir}/tensorflow/python/framework/python_op_gen.h"
+)
+
+add_library(tf_python_op_gen_main OBJECT ${tf_python_op_gen_main_srcs})
+
+add_dependencies(tf_python_op_gen_main tf_core_framework)
+
+target_include_directories(tf_python_op_gen_main PRIVATE
+ ${tensorflow_source_dir}
+ ${eigen_INCLUDE_DIRS}
+)
+
+target_compile_options(tf_python_op_gen_main PRIVATE
+ -fno-exceptions
+ -DEIGEN_AVOID_STL_ARRAY
+)
+
+# C++11
+target_compile_features(tf_python_op_gen_main PRIVATE
+ cxx_rvalue_references
+)
+
+# create directory for ops generated files
+set(python_ops_target_dir ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/ops)
+
+set(tf_python_ops_generated_files)
+
+set(tf_python_op_lib_names
+ ${tf_op_lib_names}
+ "user_ops"
+)
+
+function(GENERATE_PYTHON_OP_LIB tf_python_op_lib_name)
+ set(oneValueArgs DESTINATION)
+ set(multiValueArgs ADDITIONAL_LIBRARIES)
+ cmake_parse_arguments(GENERATE_PYTHON_OP_LIB
+ "" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
+ if(NOT DEFINED GENERATE_PYTHON_OP_LIB_DESTINATION)
+ # Default destination is tf_python/tensorflow/python/ops/gen_<...>.py.
+ set(GENERATE_PYTHON_OP_LIB_DESTINATION
+ "${python_ops_target_dir}/gen_${tf_python_op_lib_name}.py")
+ endif()
+
+ # Create a C++ executable that links in the appropriate op
+ # registrations and generates Python wrapper code based on the
+ # registered ops.
+ add_executable(${tf_python_op_lib_name}_gen_python
+ $<TARGET_OBJECTS:tf_python_op_gen_main>
+ $<TARGET_OBJECTS:tf_${tf_python_op_lib_name}>
+ $<TARGET_OBJECTS:tf_core_lib>
+ $<TARGET_OBJECTS:tf_core_framework>
+ ${GENERATE_PYTHON_OP_LIB_ADDITIONAL_LIBRARIES}
+ )
+ target_include_directories(${tf_python_op_lib_name}_gen_python PRIVATE
+ ${tensorflow_source_dir}
+ ${eigen_INCLUDE_DIRS}
+ )
+ target_link_libraries(${tf_python_op_lib_name}_gen_python PRIVATE
+ ${CMAKE_THREAD_LIBS_INIT}
+ ${PROTOBUF_LIBRARIES}
+ tf_protos_cc
+ re2_lib
+ ${gif_STATIC_LIBRARIES}
+ ${jpeg_STATIC_LIBRARIES}
+ ${png_STATIC_LIBRARIES}
+ ${ZLIB_LIBRARIES}
+ ${jsoncpp_STATIC_LIBRARIES}
+ ${boringssl_STATIC_LIBRARIES}
+ ${CMAKE_DL_LIBS}
+ )
+ target_compile_options(${tf_python_op_lib_name}_gen_python PRIVATE
+ -fno-exceptions
+ -DEIGEN_AVOID_STL_ARRAY
+ -lm
+ )
+ # C++11
+ target_compile_features(${tf_python_op_lib_name}_gen_python PRIVATE
+ cxx_rvalue_references
+ )
+
+ # Use the generated C++ executable to create a Python file
+ # containing the wrappers.
+ add_custom_command(
+ OUTPUT ${GENERATE_PYTHON_OP_LIB_DESTINATION}
+ COMMAND ${tf_python_op_lib_name}_gen_python @${tensorflow_source_dir}/tensorflow/python/ops/hidden_ops.txt 1 > ${GENERATE_PYTHON_OP_LIB_DESTINATION}
+ DEPENDS ${tf_python_op_lib_name}_gen_python
+ )
+
+ set(tf_python_ops_generated_files ${tf_python_ops_generated_files}
+ ${GENERATE_PYTHON_OP_LIB_DESTINATION} PARENT_SCOPE)
+endfunction()
+
+GENERATE_PYTHON_OP_LIB("array_ops")
+GENERATE_PYTHON_OP_LIB("math_ops")
+GENERATE_PYTHON_OP_LIB("functional_ops")
+GENERATE_PYTHON_OP_LIB("candidate_sampling_ops")
+GENERATE_PYTHON_OP_LIB("control_flow_ops"
+ ADDITIONAL_LIBRARIES $<TARGET_OBJECTS:tf_no_op>)
+GENERATE_PYTHON_OP_LIB("ctc_ops")
+GENERATE_PYTHON_OP_LIB("data_flow_ops")
+GENERATE_PYTHON_OP_LIB("image_ops")
+GENERATE_PYTHON_OP_LIB("io_ops")
+GENERATE_PYTHON_OP_LIB("linalg_ops")
+GENERATE_PYTHON_OP_LIB("logging_ops")
+GENERATE_PYTHON_OP_LIB("nn_ops")
+GENERATE_PYTHON_OP_LIB("parsing_ops")
+GENERATE_PYTHON_OP_LIB("random_ops")
+GENERATE_PYTHON_OP_LIB("script_ops")
+GENERATE_PYTHON_OP_LIB("state_ops")
+GENERATE_PYTHON_OP_LIB("sparse_ops")
+GENERATE_PYTHON_OP_LIB("string_ops")
+GENERATE_PYTHON_OP_LIB("user_ops")
+GENERATE_PYTHON_OP_LIB("training_ops"
+ DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/training/gen_training_ops.py)
+
+add_custom_target(tf_python_ops SOURCES ${tf_python_ops_generated_files})
+add_dependencies(tf_python_ops tf_python_op_gen_main)
+
+
+############################################################
+# Build the SWIG-wrapped library for the TensorFlow runtime.
+############################################################
+
+# python_deps is a shared library containing all of the TensorFlow
+# runtime and the standard ops and kernels. These are installed into
+# tf_python/tensorflow/python/.
+# TODO(mrry): Refactor this to expose a framework library that
+# facilitates `tf.load_op_library()`.
+add_library(python_deps SHARED
+ "${tensorflow_source_dir}/tensorflow/python/client/tf_session_helper.h"
+ "${tensorflow_source_dir}/tensorflow/python/client/tf_session_helper.cc"
+ "${tensorflow_source_dir}/tensorflow/python/framework/cpp_shape_inference.h"
+ "${tensorflow_source_dir}/tensorflow/python/framework/cpp_shape_inference.cc"
+ "${tensorflow_source_dir}/tensorflow/python/framework/python_op_gen.h"
+ "${tensorflow_source_dir}/tensorflow/python/framework/python_op_gen.cc"
+ "${tensorflow_source_dir}/tensorflow/python/lib/core/numpy.h"
+ "${tensorflow_source_dir}/tensorflow/python/lib/core/numpy.cc"
+ "${tensorflow_source_dir}/tensorflow/python/lib/core/py_func.h"
+ "${tensorflow_source_dir}/tensorflow/python/lib/core/py_func.cc"
+ "${tensorflow_source_dir}/tensorflow/python/lib/io/py_record_reader.h"
+ "${tensorflow_source_dir}/tensorflow/python/lib/io/py_record_reader.cc"
+ "${tensorflow_source_dir}/tensorflow/python/lib/io/py_record_writer.h"
+ "${tensorflow_source_dir}/tensorflow/python/lib/io/py_record_writer.cc"
+ "${tensorflow_source_dir}/tensorflow/c/c_api.cc"
+ "${tensorflow_source_dir}/tensorflow/c/c_api.h"
+ "${tensorflow_source_dir}/tensorflow/c/checkpoint_reader.cc"
+ "${tensorflow_source_dir}/tensorflow/c/checkpoint_reader.h"
+ "${tensorflow_source_dir}/tensorflow/c/tf_status_helper.cc"
+ "${tensorflow_source_dir}/tensorflow/c/tf_status_helper.h"
+ $<TARGET_OBJECTS:tf_core_lib>
+ $<TARGET_OBJECTS:tf_core_cpu>
+ $<TARGET_OBJECTS:tf_core_framework>
+ $<TARGET_OBJECTS:tf_core_ops>
+ $<TARGET_OBJECTS:tf_core_direct_session>
+ $<TARGET_OBJECTS:tf_core_distributed_runtime>
+ $<TARGET_OBJECTS:tf_core_kernels>
+)
+target_link_libraries(python_deps
+ ${CMAKE_THREAD_LIBS_INIT}
+ tf_protos_cc
+ ${GRPC_LIBRARIES}
+ ${PROTOBUF_LIBRARY}
+ re2_lib
+ ${boringssl_STATIC_LIBRARIES}
+ ${farmhash_STATIC_LIBRARIES}
+ ${gif_STATIC_LIBRARIES}
+ ${jpeg_STATIC_LIBRARIES}
+ ${jsoncpp_STATIC_LIBRARIES}
+ ${png_STATIC_LIBRARIES}
+ ${ZLIB_LIBRARIES}
+ ${CMAKE_DL_LIBS}
+)
+target_include_directories(python_deps PUBLIC
+ ${tensorflow_source_dir}
+ ${CMAKE_CURRENT_BINARY_DIR}
+ ${eigen_INCLUDE_DIRS}
+ ${PYTHON_INCLUDE_DIR}
+ ${NUMPY_INCLUDE_DIR}
+)
+# C++11
+target_compile_features(python_deps PRIVATE
+ cxx_rvalue_references
+)
+set_target_properties(python_deps PROPERTIES
+ LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/tf_python/tensorflow/python)
+
+# _pywrap_tensorflow is the target that generates the SWIG bindings
+# and compiles them as a shared library that depends on python_deps.
+set(CMAKE_SWIG_FLAGS "")
+set(CMAKE_SWIG_OUTDIR ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python)
+SET_SOURCE_FILES_PROPERTIES("${tensorflow_source_dir}/tensorflow/python/tensorflow.i"
+ PROPERTIES CPLUSPLUS ON
+)
+SET_PROPERTY(SOURCE "${tensorflow_source_dir}/tensorflow/python/tensorflow.i"
+ PROPERTY SWIG_FLAGS "-I\"${tensorflow_source_dir}\"" "-module" "pywrap_tensorflow"
+)
+SWIG_ADD_MODULE(pywrap_tensorflow python
+ "${tensorflow_source_dir}/tensorflow/python/tensorflow.i"
+)
+SWIG_LINK_LIBRARIES(pywrap_tensorflow
+ python_deps
+ ${PROTOBUF_LIBRARY}
+ ${CMAKE_DL_LIBS}
+)
+target_include_directories(_pywrap_tensorflow PUBLIC
+ ${tensorflow_source_dir}
+ ${CMAKE_CURRENT_BINARY_DIR}
+ ${eigen_INCLUDE_DIRS}
+ ${PYTHON_INCLUDE_DIR}
+ ${NUMPY_INCLUDE_DIR}
+)
+add_dependencies(_pywrap_tensorflow
+ eigen
+ tf_core_direct_session
+ tf_core_distributed_runtime
+ tf_core_framework
+ python_deps
+ tf_python_copy_scripts_to_destination
+ tf_python_ops
+ tf_python_touchup_modules
+)
+# C++11
+target_compile_features(_pywrap_tensorflow PRIVATE
+ cxx_rvalue_references
+)
+set_target_properties(_pywrap_tensorflow PROPERTIES
+ LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/tf_python/tensorflow/python)
diff --git a/tensorflow/contrib/cmake/tf_tools.cmake b/tensorflow/contrib/cmake/tf_tools.cmake
index 5022dfee0d6..f8bbfb348d7 100644
--- a/tensorflow/contrib/cmake/tf_tools.cmake
+++ b/tensorflow/contrib/cmake/tf_tools.cmake
@@ -33,6 +33,7 @@ target_link_libraries(${proto_text} PUBLIC
add_dependencies(${proto_text}
tf_core_lib
+ protobuf
# jpeg_copy_headers_to_destination
# png_copy_headers_to_destination
# re2_copy_headers_to_destination
diff --git a/tensorflow/core/distributed_runtime/graph_mgr.cc b/tensorflow/core/distributed_runtime/graph_mgr.cc
index 529881e75b7..c603c3c9dc3 100644
--- a/tensorflow/core/distributed_runtime/graph_mgr.cc
+++ b/tensorflow/core/distributed_runtime/graph_mgr.cc
@@ -119,7 +119,7 @@ Status GraphMgr::InitItem(const string& session, const GraphDef& gdef,
mutex_lock l(mu_);
return strings::StrCat(prefix, "_G", next_id_++);
};
- popts.get_incarnation = [this](const string& name) {
+ popts.get_incarnation = [this](const string& name) -> int64 {
Device* device = nullptr;
Status s = worker_env_->device_mgr->LookupDevice(name, &device);
if (s.ok()) {
diff --git a/tensorflow/core/distributed_runtime/master_session.cc b/tensorflow/core/distributed_runtime/master_session.cc
index f2925b6ea75..6d31ae3ab82 100644
--- a/tensorflow/core/distributed_runtime/master_session.cc
+++ b/tensorflow/core/distributed_runtime/master_session.cc
@@ -935,7 +935,7 @@ Status MasterSession::DoRunWithLocalExecution(CallOptions* opts,
mutex_lock l(mu_);
return strings::StrCat(prefix, "_S", next_node_id_++);
};
- popts.get_incarnation = [this](const string& name) {
+ popts.get_incarnation = [this](const string& name) -> int64 {
Device* d = devices_.FindDeviceByName(name);
if (d == nullptr) {
return PartitionOptions::kIllegalIncarnation;
diff --git a/tensorflow/core/distributed_runtime/rpc/BUILD b/tensorflow/core/distributed_runtime/rpc/BUILD
index 5c13c0e6547..21cde1efd4d 100644
--- a/tensorflow/core/distributed_runtime/rpc/BUILD
+++ b/tensorflow/core/distributed_runtime/rpc/BUILD
@@ -144,7 +144,7 @@ cc_library(
],
)
-cc_library(
+tf_cuda_library(
name = "grpc_worker_service",
srcs = ["grpc_worker_service.cc"],
hdrs = ["grpc_worker_service.h"],
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_remote_worker.cc b/tensorflow/core/distributed_runtime/rpc/grpc_remote_worker.cc
index 79d3b3e2f64..60597aab262 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_remote_worker.cc
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_remote_worker.cc
@@ -132,8 +132,8 @@ class GrpcRemoteWorker : public WorkerInterface {
// the RecvTensor response can not have been sent before
// the RecvTensor request, and must have been sent before
// it was received.
- send_start_usec =
- std::max(start_usec, response->metadata().send_start_micros());
+ send_start_usec = std::max(start_usec, static_cast<int64>(
+ response->metadata().send_start_micros()));
send_start_usec = std::min(send_start_usec, end_usec - 1);
}
const string& key = request->rendezvous_key();
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.cc b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.cc
index 9a87bbda1be..e897bfc477b 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.cc
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.cc
@@ -23,7 +23,9 @@ limitations under the License.
#include "tensorflow/core/common_runtime/device.h"
#include "tensorflow/core/common_runtime/device_mgr.h"
#include "tensorflow/core/common_runtime/dma_helper.h"
+#if GOOGLE_CUDA
#include "tensorflow/core/common_runtime/gpu/gpu_util.h"
+#endif // GOOGLE_CUDA
#include "tensorflow/core/common_runtime/local_device.h"
#include "tensorflow/core/common_runtime/process_util.h"
#include "tensorflow/core/common_runtime/step_stats_collector.h"
@@ -428,6 +430,7 @@ class GrpcWorkerService : public AsyncServiceInterface {
{
// Non-DMA cases.
if (src_dev->tensorflow_gpu_device_info() && (!on_host)) {
+#if GOOGLE_CUDA
RecvTensorResponse* tmp = new RecvTensorResponse;
tmp->set_is_dead(is_dead);
CHECK(send_dev_context)
@@ -455,6 +458,10 @@ class GrpcWorkerService : public AsyncServiceInterface {
GPUUtil::SetProtoFromGPU(val, src_dev, send_dev_context,
tmp->mutable_tensor(), is_dead,
response_ready);
+#else
+ call->SendResponse(ToGrpcStatus(
+ errors::Internal("No GPU device in process")));
+#endif // GOOGLE_CUDA
} else {
grpc::EncodeTensorToByteBuffer(is_dead, val, &call->response);
call->SendResponse(ToGrpcStatus(Status::OK()));
diff --git a/tensorflow/core/distributed_runtime/tensor_coding.cc b/tensorflow/core/distributed_runtime/tensor_coding.cc
index 24c2608ef46..d2ddf34fdd5 100644
--- a/tensorflow/core/distributed_runtime/tensor_coding.cc
+++ b/tensorflow/core/distributed_runtime/tensor_coding.cc
@@ -108,7 +108,7 @@ inline WireType GetTagWireType(uint32 tag) {
}
bool ReadVarintSizeAsInt(protobuf::io::CodedInputStream* input, int* result) {
- uint64 v;
+ protobuf::uint64 v;
if (input->ReadVarint64(&v) && v <= static_cast<uint64>(INT_MAX)) {
*result = static_cast<int>(v);
return true;
@@ -237,7 +237,7 @@ bool TensorResponse::ParseFast(Source* source) {
break;
}
case RecvTensorResponse::kSendStartMicrosFieldNumber: {
- uint64 v;
+ protobuf::uint64 v;
if ((wt != WIRETYPE_VARINT) || !input.ReadVarint64(&v)) return false;
meta_.set_send_start_micros(static_cast<int64>(v));
break;
diff --git a/tensorflow/tools/ci_build/Dockerfile.cmake b/tensorflow/tools/ci_build/Dockerfile.cmake
index 4b2e7d3c87f..8a28fe6cdf9 100644
--- a/tensorflow/tools/ci_build/Dockerfile.cmake
+++ b/tensorflow/tools/ci_build/Dockerfile.cmake
@@ -6,7 +6,6 @@ MAINTAINER Shanqing Cai <cais@google.com>
COPY install/*.sh /install/
RUN /install/install_bootstrap_deb_packages.sh
RUN /install/install_deb_packages.sh
-RUN /install/install_proto3_from_source.sh
RUN pip install --upgrade numpy