Automated rollback of change 144673014

Change: 144834404

tensorflow/c/BUILD

@@ -20,6 +20,12 @@ load(
 # -----------------------------------------------------------------------------
 # Public targets
 
+filegroup(
+    name = "headers",
+    srcs = ["c_api.h"],
+    visibility = ["//tensorflow:__subpackages__"],
+)
+
 tf_cuda_library(
     name = "c_api",
     srcs = ["c_api.cc"],

tensorflow/tools/ci_build/builds/libtensorflow.sh

@@ -46,20 +46,17 @@ function build_libtensorflow_tarball() {
   fi
   bazel clean --expunge
   yes "" | ./configure
-  
-  # TODO(ashankar): Once 
-  # https://github.com/tensorflow/tensorflow/commit/1b32b698eddc10c0d85b0b8cf838f42023394de7  
-  # can be undone, i.e., when bazel supports pkg_tar with python3+ then all of this below
-  # can be replaced with something like:
-  # bazel build ${BAZEL_OPTS} //tensorflow/tools/lib_package:libtensorflow.tar.gz
-  
-  bazel build ${BAZEL_OPTS} //tensorflow:libtensorflow.so
+
+  # Remove this test call when
+  # https://github.com/bazelbuild/bazel/issues/2352
+  # and https://github.com/bazelbuild/bazel/issues/1580
+  # have been resolved and the "manual" tags on the BUILD targets
+  # in tensorflow/tools/lib_package/BUILD are removed.
+  # Till then, must manually run the test.
+  bazel test ${BAZEL_OPTS} //tensorflow/tools/lib_package/...
+
+  bazel build ${BAZEL_OPTS} //tensorflow/tools/lib_package:libtensorflow.tar.gz
   DIR=lib_package
-  rm -rf ${DIR}
-  mkdir -p ${DIR}/build/lib
-  mkdir -p ${DIR}/build/include/tensorflow/c
-  cp bazel-bin/tensorflow/libtensorflow.so ${DIR}/build/lib
-  cp tensorflow/c/c_api.h ${DIR}/build/include/tensorflow/c
-  tar -C ${DIR}/build -cvf ${DIR}/libtensorflow${TARBALL_SUFFIX}.tar.gz include/tensorflow/c/c_api.h lib/libtensorflow.so
-  rm -rf ${DIR}/build
+  mkdir -p ${DIR}
+  cp bazel-bin/tensorflow/tools/lib_package/libtensorflow.tar.gz ${DIR}/libtensorflow${TARBALL_SUFFIX}.tar.gz
 }

tensorflow/tools/ci_build/ci_sanity.sh

@@ -292,8 +292,8 @@ do_buildifier(){
 }
 
 do_external_licenses_check(){
-  echo "Running do_external_licenses_check"
-  echo ""
+  BUILD_TARGET="$1"
+  LICENSES_TARGET="$2"
 
   EXTERNAL_LICENSES_CHECK_START_TIME=$(date +'%s')
 
@@ -302,8 +302,8 @@ do_external_licenses_check(){
   MISSING_LICENSES_FILE="$(mktemp)_missing_licenses.log"
   EXTRA_LICENSES_FILE="$(mktemp)_extra_licenses.log"
 
-  echo "Getting external dependencies for //tensorflow/tools/pip_package:build_pip_package."
- bazel query 'attr("licenses", "notice", deps(//tensorflow/tools/pip_package:build_pip_package))' --no_implicit_deps --no_host_deps --keep_going \
+  echo "Getting external dependencies for ${BUILD_TARGET}"
+ bazel query "attr('licenses', 'notice', deps(${BUILD_TARGET}))" --no_implicit_deps --no_host_deps --keep_going \
   | egrep -v "^//tensorflow" \
   | sed -e 's|:.*||' \
   | sort \
@@ -311,8 +311,8 @@ do_external_licenses_check(){
   | tee ${EXTERNAL_DEPENDENCIES_FILE}
 
   echo
-  echo "Getting list of external licenses."
-  bazel query 'deps(//tensorflow/tools/pip_package:licenses)' --no_implicit_deps --no_host_deps --keep_going \
+  echo "Getting list of external licenses mentioned in ${LICENSES_TARGET}."
+  bazel query "deps(${LICENSES_TARGET})" --no_implicit_deps --no_host_deps --keep_going \
   | egrep -v "^//tensorflow" \
   | sed -e 's|:.*||' \
   | sort \
@@ -331,7 +331,7 @@ do_external_licenses_check(){
   echo
 
   if [[ -s ${MISSING_LICENSES_FILE} ]] || [[ -s ${EXTRA_LICENSES_FILE} ]] ; then
-    echo "FAIL: pip package external dependencies vs licenses mismatch."
+    echo "FAIL: mismatch in packaged licenses and external dependencies"
     if [[ -s ${MISSING_LICENSES_FILE} ]] ; then
       echo "Missing the licenses for the following external dependencies:"
       cat ${MISSING_LICENSES_FILE}
@@ -355,6 +355,21 @@ do_external_licenses_check(){
   fi
 }
 
+do_pip_package_licenses_check() {
+  echo "Running do_pip_package_licenses_check"
+  echo ""
+  do_external_licenses_check \
+    "//tensorflow/tools/pip_package:build_pip_package" \
+    "//tensorflow/tools/pip_package:licenses"
+}
+
+do_lib_package_licenses_check() {
+  echo "Running do_lib_package_licenses_check"
+  echo ""
+  do_external_licenses_check \
+    "//tensorflow:libtensorflow.so" \
+    "//tensorflow/tools/lib_package:clicenses_generate"
+}
 
 # Run bazel build --nobuild to test the validity of the BUILD files
 do_bazel_nobuild() {
@@ -376,8 +391,8 @@ do_bazel_nobuild() {
 }
 
 # Supply all sanity step commands and descriptions
-SANITY_STEPS=("do_pylint PYTHON2" "do_pylint PYTHON3" "do_buildifier" "do_bazel_nobuild" "do_external_licenses_check")
-SANITY_STEPS_DESC=("Python 2 pylint" "Python 3 pylint" "buildifier check" "bazel nobuild" "external dependencies licenses check")
+SANITY_STEPS=("do_pylint PYTHON2" "do_pylint PYTHON3" "do_buildifier" "do_bazel_nobuild" "do_pip_package_licenses_check" "do_lib_package_licenses_check")
+SANITY_STEPS_DESC=("Python 2 pylint" "Python 3 pylint" "buildifier check" "bazel nobuild" "pip: license check for external dependencies" "C library: license check for external dependencies")
 
 INCREMENTAL_FLAG=""
 

tensorflow/tools/lib_package/BUILD

@@ -0,0 +1,107 @@
+# Packaging the TensorFlow C API into a small, standalone archive for use with
+# language bindings and installations without Python.
+#
+# TODO(ashankar): Something similar for the JNI library for Java?
+# TODO(ashankar): Something similar for the C++ API (caveat: ABI compatibility)
+
+package(default_visibility = ["//visibility:private"])
+
+load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar")
+
+pkg_tar(
+    name = "libtensorflow",
+    extension = "tar.gz",
+    # Mark as "manual" till
+    # https://github.com/bazelbuild/bazel/issues/2352
+    # and https://github.com/bazelbuild/bazel/issues/1580
+    # are resolved, otherwise these rules break when built
+    # with Python 3.
+    tags = ["manual"],
+    deps = [
+        ":cheaders",
+        ":clib",
+        ":clicenses",
+    ],
+)
+
+pkg_tar(
+    name = "cheaders",
+    files = ["//tensorflow/c:headers"],
+    package_dir = "include/tensorflow/c",
+    # Mark as "manual" till
+    # https://github.com/bazelbuild/bazel/issues/2352
+    # and https://github.com/bazelbuild/bazel/issues/1580
+    # are resolved, otherwise these rules break when built
+    # with Python 3.
+    tags = ["manual"],
+)
+
+pkg_tar(
+    name = "clib",
+    files = ["//tensorflow:libtensorflow.so"],
+    package_dir = "lib",
+    # Mark as "manual" till
+    # https://github.com/bazelbuild/bazel/issues/2352
+    # and https://github.com/bazelbuild/bazel/issues/1580
+    # are resolved, otherwise these rules break when built
+    # with Python 3.
+    tags = ["manual"],
+)
+
+pkg_tar(
+    name = "clicenses",
+    files = [":include/tensorflow/c/LICENSE"],
+    package_dir = "include/tensorflow/c",
+    # Mark as "manual" till
+    # https://github.com/bazelbuild/bazel/issues/2352
+    # and https://github.com/bazelbuild/bazel/issues/1580
+    # are resolved, otherwise these rules break when built
+    # with Python 3.
+    tags = ["manual"],
+)
+
+genrule(
+    name = "clicenses_generate",
+    srcs = [
+        "//third_party/hadoop:LICENSE.txt",
+        "//third_party/eigen3:LICENSE",
+        "@boringssl//:LICENSE",
+        "@com_googlesource_code_re2//:LICENSE",
+        "@curl//:COPYING",
+        "@eigen_archive//:COPYING.MPL2",
+        "@farmhash_archive//:COPYING",
+        "@gemmlowp//:LICENSE",
+        "@gif_archive//:COPYING",
+        "@grpc//:LICENSE",
+        "@highwayhash//:LICENSE",
+        "@jemalloc//:COPYING",
+        "@jpeg//:LICENSE.md",
+        "@libxsmm_archive//:LICENSE",
+        "@local_config_sycl//sycl:LICENSE.text",
+        "@nanopb_git//:LICENSE.txt",
+        "@png_archive//:LICENSE",
+        "@protobuf//:LICENSE",
+        "@zlib_archive//:zlib.h",
+    ],
+    outs = ["include/tensorflow/c/LICENSE"],
+    cmd = "$(location :concat_licenses.sh) $(SRCS) >$@",
+    tools = [":concat_licenses.sh"],
+)
+
+sh_test(
+    name = "libtensorflow_test",
+    size = "small",
+    srcs = ["libtensorflow_test.sh"],
+    data = [
+        "libtensorflow_test.c",
+        ":libtensorflow.tar.gz",
+    ],
+    # Mark as "manual" till
+    # https://github.com/bazelbuild/bazel/issues/2352
+    # and https://github.com/bazelbuild/bazel/issues/1580
+    # are resolved, otherwise these rules break when built
+    # with Python 3.
+    # Till then, this test is explicitly executed when building
+    # the release by tensorflow/tools/ci_build/builds/libtensorflow.sh
+    tags = ["manual"],
+)

tensorflow/tools/lib_package/README.md

@@ -0,0 +1,31 @@
+Bazel rules to package the TensorFlow C-library and [header
+files](https://www.tensorflow.org/code/tensorflow/c/c_api.h)
+into an archive.
+
+## TensorFlow C library
+
+The TensorFlow [C
+API](https://www.tensorflow.org/code/tensorflow/c/c_api.h)
+is typically a requirement of TensorFlow APIs in other languages such as
+[Go](https://www.tensorflow.org/code/tensorflow/go)
+and [Rust](https://github.com/tensorflow/rust).
+
+The command:
+
+```sh
+bazel build -c opt //tensorflow/tools/lib_package:libtensorflow
+```
+
+produces `bazel-bin/tensorflow/tools/lib_package/libtensorflow.tar.gz`, which
+can be distributed and installed using something like:
+
+```sh
+tar -C /usr/local -xzf libtensorflow.tar.gz
+```
+
+## Release
+
+Scripts to generate archives using these rules for release are in
+[tensorflow/tools/ci_build/linux](https://www.tensorflow.org/code/tensorflow/tools/ci_build/linux)
+and
+[tensorflow/tools/ci_build/osx](https://www.tensorflow.org/code/tensorflow/tools/ci_build/osx)

tensorflow/tools/lib_package/concat_licenses.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+#
+# Script aimed to combining multiple license files into a single one.
+
+for f in $@
+do
+  echo "--------------------------------------------------------------------------------"
+  echo "BEGIN LICENSE FOR $f"
+  echo "--------------------------------------------------------------------------------"
+  cat $f
+  echo "--------------------------------------------------------------------------------"
+  echo "END LICENSE FOR $f"
+  echo "--------------------------------------------------------------------------------"
+done

tensorflow/tools/lib_package/libtensorflow_test.c

@@ -0,0 +1,28 @@
+/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// Companion source file for libtensorflow_test.sh
+
+#include <tensorflow/c/c_api.h>
+
+int main() {
+  TF_Status* s = TF_NewStatus();
+  TF_SetStatus(s, TF_UNKNOWN, "Some error");
+  if (TF_GetCode(s) != TF_UNKNOWN) {
+    return 1;
+  }
+  TF_DeleteStatus(s);
+  return 0;
+}

tensorflow/tools/lib_package/libtensorflow_test.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+set -ex
+
+# Sanity test for the package C-library archive.
+# - Unarchive
+# - Compile a trivial C file that uses the archive
+# - Run it
+
+# Tools needed: A C-compiler and tar
+CC="${CC}"
+TAR="${TAR}"
+
+[ -z "${CC}" ] && CC="/usr/bin/gcc"
+[ -z "${TAR}"] && TAR="tar"
+
+# bazel tests run with ${PWD} set to the root of the bazel workspace
+TARFILE="${PWD}/tensorflow/tools/lib_package/libtensorflow.tar.gz"
+CFILE="${PWD}/tensorflow/tools/lib_package/libtensorflow_test.c"
+
+cd ${TEST_TMPDIR}
+
+# Extract the archive into tensorflow/
+mkdir tensorflow
+${TAR} -xzf ${TARFILE} -Ctensorflow
+
+# Compile the test .c file
+${CC} ${CFILE} -Itensorflow/include -Ltensorflow/lib -ltensorflow -oa.out
+
+# Execute it, with the shared library available.
+# DYLD_LIBRARY_PATH is used on OS X, LD_LIBRARY_PATH on Linux
+export DYLD_LIBRARY_PATH=tensorflow/lib
+export LD_LIBRARY_PATH=tensorflow/lib
+./a.out