Java: Scripts to build release artifacts.

Change: 147202107

tensorflow/java/BUILD

@@ -107,6 +107,7 @@ filegroup(
         "//tensorflow:darwin": [":libtensorflow_jni.dylib"],
         "//conditions:default": [":libtensorflow_jni.so"],
     }),
+    visibility = ["//visibility:public"],
 )
 
 LINKER_VERSION_SCRIPT = ":config/version_script.lds"

tensorflow/tools/ci_build/builds/libtensorflow.sh

@@ -14,19 +14,26 @@
 # limitations under the License.
 # ==============================================================================
 #
-# Script to generate a tarball containing the TensorFlow C-library which
-# consists of the C API header file and libtensorflow.so.
+# Script to generate tarballs:
+# (1) The TensorFlow C-library: Containing C API header files and libtensorflow.so
+# (2) Native library for the TensorFlow Java API: Containing libtensorflow_jni.so
+# And jars:
+# (3) Java API .jar
+# (4) Java API sources .jar
 #
-# Work in progress but this is a step towards a "binary" distribution of the
-# TensorFlow C-library allowing TensorFlow language bindings to be used
-# without having to recompile the TensorFlow framework from sources, which
-# takes a while and also introduces many other dependencies.
+# These binary distributions will allow use of TensorFlow in various languages
+# without having to compile the TensorFlow framework from sources, which takes
+# a while and also introduces many other dependencies.
 #
 # Usage:
 # - Source this file in another bash script
 # - Execute build_libtensorflow_tarball SUFFIX
 #
-# Produces: lib_package/libtensorflow${SUFFIX}.tar.gz
+# Produces:
+# - lib_package/libtensorflow${SUFFIX}.tar.gz
+# - lib_package/libtensorflow_jni${SUFFIX}.tar.gz
+# - lib_package/libtensorflow.jar
+# - lib_package/libtensorflow-src.jar
 #
 # ASSUMPTIONS:
 # - build_libtensorflow_tarball is invoked from the root of the git tree.
@@ -52,11 +59,21 @@ function build_libtensorflow_tarball() {
   # and https://github.com/bazelbuild/bazel/issues/1580
   # have been resolved and the "manual" tags on the BUILD targets
   # in tensorflow/tools/lib_package/BUILD are removed.
-  # Till then, must manually run the test.
-  bazel test ${BAZEL_OPTS} //tensorflow/tools/lib_package:libtensorflow_test
+  # Till then, must manually run the test since these tests are
+  # not covered by the continuous integration.
+  bazel test ${BAZEL_OPTS} \
+    //tensorflow/tools/lib_package:libtensorflow_test \
+    //tensorflow/tools/lib_package:libtensorflow_java_test
+
+  bazel build ${BAZEL_OPTS} \
+    //tensorflow/tools/lib_package:libtensorflow.tar.gz \
+    //tensorflow/tools/lib_package:libtensorflow_jni.tar.gz \
+    //tensorflow/java:libtensorflow.jar \
+    //tensorflow/java:libtensorflow-src.jar
 
-  bazel build ${BAZEL_OPTS} //tensorflow/tools/lib_package:libtensorflow.tar.gz
   DIR=lib_package
   mkdir -p ${DIR}
   cp bazel-bin/tensorflow/tools/lib_package/libtensorflow.tar.gz ${DIR}/libtensorflow${TARBALL_SUFFIX}.tar.gz
+  cp bazel-bin/tensorflow/tools/lib_package/libtensorflow_jni.tar.gz ${DIR}/libtensorflow_jni${TARBALL_SUFFIX}.tar.gz
+  cp bazel-bin/tensorflow/java/libtensorflow.jar bazel-bin/tensorflow/java/libtensorflow-src.jar ${DIR}
 }

tensorflow/tools/ci_build/ci_sanity.sh

@@ -371,6 +371,14 @@ do_lib_package_licenses_check() {
     "//tensorflow/tools/lib_package:clicenses_generate"
 }
 
+do_java_package_licenses_check() {
+  echo "Running do_java_package_licenses_check"
+  echo ""
+  do_external_licenses_check \
+    "//tensorflow/java:libtensorflow_jni.so" \
+    "//tensorflow/tools/lib_package:jnilicenses_generate"
+}
+
 # Run bazel build --nobuild to test the validity of the BUILD files
 do_bazel_nobuild() {
   BUILD_TARGET="//tensorflow/..."
@@ -391,8 +399,8 @@ do_bazel_nobuild() {
 }
 
 # Supply all sanity step commands and descriptions
-SANITY_STEPS=("do_pylint PYTHON2" "do_pylint PYTHON3" "do_buildifier" "do_bazel_nobuild" "do_pip_package_licenses_check" "do_lib_package_licenses_check")
-SANITY_STEPS_DESC=("Python 2 pylint" "Python 3 pylint" "buildifier check" "bazel nobuild" "pip: license check for external dependencies" "C library: license check for external dependencies")
+SANITY_STEPS=("do_pylint PYTHON2" "do_pylint PYTHON3" "do_buildifier" "do_bazel_nobuild" "do_pip_package_licenses_check" "do_lib_package_licenses_check" "do_java_package_licenses_check")
+SANITY_STEPS_DESC=("Python 2 pylint" "Python 3 pylint" "buildifier check" "bazel nobuild" "pip: license check for external dependencies" "C library: license check for external dependencies" "Java Native Library: license check for external dependencies")
 
 INCREMENTAL_FLAG=""
 

tensorflow/tools/ci_build/linux/libtensorflow.sh

@@ -14,9 +14,8 @@
 # limitations under the License.
 # ==============================================================================
 #
-# Script to produce a tarball release of the C-library and associated C API
-# header file. Intended to be run inside a docker container. See
-# libtensorflow_docker.sh
+# Script to produce binary releases for libtensorflow (C API, Java jars etc.).
+# Intended to be run inside a docker container. See libtensorflow_docker.sh
 
 set -ex
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

tensorflow/tools/ci_build/linux/libtensorflow_cpu.sh

@@ -14,8 +14,7 @@
 # limitations under the License.
 # ==============================================================================
 #
-# Script to build a binary release tarball for the TensorFlow C-library without
-# GPU support.
+# Script to build a binary releases of libtensorflow without GPU support.
 
 set -ex
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

tensorflow/tools/ci_build/linux/libtensorflow_docker.sh

@@ -14,9 +14,9 @@
 # limitations under the License.
 # ==============================================================================
 #
-# Script to produce a tarball release of the C-library and associated C API
-# header file. Builds a docker container and then builds the C-library in
-# said container.
+# Script to produce a tarball release of the C-library, Java native library
+# and Java .jars.
+# Builds a docker container and then builds in said container.
 #
 # See libtensorflow_cpu.sh and libtensorflow_gpu.sh
 
@@ -29,9 +29,9 @@ DOCKER_IMAGE="tf-libtensorflow-cpu"
 DOCKER_FILE="Dockerfile.cpu"
 DOCKER_BINARY="docker"
 if [ "${TF_NEED_CUDA}" == "1" ]; then
-	DOCKER_IMAGE="tf-tensorflow-gpu"
-	DOCKER_BINARY="nvidia-docker"
-	DOCKER_FILE="Dockerfile.gpu"
+  DOCKER_IMAGE="tf-tensorflow-gpu"
+  DOCKER_BINARY="nvidia-docker"
+  DOCKER_FILE="Dockerfile.gpu"
 fi
 
 docker build \

tensorflow/tools/ci_build/linux/libtensorflow_gpu.sh

@@ -14,8 +14,7 @@
 # limitations under the License.
 # ==============================================================================
 #
-# Script to build a binary release tarball for the TensorFlow C-library without
-# GPU support.
+# Script to build a binary releases of libtensorflow with GPU support.
 
 set -ex
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

tensorflow/tools/ci_build/osx/libtensorflow_cpu.sh

@@ -14,9 +14,7 @@
 # limitations under the License.
 # ==============================================================================
 #
-# Script to produce a tarball release of the C-library and associated C API
-# header file.
-# Produces: lib_package/libtensorflow-gpu-darwin-x86_64.tar.gz
+# Script to produce binary release of libtensorflow (C API, Java jars etc.).
 
 set -ex
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

tensorflow/tools/ci_build/osx/libtensorflow_gpu.sh

@@ -14,9 +14,7 @@
 # limitations under the License.
 # ==============================================================================
 #
-# Script to produce a tarball release of the C-library and associated C API
-# header file.
-# Produces: lib_package/libtensorflow-gpu-darwin-x86_64.tar.gz
+# Script to produce binary release of libtensorflow (C API, Java jars etc.).
 
 set -ex
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

tensorflow/tools/lib_package/BUILD

@@ -1,7 +1,6 @@
-# Packaging the TensorFlow C API into a small, standalone archive for use with
-# language bindings and installations without Python.
+# Packaging the TensorFlow C API and Java libraries into standalone archives
+# for use with language bindings and installations without Python.
 #
-# TODO(ashankar): Something similar for the JNI library for Java?
 # TODO(ashankar): Something similar for the C++ API (caveat: ABI compatibility)
 
 package(default_visibility = ["//visibility:private"])
@@ -24,6 +23,21 @@ pkg_tar(
     ],
 )
 
+pkg_tar(
+    name = "libtensorflow_jni",
+    extension = "tar.gz",
+    files = [
+        "include/tensorflow/jni/LICENSE",
+        "//tensorflow/java:libtensorflow_jni",
+    ],
+    # Mark as "manual" till
+    # https://github.com/bazelbuild/bazel/issues/2352
+    # and https://github.com/bazelbuild/bazel/issues/1580
+    # are resolved, otherwise these rules break when built
+    # with Python 3.
+    tags = ["manual"],
+)
+
 pkg_tar(
     name = "cheaders",
     files = ["//tensorflow/c:headers"],
@@ -88,6 +102,34 @@ genrule(
     tools = [":concat_licenses.sh"],
 )
 
+genrule(
+    name = "jnilicenses_generate",
+    srcs = [
+        "//third_party/hadoop:LICENSE.txt",
+        "//third_party/eigen3:LICENSE",
+        "@boringssl//:LICENSE",
+        "@com_googlesource_code_re2//:LICENSE",
+        "@curl//:COPYING",
+        "@eigen_archive//:COPYING.MPL2",
+        "@farmhash_archive//:COPYING",
+        "@gemmlowp//:LICENSE",
+        "@gif_archive//:COPYING",
+        "@grpc//:LICENSE",
+        "@highwayhash//:LICENSE",
+        "@jemalloc//:COPYING",
+        "@jpeg//:LICENSE.md",
+        "@libxsmm_archive//:LICENSE",
+        "@local_config_sycl//sycl:LICENSE.text",
+        "@nanopb_git//:LICENSE.txt",
+        "@png_archive//:LICENSE",
+        "@protobuf//:LICENSE",
+        "@zlib_archive//:zlib.h",
+    ],
+    outs = ["include/tensorflow/jni/LICENSE"],
+    cmd = "$(location :concat_licenses.sh) $(SRCS) >$@",
+    tools = [":concat_licenses.sh"],
+)
+
 sh_test(
     name = "libtensorflow_test",
     size = "small",
@@ -105,3 +147,22 @@ sh_test(
     # the release by tensorflow/tools/ci_build/builds/libtensorflow.sh
     tags = ["manual"],
 )
+
+sh_test(
+    name = "libtensorflow_java_test",
+    size = "small",
+    srcs = ["libtensorflow_java_test.sh"],
+    data = [
+        ":LibTensorFlowTest.java",
+        ":libtensorflow_jni.tar.gz",
+        "//tensorflow/java:libtensorflow.jar",
+    ],
+    # Mark as "manual" till
+    # https://github.com/bazelbuild/bazel/issues/2352
+    # and https://github.com/bazelbuild/bazel/issues/1580
+    # are resolved, otherwise these rules break when built
+    # with Python 3.
+    # Till then, this test is explicitly executed when building
+    # the release by tensorflow/tools/ci_build/builds/libtensorflow.sh
+    tags = ["manual"],
+)

tensorflow/tools/lib_package/LibTensorFlowTest.java

@@ -0,0 +1,24 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// Companion source file for libtensorflow_java_test.sh
+
+import org.tensorflow.TensorFlow;
+
+public class LibTensorFlowTest {
+  public static void main(String[] args) {
+    System.out.println(TensorFlow.version());
+  }
+}

tensorflow/tools/lib_package/libtensorflow_java_test.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+set -ex
+
+# Sanity test for the binary artifacts for the TensorFlow Java API.
+# - Unarchive
+# - Compile a trivial Java file that excercises the Java API and underlying
+#   native library.
+# - Run it
+
+# Tools needed: java, javac, tar
+JAVA="${JAVA}"
+JAVAC="${JAVAC}"
+TAR="${TAR}"
+
+[ -z "${JAVA}" ] && JAVA="java"
+[ -z "${JAVAC}" ] && JAVAC="javac"
+[ -z "${TAR}"] && TAR="tar"
+
+# bazel tests run with ${PWD} set to the root of the bazel workspace
+TARFILE="${PWD}/tensorflow/tools/lib_package/libtensorflow_jni.tar.gz"
+JAVAFILE="${PWD}/tensorflow/tools/lib_package/LibTensorFlowTest.java"
+JARFILE="${PWD}/tensorflow/java/libtensorflow.jar"
+
+cd ${TEST_TMPDIR}
+
+# Extract the archive into a subdirectory 'jni'
+mkdir jni
+${TAR} -xzf ${TARFILE} -Cjni
+
+# Compile and run the .java file
+${JAVAC} -cp ${JARFILE} -d . ${JAVAFILE}
+OUTPUT=$(${JAVA} \
+  -cp "${JARFILE}:." \
+  -Djava.library.path=jni \
+  LibTensorFlowTest)
+if [ -z "${OUTPUT}" ]
+then
+  echo "Empty output, expecting version number"
+  exit 1
+fi