Automatically build SWIG from source

This change allows Bazel to fetch and build SWIG rather than getting it
from the system. This change also improves the i/o performance of the
SWIG build, makes it hermetically sealed, and ensures tf_py_wrap_cc()
can function correctly across Bazel repositories.

CC: 
Change: 136783531
This commit is contained in:
Justine Tunney 2016-10-20 16:48:45 -08:00 committed by TensorFlower Gardener
parent 2da2ae76cf
commit 1dba78b997
12 changed files with 471 additions and 84 deletions

14
configure vendored
View File

@ -107,20 +107,6 @@ else
perl -pi -e "s,WITH_HDFS_SUPPORT = (False|True),WITH_HDFS_SUPPORT = False,s" tensorflow/core/platform/default/build_config.bzl
fi
## Find swig path
if [ -z "$SWIG_PATH" ]; then
SWIG_PATH=`type -p swig 2> /dev/null || true`
fi
if [[ ! -e "$SWIG_PATH" ]]; then
echo "Can't find swig. Ensure swig is in \$PATH or set \$SWIG_PATH."
exit 1
fi
# Convert swig path to Windows style before writing into swig_path
if is_windows; then
SWIG_PATH="$(cygpath -m "$SWIG_PATH")"
fi
echo "$SWIG_PATH" > tensorflow/tools/swig/swig_path
# Invoke python_config and set up symlinks to python includes
./util/python/python_config.sh --setup "$PYTHON_BIN_PATH"

View File

@ -63,12 +63,6 @@ package_group(
packages = ["//tensorflow/..."],
)
sh_binary(
name = "swig",
srcs = ["tools/swig/swig.sh"],
data = glob(["tools/swig/**"]),
)
filegroup(
name = "all_files",
srcs = glob(

View File

@ -568,9 +568,9 @@ binary path.
```bash
# For Python 2.7:
$ sudo apt-get install python-numpy swig python-dev python-wheel
$ sudo apt-get install python-numpy python-dev python-wheel
# For Python 3.x:
$ sudo apt-get install python3-numpy swig python3-dev python3-wheel
$ sudo apt-get install python3-numpy python3-dev python3-wheel
```
#### Optional: Install CUDA (GPUs on Linux)
@ -617,20 +617,16 @@ sudo chmod a+r /usr/local/cuda/include/cudnn.h /usr/local/cuda/lib64/libcudnn*
### Prepare environment for Mac OS X
We recommend using [homebrew](http://brew.sh) to install the bazel and SWIG
dependencies, and installing python dependencies using easy_install or pip.
Of course you can also install Swig from source without using homebrew. In that
case, be sure to install its dependency [PCRE](http://www.pcre.org) and not
PCRE2.
We recommend using [homebrew](http://brew.sh) to install the bazel dependency,
and installing python dependencies using easy_install or pip.
#### Dependencies
Follow instructions [here](http://bazel.io/docs/install.html) to install the
dependencies for bazel. You can then use homebrew to install bazel and SWIG:
dependencies for bazel. You can then use homebrew to install bazel:
```bash
$ brew install bazel swig
$ brew install bazel
```
You can install the python dependencies using easy_install or pip. Using

View File

@ -275,5 +275,8 @@ string ReadFromStream(tensorflow::io::BufferedInputStream* stream,
%unignoreall
%include "tensorflow/c/tf_status_helper.h"
%ignore tensorflow::io::internal::JoinPathImpl;
%include "tensorflow/core/lib/io/path.h"
%include "tensorflow/core/platform/file_statistics.h"
%include "tensorflow/core/platform/file_statistics.h"

View File

@ -557,33 +557,30 @@ def _py_wrap_cc_impl(ctx):
if len(srcs) != 1:
fail("Exactly one SWIG source file label must be specified.", "srcs")
module_name = ctx.attr.module_name
cc_out = ctx.outputs.cc_out
py_out = ctx.outputs.py_out
src = ctx.files.srcs[0]
args = ["-c++", "-python"]
args += ["-module", module_name]
args += ["-l" + f.path for f in ctx.files.swig_includes]
cc_include_dirs = set()
cc_includes = set()
inputs = set([src])
inputs += ctx.files.swig_includes
for dep in ctx.attr.deps:
cc_include_dirs += [h.dirname for h in dep.cc.transitive_headers]
cc_includes += dep.cc.transitive_headers
args += ["-I" + x for x in cc_include_dirs]
args += ["-I" + ctx.label.workspace_root]
args += ["-o", cc_out.path]
args += ["-outdir", py_out.dirname]
inputs += dep.cc.transitive_headers
inputs += ctx.files._swiglib
swig_include_dirs = set([f.root.path for f in inputs if f.root.path])
swig_include_dirs += sorted([f.dirname for f in ctx.files._swiglib])
args = ["-c++",
"-python",
"-module", module_name,
"-o", ctx.outputs.cc_out.path,
"-outdir", ctx.outputs.py_out.dirname]
args += ["-l" + f.path for f in ctx.files.swig_includes]
args += ["-I" + i for i in swig_include_dirs]
args += [src.path]
outputs = [cc_out, py_out]
# TODO(pcloudy): Move args to arguments after
# https://github.com/bazelbuild/bazel/issues/1926 is fixed
ctx.action(command=" ".join(["tensorflow/tools/swig/swig.sh"] + args),
arguments=[],
mnemonic="PythonSwig",
inputs=sorted(set([src]) + cc_includes + ctx.files.swig_includes +
ctx.attr.swig_deps.files),
outputs = [ctx.outputs.cc_out,
ctx.outputs.py_out]
ctx.action(executable=ctx.executable._swig,
arguments=args,
inputs=list(inputs),
outputs=outputs,
use_default_shell_env=True,
progress_message="SWIGing {input}".format(input=src.path))
mnemonic="PythonSwig",
progress_message="SWIGing " + src.path)
return struct(files=set(outputs))
_py_wrap_cc = rule(
@ -600,11 +597,17 @@ _py_wrap_cc = rule(
allow_files = True,
providers = ["cc"],
),
"swig_deps": attr.label(default = Label(
"//tensorflow:swig", # swig_templates
)),
"module_name": attr.string(mandatory = True),
"py_module_name": attr.string(mandatory = True),
"_swig": attr.label(
default = Label("@swig//:swig"),
executable = True,
cfg = "host",
),
"_swiglib": attr.label(
default = Label("@swig//:templates"),
allow_files = True,
),
},
outputs = {
"cc_out": "%{module_name}.cc",

View File

@ -14,7 +14,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
python-dev \
rsync \
software-properties-common \
swig \
unzip \
zip \
zlib1g-dev \

View File

@ -15,7 +15,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
python-dev \
rsync \
software-properties-common \
swig \
unzip \
zip \
zlib1g-dev \

View File

@ -1,25 +0,0 @@
#!/usr/bin/env bash
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
# If possible, read swig path out of "swig_path" generated by configure
SWIG=swig
SWIG_PATH=tensorflow/tools/swig/swig_path
if [ -e $SWIG_PATH ]; then
SWIG=`cat $SWIG_PATH`
fi
# If this line fails, rerun configure to set the path to swig correctly
"$SWIG" "$@"

View File

@ -126,6 +126,22 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
actual = str(Label("//util/python:python_headers")),
)
native.new_http_archive(
name = "pcre",
sha256 = "ccdf7e788769838f8285b3ee672ed573358202305ee361cfec7a4a4fb005bbc7",
url = "http://ftp.cs.stanford.edu/pub/exim/pcre/pcre-8.39.tar.gz",
strip_prefix = "pcre-8.39",
build_file = str(Label("//third_party:pcre.BUILD")),
)
native.new_http_archive(
name = "swig",
sha256 = "a2669657cabcedc371f63c0457407a183e0b6b2ef4e7e303c1ec9a3964cc7813",
url = "http://ufpr.dl.sourceforge.net/project/swig/swig/swig-3.0.2/swig-3.0.2.tar.gz",
strip_prefix = "swig-3.0.2",
build_file = str(Label("//third_party:swig.BUILD")),
)
# grpc expects //external:protobuf_clib and //external:protobuf_compiler
# to point to the protobuf's compiler library.
native.bind(

1
third_party/BUILD vendored Normal file
View File

@ -0,0 +1 @@
licenses(["notice"]) # Apache 2.0

80
third_party/pcre.BUILD vendored Normal file
View File

@ -0,0 +1,80 @@
licenses(["notice"]) # BSD
exports_files(["LICENSE"])
cc_library(
name = "pcre",
srcs = [
"pcre_byte_order.c",
"pcre_chartables.c",
"pcre_compile.c",
"pcre_config.c",
"pcre_dfa_exec.c",
"pcre_exec.c",
"pcre_fullinfo.c",
"pcre_get.c",
"pcre_globals.c",
"pcre_internal.h",
"pcre_jit_compile.c",
"pcre_maketables.c",
"pcre_newline.c",
"pcre_ord2utf8.c",
"pcre_refcount.c",
"pcre_string_utils.c",
"pcre_study.c",
"pcre_tables.c",
"pcre_ucd.c",
"pcre_valid_utf8.c",
"pcre_version.c",
"pcre_xclass.c",
"ucp.h",
],
hdrs = [
"pcre.h",
"pcreposix.h",
],
copts = [
"-DHAVE_BCOPY=1",
"-DHAVE_INTTYPES_H=1",
"-DHAVE_MEMMOVE=1",
"-DHAVE_STDINT_H=1",
"-DHAVE_STRERROR=1",
"-DHAVE_SYS_STAT_H=1",
"-DHAVE_SYS_TYPES_H=1",
"-DHAVE_UNISTD_H=1",
"-DLINK_SIZE=2",
"-DMATCH_LIMIT=10000000",
"-DMATCH_LIMIT_RECURSION=1000",
"-DMAX_NAME_COUNT=10000",
"-DMAX_NAME_SIZE=32",
"-DNEWLINE=10",
"-DNO_RECURSE",
"-DPARENS_NEST_LIMIT=50",
"-DPCRE_STATIC=1",
"-DPOSIX_MALLOC_THRESHOLD=10",
"-DSTDC_HEADERS=1",
"-DSUPPORT_UCP",
"-DSUPPORT_UTF",
],
includes = ["."],
visibility = ["@swig//:__pkg__"], # Please use RE2
alwayslink = 1,
)
genrule(
name = "pcre_h",
srcs = ["pcre.h.in"],
outs = ["pcre.h"],
cmd = "sed -e s/@PCRE_MAJOR@/8/" +
" -e s/@PCRE_MINOR@/39/" +
" -e s/@PCRE_PRERELEASE@//" +
" -e s/@PCRE_DATE@/redacted/" +
" $< >$@",
)
genrule(
name = "pcre_chartables_c",
srcs = ["pcre_chartables.c.dist"],
outs = ["pcre_chartables.c"],
cmd = "cp $< $@",
)

335
third_party/swig.BUILD vendored Normal file
View File

@ -0,0 +1,335 @@
licenses(["restricted"]) # GPLv3
exports_files(["LICENSE"])
cc_binary(
name = "swig",
srcs = [
"Source/CParse/cparse.h",
"Source/CParse/cscanner.c",
"Source/CParse/parser.c",
"Source/CParse/parser.h",
"Source/CParse/templ.c",
"Source/CParse/util.c",
"Source/DOH/base.c",
"Source/DOH/doh.h",
"Source/DOH/dohint.h",
"Source/DOH/file.c",
"Source/DOH/fio.c",
"Source/DOH/hash.c",
"Source/DOH/list.c",
"Source/DOH/memory.c",
"Source/DOH/string.c",
"Source/DOH/void.c",
"Source/Include/swigconfig.h",
"Source/Include/swigwarn.h",
"Source/Modules/allocate.cxx",
"Source/Modules/browser.cxx",
"Source/Modules/contract.cxx",
"Source/Modules/directors.cxx",
"Source/Modules/emit.cxx",
"Source/Modules/lang.cxx",
"Source/Modules/main.cxx",
"Source/Modules/module.cxx",
"Source/Modules/nested.cxx",
"Source/Modules/overload.cxx",
"Source/Modules/python.cxx",
"Source/Modules/swigmain-lite.cxx",
"Source/Modules/swigmod.h",
"Source/Modules/typepass.cxx",
"Source/Modules/uffi.cxx",
"Source/Modules/utils.cxx",
"Source/Modules/xml.cxx",
"Source/Preprocessor/cpp.c",
"Source/Preprocessor/expr.c",
"Source/Preprocessor/preprocessor.h",
"Source/Swig/cwrap.c",
"Source/Swig/deprecate.c",
"Source/Swig/error.c",
"Source/Swig/extend.c",
"Source/Swig/fragment.c",
"Source/Swig/getopt.c",
"Source/Swig/include.c",
"Source/Swig/misc.c",
"Source/Swig/naming.c",
"Source/Swig/parms.c",
"Source/Swig/scanner.c",
"Source/Swig/stype.c",
"Source/Swig/swig.h",
"Source/Swig/swigfile.h",
"Source/Swig/swigopt.h",
"Source/Swig/swigparm.h",
"Source/Swig/swigscan.h",
"Source/Swig/swigtree.h",
"Source/Swig/swigwrap.h",
"Source/Swig/symbol.c",
"Source/Swig/tree.c",
"Source/Swig/typemap.c",
"Source/Swig/typeobj.c",
"Source/Swig/typesys.c",
"Source/Swig/wrapfunc.c",
],
copts = ["$(STACK_FRAME_UNLIMITED)"] + select({
":x64_windows_msvc": [],
"//conditions:default": [
"-Wno-parentheses",
"-Wno-unused-variable",
"-fexceptions",
],
}),
data = [":templates"],
includes = [
"Source/CParse",
"Source/DOH",
"Source/Include",
"Source/Modules",
"Source/Preprocessor",
"Source/Swig",
],
output_licenses = ["unencumbered"],
visibility = ["//visibility:public"],
deps = ["@pcre//:pcre"],
)
filegroup(
name = "templates",
srcs = [
"Lib/allkw.swg",
"Lib/attribute.i",
"Lib/carrays.i",
"Lib/cdata.i",
"Lib/cffi/cffi.swg",
"Lib/cmalloc.i",
"Lib/constraints.i",
"Lib/cpointer.i",
"Lib/cstring.i",
"Lib/cwstring.i",
"Lib/exception.i",
"Lib/intrusive_ptr.i",
"Lib/inttypes.i",
"Lib/linkruntime.c",
"Lib/math.i",
"Lib/pointer.i",
"Lib/python/argcargv.i",
"Lib/python/attribute.i",
"Lib/python/boost_shared_ptr.i",
"Lib/python/builtin.swg",
"Lib/python/carrays.i",
"Lib/python/ccomplex.i",
"Lib/python/cdata.i",
"Lib/python/cmalloc.i",
"Lib/python/cni.i",
"Lib/python/complex.i",
"Lib/python/cpointer.i",
"Lib/python/cstring.i",
"Lib/python/cwstring.i",
"Lib/python/defarg.swg",
"Lib/python/director.swg",
"Lib/python/embed.i",
"Lib/python/embed15.i",
"Lib/python/exception.i",
"Lib/python/factory.i",
"Lib/python/file.i",
"Lib/python/implicit.i",
"Lib/python/jstring.i",
"Lib/python/pyabc.i",
"Lib/python/pyapi.swg",
"Lib/python/pybackward.swg",
"Lib/python/pybuffer.i",
"Lib/python/pyclasses.swg",
"Lib/python/pycomplex.swg",
"Lib/python/pycontainer.swg",
"Lib/python/pydocs.swg",
"Lib/python/pyerrors.swg",
"Lib/python/pyfragments.swg",
"Lib/python/pyhead.swg",
"Lib/python/pyinit.swg",
"Lib/python/pyiterators.swg",
"Lib/python/pymacros.swg",
"Lib/python/pyname_compat.i",
"Lib/python/pyopers.swg",
"Lib/python/pyprimtypes.swg",
"Lib/python/pyrun.swg",
"Lib/python/pyruntime.swg",
"Lib/python/pystdcommon.swg",
"Lib/python/pystrings.swg",
"Lib/python/python.swg",
"Lib/python/pythonkw.swg",
"Lib/python/pythreads.swg",
"Lib/python/pytuplehlp.swg",
"Lib/python/pytypemaps.swg",
"Lib/python/pyuserdir.swg",
"Lib/python/pywstrings.swg",
"Lib/python/std_alloc.i",
"Lib/python/std_auto_ptr.i",
"Lib/python/std_basic_string.i",
"Lib/python/std_carray.i",
"Lib/python/std_char_traits.i",
"Lib/python/std_common.i",
"Lib/python/std_complex.i",
"Lib/python/std_container.i",
"Lib/python/std_deque.i",
"Lib/python/std_except.i",
"Lib/python/std_ios.i",
"Lib/python/std_iostream.i",
"Lib/python/std_list.i",
"Lib/python/std_map.i",
"Lib/python/std_multimap.i",
"Lib/python/std_multiset.i",
"Lib/python/std_pair.i",
"Lib/python/std_set.i",
"Lib/python/std_shared_ptr.i",
"Lib/python/std_sstream.i",
"Lib/python/std_streambuf.i",
"Lib/python/std_string.i",
"Lib/python/std_unordered_map.i",
"Lib/python/std_unordered_multimap.i",
"Lib/python/std_unordered_multiset.i",
"Lib/python/std_unordered_set.i",
"Lib/python/std_vector.i",
"Lib/python/std_vectora.i",
"Lib/python/std_wios.i",
"Lib/python/std_wiostream.i",
"Lib/python/std_wsstream.i",
"Lib/python/std_wstreambuf.i",
"Lib/python/std_wstring.i",
"Lib/python/stl.i",
"Lib/python/typemaps.i",
"Lib/python/wchar.i",
"Lib/runtime.swg",
"Lib/shared_ptr.i",
"Lib/std/_std_deque.i",
"Lib/std/std_alloc.i",
"Lib/std/std_basic_string.i",
"Lib/std/std_carray.swg",
"Lib/std/std_char_traits.i",
"Lib/std/std_common.i",
"Lib/std/std_container.i",
"Lib/std/std_deque.i",
"Lib/std/std_except.i",
"Lib/std/std_ios.i",
"Lib/std/std_iostream.i",
"Lib/std/std_list.i",
"Lib/std/std_map.i",
"Lib/std/std_multimap.i",
"Lib/std/std_multiset.i",
"Lib/std/std_pair.i",
"Lib/std/std_queue.i",
"Lib/std/std_set.i",
"Lib/std/std_sstream.i",
"Lib/std/std_stack.i",
"Lib/std/std_streambuf.i",
"Lib/std/std_string.i",
"Lib/std/std_unordered_map.i",
"Lib/std/std_unordered_multimap.i",
"Lib/std/std_unordered_multiset.i",
"Lib/std/std_unordered_set.i",
"Lib/std/std_vector.i",
"Lib/std/std_vectora.i",
"Lib/std/std_wios.i",
"Lib/std/std_wiostream.i",
"Lib/std/std_wsstream.i",
"Lib/std/std_wstreambuf.i",
"Lib/std/std_wstring.i",
"Lib/std_except.i",
"Lib/stdint.i",
"Lib/stl.i",
"Lib/swig.swg",
"Lib/swigarch.i",
"Lib/swigerrors.swg",
"Lib/swiginit.swg",
"Lib/swiglabels.swg",
"Lib/swigrun.i",
"Lib/swigrun.swg",
"Lib/swigwarn.swg",
"Lib/swigwarnings.swg",
"Lib/typemaps/attribute.swg",
"Lib/typemaps/carrays.swg",
"Lib/typemaps/cdata.swg",
"Lib/typemaps/cmalloc.swg",
"Lib/typemaps/cpointer.swg",
"Lib/typemaps/cstring.swg",
"Lib/typemaps/cstrings.swg",
"Lib/typemaps/cwstring.swg",
"Lib/typemaps/enumint.swg",
"Lib/typemaps/exception.swg",
"Lib/typemaps/factory.swg",
"Lib/typemaps/fragments.swg",
"Lib/typemaps/implicit.swg",
"Lib/typemaps/inoutlist.swg",
"Lib/typemaps/misctypes.swg",
"Lib/typemaps/primtypes.swg",
"Lib/typemaps/ptrtypes.swg",
"Lib/typemaps/std_except.swg",
"Lib/typemaps/std_string.swg",
"Lib/typemaps/std_strings.swg",
"Lib/typemaps/std_wstring.swg",
"Lib/typemaps/string.swg",
"Lib/typemaps/strings.swg",
"Lib/typemaps/swigmacros.swg",
"Lib/typemaps/swigobject.swg",
"Lib/typemaps/swigtype.swg",
"Lib/typemaps/swigtypemaps.swg",
"Lib/typemaps/traits.swg",
"Lib/typemaps/typemaps.swg",
"Lib/typemaps/valtypes.swg",
"Lib/typemaps/void.swg",
"Lib/typemaps/wstring.swg",
"Lib/wchar.i",
"Lib/windows.i",
],
licenses = ["notice"], # simple notice license for Lib/
path = "Lib",
visibility = ["//visibility:public"],
)
genrule(
name = "swigconfig",
outs = ["Source/Include/swigconfig.h"],
cmd = "cat <<EOF >$@\n" +
"#define HAVE_BOOL\n" +
"#define HAVE_PCRE\n" +
"#define HAVE_POPEN\n" +
"#define PACKAGE_BUGREPORT \"http://www.swig.org\"\n" +
"#define PACKAGE_VERSION \"3.0.2\"\n" +
"#define STDC_HEADERS\n" +
"#define SWIG_CXX \"bazel4lyfe\"\n" +
"#define SWIG_LIB \"external/swig/Lib\"\n" +
"#define SWIG_LIB_WIN_UNIX \"\"\n" +
"#define SWIG_PLATFORM \"bazel4lyfe\"\n" +
"EOF",
)
genrule(
name = "get_rid_of_stuff_we_dont_need_yet",
srcs = ["Source/Modules/swigmain.cxx"],
outs = ["Source/Modules/swigmain-lite.cxx"],
cmd = "sed -e '/swig_allegrocl/d'" +
" -e '/swig_cffi/d'" +
" -e '/swig_chicken/d'" +
" -e '/swig_clisp/d'" +
" -e '/swig_csharp/d'" +
" -e '/swig_d/d'" +
" -e '/swig_go/d'" +
" -e '/swig_guile/d'" +
" -e '/swig_java/d'" +
" -e '/swig_lua/d'" +
" -e '/swig_modula3/d'" +
" -e '/swig_mzscheme/d'" +
" -e '/swig_ocaml/d'" +
" -e '/swig_octave/d'" +
" -e '/swig_perl/d'" +
" -e '/swig_php/d'" +
" -e '/swig_pike/d'" +
" -e '/swig_r/d'" +
" -e '/swig_ruby/d'" +
" -e '/swig_sexp/d'" +
" -e '/swig_tcl/d'" +
" -e '/swig_uffi/d'" +
" $< >$@",
)
config_setting(
name = "x64_windows_msvc",
values = {"cpu": "x64_windows_msvc"},
)