Fix #2885: Improve ds-swig integration

Dockerfile

@@ -24,7 +24,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         libsox-fmt-mp3 \
         htop \
         nano \
-        swig \
         cmake \
         libboost-all-dev \
         zlib1g-dev \
@@ -203,7 +202,7 @@ WORKDIR /DeepSpeech/native_client/python
 RUN make bindings
 RUN pip3 install dist/deepspeech*
 WORKDIR /DeepSpeech/native_client/ctcdecode
-RUN make
+RUN make bindings
 RUN pip3 install dist/*.whl
 
 

native_client/README.rst

@@ -14,7 +14,10 @@ It is required to use our fork of TensorFlow since it includes fixes for common
 If you'd like to build the language bindings or the decoder package, you'll also need:
 
 
-* `SWIG >= 3.0.12 <http://www.swig.org/>`_. If you intend to build NodeJS / ElectronJS bindings you will need a patched version of SWIG. Please refer to the matching section below.
+* `SWIG >= 3.0.12 <http://www.swig.org/>`_.
+  Unfortunately, NodeJS / ElectronJS after 10.x support on SWIG is a bit behind, and while there are pending patches proposed to upstream, it is not yet merged.
+  The proper prebuilt patched version (covering linux, windows and macOS) of SWIG should get installed under `native_client/ <native_client/>`_ as soon as you build any bindings that requires it.
+
 * `node-pre-gyp <https://github.com/mapbox/node-pre-gyp>`_ (for Node.JS bindings only)
 
 Dependencies
@@ -108,10 +111,6 @@ The API mirrors the C++ API and is demonstrated in `client.py <python/client.py>
 Install NodeJS / ElectronJS bindings
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Unfortunately, JavaScript support on SWIG is a bit behind, and while there are pending patches proposed to upstream, it is not yet merged.
-You should be able to build from `our fork <https://github.com/lissyx/swig/tree/taskcluster>`_, and you can find pre-built binaries on `TaskCluster <https://community-tc.services.mozilla.com/tasks/index/project.deepspeech.swig>`_ (please look for swig fork sha1).
-Extract the `ds-swig.tar.gz` to some place in your `$HOME`, then update `$PATH` accordingly. You might need to symlink `ds-swig` as `swig`, and you will have to `export SWIG_LIB=<path/to/swig/share>` so that it contains path to `share/swig/<VERSION>/`.
-
 After following the above build and installation instructions, the Node.JS bindings can be built:
 
 .. code-block::

native_client/ctcdecode/Makefile

@@ -43,16 +43,16 @@ workspace_status.cc:
 
 # Enforce PATH here because swig calls from build_ext looses track of some
 # variables over several runs
-bindings: clean-keep-third-party workspace_status.cc
+bindings: clean-keep-third-party workspace_status.cc ds-swig
 	pip install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==39.1.0
-	PATH=$(TOOLCHAIN):$$PATH AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
+	PATH=$(DS_SWIG_BIN_PATH):$(TOOLCHAIN):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
 	find temp_build -type f -name "*.o" -delete
 	AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
 	rm -rf temp_build
 
-bindings-debug: clean-keep-third-party workspace_status.cc
+bindings-debug: clean-keep-third-party workspace_status.cc ds-swig
 	pip install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==39.1.0
-	PATH=$(TOOLCHAIN):$$PATH AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --debug --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
+	PATH=$(DS_SWIG_BIN_PATH):$(TOOLCHAIN):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --debug --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
 	$(GENERATE_DEBUG_SYMS)
 	find temp_build -type f -name "*.o" -delete
 	AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)

native_client/definitions.mk

@@ -12,7 +12,11 @@ TOOL_LD   := ld
 TOOL_LDD  := ldd
 TOOL_LIBEXE :=
 
-DEEPSPEECH_BIN       := deepspeech
+ifeq ($(findstring _NT,$(OS)),_NT)
+PLATFORM_EXE_SUFFIX := .exe
+endif
+
+DEEPSPEECH_BIN       := deepspeech$(PLATFORM_EXE_SUFFIX)
 CFLAGS_DEEPSPEECH    := -std=c++11 -o $(DEEPSPEECH_BIN)
 LINK_DEEPSPEECH      := -ldeepspeech
 LINK_PATH_DEEPSPEECH := -L${TFDIR}/bazel-bin/native_client
@@ -36,7 +40,6 @@ endif
 endif
 
 ifeq ($(TARGET),host-win)
-DEEPSPEECH_BIN  := deepspeech.exe
 TOOLCHAIN := '$(VCINSTALLDIR)\bin\amd64\'
 TOOL_CC     := cl.exe
 TOOL_CXX    := cl.exe
@@ -170,3 +173,36 @@ define copy_missing_libs
         done; \
     fi;
 endef
+
+SWIG_DIST_URL ?= 
+ifeq ($(findstring Linux,$(OS)),Linux)
+SWIG_DIST_URL := "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.linux.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118/artifacts/public/ds-swig.tar.gz"
+else ifeq ($(findstring Darwin,$(OS)),Darwin)
+SWIG_DIST_URL := "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.darwin.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118/artifacts/public/ds-swig.tar.gz"
+else ifeq ($(findstring _NT,$(OS)),_NT)
+SWIG_DIST_URL := "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.win.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118/artifacts/public/ds-swig.tar.gz"
+else
+$(error There is no prebuilt SWIG available for your platform. Please produce one and set SWIG_DIST_URL.)
+endif
+
+# Should point to native_client/ subdir by default
+SWIG_ROOT ?= $(abspath $(shell dirname "$(lastword $(MAKEFILE_LIST))"))/ds-swig
+ifeq ($(findstring _NT,$(OS)),_NT)
+SWIG_ROOT ?= $(shell cygpath -u "$(SWIG_ROOT)")
+endif
+SWIG_LIB ?= $(SWIG_ROOT)/share/swig/4.0.2/
+
+SWIG_BIN := swig$(PLATFORM_EXE_SUFFIX)
+DS_SWIG_BIN := ds-swig$(PLATFORM_EXE_SUFFIX)
+DS_SWIG_BIN_PATH := $(SWIG_ROOT)/bin
+
+DS_SWIG_ENV := SWIG_LIB="$(SWIG_LIB)" PATH="${PATH}:$(DS_SWIG_BIN_PATH)"
+
+$(DS_SWIG_BIN_PATH)/swig:
+	mkdir -p $(SWIG_ROOT)
+	wget -O - "$(SWIG_DIST_URL)" | tar -C $(SWIG_ROOT) -zxf -
+	ln -s $(DS_SWIG_BIN) $(DS_SWIG_BIN_PATH)/$(SWIG_BIN)
+
+ds-swig: $(DS_SWIG_BIN_PATH)/swig
+	$(DS_SWIG_ENV) swig -version
+	$(DS_SWIG_ENV) swig -swiglib

native_client/java/Makefile

@@ -27,5 +27,5 @@ maven-bundle: apk
 	$(GRADLE) uploadArchives
 	$(GRADLE) zipMavenArtifacts
 
-bindings: clean
+bindings: clean ds-swig
-	swig -c++ -java -package org.mozilla.deepspeech.libdeepspeech -outdir libdeepspeech/src/main/java/org/mozilla/deepspeech/libdeepspeech/ -o jni/deepspeech_wrap.cpp jni/deepspeech.i
+	$(DS_SWIG_ENV) swig -c++ -java -package org.mozilla.deepspeech.libdeepspeech -outdir libdeepspeech/src/main/java/org/mozilla/deepspeech/libdeepspeech/ -o jni/deepspeech_wrap.cpp jni/deepspeech.i

native_client/javascript/Makefile

@@ -62,6 +62,5 @@ node-wrapper: copy-deps build
 npm-pack: clean package.json index.js npm-dev
 	PATH="$(NODE_MODULES_BIN):${PATH}" tsc && $(NPM_TOOL) pack $(NODE_BUILD_VERBOSE)
 
-deepspeech_wrap.cxx: deepspeech.i
+deepspeech_wrap.cxx: deepspeech.i ds-swig
-	swig -version
+	$(DS_SWIG_ENV) swig -c++ -javascript -node deepspeech.i
-	swig -c++ -javascript -node deepspeech.i

native_client/python/Makefile

@@ -8,9 +8,9 @@ bindings-clean:
 
 # Enforce PATH here because swig calls from build_ext looses track of some
 # variables over several runs
-bindings-build:
+bindings-build: ds-swig
 	pip install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==39.1.0
-	PATH=$(TOOLCHAIN):$$PATH AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED) $(RPATH_PYTHON)" MODEL_LDFLAGS="$(LDFLAGS_DIRS)" MODEL_LIBS="$(LIBS)" $(PYTHON_PATH) $(PYTHON_SYSCONFIGDATA) $(NUMPY_INCLUDE) python ./setup.py build_ext $(PYTHON_PLATFORM_NAME)
+	PATH=$(TOOLCHAIN):$(DS_SWIG_BIN_PATH):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED) $(RPATH_PYTHON)" MODEL_LDFLAGS="$(LDFLAGS_DIRS)" MODEL_LIBS="$(LIBS)" $(PYTHON_PATH) $(PYTHON_SYSCONFIGDATA) $(NUMPY_INCLUDE) python ./setup.py build_ext $(PYTHON_PLATFORM_NAME)
 
 MANIFEST.in: bindings-build
 	> $@

taskcluster/.shared.yml

@@ -115,10 +115,6 @@ system:
   swig:
     repo: "https://github.com/lissyx/swig"
     sha1: "b5fea54d39832d1d132d7dd921b69c0c2c9d5118"
-    cache:
-      linux_amd64: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.linux.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118/artifacts/public/ds-swig.tar.gz'
-      darwin_amd64: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.darwin.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118/artifacts/public/ds-swig.tar.gz'
-      win_amd64: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.win.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118/artifacts/public/ds-swig.tar.gz'
   username: 'build-user'
   homedir:
     linux: '/home/build-user'

taskcluster/darwin-opt-base.tyml

@@ -88,10 +88,6 @@ payload:
       format: tar.gz
       content:
         url: ${system.node_gyp_cache.url}
-    - directory: ds-swig
-      format: tar.gz
-      content:
-        url: ${system.swig.cache.darwin_amd64}
     - file: home.tar.xz
       content:
         url: ${build.tensorflow}

taskcluster/linux-opt-base.tyml

@@ -48,7 +48,7 @@ then:
           adduser --system --home ${system.homedir.linux} ${system.username} &&
           apt-get -qq update && apt-get -qq -y install ${tensorflow.packages_trusty.apt} pixz pkg-config realpath unzip wget zip && ${extraSystemSetup} &&
           cd ${system.homedir.linux}/ &&
-          echo -e "#!/bin/bash\nset -xe\n env && id && (wget -O - $TENSORFLOW_BUILD_ARTIFACT | pixz -d | tar -C ${system.homedir.linux}/ -xf - ) && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && ln -s ~/DeepSpeech/ds/native_client/ ~/DeepSpeech/tf/native_client && mkdir -p ${system.homedir.linux}/.cache/node-gyp/ && wget -O - ${system.node_gyp_cache.url} | tar -C ${system.homedir.linux}/.cache/node-gyp/ -xzf - && mkdir -p ${system.homedir.linux}/ds-swig/bin/ && wget -O - ${system.swig.cache.linux_amd64} | tar -C ${system.homedir.linux}/ds-swig/ -xzf - && mkdir -p ${system.homedir.linux}/pyenv-root/ && wget -O - ${system.pyenv.linux.url} | tar -C ${system.homedir.linux}/pyenv-root/ -xzf - && if [ ! -z "${build.gradle_cache.url}" ]; then wget -O - ${build.gradle_cache.url} | tar -C ${system.homedir.linux}/ -xzf - ; fi && if [ ! -z "${build.android_cache.url}" ]; then wget -O - ${build.android_cache.url} | tar -C ${system.homedir.linux}/ -xzf - ; fi;" > /tmp/clone.sh && chmod +x /tmp/clone.sh &&
+          echo -e "#!/bin/bash\nset -xe\n env && id && (wget -O - $TENSORFLOW_BUILD_ARTIFACT | pixz -d | tar -C ${system.homedir.linux}/ -xf - ) && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && ln -s ~/DeepSpeech/ds/native_client/ ~/DeepSpeech/tf/native_client && mkdir -p ${system.homedir.linux}/.cache/node-gyp/ && wget -O - ${system.node_gyp_cache.url} | tar -C ${system.homedir.linux}/.cache/node-gyp/ -xzf - && mkdir -p ${system.homedir.linux}/pyenv-root/ && wget -O - ${system.pyenv.linux.url} | tar -C ${system.homedir.linux}/pyenv-root/ -xzf - && if [ ! -z "${build.gradle_cache.url}" ]; then wget -O - ${build.gradle_cache.url} | tar -C ${system.homedir.linux}/ -xzf - ; fi && if [ ! -z "${build.android_cache.url}" ]; then wget -O - ${build.android_cache.url} | tar -C ${system.homedir.linux}/ -xzf - ; fi;" > /tmp/clone.sh && chmod +x /tmp/clone.sh &&
           sudo -H -u ${system.username} /bin/bash /tmp/clone.sh && ${extraSystemConfig} &&
           sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.build} &&
           sudo -H -u ${system.username} /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.package}

taskcluster/tc-all-vars.sh

@@ -6,14 +6,12 @@ export OS=$(uname)
 if [ "${OS}" = "Linux" ]; then
     export DS_ROOT_TASK=${HOME}
     export PYENV_ROOT="${DS_ROOT_TASK}/pyenv-root"
-    export SWIG_ROOT="${HOME}/ds-swig"
     export DS_CPU_COUNT=$(nproc)
 fi;
 
 if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then
     export DS_ROOT_TASK=${TASKCLUSTER_TASK_DIR}
     export PYENV_ROOT="${TASKCLUSTER_TASK_DIR}/pyenv-root"
-    export SWIG_ROOT="$(cygpath ${USERPROFILE})/ds-swig"
     export PLATFORM_EXE_SUFFIX=.exe
     export DS_CPU_COUNT=$(nproc)
 
@@ -22,7 +20,6 @@ if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then
 fi;
 
 if [ "${OS}" = "Darwin" ]; then
-    export SWIG_ROOT="${TASKCLUSTER_ORIG_TASKDIR}/ds-swig"
     export DS_ROOT_TASK=${TASKCLUSTER_TASK_DIR}
     export DS_CPU_COUNT=$(sysctl hw.ncpu |cut -d' ' -f2)
     export PYENV_ROOT="${DS_ROOT_TASK}/pyenv-root"
@@ -45,19 +42,6 @@ if [ "${OS}" = "Darwin" ]; then
     fi;
 fi;
 
-SWIG_BIN=swig${PLATFORM_EXE_SUFFIX}
-DS_SWIG_BIN=ds-swig${PLATFORM_EXE_SUFFIX}
-if [ -f "${SWIG_ROOT}/bin/${DS_SWIG_BIN}" ]; then
-    export PATH=${SWIG_ROOT}/bin/:$PATH
-    export SWIG_LIB="$(find ${SWIG_ROOT}/share/swig/ -type f -name "swig.swg" | xargs dirname)"
-    # Make an alias to be more magic
-    if [ ! -L "${SWIG_ROOT}/bin/${SWIG_BIN}" ]; then
-        ln -s ${DS_SWIG_BIN} ${SWIG_ROOT}/bin/${SWIG_BIN}
-    fi;
-    swig -version
-    swig -swiglib
-fi;
-
 PY37_OPENSSL_DIR="${PYENV_ROOT}/ssl-xenial"
 export PY37_LDPATH="${PY37_OPENSSL_DIR}/usr/lib/"
 export LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH

taskcluster/test-android-opt-base.tyml

@@ -50,7 +50,7 @@ then:
           ${extraSystemSetup} && chmod 777 /dev/kvm &&
           adduser --system --home ${system.homedir.linux} ${system.username} &&
           cd ${system.homedir.linux} &&
-          echo -e "#!/bin/bash\nset -xe\n env && id && mkdir ~/DeepSpeech/ && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && mkdir -p ${system.homedir.linux}/ds-swig/bin/ && wget -O - ${system.swig.cache.linux_amd64} | tar -C ${system.homedir.linux}/ds-swig/ -xzf - && wget -O - ${build.cache.url} | tar -C ${system.homedir.linux} -xzf - && if [ ! -z "${build.gradle_cache.url}" ]; then wget -O - ${build.gradle_cache.url} | tar -C ${system.homedir.linux}/ -xzf - ; fi;" > /tmp/clone.sh && chmod +x /tmp/clone.sh &&
+          echo -e "#!/bin/bash\nset -xe\n env && id && mkdir ~/DeepSpeech/ && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && wget -O - ${build.cache.url} | tar -C ${system.homedir.linux} -xzf - && if [ ! -z "${build.gradle_cache.url}" ]; then wget -O - ${build.gradle_cache.url} | tar -C ${system.homedir.linux}/ -xzf - ; fi;" > /tmp/clone.sh && chmod +x /tmp/clone.sh &&
           sudo -H -u ${system.username} /bin/bash /tmp/clone.sh &&
           sudo -H -u ${system.username} --preserve-env /bin/bash ${build.args.tests_cmdline}
 

taskcluster/win-opt-base.tyml

@@ -85,10 +85,6 @@ payload:
       format: tar.gz
       content:
         url: ${system.node_gyp_cache.url}
-    - directory: ds-swig
-      format: tar.gz
-      content:
-        url: ${system.swig.cache.win_amd64}
 
   artifacts:
     - type: "directory"