diff --git a/Dockerfile b/Dockerfile index 9b69596f..56477601 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,7 +24,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libsox-fmt-mp3 \ htop \ nano \ - swig \ cmake \ libboost-all-dev \ zlib1g-dev \ @@ -203,7 +202,7 @@ WORKDIR /DeepSpeech/native_client/python RUN make bindings RUN pip3 install dist/deepspeech* WORKDIR /DeepSpeech/native_client/ctcdecode -RUN make +RUN make bindings RUN pip3 install dist/*.whl diff --git a/native_client/README.rst b/native_client/README.rst index 13bae794..9f9375b9 100644 --- a/native_client/README.rst +++ b/native_client/README.rst @@ -14,7 +14,10 @@ It is required to use our fork of TensorFlow since it includes fixes for common If you'd like to build the language bindings or the decoder package, you'll also need: -* `SWIG >= 3.0.12 `_. If you intend to build NodeJS / ElectronJS bindings you will need a patched version of SWIG. Please refer to the matching section below. +* `SWIG >= 3.0.12 `_. + Unfortunately, NodeJS / ElectronJS after 10.x support on SWIG is a bit behind, and while there are pending patches proposed to upstream, it is not yet merged. + The proper prebuilt patched version (covering linux, windows and macOS) of SWIG should get installed under `native_client/ `_ as soon as you build any bindings that requires it. + * `node-pre-gyp `_ (for Node.JS bindings only) Dependencies @@ -108,10 +111,6 @@ The API mirrors the C++ API and is demonstrated in `client.py Install NodeJS / ElectronJS bindings ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Unfortunately, JavaScript support on SWIG is a bit behind, and while there are pending patches proposed to upstream, it is not yet merged. -You should be able to build from `our fork `_, and you can find pre-built binaries on `TaskCluster `_ (please look for swig fork sha1). -Extract the `ds-swig.tar.gz` to some place in your `$HOME`, then update `$PATH` accordingly. You might need to symlink `ds-swig` as `swig`, and you will have to `export SWIG_LIB=` so that it contains path to `share/swig//`. - After following the above build and installation instructions, the Node.JS bindings can be built: .. code-block:: diff --git a/native_client/ctcdecode/Makefile b/native_client/ctcdecode/Makefile index 1e8aee0c..3cf76478 100644 --- a/native_client/ctcdecode/Makefile +++ b/native_client/ctcdecode/Makefile @@ -43,16 +43,16 @@ workspace_status.cc: # Enforce PATH here because swig calls from build_ext looses track of some # variables over several runs -bindings: clean-keep-third-party workspace_status.cc +bindings: clean-keep-third-party workspace_status.cc ds-swig pip install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==39.1.0 - PATH=$(TOOLCHAIN):$$PATH AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) + PATH=$(DS_SWIG_BIN_PATH):$(TOOLCHAIN):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) find temp_build -type f -name "*.o" -delete AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) rm -rf temp_build -bindings-debug: clean-keep-third-party workspace_status.cc +bindings-debug: clean-keep-third-party workspace_status.cc ds-swig pip install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==39.1.0 - PATH=$(TOOLCHAIN):$$PATH AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --debug --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) + PATH=$(DS_SWIG_BIN_PATH):$(TOOLCHAIN):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --debug --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) $(GENERATE_DEBUG_SYMS) find temp_build -type f -name "*.o" -delete AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) diff --git a/native_client/definitions.mk b/native_client/definitions.mk index fd9358a2..68da5f10 100644 --- a/native_client/definitions.mk +++ b/native_client/definitions.mk @@ -12,7 +12,11 @@ TOOL_LD := ld TOOL_LDD := ldd TOOL_LIBEXE := -DEEPSPEECH_BIN := deepspeech +ifeq ($(findstring _NT,$(OS)),_NT) +PLATFORM_EXE_SUFFIX := .exe +endif + +DEEPSPEECH_BIN := deepspeech$(PLATFORM_EXE_SUFFIX) CFLAGS_DEEPSPEECH := -std=c++11 -o $(DEEPSPEECH_BIN) LINK_DEEPSPEECH := -ldeepspeech LINK_PATH_DEEPSPEECH := -L${TFDIR}/bazel-bin/native_client @@ -36,7 +40,6 @@ endif endif ifeq ($(TARGET),host-win) -DEEPSPEECH_BIN := deepspeech.exe TOOLCHAIN := '$(VCINSTALLDIR)\bin\amd64\' TOOL_CC := cl.exe TOOL_CXX := cl.exe @@ -170,3 +173,36 @@ define copy_missing_libs done; \ fi; endef + +SWIG_DIST_URL ?= +ifeq ($(findstring Linux,$(OS)),Linux) +SWIG_DIST_URL := "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.linux.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118/artifacts/public/ds-swig.tar.gz" +else ifeq ($(findstring Darwin,$(OS)),Darwin) +SWIG_DIST_URL := "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.darwin.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118/artifacts/public/ds-swig.tar.gz" +else ifeq ($(findstring _NT,$(OS)),_NT) +SWIG_DIST_URL := "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.win.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118/artifacts/public/ds-swig.tar.gz" +else +$(error There is no prebuilt SWIG available for your platform. Please produce one and set SWIG_DIST_URL.) +endif + +# Should point to native_client/ subdir by default +SWIG_ROOT ?= $(abspath $(shell dirname "$(lastword $(MAKEFILE_LIST))"))/ds-swig +ifeq ($(findstring _NT,$(OS)),_NT) +SWIG_ROOT ?= $(shell cygpath -u "$(SWIG_ROOT)") +endif +SWIG_LIB ?= $(SWIG_ROOT)/share/swig/4.0.2/ + +SWIG_BIN := swig$(PLATFORM_EXE_SUFFIX) +DS_SWIG_BIN := ds-swig$(PLATFORM_EXE_SUFFIX) +DS_SWIG_BIN_PATH := $(SWIG_ROOT)/bin + +DS_SWIG_ENV := SWIG_LIB="$(SWIG_LIB)" PATH="${PATH}:$(DS_SWIG_BIN_PATH)" + +$(DS_SWIG_BIN_PATH)/swig: + mkdir -p $(SWIG_ROOT) + wget -O - "$(SWIG_DIST_URL)" | tar -C $(SWIG_ROOT) -zxf - + ln -s $(DS_SWIG_BIN) $(DS_SWIG_BIN_PATH)/$(SWIG_BIN) + +ds-swig: $(DS_SWIG_BIN_PATH)/swig + $(DS_SWIG_ENV) swig -version + $(DS_SWIG_ENV) swig -swiglib diff --git a/native_client/java/Makefile b/native_client/java/Makefile index 34426e0b..191b1013 100644 --- a/native_client/java/Makefile +++ b/native_client/java/Makefile @@ -27,5 +27,5 @@ maven-bundle: apk $(GRADLE) uploadArchives $(GRADLE) zipMavenArtifacts -bindings: clean - swig -c++ -java -package org.mozilla.deepspeech.libdeepspeech -outdir libdeepspeech/src/main/java/org/mozilla/deepspeech/libdeepspeech/ -o jni/deepspeech_wrap.cpp jni/deepspeech.i +bindings: clean ds-swig + $(DS_SWIG_ENV) swig -c++ -java -package org.mozilla.deepspeech.libdeepspeech -outdir libdeepspeech/src/main/java/org/mozilla/deepspeech/libdeepspeech/ -o jni/deepspeech_wrap.cpp jni/deepspeech.i diff --git a/native_client/javascript/Makefile b/native_client/javascript/Makefile index 0581d648..b634cf4b 100644 --- a/native_client/javascript/Makefile +++ b/native_client/javascript/Makefile @@ -62,6 +62,5 @@ node-wrapper: copy-deps build npm-pack: clean package.json index.js npm-dev PATH="$(NODE_MODULES_BIN):${PATH}" tsc && $(NPM_TOOL) pack $(NODE_BUILD_VERBOSE) -deepspeech_wrap.cxx: deepspeech.i - swig -version - swig -c++ -javascript -node deepspeech.i +deepspeech_wrap.cxx: deepspeech.i ds-swig + $(DS_SWIG_ENV) swig -c++ -javascript -node deepspeech.i diff --git a/native_client/python/Makefile b/native_client/python/Makefile index 5dfba785..cf17c9db 100644 --- a/native_client/python/Makefile +++ b/native_client/python/Makefile @@ -8,9 +8,9 @@ bindings-clean: # Enforce PATH here because swig calls from build_ext looses track of some # variables over several runs -bindings-build: +bindings-build: ds-swig pip install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==39.1.0 - PATH=$(TOOLCHAIN):$$PATH AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED) $(RPATH_PYTHON)" MODEL_LDFLAGS="$(LDFLAGS_DIRS)" MODEL_LIBS="$(LIBS)" $(PYTHON_PATH) $(PYTHON_SYSCONFIGDATA) $(NUMPY_INCLUDE) python ./setup.py build_ext $(PYTHON_PLATFORM_NAME) + PATH=$(TOOLCHAIN):$(DS_SWIG_BIN_PATH):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED) $(RPATH_PYTHON)" MODEL_LDFLAGS="$(LDFLAGS_DIRS)" MODEL_LIBS="$(LIBS)" $(PYTHON_PATH) $(PYTHON_SYSCONFIGDATA) $(NUMPY_INCLUDE) python ./setup.py build_ext $(PYTHON_PLATFORM_NAME) MANIFEST.in: bindings-build > $@ diff --git a/taskcluster/.shared.yml b/taskcluster/.shared.yml index 31ca23c0..a5492b82 100644 --- a/taskcluster/.shared.yml +++ b/taskcluster/.shared.yml @@ -115,10 +115,6 @@ system: swig: repo: "https://github.com/lissyx/swig" sha1: "b5fea54d39832d1d132d7dd921b69c0c2c9d5118" - cache: - linux_amd64: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.linux.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118/artifacts/public/ds-swig.tar.gz' - darwin_amd64: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.darwin.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118/artifacts/public/ds-swig.tar.gz' - win_amd64: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.win.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118/artifacts/public/ds-swig.tar.gz' username: 'build-user' homedir: linux: '/home/build-user' diff --git a/taskcluster/darwin-opt-base.tyml b/taskcluster/darwin-opt-base.tyml index bb37a6c4..4500d5fa 100644 --- a/taskcluster/darwin-opt-base.tyml +++ b/taskcluster/darwin-opt-base.tyml @@ -88,10 +88,6 @@ payload: format: tar.gz content: url: ${system.node_gyp_cache.url} - - directory: ds-swig - format: tar.gz - content: - url: ${system.swig.cache.darwin_amd64} - file: home.tar.xz content: url: ${build.tensorflow} diff --git a/taskcluster/linux-opt-base.tyml b/taskcluster/linux-opt-base.tyml index 2a697bf0..0298954c 100644 --- a/taskcluster/linux-opt-base.tyml +++ b/taskcluster/linux-opt-base.tyml @@ -48,7 +48,7 @@ then: adduser --system --home ${system.homedir.linux} ${system.username} && apt-get -qq update && apt-get -qq -y install ${tensorflow.packages_trusty.apt} pixz pkg-config realpath unzip wget zip && ${extraSystemSetup} && cd ${system.homedir.linux}/ && - echo -e "#!/bin/bash\nset -xe\n env && id && (wget -O - $TENSORFLOW_BUILD_ARTIFACT | pixz -d | tar -C ${system.homedir.linux}/ -xf - ) && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && ln -s ~/DeepSpeech/ds/native_client/ ~/DeepSpeech/tf/native_client && mkdir -p ${system.homedir.linux}/.cache/node-gyp/ && wget -O - ${system.node_gyp_cache.url} | tar -C ${system.homedir.linux}/.cache/node-gyp/ -xzf - && mkdir -p ${system.homedir.linux}/ds-swig/bin/ && wget -O - ${system.swig.cache.linux_amd64} | tar -C ${system.homedir.linux}/ds-swig/ -xzf - && mkdir -p ${system.homedir.linux}/pyenv-root/ && wget -O - ${system.pyenv.linux.url} | tar -C ${system.homedir.linux}/pyenv-root/ -xzf - && if [ ! -z "${build.gradle_cache.url}" ]; then wget -O - ${build.gradle_cache.url} | tar -C ${system.homedir.linux}/ -xzf - ; fi && if [ ! -z "${build.android_cache.url}" ]; then wget -O - ${build.android_cache.url} | tar -C ${system.homedir.linux}/ -xzf - ; fi;" > /tmp/clone.sh && chmod +x /tmp/clone.sh && + echo -e "#!/bin/bash\nset -xe\n env && id && (wget -O - $TENSORFLOW_BUILD_ARTIFACT | pixz -d | tar -C ${system.homedir.linux}/ -xf - ) && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && ln -s ~/DeepSpeech/ds/native_client/ ~/DeepSpeech/tf/native_client && mkdir -p ${system.homedir.linux}/.cache/node-gyp/ && wget -O - ${system.node_gyp_cache.url} | tar -C ${system.homedir.linux}/.cache/node-gyp/ -xzf - && mkdir -p ${system.homedir.linux}/pyenv-root/ && wget -O - ${system.pyenv.linux.url} | tar -C ${system.homedir.linux}/pyenv-root/ -xzf - && if [ ! -z "${build.gradle_cache.url}" ]; then wget -O - ${build.gradle_cache.url} | tar -C ${system.homedir.linux}/ -xzf - ; fi && if [ ! -z "${build.android_cache.url}" ]; then wget -O - ${build.android_cache.url} | tar -C ${system.homedir.linux}/ -xzf - ; fi;" > /tmp/clone.sh && chmod +x /tmp/clone.sh && sudo -H -u ${system.username} /bin/bash /tmp/clone.sh && ${extraSystemConfig} && sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.build} && sudo -H -u ${system.username} /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.package} diff --git a/taskcluster/tc-all-vars.sh b/taskcluster/tc-all-vars.sh index cb836e83..6cdc6e7c 100755 --- a/taskcluster/tc-all-vars.sh +++ b/taskcluster/tc-all-vars.sh @@ -6,14 +6,12 @@ export OS=$(uname) if [ "${OS}" = "Linux" ]; then export DS_ROOT_TASK=${HOME} export PYENV_ROOT="${DS_ROOT_TASK}/pyenv-root" - export SWIG_ROOT="${HOME}/ds-swig" export DS_CPU_COUNT=$(nproc) fi; if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then export DS_ROOT_TASK=${TASKCLUSTER_TASK_DIR} export PYENV_ROOT="${TASKCLUSTER_TASK_DIR}/pyenv-root" - export SWIG_ROOT="$(cygpath ${USERPROFILE})/ds-swig" export PLATFORM_EXE_SUFFIX=.exe export DS_CPU_COUNT=$(nproc) @@ -22,7 +20,6 @@ if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then fi; if [ "${OS}" = "Darwin" ]; then - export SWIG_ROOT="${TASKCLUSTER_ORIG_TASKDIR}/ds-swig" export DS_ROOT_TASK=${TASKCLUSTER_TASK_DIR} export DS_CPU_COUNT=$(sysctl hw.ncpu |cut -d' ' -f2) export PYENV_ROOT="${DS_ROOT_TASK}/pyenv-root" @@ -45,19 +42,6 @@ if [ "${OS}" = "Darwin" ]; then fi; fi; -SWIG_BIN=swig${PLATFORM_EXE_SUFFIX} -DS_SWIG_BIN=ds-swig${PLATFORM_EXE_SUFFIX} -if [ -f "${SWIG_ROOT}/bin/${DS_SWIG_BIN}" ]; then - export PATH=${SWIG_ROOT}/bin/:$PATH - export SWIG_LIB="$(find ${SWIG_ROOT}/share/swig/ -type f -name "swig.swg" | xargs dirname)" - # Make an alias to be more magic - if [ ! -L "${SWIG_ROOT}/bin/${SWIG_BIN}" ]; then - ln -s ${DS_SWIG_BIN} ${SWIG_ROOT}/bin/${SWIG_BIN} - fi; - swig -version - swig -swiglib -fi; - PY37_OPENSSL_DIR="${PYENV_ROOT}/ssl-xenial" export PY37_LDPATH="${PY37_OPENSSL_DIR}/usr/lib/" export LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH diff --git a/taskcluster/test-android-opt-base.tyml b/taskcluster/test-android-opt-base.tyml index b0be5330..c2911d77 100644 --- a/taskcluster/test-android-opt-base.tyml +++ b/taskcluster/test-android-opt-base.tyml @@ -50,7 +50,7 @@ then: ${extraSystemSetup} && chmod 777 /dev/kvm && adduser --system --home ${system.homedir.linux} ${system.username} && cd ${system.homedir.linux} && - echo -e "#!/bin/bash\nset -xe\n env && id && mkdir ~/DeepSpeech/ && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && mkdir -p ${system.homedir.linux}/ds-swig/bin/ && wget -O - ${system.swig.cache.linux_amd64} | tar -C ${system.homedir.linux}/ds-swig/ -xzf - && wget -O - ${build.cache.url} | tar -C ${system.homedir.linux} -xzf - && if [ ! -z "${build.gradle_cache.url}" ]; then wget -O - ${build.gradle_cache.url} | tar -C ${system.homedir.linux}/ -xzf - ; fi;" > /tmp/clone.sh && chmod +x /tmp/clone.sh && + echo -e "#!/bin/bash\nset -xe\n env && id && mkdir ~/DeepSpeech/ && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && wget -O - ${build.cache.url} | tar -C ${system.homedir.linux} -xzf - && if [ ! -z "${build.gradle_cache.url}" ]; then wget -O - ${build.gradle_cache.url} | tar -C ${system.homedir.linux}/ -xzf - ; fi;" > /tmp/clone.sh && chmod +x /tmp/clone.sh && sudo -H -u ${system.username} /bin/bash /tmp/clone.sh && sudo -H -u ${system.username} --preserve-env /bin/bash ${build.args.tests_cmdline} diff --git a/taskcluster/win-opt-base.tyml b/taskcluster/win-opt-base.tyml index 8f8acd40..97f3d78f 100644 --- a/taskcluster/win-opt-base.tyml +++ b/taskcluster/win-opt-base.tyml @@ -85,10 +85,6 @@ payload: format: tar.gz content: url: ${system.node_gyp_cache.url} - - directory: ds-swig - format: tar.gz - content: - url: ${system.swig.cache.win_amd64} artifacts: - type: "directory"