Add CTC decoder build on TaskCluster

native_client/ctcdecode/Makefile

@@ -41,17 +41,19 @@ workspace_status.cc:
 		../bazel_workspace_status_cmd.sh > bazel-out/stable-status.txt && \
 		../gen_workspace_status.sh > $@
 
+# Enforce PATH here because swig calls from build_ext looses track of some
+# variables over several runs
 bindings: clean-keep-third-party workspace_status.cc
 	pip install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==39.1.0
-	AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
+	PATH=$(TOOLCHAIN):$$PATH AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
 	find temp_build -type f -name "*.o" -delete
-	AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
+	AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
 	rm -rf temp_build
 
 bindings-debug: clean-keep-third-party workspace_status.cc
 	pip install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==39.1.0
-	AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --debug --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
+	PATH=$(TOOLCHAIN):$$PATH AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --debug --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
 	$(GENERATE_DEBUG_SYMS)
 	find temp_build -type f -name "*.o" -delete
-	AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
+	AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
 	rm -rf temp_build

native_client/ctcdecode/build_archive.py

@@ -54,6 +54,7 @@ def build_archive(srcs=[], out_name='', build_dir='temp_build/temp_build', debug
     if sys.platform.startswith('win'):
         compiler = '"{}"'.format(compiler)
     ar = os.environ.get('AR', 'ar')
+    libexe = os.environ.get('LIBEXE', 'lib.exe')
     libtool = os.environ.get('LIBTOOL', 'libtool')
     cflags = os.environ.get('CFLAGS', '') + os.environ.get('CXXFLAGS', '')
     args = ARGS + (DBG_ARGS if debug else OPT_ARGS)
@@ -100,7 +101,8 @@ def build_archive(srcs=[], out_name='', build_dir='temp_build/temp_build', debug
         print(cmd)
         subprocess.check_call(shlex.split(cmd))
     elif sys.platform.startswith('win'):
-        cmd = '"lib.exe" /OUT:"{outfile}" {infiles} /MACHINE:X64 /NOLOGO'.format(
+        cmd = '"{libexe}" /OUT:"{outfile}" {infiles} /MACHINE:X64 /NOLOGO'.format(
+            libexe=libexe,
             outfile=out_name,
             infiles=' '.join(obj_files))
         cmd = cmd.replace('\\', '/')

native_client/definitions.mk

@@ -10,6 +10,7 @@ TOOL_CC   := gcc
 TOOL_CXX  := c++
 TOOL_LD   := ld
 TOOL_LDD  := ldd
+TOOL_LIBEXE :=
 
 DEEPSPEECH_BIN       := deepspeech
 CFLAGS_DEEPSPEECH    := -std=c++11 -o $(DEEPSPEECH_BIN)
@@ -37,9 +38,10 @@ endif
 ifeq ($(TARGET),host-win)
 DEEPSPEECH_BIN  := deepspeech.exe
 TOOLCHAIN := '$(VCINSTALLDIR)\bin\amd64\'
-TOOL_CC   := cl.exe
+TOOL_CC     := cl.exe
-TOOL_CXX  := cl.exe
+TOOL_CXX    := cl.exe
-TOOL_LD   := link.exe
+TOOL_LD     := link.exe
+TOOL_LIBEXE := lib.exe
 LINK_DEEPSPEECH      := $(TFDIR)\bazel-bin\native_client\libdeepspeech.so.if.lib
 LINK_PATH_DEEPSPEECH :=
 CFLAGS_DEEPSPEECH    := -nologo -Fe$(DEEPSPEECH_BIN)
@@ -113,6 +115,7 @@ CC      := $(TOOLCHAIN)$(TOOL_CC)
 CXX     := $(TOOLCHAIN)$(TOOL_CXX)
 LD      := $(TOOLCHAIN)$(TOOL_LD)
 LDD     := $(TOOLCHAIN)$(TOOL_LDD) $(TOOLCHAIN_LDD_OPTS)
+LIBEXE  := $(TOOLCHAIN)$(TOOL_LIBEXE)
 
 RPATH_PYTHON         := '-Wl,-rpath,\$$ORIGIN/lib/' $(LDFLAGS_RPATH)
 RPATH_NODEJS         := '-Wl,-rpath,$$\$$ORIGIN/../'

native_client/python/Makefile

@@ -6,6 +6,8 @@ bindings-clean:
 	rm -rf dist temp_build deepspeech.egg-info MANIFEST.in temp_lib
 	rm -f impl_wrap.cpp impl.py
 
+# Enforce PATH here because swig calls from build_ext looses track of some
+# variables over several runs
 bindings-build:
 	pip install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==39.1.0
 	PATH=$(TOOLCHAIN):$$PATH AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED) $(RPATH_PYTHON)" MODEL_LDFLAGS="$(LDFLAGS_DIRS)" MODEL_LIBS="$(LIBS)" $(PYTHON_PATH) $(PYTHON_SYSCONFIGDATA) $(NUMPY_INCLUDE) python ./setup.py build_ext $(PYTHON_PLATFORM_NAME)

taskcluster/decoder-build.sh

@@ -6,6 +6,10 @@ source $(dirname "$0")/tc-tests-utils.sh
 
 source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh
 
-export SYSTEM_TARGET=host
+if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then
+  export SYSTEM_TARGET=host-win
+else
+  export SYSTEM_TARGET=host
+fi;
 
 do_deepspeech_decoder_build

taskcluster/scriptworker-task-github.yml

@@ -19,6 +19,7 @@ build:
     - "android-java-opt"
     - "win-amd64-cpu-opt"
     - "win-amd64-gpu-opt"
+    - "win-amd64-ctc-opt"
   allowed:
     - "tag"
   ref_match: "refs/tags/"
@@ -38,6 +39,7 @@ build:
       - "win-amd64-cpu-opt"
       - "win-amd64-tflite-opt"
       - "win-amd64-gpu-opt"
+      - "win-amd64-ctc-opt"
     javascript:
       # GPU package
       - "node-package-gpu"

taskcluster/win-amd64-ctc-opt.yml

@@ -0,0 +1,17 @@
+build:
+  template_file: win-opt-base.tyml
+  dependencies:
+    - "swig-win-amd64"
+    - "node-gyp-cache"
+    - "pyenv-win-amd64"
+  routes:
+    - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.win-ctc"
+    - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.win-ctc"
+    - "index.project.deepspeech.deepspeech.native_client.win-ctc.${event.head.sha}"
+  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.win/artifacts/public/home.tar.xz"
+  scripts:
+    build: 'taskcluster/decoder-build.sh'
+    package: 'taskcluster/decoder-package.sh'
+  metadata:
+    name: "DeepSpeech CTC Decoder Windows AMD64 CPU"
+    description: "Building DeepSpeech CTC Decoder for Windows/AMD64, CPU only, optimized version"