Building on Windows

Fixes #1793
Fixes #1794

.taskcluster.yml

@@ -26,6 +26,7 @@ tasks:
 
     scopes: [
       "queue:create-task:lowest:{{ taskcluster.docker.provisionerId }}/deepspeech-worker",
+      "queue:create-task:lowest:{{ taskcluster.docker.provisionerId }}/deepspeech-win",
       "queue:create-task:lowest:{{ taskcluster.docker.provisionerId }}/deepspeech-kvm-worker",
       "queue:create-task:lowest:deepspeech-provisioner/ds-macos-light",
       "queue:create-task:lowest:deepspeech-provisioner/ds-scriptworker",

examples/net_framework/CSharpExamples/nupkg-gpu/build/DeepSpeech.targets

@@ -1,9 +0,0 @@
-<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
-  <ItemGroup>
-    <NativeLibs Include="$(MSBuildThisFileDirectory)\*.so" />
-    <None Include="@(NativeLibs)">
-      <Link>%(FileName)%(Extension)</Link>
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-    </None>
-  </ItemGroup>
-</Project>

examples/net_framework/CSharpExamples/nupkg-gpu/deepspeech.nuspec

@@ -1,21 +0,0 @@
-<?xml version="1.0"?>
-<package>
-  <metadata>
-    <id>DeepSpeech-GPU</id>
-    <version>0.4.1</version>
-    <title>DeepSpeech</title>
-    <authors>Mozilla</authors>
-    <owners>Mozilla</owners>
-    <license type="expression">MPL-2.0</license>
-    <projectUrl>http://github.com/mozilla/DeepSpeech</projectUrl>
-    <requireLicenseAcceptance>false</requireLicenseAcceptance>
-    <description>A library for running inference with a DeepSpeech model</description>
-    <copyright>Copyright (c) 2019 Mozilla Corporation</copyright>
-    <tags>native speech speech_recognition</tags>
-  </metadata>
-  <files>
-    <file src="build\**" target="build/"/>
-    <file src="lib\**" target="lib/"/>
-    <file src="tools\**" target="tools/"/>
-  </files>
-</package>

examples/net_framework/CSharpExamples/nupkg/deepspeech.nuspec.in

@@ -1,8 +1,8 @@
 <?xml version="1.0"?>
 <package>
   <metadata>
-    <id>DeepSpeech</id>
+    <id>$NUPKG_ID</id>
-    <version>0.4.1</version>
+    <version>$NUPKG_VERSION</version>
     <title>DeepSpeech</title>
     <authors>Mozilla</authors>
     <owners>Mozilla</owners>

native_client/Makefile

@@ -13,25 +13,25 @@
 
 include definitions.mk
 
-default: deepspeech
+default: $(DEEPSPEECH_BIN)
 
-clean:
+clean: bindings-clean
 	rm -f deepspeech
 
-deepspeech: client.cc
+$(DEEPSPEECH_BIN): client.cc
-	$(CXX) -std=c++11 -o deepspeech $(CFLAGS) $(SOX_CFLAGS) client.cc $(LDFLAGS) $(SOX_LDFLAGS)
+	$(CXX) $(CFLAGS) $(CFLAGS_DEEPSPEECH) $(SOX_CFLAGS) client.cc $(LDFLAGS) $(SOX_LDFLAGS)
 ifeq ($(OS),Darwin)
 	install_name_tool -change $$TASKCLUSTER_TASK_DIR/homebrew/opt/sox/lib/libsox.3.dylib @rpath/libsox.3.dylib deepspeech
 	install_name_tool -change bazel-out/local-opt/bin/native_client/libdeepspeech.so @rpath/libdeepspeech.so deepspeech
 endif
 
-run: deepspeech
+run: $(DEEPSPEECH_BIN)
 	${META_LD_LIBRARY_PATH}=${TFDIR}/bazel-bin/native_client:${${META_LD_LIBRARY_PATH}} ./deepspeech ${ARGS}
 
-debug: deepspeech
+debug: $(DEEPSPEECH_BIN)
 	${META_LD_LIBRARY_PATH}=${TFDIR}/bazel-bin/native_client:${${META_LD_LIBRARY_PATH}} gdb --args ./deepspeech ${ARGS}
 
-install: deepspeech
+install: $(DEEPSPEECH_BIN)
 	install -d ${PREFIX}/lib
 	install -m 0644 ${TFDIR}/bazel-bin/native_client/libdeepspeech.so ${PREFIX}/lib/
 	install -d ${PREFIX}/bin

native_client/args.h

@@ -1,7 +1,11 @@
 #ifndef __ARGS_H__
 #define __ARGS_H__
 
+#if defined(_MSC_VER)
+#include "getopt_win.h"
+#else
 #include <getopt.h>
+#endif
 #include <iostream>
 
 #include "deepspeech.h"

native_client/client.cc

@@ -2,22 +2,33 @@
 #include <stdio.h>
 
 #include <assert.h>
-#include <dirent.h>
 #include <errno.h>
 #include <math.h>
 #include <string.h>
-#ifndef __ANDROID__
-#include <sox.h>
-#endif // __ANDROID__
 #include <time.h>
-#include <unistd.h>
-
 #include <sys/types.h>
 #include <sys/stat.h>
 
 #include <sstream>
 #include <string>
 
+#if defined(__ANDROID__) || defined(_MSC_VER)
+#define NO_SOX
+#endif
+
+#if defined(_MSC_VER)
+#define NO_DIR
+#endif
+
+#ifndef NO_SOX
+#include <sox.h>
+#endif
+
+#ifndef NO_DIR
+#include <dirent.h>
+#include <unistd.h>
+#endif // NO_DIR
+
 #include "deepspeech.h"
 #include "args.h"
 
@@ -61,7 +72,7 @@ GetAudioBuffer(const char* path)
 {
   ds_audio_buffer res = {0};
 
-#ifndef __ANDROID__
+#ifndef NO_SOX
   sox_format_t* input = sox_open_read(path, NULL, NULL, NULL);
   assert(input);
 
@@ -150,9 +161,9 @@ GetAudioBuffer(const char* path)
   // Close sox handles
   sox_close(output);
   sox_close(input);
-#endif // __ANDROID__
+#endif // NO_SOX
 
-#ifdef __ANDROID__
+#ifdef NO_SOX
   // FIXME: Hack and support only 16kHz mono 16-bits PCM
   FILE* wave = fopen(path, "r");
 
@@ -160,19 +171,15 @@ GetAudioBuffer(const char* path)
 
   unsigned short audio_format;
   fseek(wave, 20, SEEK_SET); rv = fread(&audio_format, 2, 1, wave);
-  assert(rv == 2);
 
   unsigned short num_channels;
   fseek(wave, 22, SEEK_SET); rv = fread(&num_channels, 2, 1, wave);
-  assert(rv == 2);
 
   unsigned int sample_rate;
   fseek(wave, 24, SEEK_SET); rv = fread(&sample_rate, 4, 1, wave);
-  assert(rv == 2);
 
   unsigned short bits_per_sample;
   fseek(wave, 34, SEEK_SET); rv = fread(&bits_per_sample, 2, 1, wave);
-  assert(rv == 2);
 
   assert(audio_format == 1); // 1 is PCM
   assert(num_channels == 1); // MONO
@@ -185,16 +192,14 @@ GetAudioBuffer(const char* path)
   fprintf(stderr, "bits_per_sample=%d\n", bits_per_sample);
 
   fseek(wave, 40, SEEK_SET); rv = fread(&res.buffer_size, 4, 1, wave);
-  assert(rv == 2);
   fprintf(stderr, "res.buffer_size=%ld\n", res.buffer_size);
 
   fseek(wave, 44, SEEK_SET);
   res.buffer = (char*)malloc(sizeof(char) * res.buffer_size);
   rv = fread(res.buffer, sizeof(char), res.buffer_size, wave);
-  assert(rv == res.buffer_size);
 
   fclose(wave);
-#endif // __ANDROID__
+#endif // NO_SOX
 
 #ifdef __APPLE__
   res.buffer_size = (size_t)(output->olength * 2);
@@ -261,8 +266,10 @@ main(int argc, char **argv)
     }
   }
 
+#ifndef NO_SOX
   // Initialise SOX
   assert(sox_init() == SOX_SUCCESS);
+#endif
 
   struct stat wav_info;
   if (0 != stat(audio, &wav_info)) {
@@ -270,11 +277,14 @@ main(int argc, char **argv)
   }
 
   switch (wav_info.st_mode & S_IFMT) {
+#ifndef _MSC_VER
     case S_IFLNK:
+#endif
     case S_IFREG:
         ProcessFile(ctx, audio, show_times);
       break;
 
+#ifndef NO_DIR
     case S_IFDIR:
         {
           printf("Running on directory %s\n", audio);
@@ -297,16 +307,17 @@ main(int argc, char **argv)
           closedir(wav_dir);
         }
       break;
+#endif
 
     default:
         printf("Unexpected type for %s: %d\n", audio, (wav_info.st_mode & S_IFMT));
       break;
   }
 
-#ifndef __ANDROID__
+#ifndef NO_SOX
   // Deinitialise and quit
   sox_quit();
-#endif // __ANDROID__
+#endif // NO_SOX
 
   DS_DestroyModel(ctx);
 

native_client/definitions.mk

@@ -5,6 +5,17 @@ TFDIR     ?= $(abspath $(NC_DIR)/../../tensorflow)
 PREFIX    ?= /usr/local
 SO_SEARCH ?= $(TFDIR)/bazel-bin/
 
+TOOL_AS   := as
+TOOL_CC   := gcc
+TOOL_CXX  := c++
+TOOL_LD   := ld
+TOOL_LDD  := ldd
+
+DEEPSPEECH_BIN       := deepspeech
+CFLAGS_DEEPSPEECH    := -std=c++11 -o $(DEEPSPEECH_BIN)
+LINK_DEEPSPEECH      := -ldeepspeech
+LINK_PATH_DEEPSPEECH := -L${TFDIR}/bazel-bin/native_client
+
 ifeq ($(TARGET),host)
 TOOLCHAIN       :=
 CFLAGS          :=
@@ -18,6 +29,19 @@ PYTHON_PLATFORM_NAME := --plat-name manylinux1_x86_64
 endif
 endif
 
+ifeq ($(TARGET),host-win)
+DEEPSPEECH_BIN  := deepspeech.exe
+TOOLCHAIN := '$(VCINSTALLDIR)\bin\amd64\'
+TOOL_CC   := cl.exe
+TOOL_CXX  := cl.exe
+TOOL_LD   := link.exe
+LINK_DEEPSPEECH      := $(TFDIR)\bazel-bin\native_client\libdeepspeech.so.if.lib
+LINK_PATH_DEEPSPEECH :=
+CFLAGS_DEEPSPEECH    := -nologo -Fe$(DEEPSPEECH_BIN)
+SOX_CFLAGS      :=
+SOX_LDFLAGS     :=
+endif
+
 ifeq ($(TARGET),rpi3)
 TOOLCHAIN   ?= ${TFDIR}/bazel-$(shell basename "${TFDIR}")/external/LinaroArmGcc72/bin/arm-linux-gnueabihf-
 RASPBIAN    ?= $(abspath $(NC_DIR)/../multistrap-raspbian-stretch)
@@ -72,15 +96,15 @@ endif
 
 CFLAGS   += $(EXTRA_CFLAGS)
 CXXFLAGS += $(EXTRA_CXXFLAGS)
-LIBS     := -ldeepspeech $(EXTRA_LIBS)
+LIBS     := $(LINK_DEEPSPEECH) $(EXTRA_LIBS)
-LDFLAGS_DIRS := -L${TFDIR}/bazel-bin/native_client $(EXTRA_LDFLAGS)
+LDFLAGS_DIRS := $(LINK_PATH_DEEPSPEECH) $(EXTRA_LDFLAGS)
 LDFLAGS  += $(LDFLAGS_NEEDED) $(LDFLAGS_RPATH) $(LDFLAGS_DIRS) $(LIBS)
 
-AS      := $(TOOLCHAIN)as
+AS      := $(TOOLCHAIN)$(TOOL_AS)
-CC      := $(TOOLCHAIN)gcc
+CC      := $(TOOLCHAIN)$(TOOL_CC)
-CXX     := $(TOOLCHAIN)c++
+CXX     := $(TOOLCHAIN)$(TOOL_CXX)
-LD      := $(TOOLCHAIN)ld
+LD      := $(TOOLCHAIN)$(TOOL_LD)
-LDD     := $(TOOLCHAIN)ldd $(TOOLCHAIN_LDD_OPTS)
+LDD     := $(TOOLCHAIN)$(TOOL_LDD) $(TOOLCHAIN_LDD_OPTS)
 
 RPATH_PYTHON         := '-Wl,-rpath,\$$ORIGIN/lib/' $(LDFLAGS_RPATH)
 RPATH_NODEJS         := '-Wl,-rpath,$$\$$ORIGIN/../'

native_client/ds_git_version.sh

@@ -4,6 +4,10 @@ if [ `uname` = "Darwin" ]; then
    export PATH="/Users/build-user/TaskCluster/Workdir/tasks/tc-workdir/homebrew/opt/coreutils/libexec/gnubin:${PATH}"
 fi
 
+if [ `uname -o` = "Msys" ]; then
+   export PATH="/c/Program Files/Git/bin/:${PATH}"
+fi
+
 DS_GIT_DIR="$(realpath "$(dirname "$(realpath "$0")")/../.git/")"
 if [ ! -d "${DS_GIT_DIR}" ]; then
    return 1

native_client/getopt_win.h

@@ -0,0 +1,653 @@
+#ifndef __GETOPT_H__
+/**
+ * DISCLAIMER
+ * This file is part of the mingw-w64 runtime package.
+ *
+ * The mingw-w64 runtime package and its code is distributed in the hope that it 
+ * will be useful but WITHOUT ANY WARRANTY.  ALL WARRANTIES, EXPRESSED OR 
+ * IMPLIED ARE HEREBY DISCLAIMED.  This includes but is not limited to 
+ * warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ */
+ /*
+ * Copyright (c) 2002 Todd C. Miller <Todd.Miller@courtesan.com>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ *
+ * Sponsored in part by the Defense Advanced Research Projects
+ * Agency (DARPA) and Air Force Research Laboratory, Air Force
+ * Materiel Command, USAF, under agreement number F39502-99-1-0512.
+ */
+/*-
+ * Copyright (c) 2000 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Dieter Baron and Thomas Klausner.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma warning(disable:4996);
+
+#define __GETOPT_H__
+
+/* All the headers include this file. */
+#include <crtdefs.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <windows.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define	REPLACE_GETOPT		/* use this getopt as the system getopt(3) */
+
+#ifdef REPLACE_GETOPT
+int	opterr = 1;		/* if error message should be printed */
+int	optind = 1;		/* index into parent argv vector */
+int	optopt = '?';		/* character checked for validity */
+#undef	optreset		/* see getopt.h */
+#define	optreset		__mingw_optreset
+int	optreset;		/* reset getopt */
+char    *optarg;		/* argument associated with option */
+#endif
+
+//extern int optind;		/* index of first non-option in argv      */
+//extern int optopt;		/* single option character, as parsed     */
+//extern int opterr;		/* flag to enable built-in diagnostics... */
+//				/* (user may set to zero, to suppress)    */
+//
+//extern char *optarg;		/* pointer to argument of current option  */
+
+#define PRINT_ERROR	((opterr) && (*options != ':'))
+
+#define FLAG_PERMUTE	0x01	/* permute non-options to the end of argv */
+#define FLAG_ALLARGS	0x02	/* treat non-options as args to option "-1" */
+#define FLAG_LONGONLY	0x04	/* operate as getopt_long_only */
+
+/* return values */
+#define	BADCH		(int)'?'
+#define	BADARG		((*options == ':') ? (int)':' : (int)'?')
+#define	INORDER 	(int)1
+
+#ifndef __CYGWIN__
+#define __progname __argv[0]
+#else
+extern char __declspec(dllimport) *__progname;
+#endif
+
+#ifdef __CYGWIN__
+static char EMSG[] = "";
+#else
+#define	EMSG		""
+#endif
+
+static int getopt_internal(int, char * const *, const char *,
+			   const struct option *, int *, int);
+static int parse_long_options(char * const *, const char *,
+			      const struct option *, int *, int);
+static int gcd(int, int);
+static void permute_args(int, int, int, char * const *);
+
+static char *place = EMSG; /* option letter processing */
+
+/* XXX: set optreset to 1 rather than these two */
+static int nonopt_start = -1; /* first non option argument (for permute) */
+static int nonopt_end = -1;   /* first option after non options (for permute) */
+
+/* Error messages */
+static const char recargchar[] = "option requires an argument -- %c";
+static const char recargstring[] = "option requires an argument -- %s";
+static const char ambig[] = "ambiguous option -- %.*s";
+static const char noarg[] = "option doesn't take an argument -- %.*s";
+static const char illoptchar[] = "unknown option -- %c";
+static const char illoptstring[] = "unknown option -- %s";
+
+static void
+_vwarnx(const char *fmt,va_list ap)
+{
+  (void)fprintf(stderr,"%s: ",__progname);
+  if (fmt != NULL)
+    (void)vfprintf(stderr,fmt,ap);
+  (void)fprintf(stderr,"\n");
+}
+
+static void
+warnx(const char *fmt,...)
+{
+  va_list ap;
+  va_start(ap,fmt);
+  _vwarnx(fmt,ap);
+  va_end(ap);
+}
+
+/*
+ * Compute the greatest common divisor of a and b.
+ */
+static int
+gcd(int a, int b)
+{
+	int c;
+
+	c = a % b;
+	while (c != 0) {
+		a = b;
+		b = c;
+		c = a % b;
+	}
+
+	return (b);
+}
+
+/*
+ * Exchange the block from nonopt_start to nonopt_end with the block
+ * from nonopt_end to opt_end (keeping the same order of arguments
+ * in each block).
+ */
+static void
+permute_args(int panonopt_start, int panonopt_end, int opt_end,
+	char * const *nargv)
+{
+	int cstart, cyclelen, i, j, ncycle, nnonopts, nopts, pos;
+	char *swap;
+
+	/*
+	 * compute lengths of blocks and number and size of cycles
+	 */
+	nnonopts = panonopt_end - panonopt_start;
+	nopts = opt_end - panonopt_end;
+	ncycle = gcd(nnonopts, nopts);
+	cyclelen = (opt_end - panonopt_start) / ncycle;
+
+	for (i = 0; i < ncycle; i++) {
+		cstart = panonopt_end+i;
+		pos = cstart;
+		for (j = 0; j < cyclelen; j++) {
+			if (pos >= panonopt_end)
+				pos -= nnonopts;
+			else
+				pos += nopts;
+			swap = nargv[pos];
+			/* LINTED const cast */
+			((char **) nargv)[pos] = nargv[cstart];
+			/* LINTED const cast */
+			((char **)nargv)[cstart] = swap;
+		}
+	}
+}
+
+#ifdef REPLACE_GETOPT
+/*
+ * getopt --
+ *	Parse argc/argv argument vector.
+ *
+ * [eventually this will replace the BSD getopt]
+ */
+int
+getopt(int nargc, char * const *nargv, const char *options)
+{
+
+	/*
+	 * We don't pass FLAG_PERMUTE to getopt_internal() since
+	 * the BSD getopt(3) (unlike GNU) has never done this.
+	 *
+	 * Furthermore, since many privileged programs call getopt()
+	 * before dropping privileges it makes sense to keep things
+	 * as simple (and bug-free) as possible.
+	 */
+	return (getopt_internal(nargc, nargv, options, NULL, NULL, 0));
+}
+#endif /* REPLACE_GETOPT */
+
+//extern int getopt(int nargc, char * const *nargv, const char *options);
+
+#ifdef _BSD_SOURCE
+/*
+ * BSD adds the non-standard `optreset' feature, for reinitialisation
+ * of `getopt' parsing.  We support this feature, for applications which
+ * proclaim their BSD heritage, before including this header; however,
+ * to maintain portability, developers are advised to avoid it.
+ */
+# define optreset  __mingw_optreset
+extern int optreset;
+#endif
+#ifdef __cplusplus
+}
+#endif
+/*
+ * POSIX requires the `getopt' API to be specified in `unistd.h';
+ * thus, `unistd.h' includes this header.  However, we do not want
+ * to expose the `getopt_long' or `getopt_long_only' APIs, when
+ * included in this manner.  Thus, close the standard __GETOPT_H__
+ * declarations block, and open an additional __GETOPT_LONG_H__
+ * specific block, only when *not* __UNISTD_H_SOURCED__, in which
+ * to declare the extended API.
+ */
+#endif /* !defined(__GETOPT_H__) */
+
+#if !defined(__UNISTD_H_SOURCED__) && !defined(__GETOPT_LONG_H__)
+#define __GETOPT_LONG_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct option		/* specification for a long form option...	*/
+{
+  const char *name;		/* option name, without leading hyphens */
+  int         has_arg;		/* does it take an argument?		*/
+  int        *flag;		/* where to save its status, or NULL	*/
+  int         val;		/* its associated status value		*/
+};
+
+enum    		/* permitted values for its `has_arg' field...	*/
+{
+  no_argument = 0,      	/* option never takes an argument	*/
+  required_argument,		/* option always requires an argument	*/
+  optional_argument		/* option may take an argument		*/
+};
+
+/*
+ * parse_long_options --
+ *	Parse long options in argc/argv argument vector.
+ * Returns -1 if short_too is set and the option does not match long_options.
+ */
+static int
+parse_long_options(char * const *nargv, const char *options,
+	const struct option *long_options, int *idx, int short_too)
+{
+	char *current_argv, *has_equal;
+	size_t current_argv_len;
+	int i, ambiguous, match;
+
+#define IDENTICAL_INTERPRETATION(_x, _y)                                \
+	(long_options[(_x)].has_arg == long_options[(_y)].has_arg &&    \
+	 long_options[(_x)].flag == long_options[(_y)].flag &&          \
+	 long_options[(_x)].val == long_options[(_y)].val)
+
+	current_argv = place;
+	match = -1;
+	ambiguous = 0;
+
+	optind++;
+
+	if ((has_equal = strchr(current_argv, '=')) != NULL) {
+		/* argument found (--option=arg) */
+		current_argv_len = has_equal - current_argv;
+		has_equal++;
+	} else
+		current_argv_len = strlen(current_argv);
+
+	for (i = 0; long_options[i].name; i++) {
+		/* find matching long option */
+		if (strncmp(current_argv, long_options[i].name,
+		    current_argv_len))
+			continue;
+
+		if (strlen(long_options[i].name) == current_argv_len) {
+			/* exact match */
+			match = i;
+			ambiguous = 0;
+			break;
+		}
+		/*
+		 * If this is a known short option, don't allow
+		 * a partial match of a single character.
+		 */
+		if (short_too && current_argv_len == 1)
+			continue;
+
+		if (match == -1)	/* partial match */
+			match = i;
+		else if (!IDENTICAL_INTERPRETATION(i, match))
+			ambiguous = 1;
+	}
+	if (ambiguous) {
+		/* ambiguous abbreviation */
+		if (PRINT_ERROR)
+			warnx(ambig, (int)current_argv_len,
+			     current_argv);
+		optopt = 0;
+		return (BADCH);
+	}
+	if (match != -1) {		/* option found */
+		if (long_options[match].has_arg == no_argument
+		    && has_equal) {
+			if (PRINT_ERROR)
+				warnx(noarg, (int)current_argv_len,
+				     current_argv);
+			/*
+			 * XXX: GNU sets optopt to val regardless of flag
+			 */
+			if (long_options[match].flag == NULL)
+				optopt = long_options[match].val;
+			else
+				optopt = 0;
+			return (BADARG);
+		}
+		if (long_options[match].has_arg == required_argument ||
+		    long_options[match].has_arg == optional_argument) {
+			if (has_equal)
+				optarg = has_equal;
+			else if (long_options[match].has_arg ==
+			    required_argument) {
+				/*
+				 * optional argument doesn't use next nargv
+				 */
+				optarg = nargv[optind++];
+			}
+		}
+		if ((long_options[match].has_arg == required_argument)
+		    && (optarg == NULL)) {
+			/*
+			 * Missing argument; leading ':' indicates no error
+			 * should be generated.
+			 */
+			if (PRINT_ERROR)
+				warnx(recargstring,
+				    current_argv);
+			/*
+			 * XXX: GNU sets optopt to val regardless of flag
+			 */
+			if (long_options[match].flag == NULL)
+				optopt = long_options[match].val;
+			else
+				optopt = 0;
+			--optind;
+			return (BADARG);
+		}
+	} else {			/* unknown option */
+		if (short_too) {
+			--optind;
+			return (-1);
+		}
+		if (PRINT_ERROR)
+			warnx(illoptstring, current_argv);
+		optopt = 0;
+		return (BADCH);
+	}
+	if (idx)
+		*idx = match;
+	if (long_options[match].flag) {
+		*long_options[match].flag = long_options[match].val;
+		return (0);
+	} else
+		return (long_options[match].val);
+#undef IDENTICAL_INTERPRETATION
+}
+
+/*
+ * getopt_internal --
+ *	Parse argc/argv argument vector.  Called by user level routines.
+ */
+static int
+getopt_internal(int nargc, char * const *nargv, const char *options,
+	const struct option *long_options, int *idx, int flags)
+{
+	char *oli;				/* option letter list index */
+	int optchar, short_too;
+	static int posixly_correct = -1;
+
+	if (options == NULL)
+		return (-1);
+
+	/*
+	 * XXX Some GNU programs (like cvs) set optind to 0 instead of
+	 * XXX using optreset.  Work around this braindamage.
+	 */
+	if (optind == 0)
+		optind = optreset = 1;
+
+	/*
+	 * Disable GNU extensions if POSIXLY_CORRECT is set or options
+	 * string begins with a '+'.
+	 *
+	 * CV, 2009-12-14: Check POSIXLY_CORRECT anew if optind == 0 or
+	 *                 optreset != 0 for GNU compatibility.
+	 */
+	if (posixly_correct == -1 || optreset != 0)
+		posixly_correct = (getenv("POSIXLY_CORRECT") != NULL);
+	if (*options == '-')
+		flags |= FLAG_ALLARGS;
+	else if (posixly_correct || *options == '+')
+		flags &= ~FLAG_PERMUTE;
+	if (*options == '+' || *options == '-')
+		options++;
+
+	optarg = NULL;
+	if (optreset)
+		nonopt_start = nonopt_end = -1;
+start:
+	if (optreset || !*place) {		/* update scanning pointer */
+		optreset = 0;
+		if (optind >= nargc) {          /* end of argument vector */
+			place = EMSG;
+			if (nonopt_end != -1) {
+				/* do permutation, if we have to */
+				permute_args(nonopt_start, nonopt_end,
+				    optind, nargv);
+				optind -= nonopt_end - nonopt_start;
+			}
+			else if (nonopt_start != -1) {
+				/*
+				 * If we skipped non-options, set optind
+				 * to the first of them.
+				 */
+				optind = nonopt_start;
+			}
+			nonopt_start = nonopt_end = -1;
+			return (-1);
+		}
+		if (*(place = nargv[optind]) != '-' ||
+		    (place[1] == '\0' && strchr(options, '-') == NULL)) {
+			place = EMSG;		/* found non-option */
+			if (flags & FLAG_ALLARGS) {
+				/*
+				 * GNU extension:
+				 * return non-option as argument to option 1
+				 */
+				optarg = nargv[optind++];
+				return (INORDER);
+			}
+			if (!(flags & FLAG_PERMUTE)) {
+				/*
+				 * If no permutation wanted, stop parsing
+				 * at first non-option.
+				 */
+				return (-1);
+			}
+			/* do permutation */
+			if (nonopt_start == -1)
+				nonopt_start = optind;
+			else if (nonopt_end != -1) {
+				permute_args(nonopt_start, nonopt_end,
+				    optind, nargv);
+				nonopt_start = optind -
+				    (nonopt_end - nonopt_start);
+				nonopt_end = -1;
+			}
+			optind++;
+			/* process next argument */
+			goto start;
+		}
+		if (nonopt_start != -1 && nonopt_end == -1)
+			nonopt_end = optind;
+
+		/*
+		 * If we have "-" do nothing, if "--" we are done.
+		 */
+		if (place[1] != '\0' && *++place == '-' && place[1] == '\0') {
+			optind++;
+			place = EMSG;
+			/*
+			 * We found an option (--), so if we skipped
+			 * non-options, we have to permute.
+			 */
+			if (nonopt_end != -1) {
+				permute_args(nonopt_start, nonopt_end,
+				    optind, nargv);
+				optind -= nonopt_end - nonopt_start;
+			}
+			nonopt_start = nonopt_end = -1;
+			return (-1);
+		}
+	}
+
+	/*
+	 * Check long options if:
+	 *  1) we were passed some
+	 *  2) the arg is not just "-"
+	 *  3) either the arg starts with -- we are getopt_long_only()
+	 */
+	if (long_options != NULL && place != nargv[optind] &&
+	    (*place == '-' || (flags & FLAG_LONGONLY))) {
+		short_too = 0;
+		if (*place == '-')
+			place++;		/* --foo long option */
+		else if (*place != ':' && strchr(options, *place) != NULL)
+			short_too = 1;		/* could be short option too */
+
+		optchar = parse_long_options(nargv, options, long_options,
+		    idx, short_too);
+		if (optchar != -1) {
+			place = EMSG;
+			return (optchar);
+		}
+	}
+
+	if ((optchar = (int)*place++) == (int)':' ||
+	    (optchar == (int)'-' && *place != '\0') ||
+	    (oli = (char*)strchr(options, optchar)) == NULL) {
+		/*
+		 * If the user specified "-" and  '-' isn't listed in
+		 * options, return -1 (non-option) as per POSIX.
+		 * Otherwise, it is an unknown option character (or ':').
+		 */
+		if (optchar == (int)'-' && *place == '\0')
+			return (-1);
+		if (!*place)
+			++optind;
+		if (PRINT_ERROR)
+			warnx(illoptchar, optchar);
+		optopt = optchar;
+		return (BADCH);
+	}
+	if (long_options != NULL && optchar == 'W' && oli[1] == ';') {
+		/* -W long-option */
+		if (*place)			/* no space */
+			/* NOTHING */;
+		else if (++optind >= nargc) {	/* no arg */
+			place = EMSG;
+			if (PRINT_ERROR)
+				warnx(recargchar, optchar);
+			optopt = optchar;
+			return (BADARG);
+		} else				/* white space */
+			place = nargv[optind];
+		optchar = parse_long_options(nargv, options, long_options,
+		    idx, 0);
+		place = EMSG;
+		return (optchar);
+	}
+	if (*++oli != ':') {			/* doesn't take argument */
+		if (!*place)
+			++optind;
+	} else {				/* takes (optional) argument */
+		optarg = NULL;
+		if (*place)			/* no white space */
+			optarg = place;
+		else if (oli[1] != ':') {	/* arg not optional */
+			if (++optind >= nargc) {	/* no arg */
+				place = EMSG;
+				if (PRINT_ERROR)
+					warnx(recargchar, optchar);
+				optopt = optchar;
+				return (BADARG);
+			} else
+				optarg = nargv[optind];
+		}
+		place = EMSG;
+		++optind;
+	}
+	/* dump back option letter */
+	return (optchar);
+}
+
+/*
+ * getopt_long --
+ *	Parse argc/argv argument vector.
+ */
+int
+getopt_long(int nargc, char * const *nargv, const char *options,
+    const struct option *long_options, int *idx)
+{
+
+	return (getopt_internal(nargc, nargv, options, long_options, idx,
+	    FLAG_PERMUTE));
+}
+
+/*
+ * getopt_long_only --
+ *	Parse argc/argv argument vector.
+ */
+int
+getopt_long_only(int nargc, char * const *nargv, const char *options,
+    const struct option *long_options, int *idx)
+{
+
+	return (getopt_internal(nargc, nargv, options, long_options, idx,
+	    FLAG_PERMUTE|FLAG_LONGONLY));
+}
+
+//extern int getopt_long(int nargc, char * const *nargv, const char *options,
+//    const struct option *long_options, int *idx);
+//extern int getopt_long_only(int nargc, char * const *nargv, const char *options,
+//    const struct option *long_options, int *idx);
+/*
+ * Previous MinGW implementation had...
+ */
+#ifndef HAVE_DECL_GETOPT
+/*
+ * ...for the long form API only; keep this for compatibility.
+ */
+# define HAVE_DECL_GETOPT	1
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* !defined(__UNISTD_H_SOURCED__) && !defined(__GETOPT_LONG_H__) */

native_client/javascript/Makefile

@@ -6,6 +6,10 @@ PROJECT_VERSION    ?= $(shell cat ../../VERSION | tr -d '\n')
 
 include ../definitions.mk
 
+ifeq ($(TARGET),host-win)
+LIBS := '$(shell cygpath -w $(subst .lib,,$(LIBS)))'
+endif
+
 default: build
 
 clean:
@@ -32,7 +36,7 @@ configure: deepspeech_wrap.cxx package.json
 	$(NODE_BUILD_TOOL) configure $(NODE_BUILD_VERBOSE)
 
 build: configure deepspeech_wrap.cxx
-	AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" CXXFLAGS="$(CXXFLAGS)" LDFLAGS="$(RPATH_NODEJS) $(LDFLAGS)" LIBS="$(LIBS)" $(NODE_BUILD_TOOL) $(NODE_PLATFORM_TARGET) $(NODE_ABI_TARGET) rebuild $(NODE_BUILD_VERBOSE)
+	AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" CXXFLAGS="$(CXXFLAGS)" LDFLAGS="$(RPATH_NODEJS) $(LDFLAGS)" LIBS=$(LIBS) $(NODE_BUILD_TOOL) $(NODE_PLATFORM_TARGET) $(NODE_ABI_TARGET) rebuild $(NODE_BUILD_VERBOSE)
 
 copy-deps: build
 	$(call copy_missing_libs,lib/binding/*/*/*/deepspeech.node,lib/binding/*/*/)

native_client/javascript/binding.gyp

@@ -4,7 +4,7 @@
       "target_name": "deepspeech",
       "sources": [ "deepspeech_wrap.cxx" ],
       "libraries": [
-        "${LIBS}"
+        "$(LIBS)"
       ],
       "include_dirs": [
         "../"

native_client/python/Makefile

@@ -8,7 +8,7 @@ bindings-clean:
 
 bindings-build:
 	pip install --quiet $(PYTHON_PACKAGES) wheel==0.31.0 setuptools==39.1.0
-	AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED) $(RPATH_PYTHON)" MODEL_LDFLAGS="$(LDFLAGS_DIRS)" MODEL_LIBS="$(LIBS)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext $(PYTHON_PLATFORM_NAME)
+	PATH=$(TOOLCHAIN):$$PATH AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED) $(RPATH_PYTHON)" MODEL_LDFLAGS="$(LDFLAGS_DIRS)" MODEL_LIBS="$(LIBS)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext $(PYTHON_PLATFORM_NAME)
 
 MANIFEST.in: bindings-build
 	> $@

native_client/python/setup.py

@@ -39,11 +39,27 @@ class BuildExtFirst(build):
                     ('build_clib', build.has_c_libraries),
                     ('build_scripts', build.has_scripts)]
 
+# Properly pass arguments for linking, setuptools will perform some checks
+def lib_dirs_split(a):
+    if os.name == 'posix':
+        return a.split('-L')[1:]
+
+    if os.name == 'nt':
+        return []
+
+def libs_split(a):
+    if os.name == 'posix':
+        return a.split('-l')[1:]
+
+    if os.name == 'nt':
+        return a.split('.lib')[0:1]
+
 ds_ext = Extension('deepspeech._impl',
          ['impl.i'],
          include_dirs = [ numpy_include, '../' ],
-         library_dirs = list(map(lambda x: x.strip(), os.getenv('MODEL_LDFLAGS', '').split('-L')[1:])),
+         library_dirs = list(map(lambda x: x.strip(), lib_dirs_split(os.getenv('MODEL_LDFLAGS', '')))),
-         libraries = list(map(lambda x: x.strip(), os.getenv('MODEL_LIBS', '').split('-l')[1:])))
+         libraries = list(map(lambda x: x.strip(), libs_split(os.getenv('MODEL_LIBS', ''))))
+         )
 
 setup(name = project_name,
       description = 'A library for running inference on a DeepSpeech model',

taskcluster/.build.yml

@@ -23,4 +23,4 @@ build:
     tests_cmdline: ''
   convert_graphdef: ''
   benchmark_model_bin: ''
-  tensorflow_git_desc: 'TensorFlow: v1.12.0-14-g943a6c3'
+  tensorflow_git_desc: 'TensorFlow: v1.12.0-20-g0259e0a'

taskcluster/test-cpp-win-amd64-opt.yml

@@ -0,0 +1,10 @@
+build:
+  template_file: test-win-opt-base.tyml
+  dependencies:
+    - "win-amd64-cpu-opt"
+    - "test-training_upstream-linux-amd64-py27mu-opt"
+  args:
+    tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/tc-cppwin-ds-tests.sh"
+  metadata:
+    name: "DeepSpeech Windows AMD64 CPU C++ tests"
+    description: "Testing DeepSpeech C++ for Windows/AMD64, CPU only, optimized version"

taskcluster/test-netframework-win-amd64-opt.yml

@@ -0,0 +1,10 @@
+build:
+  template_file: test-win-opt-base.tyml
+  dependencies:
+    - "win-amd64-cpu-opt"
+    - "test-training_upstream-linux-amd64-py27mu-opt"
+  args:
+    tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/tc-netframework-ds-tests.sh"
+  metadata:
+    name: "DeepSpeech Windows AMD64 CPU .Net Framework tests"
+    description: "Testing DeepSpeech .Net Framework for Windows/AMD64, CPU only, optimized version"

taskcluster/test-win-opt-base.tyml

@@ -0,0 +1,80 @@
+$if: '(event.event != "push") && (event.event != "tag")'
+then:
+  taskId: ${taskcluster.taskId}
+  provisionerId: ${taskcluster.docker.provisionerId}
+  workerType: ${taskcluster.docker.workerTypeWin}
+  taskGroupId: ${taskcluster.taskGroupId}
+  schedulerId: ${taskcluster.schedulerId}
+  dependencies:
+    $map: { $eval: build.dependencies }
+    each(b):
+      $eval: as_slugid(b)
+  created: { $fromNow: '0 sec' }
+  deadline: { $fromNow: '1 day' }
+  expires: { $fromNow: '7 days' }
+
+  extra:
+    github:
+      { $eval: taskcluster.github_events.pull_request }
+
+  routes:
+    - "notify.irc-channel.${notifications.irc}.on-exception"
+    - "notify.irc-channel.${notifications.irc}.on-failed"
+
+  scopes: [
+    "queue:route:notify.irc-channel.*",
+  ]
+
+  payload:
+    maxRunTime: { $eval: to_int(build.maxRunTime) }
+    mounts:
+      - file: msys2-base-x86_64.tar.xz
+        content:
+          sha256: 4e799b5c3efcf9efcb84923656b7bcff16f75a666911abd6620ea8e5e1e9870c
+          url: >-
+            https://sourceforge.net/projects/msys2/files/Base/x86_64/msys2-base-x86_64-20180531.tar.xz/download
+
+    env:
+      $let:
+        training: { $eval: as_slugid("test-training_upstream-linux-amd64-py27mu-opt") }
+        win_amd64_build: { $eval: as_slugid("win-amd64-cpu-opt") }
+        node_package: { $eval: as_slugid("node-package") }
+      in:
+        DEEPSPEECH_ARTIFACTS_ROOT: https://queue.taskcluster.net/v1/task/${win_amd64_build}/artifacts/public
+        DEEPSPEECH_NODEJS: https://queue.taskcluster.net/v1/task/${node_package}/artifacts/public
+        DEEPSPEECH_TEST_MODEL: https://queue.taskcluster.net/v1/task/${training}/artifacts/public/output_graph.pb
+        DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.2.0-prod-ctcdecode/output_graph.pb
+        DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.2.0-prod-ctcdecode/output_graph.pbmm
+        EXPECTED_TENSORFLOW_VERSION: "${build.tensorflow_git_desc}"
+        TC_MSYS_VERSION: 'MSYS_NT-6.3'
+        MSYS: 'winsymlinks:nativestrict'
+
+    command:
+      - >-
+        "C:\Program Files\7-zip\7z.exe" x -txz -so msys2-base-x86_64.tar.xz |
+        "C:\Program Files\7-zip\7z.exe" x -o%USERPROFILE% -ttar -aoa -si
+      - .\msys64\usr\bin\bash.exe --login -cx "exit"
+      - .\msys64\usr\bin\bash.exe --login -cx "pacman --noconfirm -Syu"
+      - .\msys64\usr\bin\bash.exe --login -cxe "
+        export LC_ALL=C &&
+        export PATH=\"/c/builds/tc-workdir/msys64/usr/bin:/c/Python36:/c/Program Files/Git/bin:/c/Program Files/7-Zip/:$PATH\" &&
+        export TASKCLUSTER_ARTIFACTS=\"$USERPROFILE/public\" &&
+        export TASKCLUSTER_TASK_DIR=\"/c/builds/tc-workdir/\" &&
+        export TASKCLUSTER_TMP_DIR="$TASKCLUSTER_TASK_DIR/tmp" &&
+        export PIP_DEFAULT_TIMEOUT=60 &&
+        (mkdir $TASKCLUSTER_TASK_DIR || rm -fr $TASKCLUSTER_TASK_DIR/*) && cd $TASKCLUSTER_TASK_DIR &&
+        env &&
+        ln -s $USERPROFILE/msys64 $TASKCLUSTER_TASK_DIR/msys64 &&
+        git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ &&
+        cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git checkout --quiet ${event.head.sha} &&
+        cd $TASKCLUSTER_TASK_DIR &&
+        pacman --noconfirm -R bsdtar &&
+        pacman --noconfirm -S tar vim &&
+        /bin/bash ${build.args.tests_cmdline} ;
+        cd $TASKCLUSTER_TASK_DIR/../ && rm -fr tc-workdir/ && exit $TASKCLUSTER_TASK_EXIT_CODE"
+
+  metadata:
+    name: ${build.metadata.name}
+    description: ${build.metadata.description}
+    owner: ${event.head.user.email}
+    source: ${event.head.repo.url}

taskcluster/win-amd64-cpu-opt.yml

@@ -0,0 +1,17 @@
+build:
+  template_file: win-opt-base.tyml
+  routes:
+    - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.win"
+    - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.win"
+    - "index.project.deepspeech.deepspeech.native_client.win.${event.head.sha}"
+    - "notify.irc-channel.${notifications.irc}.on-exception"
+    - "notify.irc-channel.${notifications.irc}.on-failed"
+  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.12.943a6c332331c0ceeba981b51c24abfed2cd6ffa.win/artifacts/public/home.tar.xz"
+  scripts:
+    build: "taskcluster/win-build.sh"
+    package: "taskcluster/win-package.sh"
+  nc_asset_name: "native_client.amd64.cpu.win.tar.xz"
+  maxRunTime: 14400
+  metadata:
+    name: "DeepSpeech Windows AMD64 CPU"
+    description: "Building DeepSpeech for Windows AMD64, CPU only, optimized version"

taskcluster/win-amd64-gpu-opt.yml

@@ -0,0 +1,17 @@
+build:
+  template_file: win-opt-base.tyml
+  routes:
+    - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.win-cuda"
+    - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.win-cuda"
+    - "index.project.deepspeech.deepspeech.native_client.win-cuda.${event.head.sha}"
+    - "notify.irc-channel.${notifications.irc}.on-exception"
+    - "notify.irc-channel.${notifications.irc}.on-failed"
+  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.12.943a6c332331c0ceeba981b51c24abfed2cd6ffa.win-cuda/artifacts/public/home.tar.xz"
+  scripts:
+    build: "taskcluster/win-build.sh --cuda"
+    package: "taskcluster/win-package.sh"
+  nc_asset_name: "native_client.amd64.gpu.win.tar.xz"
+  maxRunTime: 14400
+  metadata:
+    name: "DeepSpeech Windows AMD64 CUDA"
+    description: "Building DeepSpeech for Windows AMD64, CUDA-enabled, optimized version"

taskcluster/win-build.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+set -xe
+
+cuda=$1
+
+source $(dirname "$0")/../tc-tests-utils.sh
+
+source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh
+
+BAZEL_TARGETS="
+//native_client:libdeepspeech.so
+//native_client:generate_trie
+"
+
+if [ "${cuda}" = "--cuda" ]; then
+    BAZEL_ENV_FLAGS="TF_NEED_CUDA=1 ${TF_CUDA_FLAGS}"
+    BAZEL_BUILD_FLAGS="${BAZEL_CUDA_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BAZEL_OPT_FLAGS}"
+    PROJECT_NAME="DeepSpeech-GPU"
+else
+    PROJECT_NAME="DeepSpeech"
+    BAZEL_BUILD_FLAGS="${BAZEL_OPT_FLAGS} ${BAZEL_EXTRA_FLAGS}"
+    BAZEL_ENV_FLAGS="TF_NEED_CUDA=0"
+fi
+
+SYSTEM_TARGET=host-win
+
+do_bazel_build
+
+if [ "${cuda}" = "--cuda" ]; then
+    cp ${DS_ROOT_TASK}/DeepSpeech/tf/bazel-bin/native_client/liblibdeepspeech.so.ifso ${DS_ROOT_TASK}/DeepSpeech/tf/bazel-bin/native_client/libdeepspeech.so.if.lib
+fi
+
+export PATH=$PATH:$(cygpath ${ChocolateyInstall})/bin
+
+do_deepspeech_binary_build
+
+do_deepspeech_netframework_build
+
+do_nuget_build "${PROJECT_NAME}"
+
+shutdown_bazel

taskcluster/win-opt-base.tyml

@@ -0,0 +1,89 @@
+taskId: ${taskcluster.taskId}
+provisionerId: ${taskcluster.docker.provisionerId}
+workerType: ${taskcluster.docker.workerTypeWin}
+taskGroupId: ${taskcluster.taskGroupId}
+schedulerId: ${taskcluster.schedulerId}
+dependencies:
+  $map: { $eval: build.dependencies }
+  each(b):
+    $eval: as_slugid(b)
+created: { $fromNow: '0 sec' }
+deadline: { $fromNow: '1 day' }
+expires:
+  $if: '(event.event == "push") || (event.event == "tag")'
+  then: { $fromNow: '6 months' }
+  else: { $fromNow: '7 days' }
+
+extra:
+  nc_asset_name: { $eval: build.nc_asset_name }
+  github:
+    $if: '(event.event == "push") || (event.event == "tag")'
+    then: { $eval: taskcluster.github_events.merge }
+    else: { $eval: taskcluster.github_events.pull_request }
+
+routes:
+  $if: '(event.event == "push") || (event.event == "tag")'
+  then:
+    { $eval: build.routes }
+
+payload:
+  maxRunTime: { $eval: to_int(build.maxRunTime) }
+  mounts:
+    - file: msys2-base-x86_64.tar.xz
+      content:
+        sha256: 4e799b5c3efcf9efcb84923656b7bcff16f75a666911abd6620ea8e5e1e9870c
+        url: >-
+          https://sourceforge.net/projects/msys2/files/Base/x86_64/msys2-base-x86_64-20180531.tar.xz/download
+
+  env:
+    $let:
+      training: { $eval: as_slugid("test-training_upstream-linux-amd64-py27mu-opt") }
+    in:
+      TC_MSYS_VERSION: 'MSYS_NT-6.3'
+      MSYS: 'winsymlinks:nativestrict'
+      TENSORFLOW_BUILD_ARTIFACT: ${build.tensorflow}
+      DEEPSPEECH_TEST_MODEL: https://queue.taskcluster.net/v1/task/${training}/artifacts/public/output_graph.pb
+
+  command:
+    - >-
+      "C:\Program Files\7-zip\7z.exe" x -txz -so msys2-base-x86_64.tar.xz |
+      "C:\Program Files\7-zip\7z.exe" x -o%USERPROFILE% -ttar -aoa -si
+    - .\msys64\usr\bin\bash.exe --login -cx "exit"
+    - .\msys64\usr\bin\bash.exe --login -cx "pacman --noconfirm -Syu"
+    - echo .\msys64\usr\bin\bash.exe --login -cxe "
+      export LC_ALL=C &&
+      export PATH=\"/c/builds/tc-workdir/msys64/usr/bin:/c/Python36:/c/Program Files/Git/bin:/c/Program Files/7-Zip/:$PATH\" &&
+      export TASKCLUSTER_ARTIFACTS=\"$USERPROFILE/public\" &&
+      export TASKCLUSTER_TASK_DIR=\"/c/builds/tc-workdir/\" &&
+      (mkdir $TASKCLUSTER_TASK_DIR || rm -fr $TASKCLUSTER_TASK_DIR/*) && cd $TASKCLUSTER_TASK_DIR &&
+      env &&
+      ln -s $USERPROFILE/msys64 $TASKCLUSTER_TASK_DIR/msys64 &&
+      (wget -O - $TENSORFLOW_BUILD_ARTIFACT | 7z x -txz -si -so | 7z x -o$TASKCLUSTER_TASK_DIR -aoa -ttar -si ) &&
+      git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ &&
+      cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git checkout --quiet ${event.head.sha} &&
+      ln -s $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/native_client/ $TASKCLUSTER_TASK_DIR/DeepSpeech/tf/native_client &&
+      cd $TASKCLUSTER_TASK_DIR &&
+      pacman --noconfirm -R bsdtar &&
+      pacman --noconfirm -S tar make &&
+      $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.build} &&
+      $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.package} ;
+      echo \"export TASKCLUSTER_TASK_EXIT_CODE=$?\" > $USERPROFILE/tc-exit.sh &&
+      cd $TASKCLUSTER_TASK_DIR/../ && rm -fr tc-workdir/ && exit $TASKCLUSTER_TASK_EXIT_CODE" | cmd
+      /k ""C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"" x64
+
+    - .\msys64\usr\bin\bash.exe --login -cxe "source $USERPROFILE/tc-exit.sh &&
+      exit $TASKCLUSTER_TASK_EXIT_CODE"
+
+  artifacts:
+    - type: "directory"
+      path: "public/"
+      expires:
+        $if: '(event.event == "push") || (event.event == "tag")'
+        then: { $fromNow: '6 months' }
+        else: { $fromNow: '7 days' }
+
+metadata:
+  name: ${build.metadata.name}
+  description: ${build.metadata.description}
+  owner: ${event.head.user.email}
+  source: ${event.head.repo.url}

taskcluster/win-package.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -xe
+
+arm_flavor=$1
+
+source $(dirname "$0")/../tc-tests-utils.sh
+
+mkdir -p ${TASKCLUSTER_ARTIFACTS} || true
+
+cp ${DS_ROOT_TASK}/DeepSpeech/tf/bazel*.log ${TASKCLUSTER_ARTIFACTS}/
+
+package_native_client "native_client.tar.xz"
+
+cp ${DS_ROOT_TASK}/DeepSpeech/ds/examples/net_framework/CSharpExamples/*.nupkg ${TASKCLUSTER_ARTIFACTS}/
+
+cp ${DS_ROOT_TASK}/DeepSpeech/ds/examples/net_framework/CSharpExamples/DeepSpeechConsole/bin/x64/Release/DeepSpeechConsole.exe ${TASKCLUSTER_ARTIFACTS}/

taskcluster/worker.cyml

@@ -4,6 +4,7 @@ taskcluster:
     provisionerId: aws-provisioner-v1
     workerType: deepspeech-worker
     workerTypeKvm: deepspeech-kvm-worker
+    workerTypeWin: deepspeech-win
   dockerrpi3:
     provisionerId: deepspeech-provisioner
     workerType: ds-rpi3

tc-cppwin-ds-tests.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -xe
+
+source $(dirname "$0")/tc-tests-utils.sh
+
+download_material "${TASKCLUSTER_TMP_DIR}/ds"
+
+export PATH=${TASKCLUSTER_TMP_DIR}/ds/:$PATH
+
+check_tensorflow_version
+
+run_basic_inference_tests

tc-netframework-ds-tests.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -xe
+
+cuda=$1
+
+source $(dirname "$0")/tc-tests-utils.sh
+
+if [ "${cuda}" = "--cuda" ]; then
+    PROJECT_NAME="DeepSpeech-GPU"
+else
+    PROJECT_NAME="DeepSpeech"
+fi
+
+download_data
+
+install_nuget "${PROJECT_NAME}"
+
+run_netframework_inference_tests

tc-tests-utils.sh

@@ -7,6 +7,11 @@ if [ "${OS}" = "Linux" ]; then
     export DS_ROOT_TASK=${HOME}
 fi;
 
+if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then
+    export DS_ROOT_TASK=${TASKCLUSTER_TASK_DIR}
+    export PLATFORM_EXE_SUFFIX=.exe
+fi;
+
 if [ "${OS}" = "Darwin" ]; then
     export DS_ROOT_TASK=${TASKCLUSTER_TASK_DIR}
     export SWIG_LIB="$(find ${DS_ROOT_TASK}/homebrew/Cellar/swig/ -type f -name "swig.swg" | xargs dirname)"
@@ -33,6 +38,16 @@ export DS_VERSION="$(cat ${DS_DSDIR}/VERSION)"
 export ANDROID_SDK_HOME=${DS_ROOT_TASK}/DeepSpeech/Android/SDK/
 export ANDROID_NDK_HOME=${DS_ROOT_TASK}/DeepSpeech/Android/android-ndk-r18b/
 
+TAR=${TAR:-"tar"}
+XZ=${XZ:-"pixz -9"}
+UNXZ=${UNXZ:-"pixz -d"}
+
+if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then
+  TAR=/usr/bin/tar.exe
+  XZ="xz -9 -T0 -c -"
+  UNXZ="xz -9 -T0 -d"
+fi
+
 model_source="${DEEPSPEECH_TEST_MODEL}"
 model_name="$(basename "${model_source}")"
 model_name_mmap="$(basename -s ".pb" "${model_source}").pbmm"
@@ -172,11 +187,21 @@ assert_correct_ldc93s1()
   assert_correct_inference "$1" "she had your dark suit in greasy wash water all year"
 }
 
+assert_working_ldc93s1()
+{
+  assert_working_inference "$1" "she had your dark suit in greasy wash water all year"
+}
+
 assert_correct_ldc93s1_lm()
 {
   assert_correct_inference "$1" "she had your dark suit in greasy wash water all year"
 }
 
+assert_working_ldc93s1_lm()
+{
+  assert_working_inference "$1" "she had your dark suit in greasy wash water all year"
+}
+
 assert_correct_multi_ldc93s1()
 {
   assert_shows_something "$1" "/LDC93S1.wav%she had your dark suit in greasy wash water all year%"
@@ -226,7 +251,19 @@ run_tflite_basic_inference_tests()
   assert_correct_ldc93s1 "${phrase_pbmodel_nolm}"
 }
 
-run_all_inference_tests()
+run_netframework_inference_tests()
+{
+  phrase_pbmodel_nolm=$(DeepSpeechConsole.exe --model ${TASKCLUSTER_TMP_DIR}/${model_name} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1.wav)
+  assert_working_ldc93s1 "${phrase_pbmodel_nolm}"
+
+  phrase_pbmodel_nolm=$(DeepSpeechConsole.exe --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1.wav)
+  assert_working_ldc93s1 "${phrase_pbmodel_nolm}"
+
+  phrase_pbmodel_withlm=$(DeepSpeechConsole.exe --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --lm ${TASKCLUSTER_TMP_DIR}/lm.binary --trie ${TASKCLUSTER_TMP_DIR}/trie --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1.wav)
+  assert_working_ldc93s1_lm "${phrase_pbmodel_withlm}"
+}
+
+run_basic_inference_tests()
 {
   phrase_pbmodel_nolm=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1.wav)
   assert_correct_ldc93s1 "${phrase_pbmodel_nolm}"
@@ -236,6 +273,11 @@ run_all_inference_tests()
 
   phrase_pbmodel_withlm=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --lm ${TASKCLUSTER_TMP_DIR}/lm.binary --trie ${TASKCLUSTER_TMP_DIR}/trie --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1.wav)
   assert_correct_ldc93s1_lm "${phrase_pbmodel_withlm}"
+}
+
+run_all_inference_tests()
+{
+  run_basic_inference_tests
 
   phrase_pbmodel_nolm_stereo_44k=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_2_44100.wav)
   assert_correct_ldc93s1 "${phrase_pbmodel_nolm_stereo_44k}"
@@ -299,7 +341,7 @@ generic_download_tarxz()
 
   mkdir -p ${target_dir} || true
 
-  wget ${url} -O - | pixz -d | tar -C ${target_dir} -xf -
+  wget ${url} -O - | ${UNXZ} | ${TAR} -C ${target_dir} -xf -
 }
 
 download_native_client_files()
@@ -307,6 +349,45 @@ download_native_client_files()
   generic_download_tarxz "$1" "${DEEPSPEECH_ARTIFACTS_ROOT}/native_client.tar.xz"
 }
 
+install_nuget()
+{
+  PROJECT_NAME=$1
+  if [ -z "${PROJECT_NAME}" ]; then
+    exit "Please call with a valid PROJECT_NAME"
+    exit 1
+  fi;
+
+  nuget="${PROJECT_NAME}.${DS_VERSION}.nupkg"
+
+  export PATH=$PATH:$(cygpath ${ChocolateyInstall})/bin
+
+  mkdir -p "${TASKCLUSTER_TMP_DIR}/repo/"
+  mkdir -p "${TASKCLUSTER_TMP_DIR}/ds/"
+
+  wget -O - "${DEEPSPEECH_ARTIFACTS_ROOT}/${nuget}" | gunzip > "${TASKCLUSTER_TMP_DIR}/${PROJECT_NAME}.${DS_VERSION}.nupkg"
+  wget -O - "${DEEPSPEECH_ARTIFACTS_ROOT}/DeepSpeechConsole.exe" | gunzip > "${TASKCLUSTER_TMP_DIR}/ds/DeepSpeechConsole.exe"
+
+  nuget sources add -Name repo -Source $(cygpath -w "${TASKCLUSTER_TMP_DIR}/repo/")
+
+  cd "${TASKCLUSTER_TMP_DIR}"
+  nuget add $(cygpath -w "${TASKCLUSTER_TMP_DIR}/${nuget}") -source repo
+
+  cd "${TASKCLUSTER_TMP_DIR}/ds/"
+  nuget list -Source repo -Prerelease
+  nuget install ${PROJECT_NAME} -Source repo -Prerelease
+
+  ls -halR "${PROJECT_NAME}.${DS_VERSION}"
+
+  nuget install NAudio
+  cp NAudio*/lib/net35/NAudio.dll ${TASKCLUSTER_TMP_DIR}/ds/
+  cp ${PROJECT_NAME}.${DS_VERSION}/build/libdeepspeech.so ${TASKCLUSTER_TMP_DIR}/ds/
+  cp ${PROJECT_NAME}.${DS_VERSION}/lib/net462/DeepSpeechClient.dll ${TASKCLUSTER_TMP_DIR}/ds/
+
+  ls -hal ${TASKCLUSTER_TMP_DIR}/ds/
+
+  export PATH=${TASKCLUSTER_TMP_DIR}/ds/:$PATH
+}
+
 download_data()
 {
   wget -P "${TASKCLUSTER_TMP_DIR}" "${model_source}"
@@ -390,6 +471,8 @@ is_patched_bazel()
 {
   bazel_version=$(bazel version | grep 'Build label:' | cut -d':' -f2)
 
+  bazel shutdown
+
   if [ -z "${bazel_version}" ]; then
     return 0;
   else
@@ -473,6 +556,12 @@ do_bazel_build()
   verify_bazel_rebuild "${DS_ROOT_TASK}/DeepSpeech/tf/bazel_monolithic.log"
 }
 
+shutdown_bazel()
+{
+  cd ${DS_ROOT_TASK}/DeepSpeech/tf
+  bazel ${BAZEL_OUTPUT_USER_ROOT} shutdown
+}
+
 do_bazel_shared_build()
 {
   cd ${DS_ROOT_TASK}/DeepSpeech/tf
@@ -491,7 +580,7 @@ do_deepspeech_binary_build()
     EXTRA_CFLAGS="${EXTRA_LOCAL_CFLAGS}" \
     EXTRA_LDFLAGS="${EXTRA_LOCAL_LDFLAGS}" \
     EXTRA_LIBS="${EXTRA_LOCAL_LIBS}" \
-    deepspeech
+    deepspeech${PLATFORM_EXE_SUFFIX}
 }
 
 do_deepspeech_ndk_build()
@@ -509,6 +598,58 @@ do_deepspeech_ndk_build()
     TARGET_ARCH_ABI=${arch_abi}
 }
 
+do_deepspeech_netframework_build()
+{
+  cd ${DS_DSDIR}/examples/net_framework/CSharpExamples
+
+  # Setup dependencies
+  nuget install DeepSpeechConsole/packages.config -OutputDirectory packages/
+  nuget install DeepSpeechWPF/packages.config -OutputDirectory packages/
+
+  MSBUILD="$(cygpath 'C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\MSBuild\15.0\Bin\MSBuild.exe')"
+
+  # We need MSYS2_ARG_CONV_EXCL='/' otherwise the '/' of CLI parameters gets mangled and disappears
+
+  MSYS2_ARG_CONV_EXCL='/' "${MSBUILD}" \
+    DeepSpeechClient/DeepSpeechClient.csproj \
+    /p:Configuration=Release \
+    /p:Platform=x64
+
+  MSYS2_ARG_CONV_EXCL='/' "${MSBUILD}" \
+    DeepSpeechConsole/DeepSpeechConsole.csproj \
+    /p:Configuration=Release \
+    /p:Platform=x64
+
+  MSYS2_ARG_CONV_EXCL='/' "${MSBUILD}" \
+    DeepSpeechWPF/DeepSpeech.WPF.csproj \
+    /p:Configuration=Release \
+    /p:Platform=x64
+}
+
+do_nuget_build()
+{
+  PROJECT_NAME=$1
+  if [ -z "${PROJECT_NAME}" ]; then
+    exit "Please call with a valid PROJECT_NAME"
+    exit 1
+  fi;
+
+  cd ${DS_DSDIR}/examples/net_framework/CSharpExamples
+
+  cp ${DS_TFDIR}/bazel-bin/native_client/libdeepspeech.so nupkg/build
+
+  mkdir -p nupkg/lib/net462/
+  cp DeepSpeechClient/bin/x64/Release/DeepSpeechClient.dll nupkg/lib/net462/
+
+  PROJECT_VERSION=$(shell cat ../../../VERSION | tr -d '\n' | tr -d '\r')
+  sed \
+    -e "s/\$NUPKG_ID/${PROJECT_NAME}/" \
+    -e "s/\$NUPKG_VERSION/${PROJECT_VERSION}/" \
+    nupkg/deepspeech.nuspec.in > nupkg/deepspeech.nuspec && cat nupkg/deepspeech.nuspec
+
+  nuget pack nupkg/deepspeech.nuspec
+}
+
 # Hack to extract Ubuntu's 16.04 libssl 1.0.2 packages and use them during the
 # local build of Python.
 #
@@ -778,13 +919,13 @@ package_native_client()
     echo "Please specify artifact name."
   fi;
 
-  tar -cf - \
+  ${TAR} -cf - \
-    -C ${tensorflow_dir}/bazel-bin/native_client/ generate_trie \
+    -C ${tensorflow_dir}/bazel-bin/native_client/ generate_trie${PLATFORM_EXE_SUFFIX} \
     -C ${tensorflow_dir}/bazel-bin/native_client/ libdeepspeech.so \
     -C ${deepspeech_dir}/ LICENSE \
-    -C ${deepspeech_dir}/native_client/ deepspeech \
+    -C ${deepspeech_dir}/native_client/ deepspeech${PLATFORM_EXE_SUFFIX} \
     -C ${deepspeech_dir}/native_client/kenlm/ README.mozilla \
-    | pixz -9 > "${artifacts_dir}/${artifact_name}"
+    | ${XZ} > "${artifacts_dir}/${artifact_name}"
 }
 
 package_native_client_ndk()