diff --git a/.taskcluster.yml b/.taskcluster.yml index 948dd92d..371cf88e 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -26,6 +26,7 @@ tasks: scopes: [ "queue:create-task:lowest:{{ taskcluster.docker.provisionerId }}/deepspeech-worker", + "queue:create-task:lowest:{{ taskcluster.docker.provisionerId }}/deepspeech-win", "queue:create-task:lowest:{{ taskcluster.docker.provisionerId }}/deepspeech-kvm-worker", "queue:create-task:lowest:deepspeech-provisioner/ds-macos-light", "queue:create-task:lowest:deepspeech-provisioner/ds-scriptworker", diff --git a/examples/net_framework/CSharpExamples/nupkg-gpu/build/.gitpreserve b/examples/net_framework/CSharpExamples/nupkg-gpu/build/.gitpreserve deleted file mode 100644 index e69de29b..00000000 diff --git a/examples/net_framework/CSharpExamples/nupkg-gpu/build/DeepSpeech.targets b/examples/net_framework/CSharpExamples/nupkg-gpu/build/DeepSpeech.targets deleted file mode 100644 index deebff48..00000000 --- a/examples/net_framework/CSharpExamples/nupkg-gpu/build/DeepSpeech.targets +++ /dev/null @@ -1,9 +0,0 @@ - - - - - %(FileName)%(Extension) - PreserveNewest - - - \ No newline at end of file diff --git a/examples/net_framework/CSharpExamples/nupkg-gpu/deepspeech.nuspec b/examples/net_framework/CSharpExamples/nupkg-gpu/deepspeech.nuspec deleted file mode 100644 index 97eb947a..00000000 --- a/examples/net_framework/CSharpExamples/nupkg-gpu/deepspeech.nuspec +++ /dev/null @@ -1,21 +0,0 @@ - - - - DeepSpeech-GPU - 0.4.1 - DeepSpeech - Mozilla - Mozilla - MPL-2.0 - http://github.com/mozilla/DeepSpeech - false - A library for running inference with a DeepSpeech model - Copyright (c) 2019 Mozilla Corporation - native speech speech_recognition - - - - - - - diff --git a/examples/net_framework/CSharpExamples/nupkg-gpu/lib/net462/.gitpreserve b/examples/net_framework/CSharpExamples/nupkg-gpu/lib/net462/.gitpreserve deleted file mode 100644 index e69de29b..00000000 diff --git a/examples/net_framework/CSharpExamples/nupkg-gpu/tools/.gitpreserve b/examples/net_framework/CSharpExamples/nupkg-gpu/tools/.gitpreserve deleted file mode 100644 index e69de29b..00000000 diff --git a/examples/net_framework/CSharpExamples/nupkg/deepspeech.nuspec b/examples/net_framework/CSharpExamples/nupkg/deepspeech.nuspec.in similarity index 91% rename from examples/net_framework/CSharpExamples/nupkg/deepspeech.nuspec rename to examples/net_framework/CSharpExamples/nupkg/deepspeech.nuspec.in index 11b7c883..a4797177 100644 --- a/examples/net_framework/CSharpExamples/nupkg/deepspeech.nuspec +++ b/examples/net_framework/CSharpExamples/nupkg/deepspeech.nuspec.in @@ -1,8 +1,8 @@ - DeepSpeech - 0.4.1 + $NUPKG_ID + $NUPKG_VERSION DeepSpeech Mozilla Mozilla diff --git a/native_client/Makefile b/native_client/Makefile index 5084b773..d27d60f5 100644 --- a/native_client/Makefile +++ b/native_client/Makefile @@ -13,25 +13,25 @@ include definitions.mk -default: deepspeech +default: $(DEEPSPEECH_BIN) -clean: +clean: bindings-clean rm -f deepspeech -deepspeech: client.cc - $(CXX) -std=c++11 -o deepspeech $(CFLAGS) $(SOX_CFLAGS) client.cc $(LDFLAGS) $(SOX_LDFLAGS) +$(DEEPSPEECH_BIN): client.cc + $(CXX) $(CFLAGS) $(CFLAGS_DEEPSPEECH) $(SOX_CFLAGS) client.cc $(LDFLAGS) $(SOX_LDFLAGS) ifeq ($(OS),Darwin) install_name_tool -change $$TASKCLUSTER_TASK_DIR/homebrew/opt/sox/lib/libsox.3.dylib @rpath/libsox.3.dylib deepspeech install_name_tool -change bazel-out/local-opt/bin/native_client/libdeepspeech.so @rpath/libdeepspeech.so deepspeech endif -run: deepspeech +run: $(DEEPSPEECH_BIN) ${META_LD_LIBRARY_PATH}=${TFDIR}/bazel-bin/native_client:${${META_LD_LIBRARY_PATH}} ./deepspeech ${ARGS} -debug: deepspeech +debug: $(DEEPSPEECH_BIN) ${META_LD_LIBRARY_PATH}=${TFDIR}/bazel-bin/native_client:${${META_LD_LIBRARY_PATH}} gdb --args ./deepspeech ${ARGS} -install: deepspeech +install: $(DEEPSPEECH_BIN) install -d ${PREFIX}/lib install -m 0644 ${TFDIR}/bazel-bin/native_client/libdeepspeech.so ${PREFIX}/lib/ install -d ${PREFIX}/bin diff --git a/native_client/args.h b/native_client/args.h index e6d2349d..58ebe641 100644 --- a/native_client/args.h +++ b/native_client/args.h @@ -1,7 +1,11 @@ #ifndef __ARGS_H__ #define __ARGS_H__ +#if defined(_MSC_VER) +#include "getopt_win.h" +#else #include +#endif #include #include "deepspeech.h" diff --git a/native_client/client.cc b/native_client/client.cc index f8be754d..b8eab671 100644 --- a/native_client/client.cc +++ b/native_client/client.cc @@ -2,22 +2,33 @@ #include #include -#include #include #include #include -#ifndef __ANDROID__ -#include -#endif // __ANDROID__ #include -#include - #include #include #include #include +#if defined(__ANDROID__) || defined(_MSC_VER) +#define NO_SOX +#endif + +#if defined(_MSC_VER) +#define NO_DIR +#endif + +#ifndef NO_SOX +#include +#endif + +#ifndef NO_DIR +#include +#include +#endif // NO_DIR + #include "deepspeech.h" #include "args.h" @@ -61,7 +72,7 @@ GetAudioBuffer(const char* path) { ds_audio_buffer res = {0}; -#ifndef __ANDROID__ +#ifndef NO_SOX sox_format_t* input = sox_open_read(path, NULL, NULL, NULL); assert(input); @@ -150,9 +161,9 @@ GetAudioBuffer(const char* path) // Close sox handles sox_close(output); sox_close(input); -#endif // __ANDROID__ +#endif // NO_SOX -#ifdef __ANDROID__ +#ifdef NO_SOX // FIXME: Hack and support only 16kHz mono 16-bits PCM FILE* wave = fopen(path, "r"); @@ -160,19 +171,15 @@ GetAudioBuffer(const char* path) unsigned short audio_format; fseek(wave, 20, SEEK_SET); rv = fread(&audio_format, 2, 1, wave); - assert(rv == 2); unsigned short num_channels; fseek(wave, 22, SEEK_SET); rv = fread(&num_channels, 2, 1, wave); - assert(rv == 2); unsigned int sample_rate; fseek(wave, 24, SEEK_SET); rv = fread(&sample_rate, 4, 1, wave); - assert(rv == 2); unsigned short bits_per_sample; fseek(wave, 34, SEEK_SET); rv = fread(&bits_per_sample, 2, 1, wave); - assert(rv == 2); assert(audio_format == 1); // 1 is PCM assert(num_channels == 1); // MONO @@ -185,16 +192,14 @@ GetAudioBuffer(const char* path) fprintf(stderr, "bits_per_sample=%d\n", bits_per_sample); fseek(wave, 40, SEEK_SET); rv = fread(&res.buffer_size, 4, 1, wave); - assert(rv == 2); fprintf(stderr, "res.buffer_size=%ld\n", res.buffer_size); fseek(wave, 44, SEEK_SET); res.buffer = (char*)malloc(sizeof(char) * res.buffer_size); rv = fread(res.buffer, sizeof(char), res.buffer_size, wave); - assert(rv == res.buffer_size); fclose(wave); -#endif // __ANDROID__ +#endif // NO_SOX #ifdef __APPLE__ res.buffer_size = (size_t)(output->olength * 2); @@ -261,8 +266,10 @@ main(int argc, char **argv) } } +#ifndef NO_SOX // Initialise SOX assert(sox_init() == SOX_SUCCESS); +#endif struct stat wav_info; if (0 != stat(audio, &wav_info)) { @@ -270,11 +277,14 @@ main(int argc, char **argv) } switch (wav_info.st_mode & S_IFMT) { +#ifndef _MSC_VER case S_IFLNK: +#endif case S_IFREG: ProcessFile(ctx, audio, show_times); break; +#ifndef NO_DIR case S_IFDIR: { printf("Running on directory %s\n", audio); @@ -297,16 +307,17 @@ main(int argc, char **argv) closedir(wav_dir); } break; +#endif default: printf("Unexpected type for %s: %d\n", audio, (wav_info.st_mode & S_IFMT)); break; } -#ifndef __ANDROID__ +#ifndef NO_SOX // Deinitialise and quit sox_quit(); -#endif // __ANDROID__ +#endif // NO_SOX DS_DestroyModel(ctx); diff --git a/native_client/definitions.mk b/native_client/definitions.mk index 4ed675b9..5c1a6cad 100644 --- a/native_client/definitions.mk +++ b/native_client/definitions.mk @@ -5,6 +5,17 @@ TFDIR ?= $(abspath $(NC_DIR)/../../tensorflow) PREFIX ?= /usr/local SO_SEARCH ?= $(TFDIR)/bazel-bin/ +TOOL_AS := as +TOOL_CC := gcc +TOOL_CXX := c++ +TOOL_LD := ld +TOOL_LDD := ldd + +DEEPSPEECH_BIN := deepspeech +CFLAGS_DEEPSPEECH := -std=c++11 -o $(DEEPSPEECH_BIN) +LINK_DEEPSPEECH := -ldeepspeech +LINK_PATH_DEEPSPEECH := -L${TFDIR}/bazel-bin/native_client + ifeq ($(TARGET),host) TOOLCHAIN := CFLAGS := @@ -18,6 +29,19 @@ PYTHON_PLATFORM_NAME := --plat-name manylinux1_x86_64 endif endif +ifeq ($(TARGET),host-win) +DEEPSPEECH_BIN := deepspeech.exe +TOOLCHAIN := '$(VCINSTALLDIR)\bin\amd64\' +TOOL_CC := cl.exe +TOOL_CXX := cl.exe +TOOL_LD := link.exe +LINK_DEEPSPEECH := $(TFDIR)\bazel-bin\native_client\libdeepspeech.so.if.lib +LINK_PATH_DEEPSPEECH := +CFLAGS_DEEPSPEECH := -nologo -Fe$(DEEPSPEECH_BIN) +SOX_CFLAGS := +SOX_LDFLAGS := +endif + ifeq ($(TARGET),rpi3) TOOLCHAIN ?= ${TFDIR}/bazel-$(shell basename "${TFDIR}")/external/LinaroArmGcc72/bin/arm-linux-gnueabihf- RASPBIAN ?= $(abspath $(NC_DIR)/../multistrap-raspbian-stretch) @@ -72,15 +96,15 @@ endif CFLAGS += $(EXTRA_CFLAGS) CXXFLAGS += $(EXTRA_CXXFLAGS) -LIBS := -ldeepspeech $(EXTRA_LIBS) -LDFLAGS_DIRS := -L${TFDIR}/bazel-bin/native_client $(EXTRA_LDFLAGS) +LIBS := $(LINK_DEEPSPEECH) $(EXTRA_LIBS) +LDFLAGS_DIRS := $(LINK_PATH_DEEPSPEECH) $(EXTRA_LDFLAGS) LDFLAGS += $(LDFLAGS_NEEDED) $(LDFLAGS_RPATH) $(LDFLAGS_DIRS) $(LIBS) -AS := $(TOOLCHAIN)as -CC := $(TOOLCHAIN)gcc -CXX := $(TOOLCHAIN)c++ -LD := $(TOOLCHAIN)ld -LDD := $(TOOLCHAIN)ldd $(TOOLCHAIN_LDD_OPTS) +AS := $(TOOLCHAIN)$(TOOL_AS) +CC := $(TOOLCHAIN)$(TOOL_CC) +CXX := $(TOOLCHAIN)$(TOOL_CXX) +LD := $(TOOLCHAIN)$(TOOL_LD) +LDD := $(TOOLCHAIN)$(TOOL_LDD) $(TOOLCHAIN_LDD_OPTS) RPATH_PYTHON := '-Wl,-rpath,\$$ORIGIN/lib/' $(LDFLAGS_RPATH) RPATH_NODEJS := '-Wl,-rpath,$$\$$ORIGIN/../' diff --git a/native_client/ds_git_version.sh b/native_client/ds_git_version.sh index 77f5f6fb..6fb6e419 100755 --- a/native_client/ds_git_version.sh +++ b/native_client/ds_git_version.sh @@ -4,6 +4,10 @@ if [ `uname` = "Darwin" ]; then export PATH="/Users/build-user/TaskCluster/Workdir/tasks/tc-workdir/homebrew/opt/coreutils/libexec/gnubin:${PATH}" fi +if [ `uname -o` = "Msys" ]; then + export PATH="/c/Program Files/Git/bin/:${PATH}" +fi + DS_GIT_DIR="$(realpath "$(dirname "$(realpath "$0")")/../.git/")" if [ ! -d "${DS_GIT_DIR}" ]; then return 1 diff --git a/native_client/getopt_win.h b/native_client/getopt_win.h new file mode 100644 index 00000000..0cb88895 --- /dev/null +++ b/native_client/getopt_win.h @@ -0,0 +1,653 @@ +#ifndef __GETOPT_H__ +/** + * DISCLAIMER + * This file is part of the mingw-w64 runtime package. + * + * The mingw-w64 runtime package and its code is distributed in the hope that it + * will be useful but WITHOUT ANY WARRANTY. ALL WARRANTIES, EXPRESSED OR + * IMPLIED ARE HEREBY DISCLAIMED. This includes but is not limited to + * warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ + /* + * Copyright (c) 2002 Todd C. Miller + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + * + * Sponsored in part by the Defense Advanced Research Projects + * Agency (DARPA) and Air Force Research Laboratory, Air Force + * Materiel Command, USAF, under agreement number F39502-99-1-0512. + */ +/*- + * Copyright (c) 2000 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Dieter Baron and Thomas Klausner. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma warning(disable:4996); + +#define __GETOPT_H__ + +/* All the headers include this file. */ +#include +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define REPLACE_GETOPT /* use this getopt as the system getopt(3) */ + +#ifdef REPLACE_GETOPT +int opterr = 1; /* if error message should be printed */ +int optind = 1; /* index into parent argv vector */ +int optopt = '?'; /* character checked for validity */ +#undef optreset /* see getopt.h */ +#define optreset __mingw_optreset +int optreset; /* reset getopt */ +char *optarg; /* argument associated with option */ +#endif + +//extern int optind; /* index of first non-option in argv */ +//extern int optopt; /* single option character, as parsed */ +//extern int opterr; /* flag to enable built-in diagnostics... */ +// /* (user may set to zero, to suppress) */ +// +//extern char *optarg; /* pointer to argument of current option */ + +#define PRINT_ERROR ((opterr) && (*options != ':')) + +#define FLAG_PERMUTE 0x01 /* permute non-options to the end of argv */ +#define FLAG_ALLARGS 0x02 /* treat non-options as args to option "-1" */ +#define FLAG_LONGONLY 0x04 /* operate as getopt_long_only */ + +/* return values */ +#define BADCH (int)'?' +#define BADARG ((*options == ':') ? (int)':' : (int)'?') +#define INORDER (int)1 + +#ifndef __CYGWIN__ +#define __progname __argv[0] +#else +extern char __declspec(dllimport) *__progname; +#endif + +#ifdef __CYGWIN__ +static char EMSG[] = ""; +#else +#define EMSG "" +#endif + +static int getopt_internal(int, char * const *, const char *, + const struct option *, int *, int); +static int parse_long_options(char * const *, const char *, + const struct option *, int *, int); +static int gcd(int, int); +static void permute_args(int, int, int, char * const *); + +static char *place = EMSG; /* option letter processing */ + +/* XXX: set optreset to 1 rather than these two */ +static int nonopt_start = -1; /* first non option argument (for permute) */ +static int nonopt_end = -1; /* first option after non options (for permute) */ + +/* Error messages */ +static const char recargchar[] = "option requires an argument -- %c"; +static const char recargstring[] = "option requires an argument -- %s"; +static const char ambig[] = "ambiguous option -- %.*s"; +static const char noarg[] = "option doesn't take an argument -- %.*s"; +static const char illoptchar[] = "unknown option -- %c"; +static const char illoptstring[] = "unknown option -- %s"; + +static void +_vwarnx(const char *fmt,va_list ap) +{ + (void)fprintf(stderr,"%s: ",__progname); + if (fmt != NULL) + (void)vfprintf(stderr,fmt,ap); + (void)fprintf(stderr,"\n"); +} + +static void +warnx(const char *fmt,...) +{ + va_list ap; + va_start(ap,fmt); + _vwarnx(fmt,ap); + va_end(ap); +} + +/* + * Compute the greatest common divisor of a and b. + */ +static int +gcd(int a, int b) +{ + int c; + + c = a % b; + while (c != 0) { + a = b; + b = c; + c = a % b; + } + + return (b); +} + +/* + * Exchange the block from nonopt_start to nonopt_end with the block + * from nonopt_end to opt_end (keeping the same order of arguments + * in each block). + */ +static void +permute_args(int panonopt_start, int panonopt_end, int opt_end, + char * const *nargv) +{ + int cstart, cyclelen, i, j, ncycle, nnonopts, nopts, pos; + char *swap; + + /* + * compute lengths of blocks and number and size of cycles + */ + nnonopts = panonopt_end - panonopt_start; + nopts = opt_end - panonopt_end; + ncycle = gcd(nnonopts, nopts); + cyclelen = (opt_end - panonopt_start) / ncycle; + + for (i = 0; i < ncycle; i++) { + cstart = panonopt_end+i; + pos = cstart; + for (j = 0; j < cyclelen; j++) { + if (pos >= panonopt_end) + pos -= nnonopts; + else + pos += nopts; + swap = nargv[pos]; + /* LINTED const cast */ + ((char **) nargv)[pos] = nargv[cstart]; + /* LINTED const cast */ + ((char **)nargv)[cstart] = swap; + } + } +} + +#ifdef REPLACE_GETOPT +/* + * getopt -- + * Parse argc/argv argument vector. + * + * [eventually this will replace the BSD getopt] + */ +int +getopt(int nargc, char * const *nargv, const char *options) +{ + + /* + * We don't pass FLAG_PERMUTE to getopt_internal() since + * the BSD getopt(3) (unlike GNU) has never done this. + * + * Furthermore, since many privileged programs call getopt() + * before dropping privileges it makes sense to keep things + * as simple (and bug-free) as possible. + */ + return (getopt_internal(nargc, nargv, options, NULL, NULL, 0)); +} +#endif /* REPLACE_GETOPT */ + +//extern int getopt(int nargc, char * const *nargv, const char *options); + +#ifdef _BSD_SOURCE +/* + * BSD adds the non-standard `optreset' feature, for reinitialisation + * of `getopt' parsing. We support this feature, for applications which + * proclaim their BSD heritage, before including this header; however, + * to maintain portability, developers are advised to avoid it. + */ +# define optreset __mingw_optreset +extern int optreset; +#endif +#ifdef __cplusplus +} +#endif +/* + * POSIX requires the `getopt' API to be specified in `unistd.h'; + * thus, `unistd.h' includes this header. However, we do not want + * to expose the `getopt_long' or `getopt_long_only' APIs, when + * included in this manner. Thus, close the standard __GETOPT_H__ + * declarations block, and open an additional __GETOPT_LONG_H__ + * specific block, only when *not* __UNISTD_H_SOURCED__, in which + * to declare the extended API. + */ +#endif /* !defined(__GETOPT_H__) */ + +#if !defined(__UNISTD_H_SOURCED__) && !defined(__GETOPT_LONG_H__) +#define __GETOPT_LONG_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +struct option /* specification for a long form option... */ +{ + const char *name; /* option name, without leading hyphens */ + int has_arg; /* does it take an argument? */ + int *flag; /* where to save its status, or NULL */ + int val; /* its associated status value */ +}; + +enum /* permitted values for its `has_arg' field... */ +{ + no_argument = 0, /* option never takes an argument */ + required_argument, /* option always requires an argument */ + optional_argument /* option may take an argument */ +}; + +/* + * parse_long_options -- + * Parse long options in argc/argv argument vector. + * Returns -1 if short_too is set and the option does not match long_options. + */ +static int +parse_long_options(char * const *nargv, const char *options, + const struct option *long_options, int *idx, int short_too) +{ + char *current_argv, *has_equal; + size_t current_argv_len; + int i, ambiguous, match; + +#define IDENTICAL_INTERPRETATION(_x, _y) \ + (long_options[(_x)].has_arg == long_options[(_y)].has_arg && \ + long_options[(_x)].flag == long_options[(_y)].flag && \ + long_options[(_x)].val == long_options[(_y)].val) + + current_argv = place; + match = -1; + ambiguous = 0; + + optind++; + + if ((has_equal = strchr(current_argv, '=')) != NULL) { + /* argument found (--option=arg) */ + current_argv_len = has_equal - current_argv; + has_equal++; + } else + current_argv_len = strlen(current_argv); + + for (i = 0; long_options[i].name; i++) { + /* find matching long option */ + if (strncmp(current_argv, long_options[i].name, + current_argv_len)) + continue; + + if (strlen(long_options[i].name) == current_argv_len) { + /* exact match */ + match = i; + ambiguous = 0; + break; + } + /* + * If this is a known short option, don't allow + * a partial match of a single character. + */ + if (short_too && current_argv_len == 1) + continue; + + if (match == -1) /* partial match */ + match = i; + else if (!IDENTICAL_INTERPRETATION(i, match)) + ambiguous = 1; + } + if (ambiguous) { + /* ambiguous abbreviation */ + if (PRINT_ERROR) + warnx(ambig, (int)current_argv_len, + current_argv); + optopt = 0; + return (BADCH); + } + if (match != -1) { /* option found */ + if (long_options[match].has_arg == no_argument + && has_equal) { + if (PRINT_ERROR) + warnx(noarg, (int)current_argv_len, + current_argv); + /* + * XXX: GNU sets optopt to val regardless of flag + */ + if (long_options[match].flag == NULL) + optopt = long_options[match].val; + else + optopt = 0; + return (BADARG); + } + if (long_options[match].has_arg == required_argument || + long_options[match].has_arg == optional_argument) { + if (has_equal) + optarg = has_equal; + else if (long_options[match].has_arg == + required_argument) { + /* + * optional argument doesn't use next nargv + */ + optarg = nargv[optind++]; + } + } + if ((long_options[match].has_arg == required_argument) + && (optarg == NULL)) { + /* + * Missing argument; leading ':' indicates no error + * should be generated. + */ + if (PRINT_ERROR) + warnx(recargstring, + current_argv); + /* + * XXX: GNU sets optopt to val regardless of flag + */ + if (long_options[match].flag == NULL) + optopt = long_options[match].val; + else + optopt = 0; + --optind; + return (BADARG); + } + } else { /* unknown option */ + if (short_too) { + --optind; + return (-1); + } + if (PRINT_ERROR) + warnx(illoptstring, current_argv); + optopt = 0; + return (BADCH); + } + if (idx) + *idx = match; + if (long_options[match].flag) { + *long_options[match].flag = long_options[match].val; + return (0); + } else + return (long_options[match].val); +#undef IDENTICAL_INTERPRETATION +} + +/* + * getopt_internal -- + * Parse argc/argv argument vector. Called by user level routines. + */ +static int +getopt_internal(int nargc, char * const *nargv, const char *options, + const struct option *long_options, int *idx, int flags) +{ + char *oli; /* option letter list index */ + int optchar, short_too; + static int posixly_correct = -1; + + if (options == NULL) + return (-1); + + /* + * XXX Some GNU programs (like cvs) set optind to 0 instead of + * XXX using optreset. Work around this braindamage. + */ + if (optind == 0) + optind = optreset = 1; + + /* + * Disable GNU extensions if POSIXLY_CORRECT is set or options + * string begins with a '+'. + * + * CV, 2009-12-14: Check POSIXLY_CORRECT anew if optind == 0 or + * optreset != 0 for GNU compatibility. + */ + if (posixly_correct == -1 || optreset != 0) + posixly_correct = (getenv("POSIXLY_CORRECT") != NULL); + if (*options == '-') + flags |= FLAG_ALLARGS; + else if (posixly_correct || *options == '+') + flags &= ~FLAG_PERMUTE; + if (*options == '+' || *options == '-') + options++; + + optarg = NULL; + if (optreset) + nonopt_start = nonopt_end = -1; +start: + if (optreset || !*place) { /* update scanning pointer */ + optreset = 0; + if (optind >= nargc) { /* end of argument vector */ + place = EMSG; + if (nonopt_end != -1) { + /* do permutation, if we have to */ + permute_args(nonopt_start, nonopt_end, + optind, nargv); + optind -= nonopt_end - nonopt_start; + } + else if (nonopt_start != -1) { + /* + * If we skipped non-options, set optind + * to the first of them. + */ + optind = nonopt_start; + } + nonopt_start = nonopt_end = -1; + return (-1); + } + if (*(place = nargv[optind]) != '-' || + (place[1] == '\0' && strchr(options, '-') == NULL)) { + place = EMSG; /* found non-option */ + if (flags & FLAG_ALLARGS) { + /* + * GNU extension: + * return non-option as argument to option 1 + */ + optarg = nargv[optind++]; + return (INORDER); + } + if (!(flags & FLAG_PERMUTE)) { + /* + * If no permutation wanted, stop parsing + * at first non-option. + */ + return (-1); + } + /* do permutation */ + if (nonopt_start == -1) + nonopt_start = optind; + else if (nonopt_end != -1) { + permute_args(nonopt_start, nonopt_end, + optind, nargv); + nonopt_start = optind - + (nonopt_end - nonopt_start); + nonopt_end = -1; + } + optind++; + /* process next argument */ + goto start; + } + if (nonopt_start != -1 && nonopt_end == -1) + nonopt_end = optind; + + /* + * If we have "-" do nothing, if "--" we are done. + */ + if (place[1] != '\0' && *++place == '-' && place[1] == '\0') { + optind++; + place = EMSG; + /* + * We found an option (--), so if we skipped + * non-options, we have to permute. + */ + if (nonopt_end != -1) { + permute_args(nonopt_start, nonopt_end, + optind, nargv); + optind -= nonopt_end - nonopt_start; + } + nonopt_start = nonopt_end = -1; + return (-1); + } + } + + /* + * Check long options if: + * 1) we were passed some + * 2) the arg is not just "-" + * 3) either the arg starts with -- we are getopt_long_only() + */ + if (long_options != NULL && place != nargv[optind] && + (*place == '-' || (flags & FLAG_LONGONLY))) { + short_too = 0; + if (*place == '-') + place++; /* --foo long option */ + else if (*place != ':' && strchr(options, *place) != NULL) + short_too = 1; /* could be short option too */ + + optchar = parse_long_options(nargv, options, long_options, + idx, short_too); + if (optchar != -1) { + place = EMSG; + return (optchar); + } + } + + if ((optchar = (int)*place++) == (int)':' || + (optchar == (int)'-' && *place != '\0') || + (oli = (char*)strchr(options, optchar)) == NULL) { + /* + * If the user specified "-" and '-' isn't listed in + * options, return -1 (non-option) as per POSIX. + * Otherwise, it is an unknown option character (or ':'). + */ + if (optchar == (int)'-' && *place == '\0') + return (-1); + if (!*place) + ++optind; + if (PRINT_ERROR) + warnx(illoptchar, optchar); + optopt = optchar; + return (BADCH); + } + if (long_options != NULL && optchar == 'W' && oli[1] == ';') { + /* -W long-option */ + if (*place) /* no space */ + /* NOTHING */; + else if (++optind >= nargc) { /* no arg */ + place = EMSG; + if (PRINT_ERROR) + warnx(recargchar, optchar); + optopt = optchar; + return (BADARG); + } else /* white space */ + place = nargv[optind]; + optchar = parse_long_options(nargv, options, long_options, + idx, 0); + place = EMSG; + return (optchar); + } + if (*++oli != ':') { /* doesn't take argument */ + if (!*place) + ++optind; + } else { /* takes (optional) argument */ + optarg = NULL; + if (*place) /* no white space */ + optarg = place; + else if (oli[1] != ':') { /* arg not optional */ + if (++optind >= nargc) { /* no arg */ + place = EMSG; + if (PRINT_ERROR) + warnx(recargchar, optchar); + optopt = optchar; + return (BADARG); + } else + optarg = nargv[optind]; + } + place = EMSG; + ++optind; + } + /* dump back option letter */ + return (optchar); +} + +/* + * getopt_long -- + * Parse argc/argv argument vector. + */ +int +getopt_long(int nargc, char * const *nargv, const char *options, + const struct option *long_options, int *idx) +{ + + return (getopt_internal(nargc, nargv, options, long_options, idx, + FLAG_PERMUTE)); +} + +/* + * getopt_long_only -- + * Parse argc/argv argument vector. + */ +int +getopt_long_only(int nargc, char * const *nargv, const char *options, + const struct option *long_options, int *idx) +{ + + return (getopt_internal(nargc, nargv, options, long_options, idx, + FLAG_PERMUTE|FLAG_LONGONLY)); +} + +//extern int getopt_long(int nargc, char * const *nargv, const char *options, +// const struct option *long_options, int *idx); +//extern int getopt_long_only(int nargc, char * const *nargv, const char *options, +// const struct option *long_options, int *idx); +/* + * Previous MinGW implementation had... + */ +#ifndef HAVE_DECL_GETOPT +/* + * ...for the long form API only; keep this for compatibility. + */ +# define HAVE_DECL_GETOPT 1 +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* !defined(__UNISTD_H_SOURCED__) && !defined(__GETOPT_LONG_H__) */ diff --git a/native_client/javascript/Makefile b/native_client/javascript/Makefile index b0f7ed75..54e39d90 100644 --- a/native_client/javascript/Makefile +++ b/native_client/javascript/Makefile @@ -6,6 +6,10 @@ PROJECT_VERSION ?= $(shell cat ../../VERSION | tr -d '\n') include ../definitions.mk +ifeq ($(TARGET),host-win) +LIBS := '$(shell cygpath -w $(subst .lib,,$(LIBS)))' +endif + default: build clean: @@ -32,7 +36,7 @@ configure: deepspeech_wrap.cxx package.json $(NODE_BUILD_TOOL) configure $(NODE_BUILD_VERBOSE) build: configure deepspeech_wrap.cxx - AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" CXXFLAGS="$(CXXFLAGS)" LDFLAGS="$(RPATH_NODEJS) $(LDFLAGS)" LIBS="$(LIBS)" $(NODE_BUILD_TOOL) $(NODE_PLATFORM_TARGET) $(NODE_ABI_TARGET) rebuild $(NODE_BUILD_VERBOSE) + AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" CXXFLAGS="$(CXXFLAGS)" LDFLAGS="$(RPATH_NODEJS) $(LDFLAGS)" LIBS=$(LIBS) $(NODE_BUILD_TOOL) $(NODE_PLATFORM_TARGET) $(NODE_ABI_TARGET) rebuild $(NODE_BUILD_VERBOSE) copy-deps: build $(call copy_missing_libs,lib/binding/*/*/*/deepspeech.node,lib/binding/*/*/) diff --git a/native_client/javascript/binding.gyp b/native_client/javascript/binding.gyp index 81e6be49..96276690 100644 --- a/native_client/javascript/binding.gyp +++ b/native_client/javascript/binding.gyp @@ -4,7 +4,7 @@ "target_name": "deepspeech", "sources": [ "deepspeech_wrap.cxx" ], "libraries": [ - "${LIBS}" + "$(LIBS)" ], "include_dirs": [ "../" diff --git a/native_client/python/Makefile b/native_client/python/Makefile index 85c6bfe6..9758baf0 100644 --- a/native_client/python/Makefile +++ b/native_client/python/Makefile @@ -8,7 +8,7 @@ bindings-clean: bindings-build: pip install --quiet $(PYTHON_PACKAGES) wheel==0.31.0 setuptools==39.1.0 - AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED) $(RPATH_PYTHON)" MODEL_LDFLAGS="$(LDFLAGS_DIRS)" MODEL_LIBS="$(LIBS)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext $(PYTHON_PLATFORM_NAME) + PATH=$(TOOLCHAIN):$$PATH AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED) $(RPATH_PYTHON)" MODEL_LDFLAGS="$(LDFLAGS_DIRS)" MODEL_LIBS="$(LIBS)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext $(PYTHON_PLATFORM_NAME) MANIFEST.in: bindings-build > $@ diff --git a/native_client/python/setup.py b/native_client/python/setup.py index 40c73bcf..b9e68c67 100755 --- a/native_client/python/setup.py +++ b/native_client/python/setup.py @@ -39,11 +39,27 @@ class BuildExtFirst(build): ('build_clib', build.has_c_libraries), ('build_scripts', build.has_scripts)] +# Properly pass arguments for linking, setuptools will perform some checks +def lib_dirs_split(a): + if os.name == 'posix': + return a.split('-L')[1:] + + if os.name == 'nt': + return [] + +def libs_split(a): + if os.name == 'posix': + return a.split('-l')[1:] + + if os.name == 'nt': + return a.split('.lib')[0:1] + ds_ext = Extension('deepspeech._impl', ['impl.i'], include_dirs = [ numpy_include, '../' ], - library_dirs = list(map(lambda x: x.strip(), os.getenv('MODEL_LDFLAGS', '').split('-L')[1:])), - libraries = list(map(lambda x: x.strip(), os.getenv('MODEL_LIBS', '').split('-l')[1:]))) + library_dirs = list(map(lambda x: x.strip(), lib_dirs_split(os.getenv('MODEL_LDFLAGS', '')))), + libraries = list(map(lambda x: x.strip(), libs_split(os.getenv('MODEL_LIBS', '')))) + ) setup(name = project_name, description = 'A library for running inference on a DeepSpeech model', diff --git a/taskcluster/.build.yml b/taskcluster/.build.yml index 6bb0cd6a..46682174 100644 --- a/taskcluster/.build.yml +++ b/taskcluster/.build.yml @@ -23,4 +23,4 @@ build: tests_cmdline: '' convert_graphdef: '' benchmark_model_bin: '' - tensorflow_git_desc: 'TensorFlow: v1.12.0-14-g943a6c3' + tensorflow_git_desc: 'TensorFlow: v1.12.0-20-g0259e0a' diff --git a/taskcluster/test-cpp-win-amd64-opt.yml b/taskcluster/test-cpp-win-amd64-opt.yml new file mode 100644 index 00000000..8a7424fc --- /dev/null +++ b/taskcluster/test-cpp-win-amd64-opt.yml @@ -0,0 +1,10 @@ +build: + template_file: test-win-opt-base.tyml + dependencies: + - "win-amd64-cpu-opt" + - "test-training_upstream-linux-amd64-py27mu-opt" + args: + tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/tc-cppwin-ds-tests.sh" + metadata: + name: "DeepSpeech Windows AMD64 CPU C++ tests" + description: "Testing DeepSpeech C++ for Windows/AMD64, CPU only, optimized version" diff --git a/taskcluster/test-netframework-win-amd64-opt.yml b/taskcluster/test-netframework-win-amd64-opt.yml new file mode 100644 index 00000000..4a7b0729 --- /dev/null +++ b/taskcluster/test-netframework-win-amd64-opt.yml @@ -0,0 +1,10 @@ +build: + template_file: test-win-opt-base.tyml + dependencies: + - "win-amd64-cpu-opt" + - "test-training_upstream-linux-amd64-py27mu-opt" + args: + tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/tc-netframework-ds-tests.sh" + metadata: + name: "DeepSpeech Windows AMD64 CPU .Net Framework tests" + description: "Testing DeepSpeech .Net Framework for Windows/AMD64, CPU only, optimized version" diff --git a/taskcluster/test-win-opt-base.tyml b/taskcluster/test-win-opt-base.tyml new file mode 100644 index 00000000..8d302f0f --- /dev/null +++ b/taskcluster/test-win-opt-base.tyml @@ -0,0 +1,80 @@ +$if: '(event.event != "push") && (event.event != "tag")' +then: + taskId: ${taskcluster.taskId} + provisionerId: ${taskcluster.docker.provisionerId} + workerType: ${taskcluster.docker.workerTypeWin} + taskGroupId: ${taskcluster.taskGroupId} + schedulerId: ${taskcluster.schedulerId} + dependencies: + $map: { $eval: build.dependencies } + each(b): + $eval: as_slugid(b) + created: { $fromNow: '0 sec' } + deadline: { $fromNow: '1 day' } + expires: { $fromNow: '7 days' } + + extra: + github: + { $eval: taskcluster.github_events.pull_request } + + routes: + - "notify.irc-channel.${notifications.irc}.on-exception" + - "notify.irc-channel.${notifications.irc}.on-failed" + + scopes: [ + "queue:route:notify.irc-channel.*", + ] + + payload: + maxRunTime: { $eval: to_int(build.maxRunTime) } + mounts: + - file: msys2-base-x86_64.tar.xz + content: + sha256: 4e799b5c3efcf9efcb84923656b7bcff16f75a666911abd6620ea8e5e1e9870c + url: >- + https://sourceforge.net/projects/msys2/files/Base/x86_64/msys2-base-x86_64-20180531.tar.xz/download + + env: + $let: + training: { $eval: as_slugid("test-training_upstream-linux-amd64-py27mu-opt") } + win_amd64_build: { $eval: as_slugid("win-amd64-cpu-opt") } + node_package: { $eval: as_slugid("node-package") } + in: + DEEPSPEECH_ARTIFACTS_ROOT: https://queue.taskcluster.net/v1/task/${win_amd64_build}/artifacts/public + DEEPSPEECH_NODEJS: https://queue.taskcluster.net/v1/task/${node_package}/artifacts/public + DEEPSPEECH_TEST_MODEL: https://queue.taskcluster.net/v1/task/${training}/artifacts/public/output_graph.pb + DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.2.0-prod-ctcdecode/output_graph.pb + DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.2.0-prod-ctcdecode/output_graph.pbmm + EXPECTED_TENSORFLOW_VERSION: "${build.tensorflow_git_desc}" + TC_MSYS_VERSION: 'MSYS_NT-6.3' + MSYS: 'winsymlinks:nativestrict' + + command: + - >- + "C:\Program Files\7-zip\7z.exe" x -txz -so msys2-base-x86_64.tar.xz | + "C:\Program Files\7-zip\7z.exe" x -o%USERPROFILE% -ttar -aoa -si + - .\msys64\usr\bin\bash.exe --login -cx "exit" + - .\msys64\usr\bin\bash.exe --login -cx "pacman --noconfirm -Syu" + - .\msys64\usr\bin\bash.exe --login -cxe " + export LC_ALL=C && + export PATH=\"/c/builds/tc-workdir/msys64/usr/bin:/c/Python36:/c/Program Files/Git/bin:/c/Program Files/7-Zip/:$PATH\" && + export TASKCLUSTER_ARTIFACTS=\"$USERPROFILE/public\" && + export TASKCLUSTER_TASK_DIR=\"/c/builds/tc-workdir/\" && + export TASKCLUSTER_TMP_DIR="$TASKCLUSTER_TASK_DIR/tmp" && + export PIP_DEFAULT_TIMEOUT=60 && + (mkdir $TASKCLUSTER_TASK_DIR || rm -fr $TASKCLUSTER_TASK_DIR/*) && cd $TASKCLUSTER_TASK_DIR && + env && + ln -s $USERPROFILE/msys64 $TASKCLUSTER_TASK_DIR/msys64 && + git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ && + cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && + cd $TASKCLUSTER_TASK_DIR && + pacman --noconfirm -R bsdtar && + pacman --noconfirm -S tar vim && + /bin/bash ${build.args.tests_cmdline} ; + cd $TASKCLUSTER_TASK_DIR/../ && rm -fr tc-workdir/ && exit $TASKCLUSTER_TASK_EXIT_CODE" + + metadata: + name: ${build.metadata.name} + description: ${build.metadata.description} + owner: ${event.head.user.email} + source: ${event.head.repo.url} diff --git a/taskcluster/win-amd64-cpu-opt.yml b/taskcluster/win-amd64-cpu-opt.yml new file mode 100644 index 00000000..11ebbc97 --- /dev/null +++ b/taskcluster/win-amd64-cpu-opt.yml @@ -0,0 +1,17 @@ +build: + template_file: win-opt-base.tyml + routes: + - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.win" + - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.win" + - "index.project.deepspeech.deepspeech.native_client.win.${event.head.sha}" + - "notify.irc-channel.${notifications.irc}.on-exception" + - "notify.irc-channel.${notifications.irc}.on-failed" + tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.12.943a6c332331c0ceeba981b51c24abfed2cd6ffa.win/artifacts/public/home.tar.xz" + scripts: + build: "taskcluster/win-build.sh" + package: "taskcluster/win-package.sh" + nc_asset_name: "native_client.amd64.cpu.win.tar.xz" + maxRunTime: 14400 + metadata: + name: "DeepSpeech Windows AMD64 CPU" + description: "Building DeepSpeech for Windows AMD64, CPU only, optimized version" diff --git a/taskcluster/win-amd64-gpu-opt.yml b/taskcluster/win-amd64-gpu-opt.yml new file mode 100644 index 00000000..81861f45 --- /dev/null +++ b/taskcluster/win-amd64-gpu-opt.yml @@ -0,0 +1,17 @@ +build: + template_file: win-opt-base.tyml + routes: + - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.win-cuda" + - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.win-cuda" + - "index.project.deepspeech.deepspeech.native_client.win-cuda.${event.head.sha}" + - "notify.irc-channel.${notifications.irc}.on-exception" + - "notify.irc-channel.${notifications.irc}.on-failed" + tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.12.943a6c332331c0ceeba981b51c24abfed2cd6ffa.win-cuda/artifacts/public/home.tar.xz" + scripts: + build: "taskcluster/win-build.sh --cuda" + package: "taskcluster/win-package.sh" + nc_asset_name: "native_client.amd64.gpu.win.tar.xz" + maxRunTime: 14400 + metadata: + name: "DeepSpeech Windows AMD64 CUDA" + description: "Building DeepSpeech for Windows AMD64, CUDA-enabled, optimized version" diff --git a/taskcluster/win-build.sh b/taskcluster/win-build.sh new file mode 100755 index 00000000..82883ceb --- /dev/null +++ b/taskcluster/win-build.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +set -xe + +cuda=$1 + +source $(dirname "$0")/../tc-tests-utils.sh + +source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh + +BAZEL_TARGETS=" +//native_client:libdeepspeech.so +//native_client:generate_trie +" + +if [ "${cuda}" = "--cuda" ]; then + BAZEL_ENV_FLAGS="TF_NEED_CUDA=1 ${TF_CUDA_FLAGS}" + BAZEL_BUILD_FLAGS="${BAZEL_CUDA_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BAZEL_OPT_FLAGS}" + PROJECT_NAME="DeepSpeech-GPU" +else + PROJECT_NAME="DeepSpeech" + BAZEL_BUILD_FLAGS="${BAZEL_OPT_FLAGS} ${BAZEL_EXTRA_FLAGS}" + BAZEL_ENV_FLAGS="TF_NEED_CUDA=0" +fi + +SYSTEM_TARGET=host-win + +do_bazel_build + +if [ "${cuda}" = "--cuda" ]; then + cp ${DS_ROOT_TASK}/DeepSpeech/tf/bazel-bin/native_client/liblibdeepspeech.so.ifso ${DS_ROOT_TASK}/DeepSpeech/tf/bazel-bin/native_client/libdeepspeech.so.if.lib +fi + +export PATH=$PATH:$(cygpath ${ChocolateyInstall})/bin + +do_deepspeech_binary_build + +do_deepspeech_netframework_build + +do_nuget_build "${PROJECT_NAME}" + +shutdown_bazel diff --git a/taskcluster/win-opt-base.tyml b/taskcluster/win-opt-base.tyml new file mode 100644 index 00000000..f607f2cb --- /dev/null +++ b/taskcluster/win-opt-base.tyml @@ -0,0 +1,89 @@ +taskId: ${taskcluster.taskId} +provisionerId: ${taskcluster.docker.provisionerId} +workerType: ${taskcluster.docker.workerTypeWin} +taskGroupId: ${taskcluster.taskGroupId} +schedulerId: ${taskcluster.schedulerId} +dependencies: + $map: { $eval: build.dependencies } + each(b): + $eval: as_slugid(b) +created: { $fromNow: '0 sec' } +deadline: { $fromNow: '1 day' } +expires: + $if: '(event.event == "push") || (event.event == "tag")' + then: { $fromNow: '6 months' } + else: { $fromNow: '7 days' } + +extra: + nc_asset_name: { $eval: build.nc_asset_name } + github: + $if: '(event.event == "push") || (event.event == "tag")' + then: { $eval: taskcluster.github_events.merge } + else: { $eval: taskcluster.github_events.pull_request } + +routes: + $if: '(event.event == "push") || (event.event == "tag")' + then: + { $eval: build.routes } + +payload: + maxRunTime: { $eval: to_int(build.maxRunTime) } + mounts: + - file: msys2-base-x86_64.tar.xz + content: + sha256: 4e799b5c3efcf9efcb84923656b7bcff16f75a666911abd6620ea8e5e1e9870c + url: >- + https://sourceforge.net/projects/msys2/files/Base/x86_64/msys2-base-x86_64-20180531.tar.xz/download + + env: + $let: + training: { $eval: as_slugid("test-training_upstream-linux-amd64-py27mu-opt") } + in: + TC_MSYS_VERSION: 'MSYS_NT-6.3' + MSYS: 'winsymlinks:nativestrict' + TENSORFLOW_BUILD_ARTIFACT: ${build.tensorflow} + DEEPSPEECH_TEST_MODEL: https://queue.taskcluster.net/v1/task/${training}/artifacts/public/output_graph.pb + + command: + - >- + "C:\Program Files\7-zip\7z.exe" x -txz -so msys2-base-x86_64.tar.xz | + "C:\Program Files\7-zip\7z.exe" x -o%USERPROFILE% -ttar -aoa -si + - .\msys64\usr\bin\bash.exe --login -cx "exit" + - .\msys64\usr\bin\bash.exe --login -cx "pacman --noconfirm -Syu" + - echo .\msys64\usr\bin\bash.exe --login -cxe " + export LC_ALL=C && + export PATH=\"/c/builds/tc-workdir/msys64/usr/bin:/c/Python36:/c/Program Files/Git/bin:/c/Program Files/7-Zip/:$PATH\" && + export TASKCLUSTER_ARTIFACTS=\"$USERPROFILE/public\" && + export TASKCLUSTER_TASK_DIR=\"/c/builds/tc-workdir/\" && + (mkdir $TASKCLUSTER_TASK_DIR || rm -fr $TASKCLUSTER_TASK_DIR/*) && cd $TASKCLUSTER_TASK_DIR && + env && + ln -s $USERPROFILE/msys64 $TASKCLUSTER_TASK_DIR/msys64 && + (wget -O - $TENSORFLOW_BUILD_ARTIFACT | 7z x -txz -si -so | 7z x -o$TASKCLUSTER_TASK_DIR -aoa -ttar -si ) && + git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ && + cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && + ln -s $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/native_client/ $TASKCLUSTER_TASK_DIR/DeepSpeech/tf/native_client && + cd $TASKCLUSTER_TASK_DIR && + pacman --noconfirm -R bsdtar && + pacman --noconfirm -S tar make && + $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.build} && + $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.package} ; + echo \"export TASKCLUSTER_TASK_EXIT_CODE=$?\" > $USERPROFILE/tc-exit.sh && + cd $TASKCLUSTER_TASK_DIR/../ && rm -fr tc-workdir/ && exit $TASKCLUSTER_TASK_EXIT_CODE" | cmd + /k ""C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"" x64 + + - .\msys64\usr\bin\bash.exe --login -cxe "source $USERPROFILE/tc-exit.sh && + exit $TASKCLUSTER_TASK_EXIT_CODE" + + artifacts: + - type: "directory" + path: "public/" + expires: + $if: '(event.event == "push") || (event.event == "tag")' + then: { $fromNow: '6 months' } + else: { $fromNow: '7 days' } + +metadata: + name: ${build.metadata.name} + description: ${build.metadata.description} + owner: ${event.head.user.email} + source: ${event.head.repo.url} diff --git a/taskcluster/win-package.sh b/taskcluster/win-package.sh new file mode 100755 index 00000000..696e26a0 --- /dev/null +++ b/taskcluster/win-package.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -xe + +arm_flavor=$1 + +source $(dirname "$0")/../tc-tests-utils.sh + +mkdir -p ${TASKCLUSTER_ARTIFACTS} || true + +cp ${DS_ROOT_TASK}/DeepSpeech/tf/bazel*.log ${TASKCLUSTER_ARTIFACTS}/ + +package_native_client "native_client.tar.xz" + +cp ${DS_ROOT_TASK}/DeepSpeech/ds/examples/net_framework/CSharpExamples/*.nupkg ${TASKCLUSTER_ARTIFACTS}/ + +cp ${DS_ROOT_TASK}/DeepSpeech/ds/examples/net_framework/CSharpExamples/DeepSpeechConsole/bin/x64/Release/DeepSpeechConsole.exe ${TASKCLUSTER_ARTIFACTS}/ diff --git a/taskcluster/worker.cyml b/taskcluster/worker.cyml index 953b6ccf..5415f543 100644 --- a/taskcluster/worker.cyml +++ b/taskcluster/worker.cyml @@ -4,6 +4,7 @@ taskcluster: provisionerId: aws-provisioner-v1 workerType: deepspeech-worker workerTypeKvm: deepspeech-kvm-worker + workerTypeWin: deepspeech-win dockerrpi3: provisionerId: deepspeech-provisioner workerType: ds-rpi3 diff --git a/tc-cppwin-ds-tests.sh b/tc-cppwin-ds-tests.sh new file mode 100644 index 00000000..018be828 --- /dev/null +++ b/tc-cppwin-ds-tests.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -xe + +source $(dirname "$0")/tc-tests-utils.sh + +download_material "${TASKCLUSTER_TMP_DIR}/ds" + +export PATH=${TASKCLUSTER_TMP_DIR}/ds/:$PATH + +check_tensorflow_version + +run_basic_inference_tests diff --git a/tc-netframework-ds-tests.sh b/tc-netframework-ds-tests.sh new file mode 100644 index 00000000..5e56e642 --- /dev/null +++ b/tc-netframework-ds-tests.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -xe + +cuda=$1 + +source $(dirname "$0")/tc-tests-utils.sh + +if [ "${cuda}" = "--cuda" ]; then + PROJECT_NAME="DeepSpeech-GPU" +else + PROJECT_NAME="DeepSpeech" +fi + +download_data + +install_nuget "${PROJECT_NAME}" + +run_netframework_inference_tests diff --git a/tc-tests-utils.sh b/tc-tests-utils.sh index ff677e8d..eae4a7bf 100755 --- a/tc-tests-utils.sh +++ b/tc-tests-utils.sh @@ -7,6 +7,11 @@ if [ "${OS}" = "Linux" ]; then export DS_ROOT_TASK=${HOME} fi; +if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then + export DS_ROOT_TASK=${TASKCLUSTER_TASK_DIR} + export PLATFORM_EXE_SUFFIX=.exe +fi; + if [ "${OS}" = "Darwin" ]; then export DS_ROOT_TASK=${TASKCLUSTER_TASK_DIR} export SWIG_LIB="$(find ${DS_ROOT_TASK}/homebrew/Cellar/swig/ -type f -name "swig.swg" | xargs dirname)" @@ -33,6 +38,16 @@ export DS_VERSION="$(cat ${DS_DSDIR}/VERSION)" export ANDROID_SDK_HOME=${DS_ROOT_TASK}/DeepSpeech/Android/SDK/ export ANDROID_NDK_HOME=${DS_ROOT_TASK}/DeepSpeech/Android/android-ndk-r18b/ +TAR=${TAR:-"tar"} +XZ=${XZ:-"pixz -9"} +UNXZ=${UNXZ:-"pixz -d"} + +if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then + TAR=/usr/bin/tar.exe + XZ="xz -9 -T0 -c -" + UNXZ="xz -9 -T0 -d" +fi + model_source="${DEEPSPEECH_TEST_MODEL}" model_name="$(basename "${model_source}")" model_name_mmap="$(basename -s ".pb" "${model_source}").pbmm" @@ -172,11 +187,21 @@ assert_correct_ldc93s1() assert_correct_inference "$1" "she had your dark suit in greasy wash water all year" } +assert_working_ldc93s1() +{ + assert_working_inference "$1" "she had your dark suit in greasy wash water all year" +} + assert_correct_ldc93s1_lm() { assert_correct_inference "$1" "she had your dark suit in greasy wash water all year" } +assert_working_ldc93s1_lm() +{ + assert_working_inference "$1" "she had your dark suit in greasy wash water all year" +} + assert_correct_multi_ldc93s1() { assert_shows_something "$1" "/LDC93S1.wav%she had your dark suit in greasy wash water all year%" @@ -226,7 +251,19 @@ run_tflite_basic_inference_tests() assert_correct_ldc93s1 "${phrase_pbmodel_nolm}" } -run_all_inference_tests() +run_netframework_inference_tests() +{ + phrase_pbmodel_nolm=$(DeepSpeechConsole.exe --model ${TASKCLUSTER_TMP_DIR}/${model_name} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1.wav) + assert_working_ldc93s1 "${phrase_pbmodel_nolm}" + + phrase_pbmodel_nolm=$(DeepSpeechConsole.exe --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1.wav) + assert_working_ldc93s1 "${phrase_pbmodel_nolm}" + + phrase_pbmodel_withlm=$(DeepSpeechConsole.exe --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --lm ${TASKCLUSTER_TMP_DIR}/lm.binary --trie ${TASKCLUSTER_TMP_DIR}/trie --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1.wav) + assert_working_ldc93s1_lm "${phrase_pbmodel_withlm}" +} + +run_basic_inference_tests() { phrase_pbmodel_nolm=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1.wav) assert_correct_ldc93s1 "${phrase_pbmodel_nolm}" @@ -236,6 +273,11 @@ run_all_inference_tests() phrase_pbmodel_withlm=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --lm ${TASKCLUSTER_TMP_DIR}/lm.binary --trie ${TASKCLUSTER_TMP_DIR}/trie --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1.wav) assert_correct_ldc93s1_lm "${phrase_pbmodel_withlm}" +} + +run_all_inference_tests() +{ + run_basic_inference_tests phrase_pbmodel_nolm_stereo_44k=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_2_44100.wav) assert_correct_ldc93s1 "${phrase_pbmodel_nolm_stereo_44k}" @@ -299,7 +341,7 @@ generic_download_tarxz() mkdir -p ${target_dir} || true - wget ${url} -O - | pixz -d | tar -C ${target_dir} -xf - + wget ${url} -O - | ${UNXZ} | ${TAR} -C ${target_dir} -xf - } download_native_client_files() @@ -307,6 +349,45 @@ download_native_client_files() generic_download_tarxz "$1" "${DEEPSPEECH_ARTIFACTS_ROOT}/native_client.tar.xz" } +install_nuget() +{ + PROJECT_NAME=$1 + if [ -z "${PROJECT_NAME}" ]; then + exit "Please call with a valid PROJECT_NAME" + exit 1 + fi; + + nuget="${PROJECT_NAME}.${DS_VERSION}.nupkg" + + export PATH=$PATH:$(cygpath ${ChocolateyInstall})/bin + + mkdir -p "${TASKCLUSTER_TMP_DIR}/repo/" + mkdir -p "${TASKCLUSTER_TMP_DIR}/ds/" + + wget -O - "${DEEPSPEECH_ARTIFACTS_ROOT}/${nuget}" | gunzip > "${TASKCLUSTER_TMP_DIR}/${PROJECT_NAME}.${DS_VERSION}.nupkg" + wget -O - "${DEEPSPEECH_ARTIFACTS_ROOT}/DeepSpeechConsole.exe" | gunzip > "${TASKCLUSTER_TMP_DIR}/ds/DeepSpeechConsole.exe" + + nuget sources add -Name repo -Source $(cygpath -w "${TASKCLUSTER_TMP_DIR}/repo/") + + cd "${TASKCLUSTER_TMP_DIR}" + nuget add $(cygpath -w "${TASKCLUSTER_TMP_DIR}/${nuget}") -source repo + + cd "${TASKCLUSTER_TMP_DIR}/ds/" + nuget list -Source repo -Prerelease + nuget install ${PROJECT_NAME} -Source repo -Prerelease + + ls -halR "${PROJECT_NAME}.${DS_VERSION}" + + nuget install NAudio + cp NAudio*/lib/net35/NAudio.dll ${TASKCLUSTER_TMP_DIR}/ds/ + cp ${PROJECT_NAME}.${DS_VERSION}/build/libdeepspeech.so ${TASKCLUSTER_TMP_DIR}/ds/ + cp ${PROJECT_NAME}.${DS_VERSION}/lib/net462/DeepSpeechClient.dll ${TASKCLUSTER_TMP_DIR}/ds/ + + ls -hal ${TASKCLUSTER_TMP_DIR}/ds/ + + export PATH=${TASKCLUSTER_TMP_DIR}/ds/:$PATH +} + download_data() { wget -P "${TASKCLUSTER_TMP_DIR}" "${model_source}" @@ -390,6 +471,8 @@ is_patched_bazel() { bazel_version=$(bazel version | grep 'Build label:' | cut -d':' -f2) + bazel shutdown + if [ -z "${bazel_version}" ]; then return 0; else @@ -473,6 +556,12 @@ do_bazel_build() verify_bazel_rebuild "${DS_ROOT_TASK}/DeepSpeech/tf/bazel_monolithic.log" } +shutdown_bazel() +{ + cd ${DS_ROOT_TASK}/DeepSpeech/tf + bazel ${BAZEL_OUTPUT_USER_ROOT} shutdown +} + do_bazel_shared_build() { cd ${DS_ROOT_TASK}/DeepSpeech/tf @@ -491,7 +580,7 @@ do_deepspeech_binary_build() EXTRA_CFLAGS="${EXTRA_LOCAL_CFLAGS}" \ EXTRA_LDFLAGS="${EXTRA_LOCAL_LDFLAGS}" \ EXTRA_LIBS="${EXTRA_LOCAL_LIBS}" \ - deepspeech + deepspeech${PLATFORM_EXE_SUFFIX} } do_deepspeech_ndk_build() @@ -509,6 +598,58 @@ do_deepspeech_ndk_build() TARGET_ARCH_ABI=${arch_abi} } +do_deepspeech_netframework_build() +{ + cd ${DS_DSDIR}/examples/net_framework/CSharpExamples + + # Setup dependencies + nuget install DeepSpeechConsole/packages.config -OutputDirectory packages/ + nuget install DeepSpeechWPF/packages.config -OutputDirectory packages/ + + MSBUILD="$(cygpath 'C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\MSBuild\15.0\Bin\MSBuild.exe')" + + # We need MSYS2_ARG_CONV_EXCL='/' otherwise the '/' of CLI parameters gets mangled and disappears + + MSYS2_ARG_CONV_EXCL='/' "${MSBUILD}" \ + DeepSpeechClient/DeepSpeechClient.csproj \ + /p:Configuration=Release \ + /p:Platform=x64 + + MSYS2_ARG_CONV_EXCL='/' "${MSBUILD}" \ + DeepSpeechConsole/DeepSpeechConsole.csproj \ + /p:Configuration=Release \ + /p:Platform=x64 + + MSYS2_ARG_CONV_EXCL='/' "${MSBUILD}" \ + DeepSpeechWPF/DeepSpeech.WPF.csproj \ + /p:Configuration=Release \ + /p:Platform=x64 +} + +do_nuget_build() +{ + PROJECT_NAME=$1 + if [ -z "${PROJECT_NAME}" ]; then + exit "Please call with a valid PROJECT_NAME" + exit 1 + fi; + + cd ${DS_DSDIR}/examples/net_framework/CSharpExamples + + cp ${DS_TFDIR}/bazel-bin/native_client/libdeepspeech.so nupkg/build + + mkdir -p nupkg/lib/net462/ + cp DeepSpeechClient/bin/x64/Release/DeepSpeechClient.dll nupkg/lib/net462/ + + PROJECT_VERSION=$(shell cat ../../../VERSION | tr -d '\n' | tr -d '\r') + sed \ + -e "s/\$NUPKG_ID/${PROJECT_NAME}/" \ + -e "s/\$NUPKG_VERSION/${PROJECT_VERSION}/" \ + nupkg/deepspeech.nuspec.in > nupkg/deepspeech.nuspec && cat nupkg/deepspeech.nuspec + + nuget pack nupkg/deepspeech.nuspec +} + # Hack to extract Ubuntu's 16.04 libssl 1.0.2 packages and use them during the # local build of Python. # @@ -778,13 +919,13 @@ package_native_client() echo "Please specify artifact name." fi; - tar -cf - \ - -C ${tensorflow_dir}/bazel-bin/native_client/ generate_trie \ + ${TAR} -cf - \ + -C ${tensorflow_dir}/bazel-bin/native_client/ generate_trie${PLATFORM_EXE_SUFFIX} \ -C ${tensorflow_dir}/bazel-bin/native_client/ libdeepspeech.so \ -C ${deepspeech_dir}/ LICENSE \ - -C ${deepspeech_dir}/native_client/ deepspeech \ + -C ${deepspeech_dir}/native_client/ deepspeech${PLATFORM_EXE_SUFFIX} \ -C ${deepspeech_dir}/native_client/kenlm/ README.mozilla \ - | pixz -9 > "${artifacts_dir}/${artifact_name}" + | ${XZ} > "${artifacts_dir}/${artifact_name}" } package_native_client_ndk()