From 1598be81242d4e1fa36ebf7aa489a899d0b15ff7 Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Tue, 24 Mar 2020 12:51:46 +0100 Subject: [PATCH] Add CTC decoder build on TaskCluster --- native_client/ctcdecode/Makefile | 10 ++++++---- native_client/ctcdecode/build_archive.py | 4 +++- native_client/definitions.mk | 9 ++++++--- native_client/python/Makefile | 2 ++ taskcluster/decoder-build.sh | 6 +++++- taskcluster/scriptworker-task-github.yml | 2 ++ taskcluster/win-amd64-ctc-opt.yml | 17 +++++++++++++++++ 7 files changed, 41 insertions(+), 9 deletions(-) create mode 100644 taskcluster/win-amd64-ctc-opt.yml diff --git a/native_client/ctcdecode/Makefile b/native_client/ctcdecode/Makefile index 60b42927..1e8aee0c 100644 --- a/native_client/ctcdecode/Makefile +++ b/native_client/ctcdecode/Makefile @@ -41,17 +41,19 @@ workspace_status.cc: ../bazel_workspace_status_cmd.sh > bazel-out/stable-status.txt && \ ../gen_workspace_status.sh > $@ +# Enforce PATH here because swig calls from build_ext looses track of some +# variables over several runs bindings: clean-keep-third-party workspace_status.cc pip install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==39.1.0 - AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) + PATH=$(TOOLCHAIN):$$PATH AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) find temp_build -type f -name "*.o" -delete - AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) + AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) rm -rf temp_build bindings-debug: clean-keep-third-party workspace_status.cc pip install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==39.1.0 - AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --debug --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) + PATH=$(TOOLCHAIN):$$PATH AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --debug --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) $(GENERATE_DEBUG_SYMS) find temp_build -type f -name "*.o" -delete - AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) + AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) rm -rf temp_build diff --git a/native_client/ctcdecode/build_archive.py b/native_client/ctcdecode/build_archive.py index 05d8e81b..c379d6b3 100644 --- a/native_client/ctcdecode/build_archive.py +++ b/native_client/ctcdecode/build_archive.py @@ -54,6 +54,7 @@ def build_archive(srcs=[], out_name='', build_dir='temp_build/temp_build', debug if sys.platform.startswith('win'): compiler = '"{}"'.format(compiler) ar = os.environ.get('AR', 'ar') + libexe = os.environ.get('LIBEXE', 'lib.exe') libtool = os.environ.get('LIBTOOL', 'libtool') cflags = os.environ.get('CFLAGS', '') + os.environ.get('CXXFLAGS', '') args = ARGS + (DBG_ARGS if debug else OPT_ARGS) @@ -100,7 +101,8 @@ def build_archive(srcs=[], out_name='', build_dir='temp_build/temp_build', debug print(cmd) subprocess.check_call(shlex.split(cmd)) elif sys.platform.startswith('win'): - cmd = '"lib.exe" /OUT:"{outfile}" {infiles} /MACHINE:X64 /NOLOGO'.format( + cmd = '"{libexe}" /OUT:"{outfile}" {infiles} /MACHINE:X64 /NOLOGO'.format( + libexe=libexe, outfile=out_name, infiles=' '.join(obj_files)) cmd = cmd.replace('\\', '/') diff --git a/native_client/definitions.mk b/native_client/definitions.mk index 04e6d0c5..fd9358a2 100644 --- a/native_client/definitions.mk +++ b/native_client/definitions.mk @@ -10,6 +10,7 @@ TOOL_CC := gcc TOOL_CXX := c++ TOOL_LD := ld TOOL_LDD := ldd +TOOL_LIBEXE := DEEPSPEECH_BIN := deepspeech CFLAGS_DEEPSPEECH := -std=c++11 -o $(DEEPSPEECH_BIN) @@ -37,9 +38,10 @@ endif ifeq ($(TARGET),host-win) DEEPSPEECH_BIN := deepspeech.exe TOOLCHAIN := '$(VCINSTALLDIR)\bin\amd64\' -TOOL_CC := cl.exe -TOOL_CXX := cl.exe -TOOL_LD := link.exe +TOOL_CC := cl.exe +TOOL_CXX := cl.exe +TOOL_LD := link.exe +TOOL_LIBEXE := lib.exe LINK_DEEPSPEECH := $(TFDIR)\bazel-bin\native_client\libdeepspeech.so.if.lib LINK_PATH_DEEPSPEECH := CFLAGS_DEEPSPEECH := -nologo -Fe$(DEEPSPEECH_BIN) @@ -113,6 +115,7 @@ CC := $(TOOLCHAIN)$(TOOL_CC) CXX := $(TOOLCHAIN)$(TOOL_CXX) LD := $(TOOLCHAIN)$(TOOL_LD) LDD := $(TOOLCHAIN)$(TOOL_LDD) $(TOOLCHAIN_LDD_OPTS) +LIBEXE := $(TOOLCHAIN)$(TOOL_LIBEXE) RPATH_PYTHON := '-Wl,-rpath,\$$ORIGIN/lib/' $(LDFLAGS_RPATH) RPATH_NODEJS := '-Wl,-rpath,$$\$$ORIGIN/../' diff --git a/native_client/python/Makefile b/native_client/python/Makefile index 1735fe77..5dfba785 100644 --- a/native_client/python/Makefile +++ b/native_client/python/Makefile @@ -6,6 +6,8 @@ bindings-clean: rm -rf dist temp_build deepspeech.egg-info MANIFEST.in temp_lib rm -f impl_wrap.cpp impl.py +# Enforce PATH here because swig calls from build_ext looses track of some +# variables over several runs bindings-build: pip install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==39.1.0 PATH=$(TOOLCHAIN):$$PATH AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED) $(RPATH_PYTHON)" MODEL_LDFLAGS="$(LDFLAGS_DIRS)" MODEL_LIBS="$(LIBS)" $(PYTHON_PATH) $(PYTHON_SYSCONFIGDATA) $(NUMPY_INCLUDE) python ./setup.py build_ext $(PYTHON_PLATFORM_NAME) diff --git a/taskcluster/decoder-build.sh b/taskcluster/decoder-build.sh index 4bae5e27..240a57ea 100755 --- a/taskcluster/decoder-build.sh +++ b/taskcluster/decoder-build.sh @@ -6,6 +6,10 @@ source $(dirname "$0")/tc-tests-utils.sh source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh -export SYSTEM_TARGET=host +if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then + export SYSTEM_TARGET=host-win +else + export SYSTEM_TARGET=host +fi; do_deepspeech_decoder_build diff --git a/taskcluster/scriptworker-task-github.yml b/taskcluster/scriptworker-task-github.yml index 26326305..75799d40 100644 --- a/taskcluster/scriptworker-task-github.yml +++ b/taskcluster/scriptworker-task-github.yml @@ -19,6 +19,7 @@ build: - "android-java-opt" - "win-amd64-cpu-opt" - "win-amd64-gpu-opt" + - "win-amd64-ctc-opt" allowed: - "tag" ref_match: "refs/tags/" @@ -38,6 +39,7 @@ build: - "win-amd64-cpu-opt" - "win-amd64-tflite-opt" - "win-amd64-gpu-opt" + - "win-amd64-ctc-opt" javascript: # GPU package - "node-package-gpu" diff --git a/taskcluster/win-amd64-ctc-opt.yml b/taskcluster/win-amd64-ctc-opt.yml new file mode 100644 index 00000000..ebd37445 --- /dev/null +++ b/taskcluster/win-amd64-ctc-opt.yml @@ -0,0 +1,17 @@ +build: + template_file: win-opt-base.tyml + dependencies: + - "swig-win-amd64" + - "node-gyp-cache" + - "pyenv-win-amd64" + routes: + - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.win-ctc" + - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.win-ctc" + - "index.project.deepspeech.deepspeech.native_client.win-ctc.${event.head.sha}" + tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.win/artifacts/public/home.tar.xz" + scripts: + build: 'taskcluster/decoder-build.sh' + package: 'taskcluster/decoder-package.sh' + metadata: + name: "DeepSpeech CTC Decoder Windows AMD64 CPU" + description: "Building DeepSpeech CTC Decoder for Windows/AMD64, CPU only, optimized version"