From d7cca2a791288febe572d4cf9bf2020109444b63 Mon Sep 17 00:00:00 2001 From: Ryoji Yoshida Date: Mon, 23 Mar 2020 15:52:24 +0900 Subject: [PATCH 1/3] Support building ctc_decoder wheel package on Windows system --- native_client/ctcdecode/Makefile | 12 +++++- native_client/ctcdecode/build_archive.py | 55 ++++++++++++++++++------ native_client/ctcdecode/setup.py | 8 +++- 3 files changed, 58 insertions(+), 17 deletions(-) diff --git a/native_client/ctcdecode/Makefile b/native_client/ctcdecode/Makefile index fb3ca6a4..c6c7fc64 100644 --- a/native_client/ctcdecode/Makefile +++ b/native_client/ctcdecode/Makefile @@ -15,14 +15,22 @@ else GENERATE_DEBUG_SYMS := endif +ifeq ($(findstring _NT,$(OS)),_NT) + FIRST_PARTY := first_party.lib + THIRD_PARTY := third_party.lib +else + FIRST_PARTY := first_party.a + THIRD_PARTY := third_party.a +endif + all: bindings clean-keep-third-party: rm -rf dist temp_build ds_ctcdecoder.egg-info - rm -f swigwrapper_wrap.cpp swigwrapper.py first_party.a + rm -f swigwrapper_wrap.cpp swigwrapper.py $(FIRST_PARTY) clean: clean-keep-third-party - rm -f third_party.a + rm -f $(THIRD_PARTY) rm workspace_status.cc rm -fr bazel-out/ diff --git a/native_client/ctcdecode/build_archive.py b/native_client/ctcdecode/build_archive.py index 6b36ea45..05e0056c 100644 --- a/native_client/ctcdecode/build_archive.py +++ b/native_client/ctcdecode/build_archive.py @@ -9,14 +9,23 @@ import sys from multiprocessing.dummy import Pool -ARGS = ['-DKENLM_MAX_ORDER=6', '-std=c++11', '-Wno-unused-local-typedefs', '-Wno-sign-compare'] -OPT_ARGS = ['-O3', '-DNDEBUG'] -DBG_ARGS = ['-O0', '-g', '-UNDEBUG', '-DDEBUG'] +if sys.platform.startswith('win'): + ARGS = ['/nologo', '/D KENLM_MAX_ORDER=6', '/EHsc', '/source-charset:utf-8'] + OPT_ARGS = ['/O2', '/MT', '/D NDEBUG'] + DBG_ARGS = ['/Od', '/MTd', '/Zi', '/U NDEBUG', '/D DEBUG'] + OPENFST_DIR = 'third_party/openfst-1.6.9-win' +else: + ARGS = ['-DKENLM_MAX_ORDER=6', '-std=c++11', '-Wno-unused-local-typedefs', '-Wno-sign-compare'] + OPT_ARGS = ['-O3', '-DNDEBUG'] + DBG_ARGS = ['-O0', '-g', '-UNDEBUG', '-DDEBUG'] + OPENFST_DIR = 'third_party/openfst-1.6.7' + + INCLUDES = [ '..', '../kenlm', - 'third_party/openfst-1.6.7/src/include', + OPENFST_DIR + '/src/include', 'third_party/ThreadPool' ] @@ -24,7 +33,7 @@ KENLM_FILES = (glob.glob('../kenlm/util/*.cc') + glob.glob('../kenlm/lm/*.cc') + glob.glob('../kenlm/util/double-conversion/*.cc')) -KENLM_FILES += glob.glob('third_party/openfst-1.6.7/src/lib/*.cc') +KENLM_FILES += glob.glob(OPENFST_DIR + '/src/lib/*.cc') KENLM_FILES = [ fn for fn in KENLM_FILES @@ -59,14 +68,25 @@ def build_archive(srcs=[], out_name='', build_dir='temp_build/temp_build', debug if os.path.exists(outfile): return - cmd = '{cc} -fPIC -c {cflags} {args} {includes} {infile} -o {outfile}'.format( - cc=compiler, - cflags=cflags, - args=' '.join(args), - includes=' '.join('-I' + i for i in INCLUDES), - infile=file, - outfile=outfile, - ) + if sys.platform.startswith('win'): + cmd = '"{cc}" -c {cflags} {args} {includes} {infile} -Fo"{outfile}"'.format( + cc=compiler, + cflags=cflags, + args=' '.join(args), + includes=' '.join('-I' + i for i in INCLUDES), + infile=file, + outfile=outfile, + ) + cmd = cmd.replace('\\', '/') + else: + cmd = '{cc} -fPIC -c {cflags} {args} {includes} {infile} -o {outfile}'.format( + cc=compiler, + cflags=cflags, + args=' '.join(args), + includes=' '.join('-I' + i for i in INCLUDES), + infile=file, + outfile=outfile, + ) print(cmd) subprocess.check_call(shlex.split(cmd)) return outfile @@ -82,6 +102,15 @@ def build_archive(srcs=[], out_name='', build_dir='temp_build/temp_build', debug ) print(cmd) subprocess.check_call(shlex.split(cmd)) + elif sys.platform.startswith('win'): + obj_files = [s for s in obj_files if s != None] + cmd = '"lib.exe" /OUT:"{outfile}" {infiles} /MACHINE:X64 /NOLOGO'.format( + ar=ar, + outfile=out_name, + infiles=' '.join(obj_files)) + cmd = cmd.replace('\\', '/') + print(cmd) + subprocess.check_call(shlex.split(cmd)) else: cmd = '{ar} rcs {outfile} {infiles}'.format( ar=ar, diff --git a/native_client/ctcdecode/setup.py b/native_client/ctcdecode/setup.py index fb5a7114..248b7ba0 100644 --- a/native_client/ctcdecode/setup.py +++ b/native_client/ctcdecode/setup.py @@ -54,8 +54,12 @@ def maybe_rebuild(srcs, out_name, build_dir): project_version = read('../../VERSION').strip() build_dir = 'temp_build/temp_build' -third_party_build = 'third_party.a' -ctc_decoder_build = 'first_party.a' +if sys.platform.startswith('win'): + third_party_build = 'third_party.lib' + ctc_decoder_build = 'first_party.lib' +else: + third_party_build = 'third_party.a' + ctc_decoder_build = 'first_party.a' maybe_rebuild(KENLM_FILES, third_party_build, build_dir) maybe_rebuild(CTC_DECODER_FILES, ctc_decoder_build, build_dir) From 4ffbd46cea403224d4a2a139da4faff920ca030b Mon Sep 17 00:00:00 2001 From: Ryoji Yoshida Date: Wed, 25 Mar 2020 10:33:22 +0900 Subject: [PATCH 2/3] Address review comments --- native_client/ctcdecode/Makefile | 10 ++++--- native_client/ctcdecode/build_archive.py | 35 ++++++++++-------------- native_client/ctcdecode/setup.py | 11 +++++--- 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/native_client/ctcdecode/Makefile b/native_client/ctcdecode/Makefile index c6c7fc64..60b42927 100644 --- a/native_client/ctcdecode/Makefile +++ b/native_client/ctcdecode/Makefile @@ -16,13 +16,15 @@ GENERATE_DEBUG_SYMS := endif ifeq ($(findstring _NT,$(OS)),_NT) - FIRST_PARTY := first_party.lib - THIRD_PARTY := third_party.lib + ARCHIVE_EXT := lib else - FIRST_PARTY := first_party.a - THIRD_PARTY := third_party.a + ARCHIVE_EXT := a endif +FIRST_PARTY := first_party.$(ARCHIVE_EXT) +THIRD_PARTY := third_party.$(ARCHIVE_EXT) + + all: bindings clean-keep-third-party: diff --git a/native_client/ctcdecode/build_archive.py b/native_client/ctcdecode/build_archive.py index 05e0056c..8d5773f7 100644 --- a/native_client/ctcdecode/build_archive.py +++ b/native_client/ctcdecode/build_archive.py @@ -15,7 +15,7 @@ if sys.platform.startswith('win'): DBG_ARGS = ['/Od', '/MTd', '/Zi', '/U NDEBUG', '/D DEBUG'] OPENFST_DIR = 'third_party/openfst-1.6.9-win' else: - ARGS = ['-DKENLM_MAX_ORDER=6', '-std=c++11', '-Wno-unused-local-typedefs', '-Wno-sign-compare'] + ARGS = ['fPIC', '-DKENLM_MAX_ORDER=6', '-std=c++11', '-Wno-unused-local-typedefs', '-Wno-sign-compare'] OPT_ARGS = ['-O3', '-DNDEBUG'] DBG_ARGS = ['-O0', '-g', '-UNDEBUG', '-DDEBUG'] OPENFST_DIR = 'third_party/openfst-1.6.7' @@ -51,6 +51,8 @@ CTC_DECODER_FILES = [ def build_archive(srcs=[], out_name='', build_dir='temp_build/temp_build', debug=False, num_parallel=1): compiler = os.environ.get('CXX', 'g++') + if sys.platform.startswith('win'): + compiler = '"{}"'.format(compiler) ar = os.environ.get('AR', 'ar') libtool = os.environ.get('LIBTOOL', 'libtool') cflags = os.environ.get('CFLAGS', '') + os.environ.get('CXXFLAGS', '') @@ -69,24 +71,19 @@ def build_archive(srcs=[], out_name='', build_dir='temp_build/temp_build', debug return if sys.platform.startswith('win'): - cmd = '"{cc}" -c {cflags} {args} {includes} {infile} -Fo"{outfile}"'.format( - cc=compiler, - cflags=cflags, - args=' '.join(args), - includes=' '.join('-I' + i for i in INCLUDES), - infile=file, - outfile=outfile, - ) - cmd = cmd.replace('\\', '/') + file = '"{}"'.format(file.replace('\\', '/')) + output = '/Fo"{}"'.format(outfile.replace('\\', '/')) else: - cmd = '{cc} -fPIC -c {cflags} {args} {includes} {infile} -o {outfile}'.format( - cc=compiler, - cflags=cflags, - args=' '.join(args), - includes=' '.join('-I' + i for i in INCLUDES), - infile=file, - outfile=outfile, - ) + output = '-o ' + outfile + + cmd = '{cc} -c {cflags} {args} {includes} {infile} {output}'.format( + cc=compiler, + cflags=cflags, + args=' '.join(args), + includes=' '.join('-I' + i for i in INCLUDES), + infile=file, + output=output, + ) print(cmd) subprocess.check_call(shlex.split(cmd)) return outfile @@ -103,9 +100,7 @@ def build_archive(srcs=[], out_name='', build_dir='temp_build/temp_build', debug print(cmd) subprocess.check_call(shlex.split(cmd)) elif sys.platform.startswith('win'): - obj_files = [s for s in obj_files if s != None] cmd = '"lib.exe" /OUT:"{outfile}" {infiles} /MACHINE:X64 /NOLOGO'.format( - ar=ar, outfile=out_name, infiles=' '.join(obj_files)) cmd = cmd.replace('\\', '/') diff --git a/native_client/ctcdecode/setup.py b/native_client/ctcdecode/setup.py index 248b7ba0..8a3876c9 100644 --- a/native_client/ctcdecode/setup.py +++ b/native_client/ctcdecode/setup.py @@ -54,12 +54,15 @@ def maybe_rebuild(srcs, out_name, build_dir): project_version = read('../../VERSION').strip() build_dir = 'temp_build/temp_build' + if sys.platform.startswith('win'): - third_party_build = 'third_party.lib' - ctc_decoder_build = 'first_party.lib' + archive_ext = 'lib' else: - third_party_build = 'third_party.a' - ctc_decoder_build = 'first_party.a' + archive_ext = 'a' + +third_party_build = 'third_party.{}'.format(archive_ext) +ctc_decoder_build = 'first_party.{}'.format(archive_ext) + maybe_rebuild(KENLM_FILES, third_party_build, build_dir) maybe_rebuild(CTC_DECODER_FILES, ctc_decoder_build, build_dir) From c8046cbff0902ba5cf51c3680de3184a4112399e Mon Sep 17 00:00:00 2001 From: lissyx <1645737+lissyx@users.noreply.github.com> Date: Wed, 25 Mar 2020 09:18:58 +0100 Subject: [PATCH 3/3] Fix -fPIC typo --- native_client/ctcdecode/build_archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/native_client/ctcdecode/build_archive.py b/native_client/ctcdecode/build_archive.py index 8d5773f7..05d8e81b 100644 --- a/native_client/ctcdecode/build_archive.py +++ b/native_client/ctcdecode/build_archive.py @@ -15,7 +15,7 @@ if sys.platform.startswith('win'): DBG_ARGS = ['/Od', '/MTd', '/Zi', '/U NDEBUG', '/D DEBUG'] OPENFST_DIR = 'third_party/openfst-1.6.9-win' else: - ARGS = ['fPIC', '-DKENLM_MAX_ORDER=6', '-std=c++11', '-Wno-unused-local-typedefs', '-Wno-sign-compare'] + ARGS = ['-fPIC', '-DKENLM_MAX_ORDER=6', '-std=c++11', '-Wno-unused-local-typedefs', '-Wno-sign-compare'] OPT_ARGS = ['-O3', '-DNDEBUG'] DBG_ARGS = ['-O0', '-g', '-UNDEBUG', '-DDEBUG'] OPENFST_DIR = 'third_party/openfst-1.6.7'