Cache common objects in decoder build
This commit is contained in:
parent
0dcb1f87c5
commit
cfed8ccd4f
@ -11,11 +11,14 @@ endif
|
|||||||
|
|
||||||
all: bindings
|
all: bindings
|
||||||
|
|
||||||
clean:
|
clean-keep-common:
|
||||||
rm -rf dist temp_build ds_ctcdecoder.egg-info
|
rm -rf dist temp_build ds_ctcdecoder.egg-info
|
||||||
rm -f swigwrapper_wrap.cpp swigwrapper.py
|
rm -f swigwrapper_wrap.cpp swigwrapper.py
|
||||||
|
|
||||||
bindings: clean
|
clean: clean-keep-common
|
||||||
|
rm -f common.a
|
||||||
|
|
||||||
|
bindings: clean-keep-common
|
||||||
pip install --quiet $(PYTHON_PACKAGES) wheel==0.31.0 setuptools==39.1.0
|
pip install --quiet $(PYTHON_PACKAGES) wheel==0.31.0 setuptools==39.1.0
|
||||||
AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
|
AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
|
||||||
find temp_build -type f -name "*.o" -delete
|
find temp_build -type f -name "*.o" -delete
|
||||||
|
87
native_client/ctcdecode/build_common.py
Normal file
87
native_client/ctcdecode/build_common.py
Normal file
@ -0,0 +1,87 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
from __future__ import absolute_import, division, print_function
|
||||||
|
|
||||||
|
import glob
|
||||||
|
import os
|
||||||
|
import shlex
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from multiprocessing.dummy import Pool
|
||||||
|
|
||||||
|
ARGS = ['-O3', '-DNDEBUG', '-DKENLM_MAX_ORDER=6', '-std=c++11',
|
||||||
|
'-Wno-unused-local-typedef', '-Wno-sign-compare']
|
||||||
|
|
||||||
|
INCLUDES = [
|
||||||
|
'..',
|
||||||
|
'../kenlm',
|
||||||
|
'third_party/openfst-1.6.7/src/include',
|
||||||
|
'third_party/ThreadPool'
|
||||||
|
]
|
||||||
|
|
||||||
|
COMMON_FILES = (glob.glob('../kenlm/util/*.cc')
|
||||||
|
+ glob.glob('../kenlm/lm/*.cc')
|
||||||
|
+ glob.glob('../kenlm/util/double-conversion/*.cc'))
|
||||||
|
|
||||||
|
COMMON_FILES += glob.glob('third_party/openfst-1.6.7/src/lib/*.cc')
|
||||||
|
|
||||||
|
COMMON_FILES = [
|
||||||
|
fn for fn in COMMON_FILES
|
||||||
|
if not (fn.endswith('main.cc') or fn.endswith('test.cc') or fn.endswith(
|
||||||
|
'unittest.cc'))
|
||||||
|
]
|
||||||
|
|
||||||
|
COMMON_FILES += glob.glob('*.cpp')
|
||||||
|
|
||||||
|
def build_common(out_name='common.a', build_dir='temp_build/temp_build', num_parallel=1):
|
||||||
|
compiler = os.environ.get('CXX', 'g++')
|
||||||
|
ar = os.environ.get('AR', 'ar')
|
||||||
|
libtool = os.environ.get('LIBTOOL', 'libtool')
|
||||||
|
cflags = os.environ.get('CFLAGS', '') + os.environ.get('CXXFLAGS', '')
|
||||||
|
|
||||||
|
for file in COMMON_FILES:
|
||||||
|
outfile = os.path.join(build_dir, os.path.splitext(file)[0] + '.o')
|
||||||
|
outdir = os.path.dirname(outfile)
|
||||||
|
if not os.path.exists(outdir):
|
||||||
|
print('mkdir', outdir)
|
||||||
|
os.makedirs(outdir)
|
||||||
|
|
||||||
|
def build_one(file):
|
||||||
|
outfile = os.path.join(build_dir, os.path.splitext(file)[0] + '.o')
|
||||||
|
if os.path.exists(outfile):
|
||||||
|
return
|
||||||
|
|
||||||
|
cmd = '{cc} -fPIC -c {cflags} {args} {includes} {infile} -o {outfile}'.format(
|
||||||
|
cc=compiler,
|
||||||
|
cflags=cflags,
|
||||||
|
args=' '.join(ARGS),
|
||||||
|
includes=' '.join('-I' + i for i in INCLUDES),
|
||||||
|
infile=file,
|
||||||
|
outfile=outfile,
|
||||||
|
)
|
||||||
|
print(cmd)
|
||||||
|
subprocess.check_call(shlex.split(cmd))
|
||||||
|
return outfile
|
||||||
|
|
||||||
|
pool = Pool(num_parallel)
|
||||||
|
obj_files = list(pool.imap_unordered(build_one, COMMON_FILES))
|
||||||
|
|
||||||
|
if sys.platform.startswith('darwin'):
|
||||||
|
cmd = '{libtool} -static -o {outfile} {infiles}'.format(
|
||||||
|
libtool=libtool,
|
||||||
|
outfile=out_name,
|
||||||
|
infiles=' '.join(obj_files),
|
||||||
|
)
|
||||||
|
print(cmd)
|
||||||
|
subprocess.check_call(shlex.split(cmd))
|
||||||
|
else:
|
||||||
|
cmd = '{ar} rcs {outfile} {infiles}'.format(
|
||||||
|
ar=ar,
|
||||||
|
outfile=out_name,
|
||||||
|
infiles=' '.join(obj_files)
|
||||||
|
)
|
||||||
|
print(cmd)
|
||||||
|
subprocess.check_call(shlex.split(cmd))
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
build_common()
|
@ -5,12 +5,13 @@ from distutils.command.build import build
|
|||||||
from setuptools import setup, Extension, distutils
|
from setuptools import setup, Extension, distutils
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import glob
|
|
||||||
import multiprocessing.pool
|
import multiprocessing.pool
|
||||||
import os
|
import os
|
||||||
import platform
|
import platform
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
|
from build_common import *
|
||||||
|
|
||||||
try:
|
try:
|
||||||
import numpy
|
import numpy
|
||||||
try:
|
try:
|
||||||
@ -40,66 +41,25 @@ def read(fname):
|
|||||||
|
|
||||||
project_version = read('../../VERSION').strip()
|
project_version = read('../../VERSION').strip()
|
||||||
|
|
||||||
# monkey-patch for parallel compilation
|
build_dir = 'temp_build/temp_build'
|
||||||
# See: https://stackoverflow.com/a/13176803
|
common_build = 'common.a'
|
||||||
def parallelCCompile(self,
|
|
||||||
sources,
|
|
||||||
output_dir=None,
|
|
||||||
macros=None,
|
|
||||||
include_dirs=None,
|
|
||||||
debug=0,
|
|
||||||
extra_preargs=None,
|
|
||||||
extra_postargs=None,
|
|
||||||
depends=None):
|
|
||||||
# those lines are copied from distutils.ccompiler.CCompiler directly
|
|
||||||
macros, objects, extra_postargs, pp_opts, build = self._setup_compile(
|
|
||||||
output_dir, macros, include_dirs, sources, depends, extra_postargs)
|
|
||||||
cc_args = self._get_cc_args(pp_opts, debug, extra_preargs)
|
|
||||||
|
|
||||||
# parallel code
|
if not os.path.exists(common_build):
|
||||||
def _single_compile(obj):
|
if not os.path.exists(build_dir):
|
||||||
try:
|
os.makedirs(build_dir)
|
||||||
src, ext = build[obj]
|
|
||||||
except KeyError:
|
|
||||||
return
|
|
||||||
self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts)
|
|
||||||
|
|
||||||
# convert to list, imap is evaluated on-demand
|
build_common(out_name='common.a',
|
||||||
thread_pool = multiprocessing.pool.ThreadPool(args[0].num_processes)
|
build_dir=build_dir,
|
||||||
list(thread_pool.imap(_single_compile, objects))
|
num_parallel=args[0].num_processes)
|
||||||
return objects
|
|
||||||
|
|
||||||
# hack compile to support parallel compiling
|
|
||||||
distutils.ccompiler.CCompiler.compile = parallelCCompile
|
|
||||||
|
|
||||||
FILES = glob.glob('../kenlm/util/*.cc') \
|
|
||||||
+ glob.glob('../kenlm/lm/*.cc') \
|
|
||||||
+ glob.glob('../kenlm/util/double-conversion/*.cc')
|
|
||||||
|
|
||||||
FILES += glob.glob('third_party/openfst-1.6.7/src/lib/*.cc')
|
|
||||||
|
|
||||||
FILES = [
|
|
||||||
fn for fn in FILES
|
|
||||||
if not (fn.endswith('main.cc') or fn.endswith('test.cc') or fn.endswith(
|
|
||||||
'unittest.cc'))
|
|
||||||
]
|
|
||||||
|
|
||||||
ARGS = ['-O3', '-DNDEBUG', '-DKENLM_MAX_ORDER=6', '-std=c++11',
|
|
||||||
'-Wno-unused-local-typedef', '-Wno-sign-compare']
|
|
||||||
|
|
||||||
decoder_module = Extension(
|
decoder_module = Extension(
|
||||||
name='ds_ctcdecoder._swigwrapper',
|
name='ds_ctcdecoder._swigwrapper',
|
||||||
sources=['swigwrapper.i'] + FILES + glob.glob('*.cpp'),
|
sources=['swigwrapper.i'],
|
||||||
swig_opts=['-c++', '-extranative'],
|
swig_opts=['-c++', '-extranative'],
|
||||||
language='c++',
|
language='c++',
|
||||||
include_dirs=[
|
include_dirs=INCLUDES + [numpy_include],
|
||||||
numpy_include,
|
extra_compile_args=ARGS,
|
||||||
'..',
|
extra_link_args=[common_build],
|
||||||
'../kenlm',
|
|
||||||
'third_party/openfst-1.6.7/src/include',
|
|
||||||
'third_party/ThreadPool',
|
|
||||||
],
|
|
||||||
extra_compile_args=ARGS
|
|
||||||
)
|
)
|
||||||
|
|
||||||
class BuildExtFirst(build):
|
class BuildExtFirst(build):
|
||||||
|
@ -19,6 +19,11 @@ do_bazel_build
|
|||||||
|
|
||||||
do_deepspeech_binary_build
|
do_deepspeech_binary_build
|
||||||
|
|
||||||
|
# Decoder packages for Linux are built a separate task
|
||||||
|
if [ "${OS}" = "Darwin" ]; then
|
||||||
|
do_deepspeech_decoder_build
|
||||||
|
fi
|
||||||
|
|
||||||
do_deepspeech_python_build
|
do_deepspeech_python_build
|
||||||
|
|
||||||
do_deepspeech_nodejs_build
|
do_deepspeech_nodejs_build
|
||||||
|
@ -562,7 +562,7 @@ do_deepspeech_decoder_build()
|
|||||||
|
|
||||||
cp native_client/ctcdecode/dist/*.whl wheels
|
cp native_client/ctcdecode/dist/*.whl wheels
|
||||||
|
|
||||||
make -C native_client/ctcdecode clean
|
make -C native_client/ctcdecode clean-keep-common
|
||||||
|
|
||||||
unset NUMPY_BUILD_VERSION
|
unset NUMPY_BUILD_VERSION
|
||||||
unset NUMPY_DEP_VERSION
|
unset NUMPY_DEP_VERSION
|
||||||
|
@ -73,7 +73,7 @@ if __name__ == '__main__':
|
|||||||
parser.add_argument('--target', required=False,
|
parser.add_argument('--target', required=False,
|
||||||
help='Where to put the native client binary files')
|
help='Where to put the native client binary files')
|
||||||
parser.add_argument('--arch', required=False,
|
parser.add_argument('--arch', required=False,
|
||||||
help='Which architecture to download binaries for. "arm" for ARM 7 (32-bit), "gpu" for CUDA enabled x86_64 binaries, "cpu" for CPU-only x86_64 binaries, "osx" for CPU-only x86_64 OSX binaries. Optional ("cpu" by default)')
|
help='Which architecture to download binaries for. "arm" for ARM 7 (32-bit), "arm64" for ARM64, "gpu" for CUDA enabled x86_64 binaries, "cpu" for CPU-only x86_64 binaries, "osx" for CPU-only x86_64 OSX binaries. Optional ("cpu" by default)')
|
||||||
parser.add_argument('--artifact', required=False,
|
parser.add_argument('--artifact', required=False,
|
||||||
default='native_client.tar.xz',
|
default='native_client.tar.xz',
|
||||||
help='Name of the artifact to download. Defaults to "native_client.tar.xz"')
|
help='Name of the artifact to download. Defaults to "native_client.tar.xz"')
|
||||||
|
Loading…
Reference in New Issue
Block a user