From ade66b015ded24bbefee9c09bb131e578e055492 Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Wed, 20 Dec 2017 19:14:56 +0100 Subject: [PATCH] Build TensorFlow as monolithic into libdeepspeech.so Also limit the set of included Ops to avoid wasting size. --- native_client/BUILD | 66 ++++++++++++++----- native_client/Makefile | 8 +-- native_client/README.md | 7 +- native_client/deepspeech.cc | 8 ++- native_client/deepspeech.h | 2 + native_client/deepspeech_utils.cc | 1 + native_client/deepspeech_utils.h | 3 +- native_client/definitions.mk | 6 +- taskcluster/aot-test-model-build.sh | 4 +- taskcluster/cuda-build.sh | 7 +- taskcluster/darwin-amd64-cpu-aot_prod-opt.yml | 5 +- taskcluster/darwin-amd64-cpu-opt.yml | 5 +- taskcluster/darwin-opt-base.tyml | 1 - taskcluster/decoder-build.sh | 4 +- taskcluster/host-build.sh | 7 +- taskcluster/linux-amd64-cpu-aot_prod-opt.yml | 5 +- taskcluster/linux-amd64-cpu-aot_test-opt.yml | 5 +- taskcluster/linux-amd64-cpu-opt.yml | 5 +- taskcluster/linux-amd64-ctc-opt.yml | 5 +- taskcluster/linux-amd64-gpu-opt.yml | 5 +- taskcluster/linux-opt-base.tyml | 1 - taskcluster/linux-rpi3-cpu-aot_prod-opt.yml | 5 +- taskcluster/linux-rpi3-cpu-opt.yml | 5 +- taskcluster/node-package.yml | 5 +- taskcluster/rpi3-build.sh | 5 +- ...aining+deepspeech-linux-amd64-py27-opt.yml | 2 +- ...aining+deepspeech-linux-amd64-py36-opt.yml | 2 +- .../test-training-linux-amd64-py27-opt.yml | 2 +- .../test-training-linux-amd64-py34-opt.yml | 2 +- .../test-training-linux-amd64-py35-opt.yml | 2 +- .../test-training-linux-amd64-py36-opt.yml | 2 +- tc-tests-utils.sh | 29 ++++---- tc-train-tests.sh | 10 ++- 33 files changed, 127 insertions(+), 104 deletions(-) diff --git a/native_client/BUILD b/native_client/BUILD index f42f4b85..311bfd1e 100644 --- a/native_client/BUILD +++ b/native_client/BUILD @@ -1,9 +1,9 @@ # Description: Deepspeech native client library. -load("//tensorflow:tensorflow.bzl", - "if_linux_x86_64", "tf_cc_shared_object") +load("@org_tensorflow//tensorflow:tensorflow.bzl", + "if_linux_x86_64", "tf_cc_shared_object", "if_cuda") -load("//tensorflow/compiler/aot:tfcompile.bzl", +load("@org_tensorflow//tensorflow/compiler/aot:tfcompile.bzl", "tf_library") load(":deepspeech.bzl", "if_native_model") @@ -16,7 +16,7 @@ config_setting( ) tf_library( - name = "deepspeech_model", + name = "deepspeech_model_core", cpp_class = "DeepSpeech::nativeModel", # We don't need tests or benchmark binaries gen_test=False, gen_benchmark=False, @@ -45,36 +45,66 @@ genrule( cmd = "cp $(DS_MODEL_FILE) $@" ) -cc_library( - name = "deepspeech", - srcs = ["deepspeech.cc", "alphabet.h", "beam_search.h", "trie_node.h"] + +tf_cc_shared_object( + name = "libdeepspeech.so", + srcs = ["deepspeech.cc", "deepspeech.h", "deepspeech_utils.h", "alphabet.h", "beam_search.h", "trie_node.h"] + + if_native_model(["deepspeech_model_core.h"]) + glob(["kenlm/lm/*.cc", "kenlm/util/*.cc", "kenlm/util/double-conversion/*.cc", "kenlm/lm/*.hh", "kenlm/util/*.hh", "kenlm/util/double-conversion/*.h"], exclude = ["kenlm/*/*test.cc", "kenlm/*/*main.cc"]) + glob(["boost_locale/**/*.hpp"]), - hdrs = ["deepspeech.h"], # -Wno-sign-compare to silent a lot of warnings from tensorflow itself, # which makes it harder to see our own warnings - copts = ["-std=c++11", "-Wno-sign-compare"] + if_native_model([ + copts = ["-std=c++11", "-Wno-sign-compare", "-fvisibility=hidden"] + if_native_model([ "-DDS_MODEL_TIMESTEPS=$(DS_MODEL_TIMESTEPS)", "-DDS_NATIVE_MODEL=1", ]), - linkopts = select({ - "//tensorflow:darwin": [ - "-Wl,-install_name,@rpath/libdeepspeech.so" - ], - "//conditions:default": [] - }), deps = [ - "//tensorflow/core:core", + "//tensorflow/core:core_cpu", + "//tensorflow/core:direct_session", + ### "//tensorflow/core:all_kernels", + ### => Trying to be more fine-grained + ### Obtained by trial/error process ... + ### CPU only build libdeepspeech.so from 63M to 36M + "//tensorflow/core/kernels:constant_op", # Const + "//tensorflow/core/kernels:identity_op", # Identity + "//tensorflow/core/kernels:transpose_op", # Transpose + "//tensorflow/core/kernels:reshape_op", # Reshape + "//tensorflow/core/kernels:shape_ops", # Shape + "//tensorflow/core/kernels:strided_slice_op", # StridedSlice + "//tensorflow/core/kernels:pack_op", # Pack + "//tensorflow/core/kernels:reverse_op", # ReverseV2 + "//tensorflow/core/kernels:concat_op", # ConcatV2 + "//tensorflow/core/kernels:split_op", # Split + "//tensorflow/core/kernels:sparse_to_dense_op", # SparseToDense + "//tensorflow/core/kernels:relu_op", # Relu + "//tensorflow/core/kernels:bias_op", # BiasAdd + "//tensorflow/core/kernels:math", # Range, MatMul + "//tensorflow/core/kernels:tensor_array_ops", # Placeholder, TensorArrayV3 + "//tensorflow/core/kernels:control_flow_ops", # Enter + "//tensorflow/core/kernels:ctc_ops", # CTCBeamSearchDecoder + ### Needed by production model produced without "--use_seq_length False" + "//tensorflow/core/kernels:logging_ops", # Assert + "//tensorflow/core/kernels:reverse_sequence_op", # ReverseSequence + # Classic deps "//tensorflow/core/util/ctc", "//third_party/eigen3", - ":deepspeech_utils" - ] + if_native_model([":deepspeech_model", "//tensorflow/compiler/tf2xla:xla_compiled_cpu_function"]), + ] + if_native_model([ + "//tensorflow/compiler/tf2xla:xla_compiled_cpu_function", + ]) + + if_cuda([ + "//tensorflow/core:core", + "//tensorflow/core/kernels:slice_op_gpu", # Slice GPU + ]), includes = ["kenlm", "boost_locale"], defines = ["KENLM_MAX_ORDER=6"], ) +tf_cc_shared_object( + name = "libdeepspeech_model.so", + deps = [":deepspeech_model_core"] +) + # We have a single rule including c_speech_features and kissfft here as Bazel # doesn't support static linking in library targets. diff --git a/native_client/Makefile b/native_client/Makefile index 8dda7729..9593df49 100644 --- a/native_client/Makefile +++ b/native_client/Makefile @@ -46,15 +46,13 @@ bindings-package: MANIFEST.in bindings: bindings-build bindings-package run: deepspeech - ${META_LD_LIBRARY_PATH}=${TFDIR}/bazel-bin/tensorflow:${TFDIR}/bazel-bin/native_client:${${META_LD_LIBRARY_PATH}} ./deepspeech ${ARGS} + ${META_LD_LIBRARY_PATH}=${TFDIR}/bazel-bin/native_client:${${META_LD_LIBRARY_PATH}} ./deepspeech ${ARGS} debug: deepspeech - ${META_LD_LIBRARY_PATH}=${TFDIR}/bazel-bin/tensorflow:${TFDIR}/bazel-bin/native_client:${${META_LD_LIBRARY_PATH}} gdb --args ./deepspeech ${ARGS} + ${META_LD_LIBRARY_PATH}=${TFDIR}/bazel-bin/native_client:${${META_LD_LIBRARY_PATH}} gdb --args ./deepspeech ${ARGS} install: deepspeech install -d ${PREFIX}/lib - install -m 0644 ${TFDIR}/bazel-bin/tensorflow/libtensorflow_cc.so ${PREFIX}/lib/ - install -m 0644 ${TFDIR}/bazel-bin/tensorflow/libtensorflow_framework.so ${PREFIX}/lib/ install -m 0644 ${TFDIR}/bazel-bin/native_client/libdeepspeech.so ${PREFIX}/lib/ install -m 0644 ${TFDIR}/bazel-bin/native_client/libdeepspeech_utils.so ${PREFIX}/lib/ install -d ${PREFIX}/bin @@ -65,6 +63,4 @@ uninstall: rmdir --ignore-fail-on-non-empty ${PREFIX}/bin rm -f ${PREFIX}/lib/libdeepspeech_utils.so rm -f ${PREFIX}/lib/libdeepspeech.so - rm -f ${PREFIX}/lib/libtensorflow_cc.so - rm -f ${PREFIX}/lib/libtensorflow_framework.so rmdir --ignore-fail-on-non-empty ${PREFIX}/lib diff --git a/native_client/README.md b/native_client/README.md index 9f258200..5bbde5fd 100644 --- a/native_client/README.md +++ b/native_client/README.md @@ -72,7 +72,7 @@ Before building the DeepSpeech client libraries, you will need to prepare your e Then you can build the Tensorflow and DeepSpeech libraries. ``` -bazel build -c opt --copt=-O3 //tensorflow:libtensorflow_cc.so //tensorflow:libtensorflow_framework.so //native_client:deepspeech //native_client:deepspeech_utils //native_client:libctc_decoder_with_kenlm.so //native_client:generate_trie +bazel build --config=monolithic -c opt --copt=-O3 --copt=-fvisibility=hidden //native_client:libdeepspeech.so //native_client:deepspeech_utils //native_client:libctc_decoder_with_kenlm.so //native_client:generate_trie ``` Finally, you can change to the `native_client` directory and use the `Makefile`. By default, the `Makefile` will assume there is a TensorFlow checkout in a directory above the DeepSpeech checkout. If that is not the case, set the environment variable `TFDIR` to point to the right directory. @@ -96,17 +96,16 @@ Bazel defines: Bazel targets: * `//native_client:deepspeech_model`: to produce `libdeepspeech_model.so` -* `//tensorflow/compiler/aot:runtime `, `//tensorflow/compiler/xla/service/cpu:runtime_matmul`, `//tensorflow/compiler/xla:executable_run_options` In the end, the previous example becomes: ``` -bazel build -c opt --copt=-O3 --define=DS_NATIVE_MODEL=1 --define=DS_MODEL_TIMESTEPS=64 --define=DS_MODEL_FRAMESIZE=494 --define=DS_MODEL_FILE=/tmp/model.ldc93s1.pb //tensorflow:libtensorflow_cc.so //tensorflow:libtensorflow_framework.so //native_client:deepspeech_model //tensorflow/compiler/aot:runtime //tensorflow/compiler/xla/service/cpu:runtime_matmul //tensorflow/compiler/xla:executable_run_options //native_client:deepspeech //native_client:deepspeech_utils //native_client:libctc_decoder_with_kenlm.so //native_client:generate_trie +bazel build --config=monolithic -c opt --copt=-O3 --copt=-fvisibility=hidden --define=DS_NATIVE_MODEL=1 --define=DS_MODEL_TIMESTEPS=64 --define=DS_MODEL_FRAMESIZE=494 --define=DS_MODEL_FILE=/tmp/model.ldc93s1.pb //native_client:deepspeech_model //native_client:libdeepspeech.so //native_client:deepspeech_utils //native_client:libctc_decoder_with_kenlm.so //native_client:generate_trie ``` Later, when building either `deepspeech` binaries or bindings, you will have to add some extra variables to your `make` command-line (assuming `TFDIR` points to your TensorFlow's git clone): ``` -EXTRA_LDFLAGS="-L${TFDIR}/bazel-bin/tensorflow/compiler/xla/ -L${TFDIR}/bazel-bin/tensorflow/compiler/aot/ -L${TFDIR}/bazel-bin/tensorflow/compiler/xla/service/cpu/" EXTRA_LIBS="-ldeepspeech_model -lruntime -lexecutable_run_options -lruntime_matmul" +EXTRA_LIBS="-ldeepspeech_model" ``` ## Installing diff --git a/native_client/deepspeech.cc b/native_client/deepspeech.cc index f26e3510..44ee4800 100644 --- a/native_client/deepspeech.cc +++ b/native_client/deepspeech.cc @@ -4,7 +4,7 @@ #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" -#include "native_client/deepspeech_model.h" // generated +#include "native_client/deepspeech_model_core.h" // generated #endif #include @@ -37,6 +37,7 @@ class Private { bool run_aot; }; +DEEPSPEECH_EXPORT Model::Model(const char* aModelPath, int aNCep, int aNContext, const char* aAlphabetConfigPath, int aBeamWidth) { @@ -97,6 +98,7 @@ Model::Model(const char* aModelPath, int aNCep, int aNContext, } } +DEEPSPEECH_EXPORT Model::~Model() { if (mPriv->session) { @@ -109,6 +111,7 @@ Model::~Model() delete mPriv; } +DEEPSPEECH_EXPORT void Model::enableDecoderWithLM(const char* aAlphabetConfigPath, const char* aLMPath, const char* aTriePath, float aLMWeight, @@ -118,6 +121,7 @@ Model::enableDecoderWithLM(const char* aAlphabetConfigPath, const char* aLMPath, aLMWeight, aWordCountWeight, aValidWordCountWeight); } +DEEPSPEECH_EXPORT void Model::getInputVector(const short* aBuffer, unsigned int aBufferSize, int aSampleRate, float** aMfcc, int* aNFrames, @@ -204,6 +208,7 @@ Model::decode(int aNFrames, float*** aLogits) return output; } +DEEPSPEECH_EXPORT char* Model::infer(float* aMfcc, int aNFrames, int aFrameLen) { @@ -303,6 +308,7 @@ Model::infer(float* aMfcc, int aNFrames, int aFrameLen) return decode(aNFrames, input_data_mat); } +DEEPSPEECH_EXPORT char* Model::stt(const short* aBuffer, unsigned int aBufferSize, int aSampleRate) { diff --git a/native_client/deepspeech.h b/native_client/deepspeech.h index 18330f4f..ade5f17f 100644 --- a/native_client/deepspeech.h +++ b/native_client/deepspeech.h @@ -3,6 +3,8 @@ #include +#define DEEPSPEECH_EXPORT __attribute__ ((visibility("default"))) + namespace DeepSpeech { diff --git a/native_client/deepspeech_utils.cc b/native_client/deepspeech_utils.cc index aa7c68a6..ad62a3c4 100644 --- a/native_client/deepspeech_utils.cc +++ b/native_client/deepspeech_utils.cc @@ -12,6 +12,7 @@ namespace DeepSpeech { +DEEPSPEECH_EXPORT void audioToInputVector(const short* aBuffer, unsigned int aBufferSize, int aSampleRate, int aNCep, int aNContext, float** aMfcc, diff --git a/native_client/deepspeech_utils.h b/native_client/deepspeech_utils.h index f4f16b4e..617cb273 100644 --- a/native_client/deepspeech_utils.h +++ b/native_client/deepspeech_utils.h @@ -1,9 +1,10 @@ - #ifndef __DEEPSPEECH_UTILS_H__ #define __DEEPSPEECH_UTILS_H__ #include +#define DEEPSPEECH_EXPORT __attribute__ ((visibility("default"))) + namespace DeepSpeech { diff --git a/native_client/definitions.mk b/native_client/definitions.mk index 85c7b2f9..32a4d808 100644 --- a/native_client/definitions.mk +++ b/native_client/definitions.mk @@ -48,8 +48,8 @@ LDFLAGS_RPATH := -Wl,-rpath,@executable_path endif CFLAGS += $(EXTRA_CFLAGS) -LIBS := -ldeepspeech -ldeepspeech_utils -ltensorflow_cc -ltensorflow_framework $(EXTRA_LIBS) -LDFLAGS_DIRS := -L${TFDIR}/bazel-bin/tensorflow -L${TFDIR}/bazel-bin/native_client $(EXTRA_LDFLAGS) +LIBS := -ldeepspeech -ldeepspeech_utils $(EXTRA_LIBS) +LDFLAGS_DIRS := -L${TFDIR}/bazel-bin/native_client $(EXTRA_LDFLAGS) LDFLAGS += $(LDFLAGS_NEEDED) $(LDFLAGS_RPATH) $(LDFLAGS_DIRS) $(LIBS) AS := $(TOOLCHAIN)as @@ -85,7 +85,7 @@ define copy_missing_libs missing_libs=""; \ for lib in $$SRC_FILE; do \ if [ "$(OS)" = "Darwin" ]; then \ - new_missing="$$( (for f in $$(otool -L $$lib 2>/dev/null | tail -n +2 | awk '{ print $$1 }' | grep -v '$$lib'); do ls -hal $$f; done;) 2>&1 | grep 'No such' | cut -d':' -f2 | xargs basename)"; \ + new_missing="$$( (for f in $$(otool -L $$lib 2>/dev/null | tail -n +2 | awk '{ print $$1 }' | grep -v '$$lib'); do ls -hal $$f; done;) 2>&1 | grep 'No such' | cut -d':' -f2 | xargs basename -a)"; \ missing_libs="$$missing_libs $$new_missing"; \ else \ missing_libs="$$missing_libs $$($(LDD) $$lib | grep 'not found' | awk '{ print $$1 }')"; \ diff --git a/taskcluster/aot-test-model-build.sh b/taskcluster/aot-test-model-build.sh index a692a25b..bfcb6d07 100755 --- a/taskcluster/aot-test-model-build.sh +++ b/taskcluster/aot-test-model-build.sh @@ -7,7 +7,7 @@ source $(dirname "$0")/../tc-tests-utils.sh source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh BAZEL_TARGETS=" -//native_client:deepspeech +//native_client:libdeepspeech.so //native_client:deepspeech_utils ${BAZEL_AOT_TARGETS}" @@ -19,7 +19,7 @@ EXTRA_LOCAL_LDFLAGS="${EXTRA_AOT_LDFLAGS}" EXTRA_LOCAL_LIBS="${EXTRA_AOT_LIBS}" do_get_model_parameters "${DEEPSPEECH_TEST_MODEL}" AOT_MODEL_PARAMS -BAZEL_BUILD_FLAGS="${BAZEL_OPT_FLAGS} ${BAZEL_AOT_BUILD_FLAGS} ${AOT_MODEL_PARAMS}" +BAZEL_BUILD_FLAGS="${BAZEL_OPT_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BAZEL_AOT_BUILD_FLAGS} ${AOT_MODEL_PARAMS}" do_bazel_build diff --git a/taskcluster/cuda-build.sh b/taskcluster/cuda-build.sh index e02ef71e..5b8f062d 100755 --- a/taskcluster/cuda-build.sh +++ b/taskcluster/cuda-build.sh @@ -7,14 +7,13 @@ source $(dirname "$0")/../tc-tests-utils.sh source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh BAZEL_TARGETS=" -//native_client:deepspeech +//native_client:libdeepspeech.so //native_client:deepspeech_utils //native_client:generate_trie -${BAZEL_CTC_TARGETS} " BAZEL_ENV_FLAGS="TF_NEED_CUDA=1 ${TF_CUDA_FLAGS}" -BAZEL_BUILD_FLAGS="${BAZEL_CUDA_FLAGS} ${BAZEL_OPT_FLAGS}" +BAZEL_BUILD_FLAGS="${BAZEL_CUDA_FLAGS} ${BAZEL_OPT_FLAGS} ${BAZEL_EXTRA_FLAGS}" SYSTEM_TARGET=host EXTRA_LOCAL_CFLAGS="" EXTRA_LOCAL_LDFLAGS="-L${DS_ROOT_TASK}/DeepSpeech/CUDA/lib64/ -L${DS_ROOT_TASK}/DeepSpeech/CUDA/lib64/stubs/ -lcudart -lcuda" @@ -26,3 +25,5 @@ do_deepspeech_binary_build do_deepspeech_python_build rename_to_gpu do_deepspeech_nodejs_build rename_to_gpu + +$(dirname "$0")/decoder-build.sh diff --git a/taskcluster/darwin-amd64-cpu-aot_prod-opt.yml b/taskcluster/darwin-amd64-cpu-aot_prod-opt.yml index eadf9088..f4ac652f 100644 --- a/taskcluster/darwin-amd64-cpu-aot_prod-opt.yml +++ b/taskcluster/darwin-amd64-cpu-aot_prod-opt.yml @@ -4,9 +4,8 @@ build: - "index.project.deepspeech.deepspeech.native_client.${event.head.branch}.osx_aot" - "index.project.deepspeech.deepspeech.native_client.${event.head.branch}.${event.head.sha}.osx_aot" - "index.project.deepspeech.deepspeech.native_client.osx_aot.${event.head.sha}" - tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1390dc180e25b5821be80b407ddc5fad73d4ef6a.osx/artifacts/public/home.tar.xz" - summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1390dc180e25b5821be80b407ddc5fad73d4ef6a.osx/artifacts/public/summarize_graph" - libtensorflow_framework: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1390dc180e25b5821be80b407ddc5fad73d4ef6a.osx/artifacts/public/libtensorflow_framework.so" + tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.osx/artifacts/public/home.tar.xz" + summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.osx/artifacts/public/summarize_graph" scripts: build: "taskcluster/host-build.sh --aot" package: "taskcluster/package.sh" diff --git a/taskcluster/darwin-amd64-cpu-opt.yml b/taskcluster/darwin-amd64-cpu-opt.yml index e17529c4..59a267f7 100644 --- a/taskcluster/darwin-amd64-cpu-opt.yml +++ b/taskcluster/darwin-amd64-cpu-opt.yml @@ -6,9 +6,8 @@ build: - "index.project.deepspeech.deepspeech.native_client.osx.${event.head.sha}" - "notify.irc-channel.${notifications.irc}.on-exception" - "notify.irc-channel.${notifications.irc}.on-failed" - tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1390dc180e25b5821be80b407ddc5fad73d4ef6a.osx/artifacts/public/home.tar.xz" - summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1390dc180e25b5821be80b407ddc5fad73d4ef6a.osx/artifacts/public/summarize_graph" - libtensorflow_framework: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1390dc180e25b5821be80b407ddc5fad73d4ef6a.osx/artifacts/public/libtensorflow_framework.so" + tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.osx/artifacts/public/home.tar.xz" + summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.osx/artifacts/public/summarize_graph" scripts: build: "taskcluster/host-build.sh" package: "taskcluster/package.sh" diff --git a/taskcluster/darwin-opt-base.tyml b/taskcluster/darwin-opt-base.tyml index 6c2b490e..2e5f2311 100644 --- a/taskcluster/darwin-opt-base.tyml +++ b/taskcluster/darwin-opt-base.tyml @@ -40,7 +40,6 @@ payload: in: TENSORFLOW_BUILD_ARTIFACT: ${build.tensorflow} SUMMARIZE_GRAPH_BINARY: ${build.summarize_graph} - LIBTENSORFLOW_FRAMEWORK: ${build.libtensorflow_framework} DEEPSPEECH_TEST_MODEL: https://queue.taskcluster.net/v1/task/${training}/artifacts/public/output_graph.pb DEEPSPEECH_PROD_MODEL: https://github.com/lissyx/DeepSpeech/releases/download/0.0.2/tc-fake-prod.988_e120.LSTM.ldc93s1.pb diff --git a/taskcluster/decoder-build.sh b/taskcluster/decoder-build.sh index d177a567..da47cfd3 100755 --- a/taskcluster/decoder-build.sh +++ b/taskcluster/decoder-build.sh @@ -7,7 +7,7 @@ source $(dirname "$0")/../tc-tests-utils.sh source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh BAZEL_ENV_FLAGS="TF_NEED_CUDA=0" -BAZEL_BUILD_FLAGS="${BAZEL_OPT_FLAGS}" +BAZEL_BUILD_FLAGS="${BAZEL_OPT_FLAGS} ${BAZEL_EXTRA_FLAGS}" BAZEL_TARGETS="${BAZEL_CTC_TARGETS}" -do_bazel_build +do_bazel_shared_build diff --git a/taskcluster/host-build.sh b/taskcluster/host-build.sh index 6786e3f5..cae15b50 100755 --- a/taskcluster/host-build.sh +++ b/taskcluster/host-build.sh @@ -7,13 +7,12 @@ source $(dirname "$0")/../tc-tests-utils.sh source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh BAZEL_TARGETS=" -//native_client:deepspeech +//native_client:libdeepspeech.so //native_client:deepspeech_utils //native_client:generate_trie -${BAZEL_CTC_TARGETS} " -BAZEL_BUILD_FLAGS="${BAZEL_OPT_FLAGS}" +BAZEL_BUILD_FLAGS="${BAZEL_OPT_FLAGS} ${BAZEL_EXTRA_FLAGS}" BAZEL_ENV_FLAGS="TF_NEED_CUDA=0" SYSTEM_TARGET=host @@ -35,3 +34,5 @@ do_deepspeech_binary_build do_deepspeech_python_build do_deepspeech_nodejs_build + +$(dirname "$0")/decoder-build.sh diff --git a/taskcluster/linux-amd64-cpu-aot_prod-opt.yml b/taskcluster/linux-amd64-cpu-aot_prod-opt.yml index 73c96856..b3b660f2 100644 --- a/taskcluster/linux-amd64-cpu-aot_prod-opt.yml +++ b/taskcluster/linux-amd64-cpu-aot_prod-opt.yml @@ -4,9 +4,8 @@ build: - "index.project.deepspeech.deepspeech.native_client.${event.head.branch}.cpu_aot" - "index.project.deepspeech.deepspeech.native_client.${event.head.branch}.${event.head.sha}.cpu_aot" - "index.project.deepspeech.deepspeech.native_client.cpu_aot.${event.head.sha}" - tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1390dc180e25b5821be80b407ddc5fad73d4ef6a.cpu/artifacts/public/home.tar.xz" - summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1390dc180e25b5821be80b407ddc5fad73d4ef6a.cpu/artifacts/public/summarize_graph" - libtensorflow_framework: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1390dc180e25b5821be80b407ddc5fad73d4ef6a.cpu/artifacts/public/libtensorflow_framework.so" + tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.cpu/artifacts/public/home.tar.xz" + summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.cpu/artifacts/public/summarize_graph" system_setup: > ${nodejs.packages.prep_6} && apt-get -qq update && apt-get -qq -y install nodejs python-yaml && diff --git a/taskcluster/linux-amd64-cpu-aot_test-opt.yml b/taskcluster/linux-amd64-cpu-aot_test-opt.yml index 436e005e..3265ab20 100644 --- a/taskcluster/linux-amd64-cpu-aot_test-opt.yml +++ b/taskcluster/linux-amd64-cpu-aot_test-opt.yml @@ -6,9 +6,8 @@ build: template_file: linux-opt-base.tyml dependencies: - "test-training-linux-amd64-py27-opt" - tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1390dc180e25b5821be80b407ddc5fad73d4ef6a.cpu/artifacts/public/home.tar.xz" - summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1390dc180e25b5821be80b407ddc5fad73d4ef6a.cpu/artifacts/public/summarize_graph" - libtensorflow_framework: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1390dc180e25b5821be80b407ddc5fad73d4ef6a.cpu/artifacts/public/libtensorflow_framework.so" + tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.cpu/artifacts/public/home.tar.xz" + summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.cpu/artifacts/public/summarize_graph" system_setup: > ${nodejs.packages.prep_6} && apt-get -qq update && apt-get -qq -y install nodejs python-yaml && diff --git a/taskcluster/linux-amd64-cpu-opt.yml b/taskcluster/linux-amd64-cpu-opt.yml index 3e242b02..cd9f449f 100644 --- a/taskcluster/linux-amd64-cpu-opt.yml +++ b/taskcluster/linux-amd64-cpu-opt.yml @@ -13,9 +13,8 @@ build: system_config: > ${swig.patch_nodejs.linux} - tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1390dc180e25b5821be80b407ddc5fad73d4ef6a.cpu/artifacts/public/home.tar.xz" - summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1390dc180e25b5821be80b407ddc5fad73d4ef6a.cpu/artifacts/public/summarize_graph" - libtensorflow_framework: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1390dc180e25b5821be80b407ddc5fad73d4ef6a.cpu/artifacts/public/libtensorflow_framework.so" + tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.cpu/artifacts/public/home.tar.xz" + summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.cpu/artifacts/public/summarize_graph" scripts: build: "taskcluster/host-build.sh" package: "taskcluster/package.sh" diff --git a/taskcluster/linux-amd64-ctc-opt.yml b/taskcluster/linux-amd64-ctc-opt.yml index d0e31247..cd1cfa8b 100644 --- a/taskcluster/linux-amd64-ctc-opt.yml +++ b/taskcluster/linux-amd64-ctc-opt.yml @@ -4,9 +4,8 @@ build: - "pull_request.synchronize" - "pull_request.reopened" template_file: linux-opt-base.tyml - tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1390dc180e25b5821be80b407ddc5fad73d4ef6a.cpu/artifacts/public/home.tar.xz" - summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1390dc180e25b5821be80b407ddc5fad73d4ef6a.cpu/artifacts/public/summarize_graph" - libtensorflow_framework: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1390dc180e25b5821be80b407ddc5fad73d4ef6a.cpu/artifacts/public/libtensorflow_framework.so" + tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.cpu/artifacts/public/home.tar.xz" + summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.cpu/artifacts/public/summarize_graph" scripts: build: 'taskcluster/decoder-build.sh' package: 'taskcluster/decoder-package.sh' diff --git a/taskcluster/linux-amd64-gpu-opt.yml b/taskcluster/linux-amd64-gpu-opt.yml index e748b81f..67e05c07 100644 --- a/taskcluster/linux-amd64-gpu-opt.yml +++ b/taskcluster/linux-amd64-gpu-opt.yml @@ -11,9 +11,8 @@ build: system_config: > ${swig.patch_nodejs.linux} - tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1390dc180e25b5821be80b407ddc5fad73d4ef6a.gpu/artifacts/public/home.tar.xz" - summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1390dc180e25b5821be80b407ddc5fad73d4ef6a.gpu/artifacts/public/summarize_graph" - libtensorflow_framework: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1390dc180e25b5821be80b407ddc5fad73d4ef6a.gpu/artifacts/public/libtensorflow_framework.so" + tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.gpu/artifacts/public/home.tar.xz" + summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.gpu/artifacts/public/summarize_graph" maxRunTime: 14400 scripts: build: "taskcluster/cuda-build.sh" diff --git a/taskcluster/linux-opt-base.tyml b/taskcluster/linux-opt-base.tyml index 7050dffd..db27238a 100644 --- a/taskcluster/linux-opt-base.tyml +++ b/taskcluster/linux-opt-base.tyml @@ -37,7 +37,6 @@ then: in: TENSORFLOW_BUILD_ARTIFACT: ${build.tensorflow} SUMMARIZE_GRAPH_BINARY: ${build.summarize_graph} - LIBTENSORFLOW_FRAMEWORK: ${build.libtensorflow_framework} DEEPSPEECH_TEST_MODEL: https://queue.taskcluster.net/v1/task/${training}/artifacts/public/output_graph.pb DEEPSPEECH_PROD_MODEL: https://github.com/lissyx/DeepSpeech/releases/download/0.0.2/tc-fake-prod.988_e120.LSTM.ldc93s1.pb diff --git a/taskcluster/linux-rpi3-cpu-aot_prod-opt.yml b/taskcluster/linux-rpi3-cpu-aot_prod-opt.yml index e5c6ce40..15b03aa5 100644 --- a/taskcluster/linux-rpi3-cpu-aot_prod-opt.yml +++ b/taskcluster/linux-rpi3-cpu-aot_prod-opt.yml @@ -4,9 +4,8 @@ build: - "index.project.deepspeech.deepspeech.native_client.${event.head.branch}.arm_aot" - "index.project.deepspeech.deepspeech.native_client.${event.head.branch}.${event.head.sha}.arm_aot" - "index.project.deepspeech.deepspeech.native_client.arm_aot.${event.head.sha}" - tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1390dc180e25b5821be80b407ddc5fad73d4ef6a.arm/artifacts/public/home.tar.xz" - summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1390dc180e25b5821be80b407ddc5fad73d4ef6a.cpu/artifacts/public/summarize_graph" - libtensorflow_framework: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1390dc180e25b5821be80b407ddc5fad73d4ef6a.cpu/artifacts/public/libtensorflow_framework.so" + tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.arm/artifacts/public/home.tar.xz" + summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.cpu/artifacts/public/summarize_graph" ## multistrap 2.2.0-ubuntu1 is broken in 14.04: https://bugs.launchpad.net/ubuntu/+source/multistrap/+bug/1313787 system_setup: > diff --git a/taskcluster/linux-rpi3-cpu-opt.yml b/taskcluster/linux-rpi3-cpu-opt.yml index b25af837..cd22351e 100644 --- a/taskcluster/linux-rpi3-cpu-opt.yml +++ b/taskcluster/linux-rpi3-cpu-opt.yml @@ -4,9 +4,8 @@ build: - "index.project.deepspeech.deepspeech.native_client.${event.head.branch}.arm" - "index.project.deepspeech.deepspeech.native_client.${event.head.branch}.${event.head.sha}.arm" - "index.project.deepspeech.deepspeech.native_client.arm.${event.head.sha}" - tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1390dc180e25b5821be80b407ddc5fad73d4ef6a.arm/artifacts/public/home.tar.xz" - summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1390dc180e25b5821be80b407ddc5fad73d4ef6a.cpu/artifacts/public/summarize_graph" - libtensorflow_framework: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1390dc180e25b5821be80b407ddc5fad73d4ef6a.cpu/artifacts/public/libtensorflow_framework.so" + tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.arm/artifacts/public/home.tar.xz" + summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.cpu/artifacts/public/summarize_graph" ## multistrap 2.2.0-ubuntu1 is broken in 14.04: https://bugs.launchpad.net/ubuntu/+source/multistrap/+bug/1313787 system_setup: > diff --git a/taskcluster/node-package.yml b/taskcluster/node-package.yml index cb431843..35e43260 100644 --- a/taskcluster/node-package.yml +++ b/taskcluster/node-package.yml @@ -14,9 +14,8 @@ build: system_config: > ${swig.patch_nodejs.linux} - tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1390dc180e25b5821be80b407ddc5fad73d4ef6a.cpu/artifacts/public/home.tar.xz" - summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1390dc180e25b5821be80b407ddc5fad73d4ef6a.cpu/artifacts/public/summarize_graph" - libtensorflow_framework: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1390dc180e25b5821be80b407ddc5fad73d4ef6a.cpu/artifacts/public/libtensorflow_framework.so" + tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.cpu/artifacts/public/home.tar.xz" + summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.cpu/artifacts/public/summarize_graph" scripts: build: "taskcluster/node-build.sh" package: "taskcluster/node-package.sh" diff --git a/taskcluster/rpi3-build.sh b/taskcluster/rpi3-build.sh index 7a4cf695..1d3e0e1d 100755 --- a/taskcluster/rpi3-build.sh +++ b/taskcluster/rpi3-build.sh @@ -7,13 +7,12 @@ source $(dirname "$0")/../tc-tests-utils.sh source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh BAZEL_TARGETS=" -//native_client:deepspeech +//native_client:libdeepspeech.so //native_client:deepspeech_utils //native_client:generate_trie -${BAZEL_CTC_TARGETS} " -BAZEL_BUILD_FLAGS="${BAZEL_ARM_FLAGS}" +BAZEL_BUILD_FLAGS="${BAZEL_ARM_FLAGS} ${BAZEL_EXTRA_FLAGS}" BAZEL_ENV_FLAGS="TF_NEED_CUDA=0" SYSTEM_TARGET=rpi3 diff --git a/taskcluster/test-training+deepspeech-linux-amd64-py27-opt.yml b/taskcluster/test-training+deepspeech-linux-amd64-py27-opt.yml index 0c94302a..e21a629a 100644 --- a/taskcluster/test-training+deepspeech-linux-amd64-py27-opt.yml +++ b/taskcluster/test-training+deepspeech-linux-amd64-py27-opt.yml @@ -8,7 +8,7 @@ build: apt-get -qq -y install ${python.packages.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-train-tests.sh 2.7.13 mozilla deepspeech" - python_wheel: 'https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1390dc180e25b5821be80b407ddc5fad73d4ef6a.cpu/artifacts/public/tensorflow_warpctc-1.4.0-cp27-cp27mu-linux_x86_64.whl' + python_wheel: 'https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.cpu/artifacts/public/tensorflow_warpctc-1.4.0-cp27-cp27mu-linux_x86_64.whl' deepspeech_pkg_name: 'deepspeech-0.1.1-cp27-cp27mu-manylinux1_x86_64.whl' metadata: name: "DeepSpeech Linux AMD64 CPU training Py2.7 (DS)" diff --git a/taskcluster/test-training+deepspeech-linux-amd64-py36-opt.yml b/taskcluster/test-training+deepspeech-linux-amd64-py36-opt.yml index dfe318a3..317665c7 100644 --- a/taskcluster/test-training+deepspeech-linux-amd64-py36-opt.yml +++ b/taskcluster/test-training+deepspeech-linux-amd64-py36-opt.yml @@ -8,7 +8,7 @@ build: apt-get -qq -y install ${python.packages.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-train-tests.sh 3.6.2 mozilla deepspeech" - python_wheel: 'https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1390dc180e25b5821be80b407ddc5fad73d4ef6a.cpu-py36/artifacts/public/tensorflow_warpctc-1.4.0-cp36-cp36m-linux_x86_64.whl' + python_wheel: 'https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.cpu-py36/artifacts/public/tensorflow_warpctc-1.4.0-cp36-cp36m-linux_x86_64.whl' deepspeech_pkg_name: 'deepspeech-0.1.1-cp36-cp36m-manylinux1_x86_64.whl' metadata: name: "DeepSpeech Linux AMD64 CPU training Py3.6 (DS)" diff --git a/taskcluster/test-training-linux-amd64-py27-opt.yml b/taskcluster/test-training-linux-amd64-py27-opt.yml index a0b87d5e..8f056c4c 100644 --- a/taskcluster/test-training-linux-amd64-py27-opt.yml +++ b/taskcluster/test-training-linux-amd64-py27-opt.yml @@ -7,7 +7,7 @@ build: apt-get -qq -y install ${python.packages.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-train-tests.sh 2.7.13 mozilla" - python_wheel: 'https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1390dc180e25b5821be80b407ddc5fad73d4ef6a.cpu/artifacts/public/tensorflow_warpctc-1.4.0-cp27-cp27mu-linux_x86_64.whl' + python_wheel: 'https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.cpu/artifacts/public/tensorflow_warpctc-1.4.0-cp27-cp27mu-linux_x86_64.whl' metadata: name: "DeepSpeech Linux AMD64 CPU training Py2.7" description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 Python 2.7, CPU only, optimized version" diff --git a/taskcluster/test-training-linux-amd64-py34-opt.yml b/taskcluster/test-training-linux-amd64-py34-opt.yml index b223f9db..65735907 100644 --- a/taskcluster/test-training-linux-amd64-py34-opt.yml +++ b/taskcluster/test-training-linux-amd64-py34-opt.yml @@ -7,7 +7,7 @@ build: apt-get -qq -y install ${python.packages.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-train-tests.sh 3.4.6 mozilla" - python_wheel: 'https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1390dc180e25b5821be80b407ddc5fad73d4ef6a.cpu-py34/artifacts/public/tensorflow_warpctc-1.4.0-cp34-cp34m-linux_x86_64.whl' + python_wheel: 'https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.cpu-py34/artifacts/public/tensorflow_warpctc-1.4.0-cp34-cp34m-linux_x86_64.whl' metadata: name: "DeepSpeech Linux AMD64 CPU training Py3.4" description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 Python 3.4, CPU only, optimized version" diff --git a/taskcluster/test-training-linux-amd64-py35-opt.yml b/taskcluster/test-training-linux-amd64-py35-opt.yml index 3a3f60a1..eb015923 100644 --- a/taskcluster/test-training-linux-amd64-py35-opt.yml +++ b/taskcluster/test-training-linux-amd64-py35-opt.yml @@ -7,7 +7,7 @@ build: apt-get -qq -y install ${python.packages.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-train-tests.sh 3.5.3 mozilla" - python_wheel: 'https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1390dc180e25b5821be80b407ddc5fad73d4ef6a.cpu-py35/artifacts/public/tensorflow_warpctc-1.4.0-cp35-cp35m-linux_x86_64.whl' + python_wheel: 'https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.cpu-py35/artifacts/public/tensorflow_warpctc-1.4.0-cp35-cp35m-linux_x86_64.whl' metadata: name: "DeepSpeech Linux AMD64 CPU training Py3.5" description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 Python 3.5, CPU only, optimized version" diff --git a/taskcluster/test-training-linux-amd64-py36-opt.yml b/taskcluster/test-training-linux-amd64-py36-opt.yml index 6df729e9..5ec7f79c 100644 --- a/taskcluster/test-training-linux-amd64-py36-opt.yml +++ b/taskcluster/test-training-linux-amd64-py36-opt.yml @@ -7,7 +7,7 @@ build: apt-get -qq -y install ${python.packages.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-train-tests.sh 3.6.2 mozilla" - python_wheel: 'https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1390dc180e25b5821be80b407ddc5fad73d4ef6a.cpu-py36/artifacts/public/tensorflow_warpctc-1.4.0-cp36-cp36m-linux_x86_64.whl' + python_wheel: 'https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.cpu-py36/artifacts/public/tensorflow_warpctc-1.4.0-cp36-cp36m-linux_x86_64.whl' metadata: name: "DeepSpeech Linux AMD64 CPU training Py3.6" description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 Python 3.6, CPU only, optimized version" diff --git a/tc-tests-utils.sh b/tc-tests-utils.sh index 67c9627d..57237b94 100755 --- a/tc-tests-utils.sh +++ b/tc-tests-utils.sh @@ -29,20 +29,15 @@ export DS_DSDIR=${DS_ROOT_TASK}/DeepSpeech/ds export BAZEL_CTC_TARGETS="//native_client:libctc_decoder_with_kenlm.so" export EXTRA_AOT_CFLAGS="" -export EXTRA_AOT_LDFLAGS="-L${DS_TFDIR}/bazel-bin/tensorflow/compiler/xla -L${DS_TFDIR}/bazel-bin/tensorflow/compiler/tf2xla -L${DS_TFDIR}/bazel-bin/tensorflow/compiler/aot -L${DS_TFDIR}/bazel-bin/tensorflow/compiler/xla/service/cpu" -export EXTRA_AOT_LIBS="-ldeepspeech_model -lxla_compiled_cpu_function -lruntime -lruntime_matmul -lruntime_matvec -lexecutable_run_options" +export EXTRA_AOT_LDFLAGS="" +export EXTRA_AOT_LIBS="-ldeepspeech_model" # FIXME: # Previously, with r1.3, we could use timesteps of 64 # With r1.4 it seems to eat too much resources at tfcompile step export BAZEL_AOT_BUILD_FLAGS="--define=DS_NATIVE_MODEL=1 --define=DS_MODEL_TIMESTEPS=16" export BAZEL_AOT_TARGETS=" -//native_client:deepspeech_model -//tensorflow/compiler/aot:runtime -//tensorflow/compiler/xla/service/cpu:runtime_matmul -//tensorflow/compiler/xla/service/cpu:runtime_matvec -//tensorflow/compiler/xla:executable_run_options -//tensorflow/compiler/tf2xla:xla_compiled_cpu_function +//native_client:libdeepspeech_model.so " model_source=${DEEPSPEECH_TEST_MODEL} @@ -221,7 +216,6 @@ do_get_model_parameters() wget "${model_url}" -O "${model_file}" wget "${SUMMARIZE_GRAPH_BINARY}" -O "/tmp/summarize_graph" - wget "${LIBTENSORFLOW_FRAMEWORK}" -O "/tmp/libtensorflow_framework.so" chmod +x /tmp/summarize_graph @@ -236,6 +230,14 @@ do_get_model_parameters() } do_bazel_build() +{ + cd ${DS_ROOT_TASK}/DeepSpeech/tf + eval "export ${BAZEL_ENV_FLAGS}" + PATH=${DS_ROOT_TASK}/bin/:$PATH bazel ${BAZEL_OUTPUT_USER_ROOT} build \ + --config=monolithic -c opt ${BAZEL_BUILD_FLAGS} ${BAZEL_TARGETS} +} + +do_bazel_shared_build() { cd ${DS_ROOT_TASK}/DeepSpeech/tf eval "export ${BAZEL_ENV_FLAGS}" @@ -361,13 +363,6 @@ package_native_client() if [ -f "${tensorflow_dir}/bazel-bin/native_client/libdeepspeech_model.so" ]; then tar -cf - \ - -C ${tensorflow_dir}/bazel-bin/tensorflow/ libtensorflow_cc.so \ - -C ${tensorflow_dir}/bazel-bin/tensorflow/ libtensorflow_framework.so \ - -C ${tensorflow_dir}/bazel-bin/tensorflow/compiler/aot/ libruntime.so \ - -C ${tensorflow_dir}/bazel-bin/tensorflow/compiler/xla/service/cpu/ libruntime_matmul.so \ - -C ${tensorflow_dir}/bazel-bin/tensorflow/compiler/xla/service/cpu/ libruntime_matvec.so \ - -C ${tensorflow_dir}/bazel-bin/tensorflow/compiler/xla/ libexecutable_run_options.so \ - -C ${tensorflow_dir}/bazel-bin/tensorflow/compiler/tf2xla/ libxla_compiled_cpu_function.so \ -C ${tensorflow_dir}/bazel-bin/native_client/ generate_trie \ -C ${tensorflow_dir}/bazel-bin/native_client/ libctc_decoder_with_kenlm.so \ -C ${tensorflow_dir}/bazel-bin/native_client/ libdeepspeech.so \ @@ -379,8 +374,6 @@ package_native_client() | pixz -9 > "${artifacts_dir}/${artifact_name}" else tar -cf - \ - -C ${tensorflow_dir}/bazel-bin/tensorflow/ libtensorflow_cc.so \ - -C ${tensorflow_dir}/bazel-bin/tensorflow/ libtensorflow_framework.so \ -C ${tensorflow_dir}/bazel-bin/native_client/ generate_trie \ -C ${tensorflow_dir}/bazel-bin/native_client/ libctc_decoder_with_kenlm.so \ -C ${tensorflow_dir}/bazel-bin/native_client/ libdeepspeech.so \ diff --git a/tc-train-tests.sh b/tc-train-tests.sh index 5ea82248..e5b89690 100644 --- a/tc-train-tests.sh +++ b/tc-train-tests.sh @@ -42,9 +42,15 @@ fi; if [ "${ds}" = "deepspeech" ]; then pip install "${DEEPSPEECH_PYTHON_PACKAGE}" | cat python -c "import tensorflow; from deepspeech.utils import audioToInputVector" -fi; -download_ctc_kenlm "/tmp/ds" + # Since this build depends on the completion of the whole deepspeech package + # and we might get into funny situation with --config=monolithic, then let's + # be extra-cautious and leverage our dependency against the build to also + # test with libctc_decoder_with_kenlm.so that is packaged for release + download_native_client_files "/tmp/ds" +else + download_ctc_kenlm "/tmp/ds" +fi; pushd ${HOME}/DeepSpeech/ds/ time ./bin/run-tc-ldc93s1.sh