diff --git a/taskcluster/scriptworker-task-github.yml b/taskcluster/scriptworker-task-github.yml index 9729732c..a5a5626c 100644 --- a/taskcluster/scriptworker-task-github.yml +++ b/taskcluster/scriptworker-task-github.yml @@ -51,11 +51,13 @@ build: - "android-armv7-cpu-opt" - "win-amd64-cpu-opt" - "win-amd64-gpu-opt" + - "win-amd64-tflite-opt" java_aar: - "android-java-opt" nuget: - "win-amd64-cpu-opt" - "win-amd64-gpu-opt" + - "win-amd64-tflite-opt" metadata: name: "DeepSpeech GitHub Packages" description: "Trigger Uploading of DeepSpeech Packages to GitHub release page" diff --git a/taskcluster/scriptworker-task-nuget.yml b/taskcluster/scriptworker-task-nuget.yml index 40a29750..88454b1c 100644 --- a/taskcluster/scriptworker-task-nuget.yml +++ b/taskcluster/scriptworker-task-nuget.yml @@ -4,6 +4,7 @@ build: # Make sure builds are ready - "win-amd64-cpu-opt" - "win-amd64-gpu-opt" + - "win-amd64-tflite-opt" allowed: - "tag" ref_match: "refs/tags/" @@ -20,6 +21,7 @@ build: nuget: - "win-amd64-cpu-opt" - "win-amd64-gpu-opt" + - "win-amd64-tflite-opt" metadata: name: "DeepSpeech NuGet Packages" description: "Trigger Uploading of DeepSpeech .Net Framework bindings to NuGet" diff --git a/taskcluster/tc-netframework-ds-tests.sh b/taskcluster/tc-netframework-ds-tests.sh index 3e48bf25..552a4204 100644 --- a/taskcluster/tc-netframework-ds-tests.sh +++ b/taskcluster/tc-netframework-ds-tests.sh @@ -2,15 +2,21 @@ set -xe -cuda=$1 +bitrate=$1 +package_option=$2 source $(dirname "$0")/tc-tests-utils.sh bitrate=$1 set_ldc_sample_filename "${bitrate}" -if [ "${cuda}" = "--cuda" ]; then +if [ "${package_option}" = "--cuda" ]; then PROJECT_NAME="DeepSpeech-GPU" +elif [ "${package_option}" = "--tflite" ]; then + PROJECT_NAME="DeepSpeech-TFLite" + model_source=${DEEPSPEECH_TEST_MODEL//.pb/.tflite} + model_name=$(basename "${model_source}") + model_name_mmap=$(basename "${model_source}") else PROJECT_NAME="DeepSpeech" fi diff --git a/taskcluster/tc-tests-utils.sh b/taskcluster/tc-tests-utils.sh index e2cf5f17..2fef2f6b 100755 --- a/taskcluster/tc-tests-utils.sh +++ b/taskcluster/tc-tests-utils.sh @@ -614,8 +614,11 @@ install_nuget() mkdir -p "${TASKCLUSTER_TMP_DIR}/repo/" mkdir -p "${TASKCLUSTER_TMP_DIR}/ds/" - ${WGET} -O - "${DEEPSPEECH_ARTIFACTS_ROOT}/${nuget}" | gunzip > "${TASKCLUSTER_TMP_DIR}/${PROJECT_NAME}.${DS_VERSION}.nupkg" - ${WGET} -O - "${DEEPSPEECH_ARTIFACTS_ROOT}/DeepSpeechConsole.exe" | gunzip > "${TASKCLUSTER_TMP_DIR}/ds/DeepSpeechConsole.exe" + nuget_pkg_url=$(get_dep_nuget_pkg_url "${nuget}") + console_pkg_url=$(get_dep_nuget_pkg_url "DeepSpeechConsole.exe") + + ${WGET} -O - "${nuget_pkg_url}" | gunzip > "${TASKCLUSTER_TMP_DIR}/${PROJECT_NAME}.${DS_VERSION}.nupkg" + ${WGET} -O - "${console_pkg_url}" | gunzip > "${TASKCLUSTER_TMP_DIR}/ds/DeepSpeechConsole.exe" nuget sources add -Name repo -Source $(cygpath -w "${TASKCLUSTER_TMP_DIR}/repo/") @@ -1207,6 +1210,25 @@ get_dep_npm_pkg_url() exit 1 } +# Will inspect this task's dependencies for one that provides a matching NuGet package +get_dep_nuget_pkg_url() +{ + local deepspeech_pkg=$1 + local all_deps="$(curl -s https://community-tc.services.mozilla.com/api/queue/v1/task/${TASK_ID} | python -c 'import json; import sys; print(" ".join(json.loads(sys.stdin.read())["dependencies"]));')" + + for dep in ${all_deps}; do + local has_artifact=$(curl -s https://community-tc.services.mozilla.com/api/queue/v1/task/${dep}/artifacts | python -c 'import json; import sys; has_artifact = True in [ e["name"].find("'${deepspeech_pkg}'") > 0 for e in json.loads(sys.stdin.read())["artifacts"] ]; print(has_artifact)') + if [ "${has_artifact}" = "True" ]; then + echo "https://community-tc.services.mozilla.com/api/queue/v1/task/${dep}/artifacts/public/${deepspeech_pkg}" + exit 0 + fi; + done; + + echo "" + # This should not be reached, otherwise it means we could not find a matching nodejs package + exit 1 +} + extract_python_versions() { # call extract_python_versions ${pyver_full} pyver pyver_pkg py_unicode_type pyconf pyalias diff --git a/taskcluster/test-netframework-win-tflite-opt.yml b/taskcluster/test-netframework-win-tflite-opt.yml new file mode 100644 index 00000000..327bf685 --- /dev/null +++ b/taskcluster/test-netframework-win-tflite-opt.yml @@ -0,0 +1,11 @@ +build: + template_file: test-win-opt-base.tyml + dependencies: + - "win-amd64-tflite-opt" + - "test-training_16k-linux-amd64-py36m-opt" + test_model_task: "test-training_16k-linux-amd64-py36m-opt" + args: + tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-netframework-ds-tests.sh 16k --tflite" + metadata: + name: "DeepSpeech Windows AMD64 TFLite .Net Framework tests" + description: "Testing DeepSpeech .Net Framework for Windows/AMD64, TFLite, optimized version" diff --git a/taskcluster/win-amd64-tflite-opt.yml b/taskcluster/win-amd64-tflite-opt.yml index 3496e344..feb302e4 100644 --- a/taskcluster/win-amd64-tflite-opt.yml +++ b/taskcluster/win-amd64-tflite-opt.yml @@ -8,7 +8,7 @@ build: - "notify.irc-channel.${notifications.irc}.on-failed" tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.win/artifacts/public/home.tar.xz" scripts: - build: "taskcluster/win-build.sh tflite" + build: "taskcluster/win-build.sh --tflite" package: "taskcluster/win-package.sh" nc_asset_name: "native_client.amd64.tflite.win.tar.xz" maxRunTime: 14400 diff --git a/taskcluster/win-build.sh b/taskcluster/win-build.sh index 51738f45..e3a4133d 100755 --- a/taskcluster/win-build.sh +++ b/taskcluster/win-build.sh @@ -2,8 +2,7 @@ set -xe -cuda=$1 -runtime=$1 +package_option=$1 source $(dirname "$0")/tc-tests-utils.sh @@ -14,16 +13,17 @@ BAZEL_TARGETS=" //native_client:generate_trie " -if [ "${cuda}" = "--cuda" ]; then +if [ "${package_option}" = "--cuda" ]; then BAZEL_ENV_FLAGS="TF_NEED_CUDA=1 ${TF_CUDA_FLAGS}" BAZEL_BUILD_FLAGS="${BAZEL_CUDA_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BAZEL_OPT_FLAGS}" PROJECT_NAME="DeepSpeech-GPU" +elif [ "${package_option}" = "--tflite" ]; then + PROJECT_NAME="DeepSpeech-TFLite" + BAZEL_BUILD_FLAGS="--define=runtime=tflite ${BAZEL_OPT_FLAGS} ${BAZEL_EXTRA_FLAGS}" + BAZEL_ENV_FLAGS="TF_NEED_CUDA=0" else PROJECT_NAME="DeepSpeech" - if [ "${runtime}" = "tflite" ]; then - BAZEL_BUILD_TFLITE="--define=runtime=tflite" - fi; - BAZEL_BUILD_FLAGS="${BAZEL_BUILD_TFLITE} ${BAZEL_OPT_FLAGS} ${BAZEL_EXTRA_FLAGS}" + BAZEL_BUILD_FLAGS="${BAZEL_OPT_FLAGS} ${BAZEL_EXTRA_FLAGS}" BAZEL_ENV_FLAGS="TF_NEED_CUDA=0" fi @@ -31,7 +31,7 @@ SYSTEM_TARGET=host-win do_bazel_build -if [ "${cuda}" = "--cuda" ]; then +if [ "${package_option}" = "--cuda" ]; then cp ${DS_ROOT_TASK}/DeepSpeech/tf/bazel-bin/native_client/liblibdeepspeech.so.ifso ${DS_ROOT_TASK}/DeepSpeech/tf/bazel-bin/native_client/libdeepspeech.so.if.lib fi @@ -44,10 +44,10 @@ export SUPPORTED_PYTHON_VERSIONS="3.5.4 3.6.8 3.7.6 3.8.1" if [ "${runtime}" = "tflite" ]; then do_deepspeech_python_build "--tflite" else - do_deepspeech_python_build "${cuda}" + do_deepspeech_python_build "${package_option}" fi -do_deepspeech_nodejs_build "${cuda}" +do_deepspeech_nodejs_build "${package_option}" do_deepspeech_netframework_build