From c5ded5adfe086d6d3e55020582255020e305b10e Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Mon, 24 Feb 2020 13:36:58 +0100 Subject: [PATCH] Cache gradle deps and Android emulator setup --- native_client/java/Makefile | 9 ++-- native_client/java/build.gradle | 1 - taskcluster/.build.yml | 9 ++++ taskcluster/.shared.yml | 32 +++++++++++ taskcluster/android-apk-build.sh | 2 - .../android-cache-arm64-v8a-android-24.yml | 14 +++++ .../android-cache-arm64-v8a-android-25.yml | 14 +++++ .../android-cache-armeabi-v7a-android-24.yml | 14 +++++ .../android-cache-armeabi-v7a-android-25.yml | 14 +++++ taskcluster/android-cache-sdk-android-27.yml | 14 +++++ .../android-cache-x86_64-android-24.yml | 14 +++++ .../android-cache-x86_64-android-25.yml | 14 +++++ taskcluster/android-java-opt.yml | 13 +++-- taskcluster/android_cache-build.sh | 21 ++++++++ taskcluster/android_cache-opt-base.tyml | 52 ++++++++++++++++++ taskcluster/android_cache-package.sh | 18 +++++++ taskcluster/gradle-build.sh | 27 ++++++++++ taskcluster/gradle-cache.yml | 14 +++++ taskcluster/gradle-package.sh | 18 +++++++ taskcluster/linux-opt-base.tyml | 2 +- taskcluster/tc-all-vars.sh | 1 + taskcluster/tc-android-apk-tests.sh | 6 +-- taskcluster/tc-android-ds-tests.sh | 2 +- taskcluster/tc-android-utils.sh | 54 +++++++++++-------- taskcluster/test-android-opt-base.tyml | 2 +- .../test-apk-android-24-x86_64-opt.yml | 8 +++ .../test-apk-android-25-x86_64-opt.yml | 8 +++ taskcluster/test-cpp-android-24-arm64-opt.yml | 4 ++ taskcluster/test-cpp-android-24-armv7-opt.yml | 4 ++ taskcluster/test-cpp-android-25-arm64-opt.yml | 4 ++ taskcluster/test-cpp-android-25-armv7-opt.yml | 4 ++ 31 files changed, 372 insertions(+), 41 deletions(-) create mode 100644 taskcluster/android-cache-arm64-v8a-android-24.yml create mode 100644 taskcluster/android-cache-arm64-v8a-android-25.yml create mode 100644 taskcluster/android-cache-armeabi-v7a-android-24.yml create mode 100644 taskcluster/android-cache-armeabi-v7a-android-25.yml create mode 100644 taskcluster/android-cache-sdk-android-27.yml create mode 100644 taskcluster/android-cache-x86_64-android-24.yml create mode 100644 taskcluster/android-cache-x86_64-android-25.yml create mode 100755 taskcluster/android_cache-build.sh create mode 100644 taskcluster/android_cache-opt-base.tyml create mode 100755 taskcluster/android_cache-package.sh create mode 100755 taskcluster/gradle-build.sh create mode 100644 taskcluster/gradle-cache.yml create mode 100755 taskcluster/gradle-package.sh diff --git a/native_client/java/Makefile b/native_client/java/Makefile index 3dda412d..34426e0b 100644 --- a/native_client/java/Makefile +++ b/native_client/java/Makefile @@ -3,6 +3,7 @@ include ../definitions.mk ARCHS := $(shell grep 'ABI_FILTERS' libdeepspeech/gradle.properties | cut -d'=' -f2 | sed -e 's/;/ /g') +GRADLE ?= ./gradlew all: apk @@ -10,7 +11,7 @@ clean: apk-clean rm -rf *.java jni/deepspeech_wrap.cpp apk-clean: - ./gradlew clean + $(GRADLE) clean libs-clean: rm -fr libdeepspeech/libs/*/libdeepspeech.so @@ -20,11 +21,11 @@ libdeepspeech/libs/%/libdeepspeech.so: cp ${TFDIR}/bazel-out/$*-*/bin/native_client/libdeepspeech.so libdeepspeech/libs/$*/ apk: apk-clean bindings $(patsubst %,libdeepspeech/libs/%/libdeepspeech.so,$(ARCHS)) - ./gradlew build + $(GRADLE) build maven-bundle: apk - ./gradlew uploadArchives - ./gradlew zipMavenArtifacts + $(GRADLE) uploadArchives + $(GRADLE) zipMavenArtifacts bindings: clean swig -c++ -java -package org.mozilla.deepspeech.libdeepspeech -outdir libdeepspeech/src/main/java/org/mozilla/deepspeech/libdeepspeech/ -o jni/deepspeech_wrap.cpp jni/deepspeech.i diff --git a/native_client/java/build.gradle b/native_client/java/build.gradle index df7d1d27..be1f2abc 100644 --- a/native_client/java/build.gradle +++ b/native_client/java/build.gradle @@ -9,7 +9,6 @@ buildscript { dependencies { classpath 'com.android.tools.build:gradle:3.2.1' - // NOTE: Do not place your application dependencies here; they belong // in the individual module build.gradle files } diff --git a/taskcluster/.build.yml b/taskcluster/.build.yml index c1d4bf4c..7e9809e5 100644 --- a/taskcluster/.build.yml +++ b/taskcluster/.build.yml @@ -27,3 +27,12 @@ build: homebrew: url: '' namespace: '' + cache: + url: '' + namespace: '' + android_cache: + url: '' + namespace: '' + gradle_cache: + url: '' + namespace: '' diff --git a/taskcluster/.shared.yml b/taskcluster/.shared.yml index ce5445bf..e762ebd8 100644 --- a/taskcluster/.shared.yml +++ b/taskcluster/.shared.yml @@ -8,6 +8,9 @@ python: tensorflow: packages_trusty: apt: 'make build-essential gfortran git libblas-dev liblapack-dev libsox-dev libmagic-dev libgsm1-dev libltdl-dev libpng-dev python zlib1g-dev' +java: + packages_trusty: + apt: 'apt-get -qq -y install curl software-properties-common wget unzip && add-apt-repository --yes ppa:openjdk-r/ppa && apt-get -qq update && DEBIAN_FRONTEND=noninteractive apt-get -qq -y --force-yes install openjdk-8-jdk && java -version && update-ca-certificates -f' electronjs: packages_xenial: apt: 'libatk1.0-0 libatk-bridge2.0-0 libcairo2 libcups2 libdbus-1-3 libgdk-pixbuf2.0-0 libgtk-3-0 libnspr4 libnss3 libpango-1.0-0 libpangocairo-1.0-0 libx11-xcb1 libxcomposite1 libxcursor1 libxdamage1 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 xvfb' @@ -55,6 +58,35 @@ system: homebrew_tests: url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.homebrew_tests.4/artifacts/public/homebrew_tests.tar.gz' namespace: 'project.deepspeech.homebrew_tests.4' + android_cache: + arm64_v8a: + android_24: + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.android_cache.arm64-v8a.android-24.4/artifacts/public/android_cache.tar.gz' + namespace: 'project.deepspeech.android_cache.arm64-v8a.android-24.4' + android_25: + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.android_cache.arm64-v8a.android-25.4/artifacts/public/android_cache.tar.gz' + namespace: 'project.deepspeech.android_cache.arm64-v8a.android-25.4' + armeabi_v7a: + android_24: + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.android_cache.armeabi-v7a.android-24.4/artifacts/public/android_cache.tar.gz' + namespace: 'project.deepspeech.android_cache.armeabi-v7a.android-24.4' + android_25: + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.android_cache.armeabi-v7a.android-25.4/artifacts/public/android_cache.tar.gz' + namespace: 'project.deepspeech.android_cache.armeabi-v7a.android-25.4' + x86_64: + android_24: + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.android_cache.x86_64.android-24.4/artifacts/public/android_cache.tar.gz' + namespace: 'project.deepspeech.android_cache.x86_64.android-24.4' + android_25: + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.android_cache.x86_64.android-25.4/artifacts/public/android_cache.tar.gz' + namespace: 'project.deepspeech.android_cache.x86_64.android-25.4' + sdk: + android_27: + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.android_cache.sdk.android-27.4/artifacts/public/android_cache.tar.gz' + namespace: 'project.deepspeech.android_cache.sdk.android-27.4' + gradle_cache: + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.gradle.4/artifacts/public/gradle.tar.gz' + namespace: 'project.deepspeech.gradle.4' pyenv: linux: url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.pyenv.linux.5/artifacts/public/pyenv.tar.gz' diff --git a/taskcluster/android-apk-build.sh b/taskcluster/android-apk-build.sh index 2cbbee14..9259995e 100755 --- a/taskcluster/android-apk-build.sh +++ b/taskcluster/android-apk-build.sh @@ -8,6 +8,4 @@ source $(dirname "$0")/tc-tests-utils.sh source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh -android_install_sdk_platform - do_deepspeech_java_apk_build diff --git a/taskcluster/android-cache-arm64-v8a-android-24.yml b/taskcluster/android-cache-arm64-v8a-android-24.yml new file mode 100644 index 00000000..a8fef42f --- /dev/null +++ b/taskcluster/android-cache-arm64-v8a-android-24.yml @@ -0,0 +1,14 @@ +build: + template_file: android_cache-opt-base.tyml + system_setup: + > + ${java.packages_trusty.apt} + cache: + url: ${system.android_cache.arm64_v8a.android_24.url} + namespace: ${system.android_cache.arm64_v8a.android_24.namespace} + scripts: + build: "taskcluster/android_cache-build.sh arm64-v8a android-24" + package: "taskcluster/android_cache-package.sh" + metadata: + name: "Builds Android cache arm64-v8a / android-24" + description: "Setup an Android SDK / emulator cache for Android arm64-v8a / android-24" diff --git a/taskcluster/android-cache-arm64-v8a-android-25.yml b/taskcluster/android-cache-arm64-v8a-android-25.yml new file mode 100644 index 00000000..7dca05ea --- /dev/null +++ b/taskcluster/android-cache-arm64-v8a-android-25.yml @@ -0,0 +1,14 @@ +build: + template_file: android_cache-opt-base.tyml + system_setup: + > + ${java.packages_trusty.apt} + cache: + url: ${system.android_cache.arm64_v8a.android_25.url} + namespace: ${system.android_cache.arm64_v8a.android_25.namespace} + scripts: + build: "taskcluster/android_cache-build.sh arm64-v8a android-25" + package: "taskcluster/android_cache-package.sh" + metadata: + name: "Builds Android cache arm64-v8a / android-25" + description: "Setup an Android SDK / emulator cache for Android arm64-v8a / android-25" diff --git a/taskcluster/android-cache-armeabi-v7a-android-24.yml b/taskcluster/android-cache-armeabi-v7a-android-24.yml new file mode 100644 index 00000000..605f0e92 --- /dev/null +++ b/taskcluster/android-cache-armeabi-v7a-android-24.yml @@ -0,0 +1,14 @@ +build: + template_file: android_cache-opt-base.tyml + system_setup: + > + ${java.packages_trusty.apt} + cache: + url: ${system.android_cache.armeabi_v7a.android_24.url} + namespace: ${system.android_cache.armeabi_v7a.android_24.namespace} + scripts: + build: "taskcluster/android_cache-build.sh armeabi-v7a android-24" + package: "taskcluster/android_cache-package.sh" + metadata: + name: "Builds Android cache armeabi-v7a / android-24" + description: "Setup an Android SDK / emulator cache for Android armeabi-v7a / android-24" diff --git a/taskcluster/android-cache-armeabi-v7a-android-25.yml b/taskcluster/android-cache-armeabi-v7a-android-25.yml new file mode 100644 index 00000000..6089d940 --- /dev/null +++ b/taskcluster/android-cache-armeabi-v7a-android-25.yml @@ -0,0 +1,14 @@ +build: + template_file: android_cache-opt-base.tyml + system_setup: + > + ${java.packages_trusty.apt} + cache: + url: ${system.android_cache.armeabi_v7a.android_25.url} + namespace: ${system.android_cache.armeabi_v7a.android_25.namespace} + scripts: + build: "taskcluster/android_cache-build.sh armeabi-v7a android-25" + package: "taskcluster/android_cache-package.sh" + metadata: + name: "Builds Android cache armeabi-v7a / android-25" + description: "Setup an Android SDK / emulator cache for Android armeabi-v7a / android-25" diff --git a/taskcluster/android-cache-sdk-android-27.yml b/taskcluster/android-cache-sdk-android-27.yml new file mode 100644 index 00000000..7632e5b8 --- /dev/null +++ b/taskcluster/android-cache-sdk-android-27.yml @@ -0,0 +1,14 @@ +build: + template_file: android_cache-opt-base.tyml + system_setup: + > + ${java.packages_trusty.apt} + cache: + url: ${system.android_cache.sdk.android_27.url} + namespace: ${system.android_cache.sdk.android_27.namespace} + scripts: + build: "taskcluster/android_cache-build.sh sdk android-27" + package: "taskcluster/android_cache-package.sh" + metadata: + name: "Builds Android cache sdk / android-27" + description: "Setup an Android SDK / emulator cache for Android sdk / android-27" diff --git a/taskcluster/android-cache-x86_64-android-24.yml b/taskcluster/android-cache-x86_64-android-24.yml new file mode 100644 index 00000000..38013d08 --- /dev/null +++ b/taskcluster/android-cache-x86_64-android-24.yml @@ -0,0 +1,14 @@ +build: + template_file: android_cache-opt-base.tyml + system_setup: + > + ${java.packages_trusty.apt} + cache: + url: ${system.android_cache.x86_64.android_24.url} + namespace: ${system.android_cache.x86_64.android_24.namespace} + scripts: + build: "taskcluster/android_cache-build.sh x86_64 android-24" + package: "taskcluster/android_cache-package.sh" + metadata: + name: "Builds Android cache x86_64 / android-24" + description: "Setup an Android SDK / emulator cache for Android x86_64 / android-24" diff --git a/taskcluster/android-cache-x86_64-android-25.yml b/taskcluster/android-cache-x86_64-android-25.yml new file mode 100644 index 00000000..6f57be42 --- /dev/null +++ b/taskcluster/android-cache-x86_64-android-25.yml @@ -0,0 +1,14 @@ +build: + template_file: android_cache-opt-base.tyml + system_setup: + > + ${java.packages_trusty.apt} + cache: + url: ${system.android_cache.x86_64.android_25.url} + namespace: ${system.android_cache.x86_64.android_25.namespace} + scripts: + build: "taskcluster/android_cache-build.sh x86_64 android-25" + package: "taskcluster/android_cache-package.sh" + metadata: + name: "Builds Android cache x86_64 / android-25" + description: "Setup an Android SDK / emulator cache for Android / x86_64 android-25" diff --git a/taskcluster/android-java-opt.yml b/taskcluster/android-java-opt.yml index 69f44dc7..7ca7d0f2 100644 --- a/taskcluster/android-java-opt.yml +++ b/taskcluster/android-java-opt.yml @@ -5,17 +5,22 @@ build: - "android-armv7-cpu-opt" - "android-x86_64-cpu-opt" - "pyenv-linux-amd64" + - "gradle-cache" + - "android-cache-sdk-android-27" routes: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.android-apk" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.android-apk" - "index.project.deepspeech.deepspeech.native_client.android-apk.${event.head.sha}" system_setup: > - apt-get -qq -y install curl software-properties-common && - add-apt-repository --yes ppa:openjdk-r/ppa && apt-get -qq update && - DEBIAN_FRONTEND=noninteractive apt-get -qq -y --force-yes install openjdk-8-jdk && - java -version && update-ca-certificates -f + ${java.packages_trusty.apt} tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.android-armv7/artifacts/public/home.tar.xz" + gradle_cache: + url: ${system.gradle_cache.url} + namespace: ${system.gradle_cache.namespace} + android_cache: + url: ${system.android_cache.sdk.android_27.url} + namespace: ${system.android_cache.sdk.android_27.namespace} scripts: build: "taskcluster/android-apk-build.sh" package: "taskcluster/android-apk-package.sh" diff --git a/taskcluster/android_cache-build.sh b/taskcluster/android_cache-build.sh new file mode 100755 index 00000000..c22ece47 --- /dev/null +++ b/taskcluster/android_cache-build.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -xe + +source $(dirname "$0")/tc-tests-utils.sh + +arm_flavor=$1 +api_level=$2 + +export ANDROID_HOME=${ANDROID_SDK_HOME} + +android_install_ndk + +android_install_sdk + +# Required for running APK tests later +android_install_sdk_platform "android-27" + +if [ "${arm_flavor}" != "sdk" ]; then + android_setup_emulator "${arm_flavor}" "${api_level}" +fi; diff --git a/taskcluster/android_cache-opt-base.tyml b/taskcluster/android_cache-opt-base.tyml new file mode 100644 index 00000000..b44778aa --- /dev/null +++ b/taskcluster/android_cache-opt-base.tyml @@ -0,0 +1,52 @@ +$if: 'event.event in build.allowed' +then: + taskId: ${taskcluster.taskId} + provisionerId: ${taskcluster.docker.provisionerId} + workerType: ${taskcluster.docker.workerType} + taskGroupId: ${taskcluster.taskGroupId} + schedulerId: ${taskcluster.schedulerId} + created: { $fromNow: '0 sec' } + deadline: { $fromNow: '1 day' } + expires: { $fromNow: '6 months' } + scopes: + - "index:insert-task:project.deepspeech.*" + + payload: + maxRunTime: { $eval: to_int(build.maxRunTime) } + image: ${build.docker_image} + + features: + taskclusterProxy: true + + command: + - "/bin/bash" + - "--login" + - "-cxe" + - $let: + extraSystemSetup: { $eval: strip(str(build.system_setup)) } + taskIndexExpire: { $fromNow: '6 months' } + in: > + (apt-get -qq -y remove --purge ubuntu-advantage-tools || true) && + apt-get -qq update && apt-get -qq -y install curl git && ${extraSystemSetup}; + cache_file=`curl -sSIL -o /dev/null -w "%{http_code}" ${build.cache.url}` && + if [ "$cache_file" != "200" ]; then + ${extraSystemSetup} && + adduser --system --home ${system.homedir.linux} ${system.username} && cd ${system.homedir.linux}/ && + mkdir -p /tmp/artifacts/ && chmod 777 /tmp/artifacts && + echo -e "#!/bin/bash\nset -xe\n env && id && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha}" > /tmp/clone.sh && chmod +x /tmp/clone.sh && + sudo -H -u ${system.username} /bin/bash /tmp/clone.sh && + sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.build} && + sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.package} ${taskIndexExpire} taskcluster ${build.cache.namespace} + fi; + + artifacts: + "public": + type: "directory" + path: "/tmp/artifacts/" + expires: { $fromNow: '6 months' } + + metadata: + name: ${build.metadata.name} + description: ${build.metadata.description} + owner: ${event.head.user.email} + source: ${event.head.repo.url} diff --git a/taskcluster/android_cache-package.sh b/taskcluster/android_cache-package.sh new file mode 100755 index 00000000..22ec767d --- /dev/null +++ b/taskcluster/android_cache-package.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -xe + +TC_EXPIRE=$1 +TC_INSTANCE=$2 +TC_INDEX=$3 + +source $(dirname "$0")/tc-tests-utils.sh + +cd $HOME/ && tar -czf $TASKCLUSTER_ARTIFACTS/android_cache.tar.gz DeepSpeech/Android/ + +if [ ! -z "${TC_EXPIRE}" -a ! -z "${TC_INSTANCE}" -a ! -z "${TC_INDEX}" ]; then + curl -sSL --fail -X PUT \ + -H "Content-Type: application/json" \ + -d "{\"taskId\":\"$TASK_ID\",\"rank\":0,\"expires\":\"${TC_EXPIRE}\",\"data\":{}}" \ + "http://${TC_INSTANCE}/index/v1/task/${TC_INDEX}" +fi; diff --git a/taskcluster/gradle-build.sh b/taskcluster/gradle-build.sh new file mode 100755 index 00000000..48675bb6 --- /dev/null +++ b/taskcluster/gradle-build.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -xe + +source $(dirname "$0")/tc-tests-utils.sh + +if [ -z "${GRADLE_USER_HOME}" ]; then + echo "Unable to generate cache without an emplacement" + exit 1 +fi; + +mkdir -p ${GRADLE_USER_HOME} + +export ANDROID_HOME=${ANDROID_SDK_HOME} + +# Gradle likes to play with us. +android_install_ndk +android_install_sdk + +pushd ${DS_ROOT_TASK}/DeepSpeech/ds/native_client/java/ + ./gradlew androidDependencies dependentComponents dependencies + # we need this for aapt2 binary + ./gradlew buildNeeded || true # will try javac which is doomed to fail + ./gradlew --refresh-dependencies +popd + +du -hs ${GRADLE_USER_HOME}/* diff --git a/taskcluster/gradle-cache.yml b/taskcluster/gradle-cache.yml new file mode 100644 index 00000000..334545c3 --- /dev/null +++ b/taskcluster/gradle-cache.yml @@ -0,0 +1,14 @@ +build: + template_file: android_cache-opt-base.tyml + system_setup: + > + ${java.packages_trusty.apt} + cache: + url: ${system.gradle_cache.url} + namespace: ${system.gradle_cache.namespace} + scripts: + build: "taskcluster/gradle-build.sh" + package: "taskcluster/gradle-package.sh" + metadata: + name: "Builds Gradle cache" + description: "Setup a Gradle cache for Android" diff --git a/taskcluster/gradle-package.sh b/taskcluster/gradle-package.sh new file mode 100755 index 00000000..495c05e8 --- /dev/null +++ b/taskcluster/gradle-package.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -xe + +TC_EXPIRE=$1 +TC_INSTANCE=$2 +TC_INDEX=$3 + +source $(dirname "$0")/tc-tests-utils.sh + +cd ${GRADLE_USER_HOME}/../ && tar -czf $TASKCLUSTER_ARTIFACTS/gradle.tar.gz gradle-cache/ + +if [ ! -z "${TC_EXPIRE}" -a ! -z "${TC_INSTANCE}" -a ! -z "${TC_INDEX}" ]; then + curl -sSL --fail -X PUT \ + -H "Content-Type: application/json" \ + -d "{\"taskId\":\"$TASK_ID\",\"rank\":0,\"expires\":\"${TC_EXPIRE}\",\"data\":{}}" \ + "http://${TC_INSTANCE}/index/v1/task/${TC_INDEX}" +fi; diff --git a/taskcluster/linux-opt-base.tyml b/taskcluster/linux-opt-base.tyml index dd50397a..2a697bf0 100644 --- a/taskcluster/linux-opt-base.tyml +++ b/taskcluster/linux-opt-base.tyml @@ -48,7 +48,7 @@ then: adduser --system --home ${system.homedir.linux} ${system.username} && apt-get -qq update && apt-get -qq -y install ${tensorflow.packages_trusty.apt} pixz pkg-config realpath unzip wget zip && ${extraSystemSetup} && cd ${system.homedir.linux}/ && - echo -e "#!/bin/bash\nset -xe\n env && id && (wget -O - $TENSORFLOW_BUILD_ARTIFACT | pixz -d | tar -C ${system.homedir.linux}/ -xf - ) && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && ln -s ~/DeepSpeech/ds/native_client/ ~/DeepSpeech/tf/native_client && mkdir -p ${system.homedir.linux}/.cache/node-gyp/ && wget -O - ${system.node_gyp_cache.url} | tar -C ${system.homedir.linux}/.cache/node-gyp/ -xzf - && mkdir -p ${system.homedir.linux}/ds-swig/bin/ && wget -O - ${system.swig.cache.linux_amd64} | tar -C ${system.homedir.linux}/ds-swig/ -xzf - && mkdir -p ${system.homedir.linux}/pyenv-root/ && wget -O - ${system.pyenv.linux.url} | tar -C ${system.homedir.linux}/pyenv-root/ -xzf -" > /tmp/clone.sh && chmod +x /tmp/clone.sh && + echo -e "#!/bin/bash\nset -xe\n env && id && (wget -O - $TENSORFLOW_BUILD_ARTIFACT | pixz -d | tar -C ${system.homedir.linux}/ -xf - ) && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && ln -s ~/DeepSpeech/ds/native_client/ ~/DeepSpeech/tf/native_client && mkdir -p ${system.homedir.linux}/.cache/node-gyp/ && wget -O - ${system.node_gyp_cache.url} | tar -C ${system.homedir.linux}/.cache/node-gyp/ -xzf - && mkdir -p ${system.homedir.linux}/ds-swig/bin/ && wget -O - ${system.swig.cache.linux_amd64} | tar -C ${system.homedir.linux}/ds-swig/ -xzf - && mkdir -p ${system.homedir.linux}/pyenv-root/ && wget -O - ${system.pyenv.linux.url} | tar -C ${system.homedir.linux}/pyenv-root/ -xzf - && if [ ! -z "${build.gradle_cache.url}" ]; then wget -O - ${build.gradle_cache.url} | tar -C ${system.homedir.linux}/ -xzf - ; fi && if [ ! -z "${build.android_cache.url}" ]; then wget -O - ${build.android_cache.url} | tar -C ${system.homedir.linux}/ -xzf - ; fi;" > /tmp/clone.sh && chmod +x /tmp/clone.sh && sudo -H -u ${system.username} /bin/bash /tmp/clone.sh && ${extraSystemConfig} && sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.build} && sudo -H -u ${system.username} /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.package} diff --git a/taskcluster/tc-all-vars.sh b/taskcluster/tc-all-vars.sh index 8740725f..cb836e83 100755 --- a/taskcluster/tc-all-vars.sh +++ b/taskcluster/tc-all-vars.sh @@ -75,6 +75,7 @@ export DS_EXAMPLEDIR=${DS_ROOT_TASK}/DeepSpeech/examples export DS_VERSION="$(cat ${DS_DSDIR}/VERSION)" +export GRADLE_USER_HOME=${DS_ROOT_TASK}/gradle-cache export ANDROID_SDK_HOME=${DS_ROOT_TASK}/DeepSpeech/Android/SDK/ export ANDROID_NDK_HOME=${DS_ROOT_TASK}/DeepSpeech/Android/android-ndk-r18b/ diff --git a/taskcluster/tc-android-apk-tests.sh b/taskcluster/tc-android-apk-tests.sh index fec910bd..7074aa85 100644 --- a/taskcluster/tc-android-apk-tests.sh +++ b/taskcluster/tc-android-apk-tests.sh @@ -15,12 +15,10 @@ model_name=$(basename "${model_source}") download_data -android_install_ndk - -android_setup_emulator "${arm_flavor}" "${api_level}" - force_java_apk_x86_64 +android_start_emulator "${arm_flavor}" "${api_level}" + # Required, because of "gradle connectedAndroidTest" deps do_deepspeech_java_apk_build diff --git a/taskcluster/tc-android-ds-tests.sh b/taskcluster/tc-android-ds-tests.sh index b9c70d03..257a9496 100755 --- a/taskcluster/tc-android-ds-tests.sh +++ b/taskcluster/tc-android-ds-tests.sh @@ -24,7 +24,7 @@ fi download_material "${TASKCLUSTER_TMP_DIR}/ds" -android_setup_emulator "${arm_flavor}" "${api_level}" +android_start_emulator "${arm_flavor}" "${api_level}" android_setup_ndk_data diff --git a/taskcluster/tc-android-utils.sh b/taskcluster/tc-android-utils.sh index dfd75212..4deffd5a 100755 --- a/taskcluster/tc-android-utils.sh +++ b/taskcluster/tc-android-utils.sh @@ -43,9 +43,9 @@ do_deepspeech_java_apk_build() fi; done; - make -C native_client/java/ + make GRADLE="./gradlew " -C native_client/java/ - make -C native_client/java/ maven-bundle + make GRADLE="./gradlew " -C native_client/java/ maven-bundle } android_run_tests() @@ -100,8 +100,6 @@ android_install_ndk() android_setup_emulator() { - android_install_sdk - if [ -z "${ANDROID_SDK_HOME}" ]; then echo "No Android SDK home available, aborting." exit 1 @@ -112,8 +110,30 @@ android_setup_emulator() exit 1 fi; - flavor=$1 - api_level=${2:-android-25} + local _flavor=$1 + local _api_level=${2:-android-25} + + export PATH=${ANDROID_SDK_HOME}/tools/bin/:${ANDROID_SDK_HOME}/platform-tools/:$PATH + export DS_BINARY_PREFIX="adb shell LD_LIBRARY_PATH=${ANDROID_TMP_DIR}/ds/ ${ANDROID_TMP_DIR}/ds/" + + # Pipe yes in case of license being shown + yes | sdkmanager --update + yes | sdkmanager --install "emulator" + + android_install_sdk_platform "${_api_level}" + + # Same, yes in case of license + yes | sdkmanager --install "system-images;${_api_level};google_apis;${_flavor}" + + android_sdk_accept_licenses + + avdmanager create avd --name "${_flavor}-ds-pixel-${_api_level}" --device 17 --package "system-images;${_api_level};google_apis;${_flavor}" +} + +android_start_emulator() +{ + local _flavor=$1 + local _api_level=${2:-android-25} export PATH=${ANDROID_SDK_HOME}/tools/bin/:${ANDROID_SDK_HOME}/platform-tools/:$PATH export DS_BINARY_PREFIX="adb shell LD_LIBRARY_PATH=${ANDROID_TMP_DIR}/ds/ ${ANDROID_TMP_DIR}/ds/" @@ -121,25 +141,11 @@ android_setup_emulator() # minutes (2 minutes by default) export ADB_INSTALL_TIMEOUT=8 - # Pipe yes in case of license being shown - yes | sdkmanager --update - yes | sdkmanager --install "emulator" - - android_install_sdk_platform "${api_level}" - - # Same, yes in case of license - yes | sdkmanager --install "system-images;${api_level};google_apis;${flavor}" - - android_sdk_accept_licenses - - avdmanager create avd --name "ds-pixel" --device 17 --package "system-images;${api_level};google_apis;${flavor}" - # Use xvfb because: # > emulator: INFO: QtLogger.cpp:68: Warning: could not connect to display ((null):0, (null)) - # -accel on is needed otherwise it is too slow, but it will require KVM support exposed pushd ${ANDROID_SDK_HOME} - xvfb-run ./tools/emulator -verbose -avd ds-pixel -no-skin -no-audio -no-window -no-boot-anim -accel off & + xvfb-run ./tools/emulator -verbose -avd "${_flavor}-ds-pixel-${_api_level}" -no-skin -no-audio -no-window -no-boot-anim -accel off & emulator_rc=$? export ANDROID_DEVICE_EMULATOR=$! popd @@ -159,7 +165,7 @@ android_setup_emulator() android_install_sdk_platform() { - api_level=${1:-android-27} + local _api_level=${1:-android-27} if [ -z "${ANDROID_SDK_HOME}" ]; then echo "No Android SDK home available, aborting." @@ -170,8 +176,10 @@ android_install_sdk_platform() # Pipe yes in case of license being shown yes | sdkmanager --update + yes | sdkmanager --install "build-tools;28.0.3" + yes | sdkmanager --install "cmake;3.6.4111459" yes | sdkmanager --install "platform-tools" - yes | sdkmanager --install "platforms;${api_level}" + yes | sdkmanager --install "platforms;${_api_level}" android_sdk_accept_licenses } diff --git a/taskcluster/test-android-opt-base.tyml b/taskcluster/test-android-opt-base.tyml index cc5d2b56..dda7e963 100644 --- a/taskcluster/test-android-opt-base.tyml +++ b/taskcluster/test-android-opt-base.tyml @@ -44,7 +44,7 @@ then: ${extraSystemSetup} && adduser --system --home ${system.homedir.linux} ${system.username} && cd ${system.homedir.linux} && - echo -e "#!/bin/bash\nset -xe\n env && id && mkdir ~/DeepSpeech/ && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && mkdir -p ${system.homedir.linux}/ds-swig/bin/ && wget -O - ${system.swig.cache.linux_amd64} | tar -C ${system.homedir.linux}/ds-swig/ -xzf -" > /tmp/clone.sh && chmod +x /tmp/clone.sh && + echo -e "#!/bin/bash\nset -xe\n env && id && mkdir ~/DeepSpeech/ && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && mkdir -p ${system.homedir.linux}/ds-swig/bin/ && wget -O - ${system.swig.cache.linux_amd64} | tar -C ${system.homedir.linux}/ds-swig/ -xzf - && wget -O - ${build.cache.url} | tar -C ${system.homedir.linux} -xzf - && if [ ! -z "${build.gradle_cache.url}" ]; then wget -O - ${build.gradle_cache.url} | tar -C ${system.homedir.linux}/ -xzf - ; fi;" > /tmp/clone.sh && chmod +x /tmp/clone.sh && sudo -H -u ${system.username} /bin/bash /tmp/clone.sh && sudo -H -u ${system.username} --preserve-env /bin/bash ${build.args.tests_cmdline} diff --git a/taskcluster/test-apk-android-24-x86_64-opt.yml b/taskcluster/test-apk-android-24-x86_64-opt.yml index e311790f..ead96e01 100644 --- a/taskcluster/test-apk-android-24-x86_64-opt.yml +++ b/taskcluster/test-apk-android-24-x86_64-opt.yml @@ -4,10 +4,18 @@ build: - "swig-linux-amd64" - "android-x86_64-cpu-opt" - "test-training_16k-linux-amd64-py36m-opt" + - "gradle-cache" + - "android-cache-x86_64-android-24" test_model_task: "test-training_16k-linux-amd64-py36m-opt" system_setup: > apt-get -qq -y install curl make python + cache: + url: ${system.android_cache.x86_64.android_24.url} + namespace: ${system.android_cache.x86_64.android_24.namespace} + gradle_cache: + url: ${system.gradle_cache.url} + namespace: ${system.gradle_cache.namespace} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-android-apk-tests.sh x86_64 android-24 16k" metadata: diff --git a/taskcluster/test-apk-android-25-x86_64-opt.yml b/taskcluster/test-apk-android-25-x86_64-opt.yml index 1273b339..fbf2c655 100644 --- a/taskcluster/test-apk-android-25-x86_64-opt.yml +++ b/taskcluster/test-apk-android-25-x86_64-opt.yml @@ -4,10 +4,18 @@ build: - "swig-linux-amd64" - "android-x86_64-cpu-opt" - "test-training_16k-linux-amd64-py36m-opt" + - "gradle-cache" + - "android-cache-x86_64-android-25" test_model_task: "test-training_16k-linux-amd64-py36m-opt" system_setup: > apt-get -qq -y install curl make python + cache: + url: ${system.android_cache.x86_64.android_25.url} + namespace: ${system.android_cache.x86_64.android_25.namespace} + gradle_cache: + url: ${system.gradle_cache.url} + namespace: ${system.gradle_cache.namespace} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-android-apk-tests.sh x86_64 android-25 16k" metadata: diff --git a/taskcluster/test-cpp-android-24-arm64-opt.yml b/taskcluster/test-cpp-android-24-arm64-opt.yml index abd21dc9..5125244f 100644 --- a/taskcluster/test-cpp-android-24-arm64-opt.yml +++ b/taskcluster/test-cpp-android-24-arm64-opt.yml @@ -3,6 +3,10 @@ build: dependencies: - "android-arm64-cpu-opt" - "test-training_16k-linux-amd64-py36m-opt" + - "android-cache-arm64-v8a-android-24" + cache: + url: ${system.android_cache.arm64_v8a.android_24.url} + namespace: ${system.android_cache.arm64_v8a.android_24.namespace} test_model_task: "test-training_16k-linux-amd64-py36m-opt" args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-android-ds-tests.sh arm64-v8a android-24 16k" diff --git a/taskcluster/test-cpp-android-24-armv7-opt.yml b/taskcluster/test-cpp-android-24-armv7-opt.yml index af5b1c6e..37a45300 100644 --- a/taskcluster/test-cpp-android-24-armv7-opt.yml +++ b/taskcluster/test-cpp-android-24-armv7-opt.yml @@ -3,6 +3,10 @@ build: dependencies: - "android-armv7-cpu-opt" - "test-training_16k-linux-amd64-py36m-opt" + - "android-cache-armeabi-v7a-android-24" + cache: + url: ${system.android_cache.armeabi_v7a.android_24.url} + namespace: ${system.android_cache.armeabi_v7a.android_24.namespace} test_model_task: "test-training_16k-linux-amd64-py36m-opt" args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-android-ds-tests.sh armeabi-v7a android-24 16k" diff --git a/taskcluster/test-cpp-android-25-arm64-opt.yml b/taskcluster/test-cpp-android-25-arm64-opt.yml index 7183105e..bc450863 100644 --- a/taskcluster/test-cpp-android-25-arm64-opt.yml +++ b/taskcluster/test-cpp-android-25-arm64-opt.yml @@ -3,6 +3,10 @@ build: dependencies: - "android-arm64-cpu-opt" - "test-training_16k-linux-amd64-py36m-opt" + - "android-cache-arm64-v8a-android-25" + cache: + url: ${system.android_cache.arm64_v8a.android_25.url} + namespace: ${system.android_cache.arm64_v8a.android_25.namespace} test_model_task: "test-training_16k-linux-amd64-py36m-opt" args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-android-ds-tests.sh arm64-v8a android-25 16k" diff --git a/taskcluster/test-cpp-android-25-armv7-opt.yml b/taskcluster/test-cpp-android-25-armv7-opt.yml index 8bb1c8f1..85818edf 100644 --- a/taskcluster/test-cpp-android-25-armv7-opt.yml +++ b/taskcluster/test-cpp-android-25-armv7-opt.yml @@ -3,6 +3,10 @@ build: dependencies: - "android-armv7-cpu-opt" - "test-training_16k-linux-amd64-py36m-opt" + - "android-cache-armeabi-v7a-android-25" + cache: + url: ${system.android_cache.armeabi_v7a.android_25.url} + namespace: ${system.android_cache.armeabi_v7a.android_25.namespace} test_model_task: "test-training_16k-linux-amd64-py36m-opt" args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-android-ds-tests.sh armeabi-v7a android-25 16k"