From 4f7842c96657c24c3bc3f0e7cb6ae9efd848e15a Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Tue, 16 Jun 2020 17:09:14 +0200 Subject: [PATCH 1/4] Fix #3068: More generic TaskCluster build/caching tasks --- taskcluster/.build.yml | 4 ++ taskcluster/.shared.yml | 10 ++++ .../android-cache-arm64-v8a-android-24.yml | 6 +- .../android-cache-arm64-v8a-android-25.yml | 6 +- .../android-cache-armeabi-v7a-android-24.yml | 6 +- .../android-cache-armeabi-v7a-android-25.yml | 6 +- taskcluster/android-cache-sdk-android-27.yml | 6 +- .../android-cache-x86_64-android-24.yml | 6 +- .../android-cache-x86_64-android-25.yml | 6 +- .../android-cache-x86_64-android-26.yml | 6 +- .../android-cache-x86_64-android-28.yml | 6 +- .../android-cache-x86_64-android-29.yml | 6 +- taskcluster/android_cache-opt-base.tyml | 52 ---------------- taskcluster/android_cache-package.sh | 11 ---- ...> generic_tc_caching-darwin-opt-base.tyml} | 13 ++-- .../generic_tc_caching-linux-opt-base.tyml | 50 ++++++++++++++++ .../generic_tc_caching-win-opt-base.tyml | 57 ++++++++++++++++++ taskcluster/gradle-cache.yml | 8 +-- taskcluster/gradle-package.sh | 11 ---- taskcluster/homebrew-darwin-opt-base.tyml | 59 ------------------- taskcluster/homebrew_builds-darwin-amd64.yml | 8 +-- taskcluster/homebrew_tests-darwin-amd64.yml | 8 +-- taskcluster/node-gyp-cache-base.tyml | 57 ------------------ taskcluster/node-gyp-cache.yml | 9 ++- taskcluster/node-gyp-populate.sh | 4 +- taskcluster/pyenv-darwin-amd64.yml | 5 +- taskcluster/pyenv-linux-amd64.yml | 5 +- taskcluster/pyenv-linux-opt-base.tyml | 52 ---------------- taskcluster/pyenv-package.sh | 11 ---- taskcluster/pyenv-win-amd64.yml | 5 +- taskcluster/pyenv-win-opt-base.tyml | 59 ------------------- taskcluster/swig-darwin-amd64.yml | 11 +++- taskcluster/swig-darwin-opt-base.tyml | 56 ------------------ taskcluster/swig-linux-amd64.yml | 11 +++- taskcluster/swig-linux-opt-base.tyml | 54 ----------------- taskcluster/swig-win-amd64.yml | 11 +++- taskcluster/tc-update-index.sh | 18 ++++++ 37 files changed, 235 insertions(+), 484 deletions(-) delete mode 100644 taskcluster/android_cache-opt-base.tyml rename taskcluster/{pyenv-darwin-opt-base.tyml => generic_tc_caching-darwin-opt-base.tyml} (66%) create mode 100644 taskcluster/generic_tc_caching-linux-opt-base.tyml create mode 100644 taskcluster/generic_tc_caching-win-opt-base.tyml delete mode 100644 taskcluster/homebrew-darwin-opt-base.tyml delete mode 100644 taskcluster/node-gyp-cache-base.tyml delete mode 100644 taskcluster/pyenv-linux-opt-base.tyml delete mode 100644 taskcluster/pyenv-win-opt-base.tyml delete mode 100644 taskcluster/swig-darwin-opt-base.tyml delete mode 100644 taskcluster/swig-linux-opt-base.tyml create mode 100755 taskcluster/tc-update-index.sh diff --git a/taskcluster/.build.yml b/taskcluster/.build.yml index 7e9809e5..fee2c1d6 100644 --- a/taskcluster/.build.yml +++ b/taskcluster/.build.yml @@ -36,3 +36,7 @@ build: gradle_cache: url: '' namespace: '' + build_or_cache: + repo: "${event.head.repo.url}" + sha: "${event.head.sha}" + dir: "DeepSpeech/ds" diff --git a/taskcluster/.shared.yml b/taskcluster/.shared.yml index dadb1094..b04ff9de 100644 --- a/taskcluster/.shared.yml +++ b/taskcluster/.shared.yml @@ -119,6 +119,16 @@ system: swig: repo: "https://github.com/lissyx/swig" sha1: "b5fea54d39832d1d132d7dd921b69c0c2c9d5118" + swig_build: + linux: + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.linux.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118/artifacts/public/ds-swig.tar.gz" + namespace: "project.deepspeech.swig.linux.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118" + osx: + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.darwin.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118/artifacts/public/ds-swig.tar.gz" + namespace: "project.deepspeech.swig.darwin.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118" + win: + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.win.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118/artifacts/public/ds-swig.tar.gz" + namespace: "project.deepspeech.swig.win.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118" username: 'build-user' homedir: linux: '/home/build-user' diff --git a/taskcluster/android-cache-arm64-v8a-android-24.yml b/taskcluster/android-cache-arm64-v8a-android-24.yml index a8fef42f..8fa8e927 100644 --- a/taskcluster/android-cache-arm64-v8a-android-24.yml +++ b/taskcluster/android-cache-arm64-v8a-android-24.yml @@ -1,11 +1,11 @@ build: - template_file: android_cache-opt-base.tyml + template_file: generic_tc_caching-linux-opt-base.tyml system_setup: > ${java.packages_trusty.apt} cache: - url: ${system.android_cache.arm64_v8a.android_24.url} - namespace: ${system.android_cache.arm64_v8a.android_24.namespace} + artifact_url: ${system.android_cache.arm64_v8a.android_24.url} + artifact_namespace: ${system.android_cache.arm64_v8a.android_24.namespace} scripts: build: "taskcluster/android_cache-build.sh arm64-v8a android-24" package: "taskcluster/android_cache-package.sh" diff --git a/taskcluster/android-cache-arm64-v8a-android-25.yml b/taskcluster/android-cache-arm64-v8a-android-25.yml index 7dca05ea..36638726 100644 --- a/taskcluster/android-cache-arm64-v8a-android-25.yml +++ b/taskcluster/android-cache-arm64-v8a-android-25.yml @@ -1,11 +1,11 @@ build: - template_file: android_cache-opt-base.tyml + template_file: generic_tc_caching-linux-opt-base.tyml system_setup: > ${java.packages_trusty.apt} cache: - url: ${system.android_cache.arm64_v8a.android_25.url} - namespace: ${system.android_cache.arm64_v8a.android_25.namespace} + artifact_url: ${system.android_cache.arm64_v8a.android_25.url} + artifact_namespace: ${system.android_cache.arm64_v8a.android_25.namespace} scripts: build: "taskcluster/android_cache-build.sh arm64-v8a android-25" package: "taskcluster/android_cache-package.sh" diff --git a/taskcluster/android-cache-armeabi-v7a-android-24.yml b/taskcluster/android-cache-armeabi-v7a-android-24.yml index 605f0e92..3536ab3f 100644 --- a/taskcluster/android-cache-armeabi-v7a-android-24.yml +++ b/taskcluster/android-cache-armeabi-v7a-android-24.yml @@ -1,11 +1,11 @@ build: - template_file: android_cache-opt-base.tyml + template_file: generic_tc_caching-linux-opt-base.tyml system_setup: > ${java.packages_trusty.apt} cache: - url: ${system.android_cache.armeabi_v7a.android_24.url} - namespace: ${system.android_cache.armeabi_v7a.android_24.namespace} + artifact_url: ${system.android_cache.armeabi_v7a.android_24.url} + artifact_namespace: ${system.android_cache.armeabi_v7a.android_24.namespace} scripts: build: "taskcluster/android_cache-build.sh armeabi-v7a android-24" package: "taskcluster/android_cache-package.sh" diff --git a/taskcluster/android-cache-armeabi-v7a-android-25.yml b/taskcluster/android-cache-armeabi-v7a-android-25.yml index 6089d940..e4f78de7 100644 --- a/taskcluster/android-cache-armeabi-v7a-android-25.yml +++ b/taskcluster/android-cache-armeabi-v7a-android-25.yml @@ -1,11 +1,11 @@ build: - template_file: android_cache-opt-base.tyml + template_file: generic_tc_caching-linux-opt-base.tyml system_setup: > ${java.packages_trusty.apt} cache: - url: ${system.android_cache.armeabi_v7a.android_25.url} - namespace: ${system.android_cache.armeabi_v7a.android_25.namespace} + artifact_url: ${system.android_cache.armeabi_v7a.android_25.url} + artifact_namespace: ${system.android_cache.armeabi_v7a.android_25.namespace} scripts: build: "taskcluster/android_cache-build.sh armeabi-v7a android-25" package: "taskcluster/android_cache-package.sh" diff --git a/taskcluster/android-cache-sdk-android-27.yml b/taskcluster/android-cache-sdk-android-27.yml index 7632e5b8..668e142e 100644 --- a/taskcluster/android-cache-sdk-android-27.yml +++ b/taskcluster/android-cache-sdk-android-27.yml @@ -1,11 +1,11 @@ build: - template_file: android_cache-opt-base.tyml + template_file: generic_tc_caching-linux-opt-base.tyml system_setup: > ${java.packages_trusty.apt} cache: - url: ${system.android_cache.sdk.android_27.url} - namespace: ${system.android_cache.sdk.android_27.namespace} + artifact_url: ${system.android_cache.sdk.android_27.url} + artifact_namespace: ${system.android_cache.sdk.android_27.namespace} scripts: build: "taskcluster/android_cache-build.sh sdk android-27" package: "taskcluster/android_cache-package.sh" diff --git a/taskcluster/android-cache-x86_64-android-24.yml b/taskcluster/android-cache-x86_64-android-24.yml index 38013d08..ff06f551 100644 --- a/taskcluster/android-cache-x86_64-android-24.yml +++ b/taskcluster/android-cache-x86_64-android-24.yml @@ -1,11 +1,11 @@ build: - template_file: android_cache-opt-base.tyml + template_file: generic_tc_caching-linux-opt-base.tyml system_setup: > ${java.packages_trusty.apt} cache: - url: ${system.android_cache.x86_64.android_24.url} - namespace: ${system.android_cache.x86_64.android_24.namespace} + artifact_url: ${system.android_cache.x86_64.android_24.url} + artifact_namespace: ${system.android_cache.x86_64.android_24.namespace} scripts: build: "taskcluster/android_cache-build.sh x86_64 android-24" package: "taskcluster/android_cache-package.sh" diff --git a/taskcluster/android-cache-x86_64-android-25.yml b/taskcluster/android-cache-x86_64-android-25.yml index 6f57be42..8ccac4f9 100644 --- a/taskcluster/android-cache-x86_64-android-25.yml +++ b/taskcluster/android-cache-x86_64-android-25.yml @@ -1,11 +1,11 @@ build: - template_file: android_cache-opt-base.tyml + template_file: generic_tc_caching-linux-opt-base.tyml system_setup: > ${java.packages_trusty.apt} cache: - url: ${system.android_cache.x86_64.android_25.url} - namespace: ${system.android_cache.x86_64.android_25.namespace} + artifact_url: ${system.android_cache.x86_64.android_25.url} + artifact_namespace: ${system.android_cache.x86_64.android_25.namespace} scripts: build: "taskcluster/android_cache-build.sh x86_64 android-25" package: "taskcluster/android_cache-package.sh" diff --git a/taskcluster/android-cache-x86_64-android-26.yml b/taskcluster/android-cache-x86_64-android-26.yml index ec711ede..0a74e78e 100644 --- a/taskcluster/android-cache-x86_64-android-26.yml +++ b/taskcluster/android-cache-x86_64-android-26.yml @@ -1,11 +1,11 @@ build: - template_file: android_cache-opt-base.tyml + template_file: generic_tc_caching-linux-opt-base.tyml system_setup: > ${java.packages_trusty.apt} cache: - url: ${system.android_cache.x86_64.android_26.url} - namespace: ${system.android_cache.x86_64.android_26.namespace} + artifact_url: ${system.android_cache.x86_64.android_26.url} + artifact_namespace: ${system.android_cache.x86_64.android_26.namespace} scripts: build: "taskcluster/android_cache-build.sh x86_64 android-26" package: "taskcluster/android_cache-package.sh" diff --git a/taskcluster/android-cache-x86_64-android-28.yml b/taskcluster/android-cache-x86_64-android-28.yml index 471f33b9..4fa124fd 100644 --- a/taskcluster/android-cache-x86_64-android-28.yml +++ b/taskcluster/android-cache-x86_64-android-28.yml @@ -1,11 +1,11 @@ build: - template_file: android_cache-opt-base.tyml + template_file: generic_tc_caching-linux-opt-base.tyml system_setup: > ${java.packages_trusty.apt} cache: - url: ${system.android_cache.x86_64.android_28.url} - namespace: ${system.android_cache.x86_64.android_28.namespace} + artifact_url: ${system.android_cache.x86_64.android_28.url} + artifact_namespace: ${system.android_cache.x86_64.android_28.namespace} scripts: build: "taskcluster/android_cache-build.sh x86_64 android-28" package: "taskcluster/android_cache-package.sh" diff --git a/taskcluster/android-cache-x86_64-android-29.yml b/taskcluster/android-cache-x86_64-android-29.yml index 835453f9..57c194f2 100644 --- a/taskcluster/android-cache-x86_64-android-29.yml +++ b/taskcluster/android-cache-x86_64-android-29.yml @@ -1,11 +1,11 @@ build: - template_file: android_cache-opt-base.tyml + template_file: generic_tc_caching-linux-opt-base.tyml system_setup: > ${java.packages_trusty.apt} cache: - url: ${system.android_cache.x86_64.android_29.url} - namespace: ${system.android_cache.x86_64.android_29.namespace} + artifact_url: ${system.android_cache.x86_64.android_29.url} + artifact_namespace: ${system.android_cache.x86_64.android_29.namespace} scripts: build: "taskcluster/android_cache-build.sh x86_64 android-29" package: "taskcluster/android_cache-package.sh" diff --git a/taskcluster/android_cache-opt-base.tyml b/taskcluster/android_cache-opt-base.tyml deleted file mode 100644 index b44778aa..00000000 --- a/taskcluster/android_cache-opt-base.tyml +++ /dev/null @@ -1,52 +0,0 @@ -$if: 'event.event in build.allowed' -then: - taskId: ${taskcluster.taskId} - provisionerId: ${taskcluster.docker.provisionerId} - workerType: ${taskcluster.docker.workerType} - taskGroupId: ${taskcluster.taskGroupId} - schedulerId: ${taskcluster.schedulerId} - created: { $fromNow: '0 sec' } - deadline: { $fromNow: '1 day' } - expires: { $fromNow: '6 months' } - scopes: - - "index:insert-task:project.deepspeech.*" - - payload: - maxRunTime: { $eval: to_int(build.maxRunTime) } - image: ${build.docker_image} - - features: - taskclusterProxy: true - - command: - - "/bin/bash" - - "--login" - - "-cxe" - - $let: - extraSystemSetup: { $eval: strip(str(build.system_setup)) } - taskIndexExpire: { $fromNow: '6 months' } - in: > - (apt-get -qq -y remove --purge ubuntu-advantage-tools || true) && - apt-get -qq update && apt-get -qq -y install curl git && ${extraSystemSetup}; - cache_file=`curl -sSIL -o /dev/null -w "%{http_code}" ${build.cache.url}` && - if [ "$cache_file" != "200" ]; then - ${extraSystemSetup} && - adduser --system --home ${system.homedir.linux} ${system.username} && cd ${system.homedir.linux}/ && - mkdir -p /tmp/artifacts/ && chmod 777 /tmp/artifacts && - echo -e "#!/bin/bash\nset -xe\n env && id && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha}" > /tmp/clone.sh && chmod +x /tmp/clone.sh && - sudo -H -u ${system.username} /bin/bash /tmp/clone.sh && - sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.build} && - sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.package} ${taskIndexExpire} taskcluster ${build.cache.namespace} - fi; - - artifacts: - "public": - type: "directory" - path: "/tmp/artifacts/" - expires: { $fromNow: '6 months' } - - metadata: - name: ${build.metadata.name} - description: ${build.metadata.description} - owner: ${event.head.user.email} - source: ${event.head.repo.url} diff --git a/taskcluster/android_cache-package.sh b/taskcluster/android_cache-package.sh index 22ec767d..8c73070d 100755 --- a/taskcluster/android_cache-package.sh +++ b/taskcluster/android_cache-package.sh @@ -2,17 +2,6 @@ set -xe -TC_EXPIRE=$1 -TC_INSTANCE=$2 -TC_INDEX=$3 - source $(dirname "$0")/tc-tests-utils.sh cd $HOME/ && tar -czf $TASKCLUSTER_ARTIFACTS/android_cache.tar.gz DeepSpeech/Android/ - -if [ ! -z "${TC_EXPIRE}" -a ! -z "${TC_INSTANCE}" -a ! -z "${TC_INDEX}" ]; then - curl -sSL --fail -X PUT \ - -H "Content-Type: application/json" \ - -d "{\"taskId\":\"$TASK_ID\",\"rank\":0,\"expires\":\"${TC_EXPIRE}\",\"data\":{}}" \ - "http://${TC_INSTANCE}/index/v1/task/${TC_INDEX}" -fi; diff --git a/taskcluster/pyenv-darwin-opt-base.tyml b/taskcluster/generic_tc_caching-darwin-opt-base.tyml similarity index 66% rename from taskcluster/pyenv-darwin-opt-base.tyml rename to taskcluster/generic_tc_caching-darwin-opt-base.tyml index 063c6613..e6777f22 100644 --- a/taskcluster/pyenv-darwin-opt-base.tyml +++ b/taskcluster/generic_tc_caching-darwin-opt-base.tyml @@ -35,12 +35,13 @@ payload: export SDKROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX10.14.sdk/ && env && mkdir -p $TASKCLUSTER_ARTIFACTS/ && - swig_bin=`curl -sSIL -o /dev/null -w "%{http_code}" ${system.pyenv.osx.url}` && - if [ "$swig_bin" != "200" ]; then - git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ && - cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && - $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.build} && - $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.package} ${taskIndexExpire} 127.0.0.1:8080 ${system.pyenv.osx.namespace} + cache_artifact=`curl -sSIL -o /dev/null -w "%{http_code}" ${build.cache.artifact_url}` && + if [ "$cache_artifact" != "200" ]; then + git clone --quiet ${build.build_or_cache.repo} $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir} && + cd $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir} && git checkout --quiet ${build.build_or_cache.sha} && + $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/${build.scripts.build} && + $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/${build.scripts.package} && + $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/taskcluster/tc-update-index.sh ${taskIndexExpire} 127.0.0.1:8080 ${build.cache.artifact_namespace} fi; artifacts: diff --git a/taskcluster/generic_tc_caching-linux-opt-base.tyml b/taskcluster/generic_tc_caching-linux-opt-base.tyml new file mode 100644 index 00000000..36668a0c --- /dev/null +++ b/taskcluster/generic_tc_caching-linux-opt-base.tyml @@ -0,0 +1,50 @@ +taskId: ${taskcluster.taskId} +provisionerId: ${taskcluster.docker.provisionerId} +workerType: ${taskcluster.docker.workerType} +taskGroupId: ${taskcluster.taskGroupId} +schedulerId: ${taskcluster.schedulerId} +created: { $fromNow: '0 sec' } +deadline: { $fromNow: '1 day' } +expires: { $fromNow: '6 months' } +scopes: + - "index:insert-task:project.deepspeech.*" + +payload: + maxRunTime: { $eval: to_int(build.maxRunTime) } + image: ${build.docker_image} + + features: + taskclusterProxy: true + + command: + - "/bin/bash" + - "--login" + - "-cxe" + - $let: + extraSystemSetup: { $eval: strip(str(build.system_setup)) } + taskIndexExpire: { $fromNow: '6 months' } + in: > + (apt-get -qq -y remove --purge ubuntu-advantage-tools || true) && + apt-get -qq update && apt-get -qq -y install curl git && ${extraSystemSetup}; + cache_artifact=`curl -sSIL -o /dev/null -w "%{http_code}" ${build.cache.artifact_url}` && + if [ "$cache_artifact" != "200" ]; then + adduser --system --home ${system.homedir.linux} ${system.username} && cd ${system.homedir.linux}/ && + mkdir -p /tmp/artifacts/ && chmod 777 /tmp/artifacts && + echo -e "#!/bin/bash\nset -xe\n env && id && (git clone --quiet ${build.build_or_cache.repo} ~/${build.build_or_cache.dir}/ && cd ~/${build.build_or_cache.dir}/ && git checkout --quiet ${build.build_or_cache.sha})" > /tmp/clone.sh && chmod +x /tmp/clone.sh && + sudo -H -u ${system.username} /bin/bash /tmp/clone.sh && + sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/${build.build_or_cache.dir}/${build.scripts.build} && + sudo -H -u ${system.username} /bin/bash ${system.homedir.linux}/${build.build_or_cache.dir}/${build.scripts.package} && + sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/${build.build_or_cache.dir}/taskcluster/tc-update-index.sh ${taskIndexExpire} taskcluster ${build.cache.artifact_namespace} + fi; + + artifacts: + "public": + type: "directory" + path: "/tmp/artifacts/" + expires: { $fromNow: '6 months' } + +metadata: + name: ${build.metadata.name} + description: ${build.metadata.description} + owner: ${event.head.user.email} + source: ${event.head.repo.url} diff --git a/taskcluster/generic_tc_caching-win-opt-base.tyml b/taskcluster/generic_tc_caching-win-opt-base.tyml new file mode 100644 index 00000000..a4110318 --- /dev/null +++ b/taskcluster/generic_tc_caching-win-opt-base.tyml @@ -0,0 +1,57 @@ +taskId: ${taskcluster.taskId} +provisionerId: ${taskcluster.docker.provisionerId} +workerType: ${taskcluster.docker.workerTypeWin} +taskGroupId: ${taskcluster.taskGroupId} +schedulerId: ${taskcluster.schedulerId} +created: { $fromNow: '0 sec' } +deadline: { $fromNow: '1 day' } +expires: { $fromNow: '6 months' } +scopes: + - "index:insert-task:project.deepspeech.*" + +payload: + maxRunTime: { $eval: to_int(build.maxRunTime) } + + features: + taskclusterProxy: true + + mounts: + - file: msys2-base-x86_64.tar.xz + content: + sha256: c4443113497acb2d2e285d40b929fc55f33f8f669902595ecdf66a655b63dc60 + url: >- + https://github.com/msys2/msys2-installer/releases/download/2020-05-17/msys2-base-x86_64-20200517.tar.xz + + env: + TC_MSYS_VERSION: 'MSYS_NT-6.3-9600' + MSYS: 'winsymlinks:nativestrict' + + command: + - >- + "C:\Program Files\7-zip\7z.exe" x -txz -so msys2-base-x86_64.tar.xz | + "C:\Program Files\7-zip\7z.exe" x -o%USERPROFILE% -ttar -aoa -si + - .\msys64\usr\bin\bash.exe --login -cx "export THIS_BASH_PID=$$; ps -ef | grep '[?]' | awk '{print $2}' | grep -v $THIS_BASH_PID | xargs -r kill; exit 0" + - .\msys64\usr\bin\bash.exe --login -cx "pacman -Syu --noconfirm" + - .\msys64\usr\bin\bash.exe --login -cx "pacman -Syu --noconfirm" + - $let: + taskIndexExpire: { $fromNow: '6 months' } + in: > + echo .\msys64\usr\bin\bash.exe --login -cxe "export LC_ALL=C && + export PATH=\"$USERPROFILE/msys64/usr/bin:/c/Python36:/c/Program Files/Git/bin:/c/Program Files/7-Zip/:$PATH\" && + export TASKCLUSTER_ARTIFACTS=\"$(cygpath -u $USERPROFILE/public)\" && + export TASKCLUSTER_TASK_DIR=\"/c/builds/tc-workdir/\" && + echo \"export TASKCLUSTER_TASK_EXIT_CODE=0\" > $USERPROFILE/tc-exit.sh && + env && pacman --noconfirm -R bsdtar && pacman --noconfirm -S tar && mkdir -p $TASKCLUSTER_ARTIFACTS/ && if [ \"`curl -sSIL -o /dev/null -w %%{http_code} ${build.cache.artifact_url}`\" != \"200\" ]; then git clone --quiet ${build.build_or_cache.repo} $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/ && cd $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir} && git checkout --quiet ${build.build_or_cache.sha} && $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/${build.scripts.build} && $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/${build.scripts.package} && $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/taskcluster/tc-update-index.sh ${taskIndexExpire} taskcluster ${build.cache.artifact_namespace}; fi; echo \"export TASKCLUSTER_TASK_EXIT_CODE=$?\" > $USERPROFILE/tc-exit.sh" | cmd /k + + - .\msys64\usr\bin\bash.exe --login -cxe "source $USERPROFILE/tc-exit.sh && exit $TASKCLUSTER_TASK_EXIT_CODE" + + artifacts: + - type: "directory" + path: "public/" + expires: { $fromNow: '6 months' } + +metadata: + name: ${build.metadata.name} + description: ${build.metadata.description} + owner: ${event.head.user.email} + source: ${event.head.repo.url} diff --git a/taskcluster/gradle-cache.yml b/taskcluster/gradle-cache.yml index 334545c3..ef589197 100644 --- a/taskcluster/gradle-cache.yml +++ b/taskcluster/gradle-cache.yml @@ -1,11 +1,11 @@ build: - template_file: android_cache-opt-base.tyml + template_file: generic_tc_caching-linux-opt-base.tyml + cache: + artifact_url: ${system.gradle_cache.url} + artifact_namespace: ${system.gradle_cache.namespace} system_setup: > ${java.packages_trusty.apt} - cache: - url: ${system.gradle_cache.url} - namespace: ${system.gradle_cache.namespace} scripts: build: "taskcluster/gradle-build.sh" package: "taskcluster/gradle-package.sh" diff --git a/taskcluster/gradle-package.sh b/taskcluster/gradle-package.sh index 495c05e8..840fffc8 100755 --- a/taskcluster/gradle-package.sh +++ b/taskcluster/gradle-package.sh @@ -2,17 +2,6 @@ set -xe -TC_EXPIRE=$1 -TC_INSTANCE=$2 -TC_INDEX=$3 - source $(dirname "$0")/tc-tests-utils.sh cd ${GRADLE_USER_HOME}/../ && tar -czf $TASKCLUSTER_ARTIFACTS/gradle.tar.gz gradle-cache/ - -if [ ! -z "${TC_EXPIRE}" -a ! -z "${TC_INSTANCE}" -a ! -z "${TC_INDEX}" ]; then - curl -sSL --fail -X PUT \ - -H "Content-Type: application/json" \ - -d "{\"taskId\":\"$TASK_ID\",\"rank\":0,\"expires\":\"${TC_EXPIRE}\",\"data\":{}}" \ - "http://${TC_INSTANCE}/index/v1/task/${TC_INDEX}" -fi; diff --git a/taskcluster/homebrew-darwin-opt-base.tyml b/taskcluster/homebrew-darwin-opt-base.tyml deleted file mode 100644 index ac6c5af3..00000000 --- a/taskcluster/homebrew-darwin-opt-base.tyml +++ /dev/null @@ -1,59 +0,0 @@ -taskId: ${taskcluster.taskId} -provisionerId: ${taskcluster.generic.provisionerId} -workerType: ${taskcluster.generic.workerType} -taskGroupId: ${taskcluster.taskGroupId} -schedulerId: ${taskcluster.schedulerId} -dependencies: - $map: { $eval: build.dependencies } - each(b): - $eval: as_slugid(b) -created: { $fromNow: '0 sec' } -deadline: { $fromNow: '1 day' } -expires: { $fromNow: '6 months' } -scopes: - - "index:insert-task:project.deepspeech.*" - -payload: - maxRunTime: { $eval: to_int(build.maxRunTime) } - - features: - taskclusterProxy: true - - command: - - - "/bin/bash" - - "--login" - - "-cxe" - - $let: - taskIndexExpire: { $fromNow: '6 months' } - in: > - export TASKCLUSTER_ARTIFACTS="$(pwd)/public/" && - export TASKCLUSTER_ORIG_TASKDIR="$(pwd)" && - (mkdir ../tc-workdir/ || rm -fr ../tc-workdir/*) && cd ../tc-workdir/ && - export TASKCLUSTER_TASK_DIR="$(pwd)" && - export LC_ALL=C && - export MACOSX_DEPLOYMENT_TARGET=10.10 && - export SDKROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX10.14.sdk/ && - env && - mkdir -p $TASKCLUSTER_ARTIFACTS/ && - swig_bin=`curl -sSIL -o /dev/null -w "%{http_code}" ${build.homebrew.url}` && - if [ "$swig_bin" != "200" ]; then - git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ && - cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && - $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.build} && - $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.package} && - curl -sSL --fail -X PUT \ - -H "Content-Type: application/json" \ - -d "{\"taskId\":\"$TASK_ID\",\"rank\":0,\"expires\":\"${taskIndexExpire}\",\"data\":{}}" \ - "http://127.0.0.1:8080/index/v1/task/${build.homebrew.namespace}" - fi; - - artifacts: - - type: "directory" - path: "public/" - expires: { $fromNow: '6 months' } - -metadata: - name: ${build.metadata.name} - description: ${build.metadata.description} - owner: ${event.head.user.email} - source: ${event.head.repo.url} diff --git a/taskcluster/homebrew_builds-darwin-amd64.yml b/taskcluster/homebrew_builds-darwin-amd64.yml index 4dac3a13..3377dcac 100644 --- a/taskcluster/homebrew_builds-darwin-amd64.yml +++ b/taskcluster/homebrew_builds-darwin-amd64.yml @@ -1,8 +1,8 @@ build: - template_file: homebrew-darwin-opt-base.tyml - homebrew: - url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.homebrew_builds.5/artifacts/public/homebrew_builds.tar.gz' - namespace: 'project.deepspeech.homebrew_builds.5' + template_file: generic_tc_caching-darwin-opt-base.tyml + cache: + artifact_url: ${system.homebrew_builds.url} + artifact_namespace: ${system.homebrew_builds.namespace} scripts: build: "taskcluster/homebrew-build.sh --builds" package: "taskcluster/homebrew-package.sh --builds" diff --git a/taskcluster/homebrew_tests-darwin-amd64.yml b/taskcluster/homebrew_tests-darwin-amd64.yml index fc9637f2..dc93c183 100644 --- a/taskcluster/homebrew_tests-darwin-amd64.yml +++ b/taskcluster/homebrew_tests-darwin-amd64.yml @@ -1,8 +1,8 @@ build: - template_file: homebrew-darwin-opt-base.tyml - homebrew: - url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.homebrew_tests.6/artifacts/public/homebrew_tests.tar.gz' - namespace: 'project.deepspeech.homebrew_tests.6' + template_file: generic_tc_caching-darwin-opt-base.tyml + cache: + artifact_url: ${system.homebrew_tests.url} + artifact_namespace: ${system.homebrew_tests.namespace} scripts: build: "taskcluster/homebrew-build.sh --tests" package: "taskcluster/homebrew-package.sh --tests" diff --git a/taskcluster/node-gyp-cache-base.tyml b/taskcluster/node-gyp-cache-base.tyml deleted file mode 100644 index 5c0e895d..00000000 --- a/taskcluster/node-gyp-cache-base.tyml +++ /dev/null @@ -1,57 +0,0 @@ -$if: 'event.event in build.allowed' -then: - taskId: ${taskcluster.taskId} - provisionerId: ${taskcluster.docker.provisionerId} - workerType: ${taskcluster.docker.workerType} - taskGroupId: ${taskcluster.taskGroupId} - schedulerId: ${taskcluster.schedulerId} - created: { $fromNow: '0 sec' } - deadline: { $fromNow: '1 day' } - expires: { $fromNow: '6 months' } - scopes: - - "index:insert-task:project.deepspeech.*" - - payload: - maxRunTime: { $eval: to_int(build.maxRunTime) } - image: "node:12" - - features: - taskclusterProxy: true - - # This task will inspect system.node_gyp_cache taskcluster index existence: - # - if the artifact does not exists, it will build it - # - if the artifact exists, it will re-mirror it (if we don't do that, new - # index gets published with no artifact and erases existing one) - command: - - "/bin/bash" - - "--login" - - "-cxe" - - $let: - extraSystemSetup: { $eval: strip(str(build.system_setup)) } - taskIndexExpire: { $fromNow: '6 months' } - in: > - apt-get -qq update && apt-get -qq -y install curl git && ${extraSystemSetup}; - cache_file=`curl -sSIL -o /dev/null -w "%{http_code}" ${system.node_gyp_cache.url}` && - if [ "$cache_file" != "200" ]; then - mkdir -p ~/DeepSpeech/ds/ && - git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && - cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && - ~/DeepSpeech/ds/${build.scripts.build} && - ~/DeepSpeech/ds/${build.scripts.package} && - curl -sSL --fail -X PUT \ - -H "Content-Type: application/json" \ - -d "{\"taskId\":\"$TASK_ID\",\"rank\":0,\"expires\":\"${taskIndexExpire}\",\"data\":{}}" \ - "http://taskcluster/index/v1/task/${system.node_gyp_cache.namespace}" - fi; - - artifacts: - "public": - type: "directory" - path: "/tmp/artifacts/" - expires: { $fromNow: '6 months' } - - metadata: - name: ${build.metadata.name} - description: ${build.metadata.description} - owner: ${event.head.user.email} - source: ${event.head.repo.url} diff --git a/taskcluster/node-gyp-cache.yml b/taskcluster/node-gyp-cache.yml index a9be39de..a0c48ba5 100644 --- a/taskcluster/node-gyp-cache.yml +++ b/taskcluster/node-gyp-cache.yml @@ -1,5 +1,12 @@ build: - template_file: node-gyp-cache-base.tyml + template_file: generic_tc_caching-linux-opt-base.tyml + docker_image: "node:12" + cache: + artifact_url: "${system.node_gyp_cache.url}" + artifact_namespace: "${system.node_gyp_cache.namespace}" + system_setup: + > + (apt-get -qq -y install sudo || true) scripts: build: "taskcluster/node-gyp-populate.sh" package: "taskcluster/node-gyp-package.sh" diff --git a/taskcluster/node-gyp-populate.sh b/taskcluster/node-gyp-populate.sh index 7043d332..333c692a 100755 --- a/taskcluster/node-gyp-populate.sh +++ b/taskcluster/node-gyp-populate.sh @@ -8,7 +8,9 @@ node --version npm --version -npm install -g node-gyp@6.x +npm install node-gyp@6.x + +export PATH=$HOME/node_modules/.bin/:$PATH devDir=$DS_ROOT_TASK/node-gyp-cache/ diff --git a/taskcluster/pyenv-darwin-amd64.yml b/taskcluster/pyenv-darwin-amd64.yml index 47cceec9..3ad055ca 100644 --- a/taskcluster/pyenv-darwin-amd64.yml +++ b/taskcluster/pyenv-darwin-amd64.yml @@ -1,5 +1,8 @@ build: - template_file: pyenv-darwin-opt-base.tyml + template_file: generic_tc_caching-darwin-opt-base.tyml + cache: + artifact_url: ${system.pyenv.osx.url} + artifact_namespace: ${system.pyenv.osx.namespace} scripts: build: "taskcluster/pyenv-build.sh" package: "taskcluster/pyenv-package.sh" diff --git a/taskcluster/pyenv-linux-amd64.yml b/taskcluster/pyenv-linux-amd64.yml index 393a8872..a8ece0b4 100644 --- a/taskcluster/pyenv-linux-amd64.yml +++ b/taskcluster/pyenv-linux-amd64.yml @@ -1,5 +1,8 @@ build: - template_file: pyenv-linux-opt-base.tyml + template_file: generic_tc_caching-linux-opt-base.tyml + cache: + artifact_url: "${system.pyenv.linux.url}" + artifact_namespace: "${system.pyenv.linux.namespace}" system_setup: > apt-get -qq update && apt-get -qq -y install python-yaml ${python.packages_trusty.apt} wget diff --git a/taskcluster/pyenv-linux-opt-base.tyml b/taskcluster/pyenv-linux-opt-base.tyml deleted file mode 100644 index 15d8f7b0..00000000 --- a/taskcluster/pyenv-linux-opt-base.tyml +++ /dev/null @@ -1,52 +0,0 @@ -$if: 'event.event in build.allowed' -then: - taskId: ${taskcluster.taskId} - provisionerId: ${taskcluster.docker.provisionerId} - workerType: ${taskcluster.docker.workerType} - taskGroupId: ${taskcluster.taskGroupId} - schedulerId: ${taskcluster.schedulerId} - created: { $fromNow: '0 sec' } - deadline: { $fromNow: '1 day' } - expires: { $fromNow: '6 months' } - scopes: - - "index:insert-task:project.deepspeech.*" - - payload: - maxRunTime: { $eval: to_int(build.maxRunTime) } - image: ${build.docker_image} - - features: - taskclusterProxy: true - - command: - - "/bin/bash" - - "--login" - - "-cxe" - - $let: - extraSystemSetup: { $eval: strip(str(build.system_setup)) } - taskIndexExpire: { $fromNow: '6 months' } - in: > - (apt-get -qq -y remove --purge ubuntu-advantage-tools || true) && - apt-get -qq update && apt-get -qq -y install curl git; - swig_bin=`curl -sSIL -o /dev/null -w "%{http_code}" ${system.pyenv.linux.url}` && - if [ "$swig_bin" != "200" ]; then - ${extraSystemSetup} && - adduser --system --home ${system.homedir.linux} ${system.username} && cd ${system.homedir.linux}/ && - mkdir -p /tmp/artifacts/ && chmod 777 /tmp/artifacts && - echo -e "#!/bin/bash\nset -xe\n env && id && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha}" > /tmp/clone.sh && chmod +x /tmp/clone.sh && - sudo -H -u ${system.username} /bin/bash /tmp/clone.sh && - sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.build} && - sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.package} ${taskIndexExpire} taskcluster ${system.pyenv.linux.namespace} - fi; - - artifacts: - "public": - type: "directory" - path: "/tmp/artifacts/" - expires: { $fromNow: '6 months' } - - metadata: - name: ${build.metadata.name} - description: ${build.metadata.description} - owner: ${event.head.user.email} - source: ${event.head.repo.url} diff --git a/taskcluster/pyenv-package.sh b/taskcluster/pyenv-package.sh index bc2c0639..73f6a542 100755 --- a/taskcluster/pyenv-package.sh +++ b/taskcluster/pyenv-package.sh @@ -2,17 +2,6 @@ set -xe -TC_EXPIRE=$1 -TC_INSTANCE=$2 -TC_INDEX=$3 - source $(dirname "$0")/tc-tests-utils.sh cd ${PYENV_ROOT}/ && $TAR -czf $TASKCLUSTER_ARTIFACTS/pyenv.tar.gz . - -if [ ! -z "${TC_EXPIRE}" -a ! -z "${TC_INSTANCE}" -a ! -z "${TC_INDEX}" ]; then - curl -sSL --fail -X PUT \ - -H "Content-Type: application/json" \ - -d "{\"taskId\":\"$TASK_ID\",\"rank\":0,\"expires\":\"${TC_EXPIRE}\",\"data\":{}}" \ - "http://${TC_INSTANCE}/index/v1/task/${TC_INDEX}" -fi; diff --git a/taskcluster/pyenv-win-amd64.yml b/taskcluster/pyenv-win-amd64.yml index d22872b3..7d05f141 100644 --- a/taskcluster/pyenv-win-amd64.yml +++ b/taskcluster/pyenv-win-amd64.yml @@ -1,5 +1,8 @@ build: - template_file: pyenv-win-opt-base.tyml + template_file: generic_tc_caching-win-opt-base.tyml + cache: + artifact_url: "${system.pyenv.win.url}" + artifact_namespace: "${system.pyenv.win.namespace}" scripts: build: "taskcluster/pyenv-build.sh" package: "taskcluster/pyenv-package.sh" diff --git a/taskcluster/pyenv-win-opt-base.tyml b/taskcluster/pyenv-win-opt-base.tyml deleted file mode 100644 index 59c584ba..00000000 --- a/taskcluster/pyenv-win-opt-base.tyml +++ /dev/null @@ -1,59 +0,0 @@ -$if: 'event.event in build.allowed' -then: - taskId: ${taskcluster.taskId} - provisionerId: ${taskcluster.docker.provisionerId} - workerType: ${taskcluster.docker.workerTypeWin} - taskGroupId: ${taskcluster.taskGroupId} - schedulerId: ${taskcluster.schedulerId} - created: { $fromNow: '0 sec' } - deadline: { $fromNow: '1 day' } - expires: { $fromNow: '6 months' } - scopes: - - "index:insert-task:project.deepspeech.*" - - payload: - maxRunTime: { $eval: to_int(build.maxRunTime) } - - features: - taskclusterProxy: true - - mounts: - - file: msys2-base-x86_64.tar.xz - content: - sha256: c4443113497acb2d2e285d40b929fc55f33f8f669902595ecdf66a655b63dc60 - url: >- - https://github.com/msys2/msys2-installer/releases/download/2020-05-17/msys2-base-x86_64-20200517.tar.xz - - env: - TC_MSYS_VERSION: 'MSYS_NT-6.3-9600' - MSYS: 'winsymlinks:nativestrict' - - command: - - >- - "C:\Program Files\7-zip\7z.exe" x -txz -so msys2-base-x86_64.tar.xz | - "C:\Program Files\7-zip\7z.exe" x -o%USERPROFILE% -ttar -aoa -si - - .\msys64\usr\bin\bash.exe --login -cx "export THIS_BASH_PID=$$; ps -ef | grep '[?]' | awk '{print $2}' | grep -v $THIS_BASH_PID | xargs -r kill; exit 0" - - .\msys64\usr\bin\bash.exe --login -cx "pacman -Syu --noconfirm" - - .\msys64\usr\bin\bash.exe --login -cx "pacman -Syu --noconfirm" - - $let: - taskIndexExpire: { $fromNow: '6 months' } - in: > - echo .\msys64\usr\bin\bash.exe --login -cxe "export LC_ALL=C && - export PATH=\"$USERPROFILE/msys64/usr/bin:/c/Python36:/c/Program Files/Git/bin:/c/Program Files/7-Zip/:$PATH\" && - export TASKCLUSTER_ARTIFACTS=\"$(cygpath -u $USERPROFILE/public)\" && - export TASKCLUSTER_TASK_DIR=\"/c/builds/tc-workdir/\" && - echo \"export TASKCLUSTER_TASK_EXIT_CODE=0\" > $USERPROFILE/tc-exit.sh && - env && pacman --noconfirm -R bsdtar && pacman --noconfirm -S tar && mkdir -p $TASKCLUSTER_ARTIFACTS/ && if [ \"`curl -sSIL -o /dev/null -w %%{http_code} ${system.pyenv.win.url}`\" != \"200\" ]; then git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ && cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.build} && $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.package} ${taskIndexExpire} taskcluster ${system.pyenv.win.namespace}; fi; echo \"export TASKCLUSTER_TASK_EXIT_CODE=$?\" > $USERPROFILE/tc-exit.sh" | cmd /k - - - .\msys64\usr\bin\bash.exe --login -cxe "source $USERPROFILE/tc-exit.sh && exit $TASKCLUSTER_TASK_EXIT_CODE" - - artifacts: - - type: "directory" - path: "public/" - expires: { $fromNow: '6 months' } - - metadata: - name: ${build.metadata.name} - description: ${build.metadata.description} - owner: ${event.head.user.email} - source: ${event.head.repo.url} diff --git a/taskcluster/swig-darwin-amd64.yml b/taskcluster/swig-darwin-amd64.yml index cb08cdef..64ca794b 100644 --- a/taskcluster/swig-darwin-amd64.yml +++ b/taskcluster/swig-darwin-amd64.yml @@ -1,7 +1,12 @@ build: - template_file: swig-darwin-opt-base.tyml - swig_system: "darwin" - swig_arch: "amd64" + template_file: generic_tc_caching-darwin-opt-base.tyml + build_or_cache: + repo: "${system.swig.repo}" + sha: "${system.swig.sha1}" + dir: "swig" + cache: + artifact_url: "${system.swig_build.osx.url}" + artifact_namespace: "${system.swig_build.osx.namespace}" scripts: build: "taskcluster/build.sh" package: "taskcluster/package.sh" diff --git a/taskcluster/swig-darwin-opt-base.tyml b/taskcluster/swig-darwin-opt-base.tyml deleted file mode 100644 index 0f08e596..00000000 --- a/taskcluster/swig-darwin-opt-base.tyml +++ /dev/null @@ -1,56 +0,0 @@ -taskId: ${taskcluster.taskId} -provisionerId: ${taskcluster.generic.provisionerId} -workerType: ${taskcluster.generic.workerType} -taskGroupId: ${taskcluster.taskGroupId} -schedulerId: ${taskcluster.schedulerId} -dependencies: - $map: { $eval: build.dependencies } - each(b): - $eval: as_slugid(b) -created: { $fromNow: '0 sec' } -deadline: { $fromNow: '1 day' } -expires: { $fromNow: '6 months' } -scopes: - - "index:insert-task:project.deepspeech.*" - -payload: - maxRunTime: { $eval: to_int(build.maxRunTime) } - - command: - - - "/bin/bash" - - "--login" - - "-cxe" - - $let: - taskIndexExpire: { $fromNow: '6 months' } - in: > - export TASKCLUSTER_ARTIFACTS="$(pwd)/public/" && - export TASKCLUSTER_TASKDIR="$(pwd)" && - export TASKCLUSTER_ORIG_TASKDIR="$(pwd)" && - export LC_ALL=C && - export MACOSX_DEPLOYMENT_TARGET=10.10 && - export SDKROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX10.14.sdk/ && - export HOMEBREW_NO_AUTO_UPDATE=1 && - env && - mkdir -p $TASKCLUSTER_ARTIFACTS/ && - swig_bin=`curl -sSIL -o /dev/null -w "%{http_code}" https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.${build.swig_system}.${build.swig_arch}.${system.swig.sha1}/artifacts/public/ds-swig.tar.gz` && - if [ "$swig_bin" != "200" ]; then - git clone --quiet ${system.swig.repo} $TASKCLUSTER_TASKDIR/swig/ && - cd $TASKCLUSTER_TASKDIR/swig/ && git checkout --quiet ${system.swig.sha1} && - $TASKCLUSTER_TASKDIR/swig/${build.scripts.build} && - $TASKCLUSTER_TASKDIR/swig/${build.scripts.package} && - curl -sSL --fail -X PUT \ - -H "Content-Type: application/json" \ - -d "{\"taskId\":\"$TASK_ID\",\"rank\":0,\"expires\":\"${taskIndexExpire}\",\"data\":{}}" \ - "http://taskcluster/index/v1/task/project.deepspeech.swig.${build.swig_system}.${build.swig_arch}.${system.swig.sha1}" - fi; - - artifacts: - - type: "directory" - path: "public/" - expires: { $fromNow: '6 months' } - -metadata: - name: ${build.metadata.name} - description: ${build.metadata.description} - owner: ${event.head.user.email} - source: ${event.head.repo.url} diff --git a/taskcluster/swig-linux-amd64.yml b/taskcluster/swig-linux-amd64.yml index 27bd058c..3b199c1a 100644 --- a/taskcluster/swig-linux-amd64.yml +++ b/taskcluster/swig-linux-amd64.yml @@ -1,8 +1,13 @@ build: - template_file: swig-linux-opt-base.tyml + template_file: generic_tc_caching-linux-opt-base.tyml docker_image: "ubuntu:14.04" - swig_system: "linux" - swig_arch: "amd64" + build_or_cache: + repo: "${system.swig.repo}" + sha: "${system.swig.sha1}" + dir: "swig" + cache: + artifact_url: "${system.swig_build.linux.url}" + artifact_namespace: "${system.swig_build.linux.namespace}" system_setup: > apt-get -qq -y install autoconf automake bison build-essential diff --git a/taskcluster/swig-linux-opt-base.tyml b/taskcluster/swig-linux-opt-base.tyml deleted file mode 100644 index fb3662e4..00000000 --- a/taskcluster/swig-linux-opt-base.tyml +++ /dev/null @@ -1,54 +0,0 @@ -$if: 'event.event in build.allowed' -then: - taskId: ${taskcluster.taskId} - provisionerId: ${taskcluster.docker.provisionerId} - workerType: ${taskcluster.docker.workerType} - taskGroupId: ${taskcluster.taskGroupId} - schedulerId: ${taskcluster.schedulerId} - created: { $fromNow: '0 sec' } - deadline: { $fromNow: '1 day' } - expires: { $fromNow: '6 months' } - scopes: - - "index:insert-task:project.deepspeech.*" - - payload: - maxRunTime: { $eval: to_int(build.maxRunTime) } - image: ${build.docker_image} - - features: - taskclusterProxy: true - - command: - - "/bin/bash" - - "--login" - - "-cxe" - - $let: - extraSystemSetup: { $eval: strip(str(build.system_setup)) } - taskIndexExpire: { $fromNow: '6 months' } - in: > - (apt-get -qq -y remove --purge ubuntu-advantage-tools || true) && - apt-get -qq update && apt-get -qq -y install curl git && ${extraSystemSetup}; - swig_bin=`curl -sSIL -o /dev/null -w "%{http_code}" https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.${build.swig_system}.${build.swig_arch}.${system.swig.sha1}/artifacts/public/ds-swig.tar.gz` && - if [ "$swig_bin" != "200" ]; then - adduser --system --home ${system.homedir.linux} ${system.username} && cd ${system.homedir.linux}/ && - echo -e "#!/bin/bash\nset -xe\n env && id && (git clone --quiet ${system.swig.repo} ~/swig/ && cd ~/swig/ && git checkout --quiet ${system.swig.sha1})" > /tmp/clone.sh && chmod +x /tmp/clone.sh && - sudo -H -u ${system.username} /bin/bash /tmp/clone.sh && - sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/swig/${build.scripts.build} && - sudo -H -u ${system.username} /bin/bash ${system.homedir.linux}/swig/${build.scripts.package} && - curl -sSL --fail -X PUT \ - -H "Content-Type: application/json" \ - -d "{\"taskId\":\"$TASK_ID\",\"rank\":0,\"expires\":\"${taskIndexExpire}\",\"data\":{}}" \ - "http://taskcluster/index/v1/task/project.deepspeech.swig.${build.swig_system}.${build.swig_arch}.${system.swig.sha1}" - fi; - - artifacts: - "public": - type: "directory" - path: "/tmp/artifacts/" - expires: { $fromNow: '6 months' } - - metadata: - name: ${build.metadata.name} - description: ${build.metadata.description} - owner: ${event.head.user.email} - source: ${event.head.repo.url} diff --git a/taskcluster/swig-win-amd64.yml b/taskcluster/swig-win-amd64.yml index 576bdf73..310a8112 100644 --- a/taskcluster/swig-win-amd64.yml +++ b/taskcluster/swig-win-amd64.yml @@ -1,8 +1,13 @@ build: - template_file: swig-linux-opt-base.tyml + template_file: generic_tc_caching-linux-opt-base.tyml docker_image: "ubuntu:18.04" - swig_system: "win" - swig_arch: "amd64" + build_or_cache: + repo: "${system.swig.repo}" + sha: "${system.swig.sha1}" + dir: "swig" + cache: + artifact_url: "${system.swig_build.win.url}" + artifact_namespace: "${system.swig_build.win.namespace}" system_setup: > apt-get -qq -y install autoconf automake bison build-essential mingw-w64 && diff --git a/taskcluster/tc-update-index.sh b/taskcluster/tc-update-index.sh new file mode 100755 index 00000000..fbc72316 --- /dev/null +++ b/taskcluster/tc-update-index.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Helper script because it is way too painful to deal with Windows' CMD.exe +# ways of escaping things when pushing JSON + +set -xe + +TC_EXPIRE=$1 +TC_INSTANCE=$2 +TC_INDEX=$3 + +source $(dirname "$0")/tc-tests-utils.sh + +if [ ! -z "${TC_EXPIRE}" -a ! -z "${TC_INSTANCE}" -a ! -z "${TC_INDEX}" ]; then + curl -sSL --fail -X PUT \ + -H "Content-Type: application/json" \ + -d "{\"taskId\":\"$TASK_ID\",\"rank\":0,\"expires\":\"${TC_EXPIRE}\",\"data\":{}}" \ + "http://${TC_INSTANCE}/index/v1/task/${TC_INDEX}" +fi; From b52139ceb6ef54036b06a7140cf565178f21f654 Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Tue, 16 Jun 2020 23:50:04 +0200 Subject: [PATCH 2/4] Fix #3075: Add Android 11 to CI --- taskcluster/.shared.yml | 3 +++ .../android-cache-x86_64-android-30.yml | 14 +++++++++++ .../test-apk-android-30-x86_64-opt.yml | 23 +++++++++++++++++++ 3 files changed, 40 insertions(+) create mode 100644 taskcluster/android-cache-x86_64-android-30.yml create mode 100644 taskcluster/test-apk-android-30-x86_64-opt.yml diff --git a/taskcluster/.shared.yml b/taskcluster/.shared.yml index b04ff9de..5b174b31 100644 --- a/taskcluster/.shared.yml +++ b/taskcluster/.shared.yml @@ -99,6 +99,9 @@ system: android_29: url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.android_cache.x86_64.android-29.0/artifacts/public/android_cache.tar.gz' namespace: 'project.deepspeech.android_cache.x86_64.android-29.0' + android_30: + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.android_cache.x86_64.android-30.0/artifacts/public/android_cache.tar.gz' + namespace: 'project.deepspeech.android_cache.x86_64.android-30.0' sdk: android_27: url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.android_cache.sdk.android-27.4/artifacts/public/android_cache.tar.gz' diff --git a/taskcluster/android-cache-x86_64-android-30.yml b/taskcluster/android-cache-x86_64-android-30.yml new file mode 100644 index 00000000..217459aa --- /dev/null +++ b/taskcluster/android-cache-x86_64-android-30.yml @@ -0,0 +1,14 @@ +build: + template_file: generic_tc_caching-linux-opt-base.tyml + system_setup: + > + ${java.packages_trusty.apt} + cache: + artifact_url: ${system.android_cache.x86_64.android_30.url} + artifact_namespace: ${system.android_cache.x86_64.android_30.namespace} + scripts: + build: "taskcluster/android_cache-build.sh x86_64 android-30" + package: "taskcluster/android_cache-package.sh" + metadata: + name: "Builds Android cache x86_64 / android-30" + description: "Setup an Android SDK / emulator cache for Android / x86_64 android-30" diff --git a/taskcluster/test-apk-android-30-x86_64-opt.yml b/taskcluster/test-apk-android-30-x86_64-opt.yml new file mode 100644 index 00000000..c3ee8265 --- /dev/null +++ b/taskcluster/test-apk-android-30-x86_64-opt.yml @@ -0,0 +1,23 @@ +build: + template_file: test-android-opt-base.tyml + dependencies: + - "android-x86_64-cpu-opt" + - "test-training_16k-linux-amd64-py36m-opt" + - "swig-linux-amd64" + - "gradle-cache" + - "android-cache-x86_64-android-30" + test_model_task: "test-training_16k-linux-amd64-py36m-opt" + system_setup: + > + apt-get -qq -y install curl make python + cache: + url: ${system.android_cache.x86_64.android_30.url} + namespace: ${system.android_cache.x86_64.android_30.namespace} + gradle_cache: + url: ${system.gradle_cache.url} + namespace: ${system.gradle_cache.namespace} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-android-apk-tests.sh x86_64 android-30" + metadata: + name: "DeepSpeech Android 11.0 x86_64 Google Pixel APK/Java tests" + description: "Testing DeepSpeech APK/Java for Android 11.0 x86_64 Google Pixel, optimized version" From 0fd28cfbdf0bd2add0a4467ade81efef5a2812ce Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Tue, 16 Jun 2020 23:10:51 +0200 Subject: [PATCH 3/4] Updating caches --- taskcluster/.shared.yml | 76 ++++++++++++++++++++--------------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/taskcluster/.shared.yml b/taskcluster/.shared.yml index 5b174b31..93679741 100644 --- a/taskcluster/.shared.yml +++ b/taskcluster/.shared.yml @@ -57,68 +57,68 @@ nodejs: prep_14: '/usr/bin/wget.exe https://nodejs.org/dist/v14.3.0/node-v14.3.0-win-x64.zip && ""C:\Program Files\7-zip\7z.exe"" x -o$TASKCLUSTER_NODE_DIR -tzip -aoa node-v14.3.0-win-x64.zip && rm node-*.zip && export PATH=$TASKCLUSTER_TASK_DIR/bin/node-v14.3.0-win-x64/:$PATH' system: node_gyp_cache: - url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.node-gyp-cache.4/artifacts/public/node-gyp-cache.tar.gz' - namespace: 'project.deepspeech.node-gyp-cache.4' + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.node-gyp-cache.6/artifacts/public/node-gyp-cache.tar.gz' + namespace: 'project.deepspeech.node-gyp-cache.6' homebrew_builds: - url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.homebrew_builds.5/artifacts/public/homebrew_builds.tar.gz' - namespace: 'project.deepspeech.homebrew_builds.5' + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.homebrew_builds.7/artifacts/public/homebrew_builds.tar.gz' + namespace: 'project.deepspeech.homebrew_builds.7' homebrew_tests: - url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.homebrew_tests.6/artifacts/public/homebrew_tests.tar.gz' - namespace: 'project.deepspeech.homebrew_tests.6' + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.homebrew_tests.8/artifacts/public/homebrew_tests.tar.gz' + namespace: 'project.deepspeech.homebrew_tests.8' android_cache: arm64_v8a: android_24: - url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.android_cache.arm64-v8a.android-24.4/artifacts/public/android_cache.tar.gz' - namespace: 'project.deepspeech.android_cache.arm64-v8a.android-24.4' + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.android_cache.arm64-v8a.android-24.6/artifacts/public/android_cache.tar.gz' + namespace: 'project.deepspeech.android_cache.arm64-v8a.android-24.6' android_25: - url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.android_cache.arm64-v8a.android-25.4/artifacts/public/android_cache.tar.gz' - namespace: 'project.deepspeech.android_cache.arm64-v8a.android-25.4' + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.android_cache.arm64-v8a.android-25.6/artifacts/public/android_cache.tar.gz' + namespace: 'project.deepspeech.android_cache.arm64-v8a.android-25.6' armeabi_v7a: android_24: - url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.android_cache.armeabi-v7a.android-24.4/artifacts/public/android_cache.tar.gz' - namespace: 'project.deepspeech.android_cache.armeabi-v7a.android-24.4' + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.android_cache.armeabi-v7a.android-24.6/artifacts/public/android_cache.tar.gz' + namespace: 'project.deepspeech.android_cache.armeabi-v7a.android-24.6' android_25: - url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.android_cache.armeabi-v7a.android-25.4/artifacts/public/android_cache.tar.gz' - namespace: 'project.deepspeech.android_cache.armeabi-v7a.android-25.4' + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.android_cache.armeabi-v7a.android-25.6/artifacts/public/android_cache.tar.gz' + namespace: 'project.deepspeech.android_cache.armeabi-v7a.android-25.6' x86_64: android_24: - url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.android_cache.x86_64.android-24.4/artifacts/public/android_cache.tar.gz' - namespace: 'project.deepspeech.android_cache.x86_64.android-24.4' + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.android_cache.x86_64.android-24.6/artifacts/public/android_cache.tar.gz' + namespace: 'project.deepspeech.android_cache.x86_64.android-24.6' android_25: - url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.android_cache.x86_64.android-25.4/artifacts/public/android_cache.tar.gz' - namespace: 'project.deepspeech.android_cache.x86_64.android-25.4' + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.android_cache.x86_64.android-25.6/artifacts/public/android_cache.tar.gz' + namespace: 'project.deepspeech.android_cache.x86_64.android-25.6' android_26: - url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.android_cache.x86_64.android-26.0/artifacts/public/android_cache.tar.gz' - namespace: 'project.deepspeech.android_cache.x86_64.android-26.0' + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.android_cache.x86_64.android-26.2/artifacts/public/android_cache.tar.gz' + namespace: 'project.deepspeech.android_cache.x86_64.android-26.2' android_27: - url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.android_cache.x86_64.android-27.0/artifacts/public/android_cache.tar.gz' - namespace: 'project.deepspeech.android_cache.x86_64.android-27.0' + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.android_cache.x86_64.android-27.2/artifacts/public/android_cache.tar.gz' + namespace: 'project.deepspeech.android_cache.x86_64.android-27.2' android_28: - url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.android_cache.x86_64.android-28.0/artifacts/public/android_cache.tar.gz' - namespace: 'project.deepspeech.android_cache.x86_64.android-28.0' + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.android_cache.x86_64.android-28.2/artifacts/public/android_cache.tar.gz' + namespace: 'project.deepspeech.android_cache.x86_64.android-28.2' android_29: - url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.android_cache.x86_64.android-29.0/artifacts/public/android_cache.tar.gz' - namespace: 'project.deepspeech.android_cache.x86_64.android-29.0' + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.android_cache.x86_64.android-29.2/artifacts/public/android_cache.tar.gz' + namespace: 'project.deepspeech.android_cache.x86_64.android-29.2' android_30: - url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.android_cache.x86_64.android-30.0/artifacts/public/android_cache.tar.gz' - namespace: 'project.deepspeech.android_cache.x86_64.android-30.0' + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.android_cache.x86_64.android-30.2/artifacts/public/android_cache.tar.gz' + namespace: 'project.deepspeech.android_cache.x86_64.android-30.2' sdk: android_27: - url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.android_cache.sdk.android-27.4/artifacts/public/android_cache.tar.gz' - namespace: 'project.deepspeech.android_cache.sdk.android-27.4' + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.android_cache.sdk.android-27.6/artifacts/public/android_cache.tar.gz' + namespace: 'project.deepspeech.android_cache.sdk.android-27.6' gradle_cache: - url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.gradle.4/artifacts/public/gradle.tar.gz' - namespace: 'project.deepspeech.gradle.4' + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.gradle.6/artifacts/public/gradle.tar.gz' + namespace: 'project.deepspeech.gradle.6' pyenv: linux: - url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.pyenv.linux.5/artifacts/public/pyenv.tar.gz' - namespace: 'project.deepspeech.pyenv.linux.5' + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.pyenv.linux.7/artifacts/public/pyenv.tar.gz' + namespace: 'project.deepspeech.pyenv.linux.7' osx: - url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.pyenv.osx.5/artifacts/public/pyenv.tar.gz' - namespace: 'project.deepspeech.pyenv.osx.5' + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.pyenv.osx.7/artifacts/public/pyenv.tar.gz' + namespace: 'project.deepspeech.pyenv.osx.7' win: - url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.pyenv.win.5/artifacts/public/pyenv.tar.gz' - namespace: 'project.deepspeech.pyenv.win.5' + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.pyenv.win.7/artifacts/public/pyenv.tar.gz' + namespace: 'project.deepspeech.pyenv.win.7' swig: repo: "https://github.com/lissyx/swig" sha1: "b5fea54d39832d1d132d7dd921b69c0c2c9d5118" From a47c9a2b8c847a621abbb46ed892b518f118f20e Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Tue, 16 Jun 2020 23:44:00 +0200 Subject: [PATCH 4/4] Request android default instead of google_apis It seems some armv7a image disappeared --- taskcluster/android-cache-armeabi-v7a-android-24.yml | 2 +- taskcluster/android_cache-build.sh | 3 ++- taskcluster/tc-android-utils.sh | 9 +++++++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/taskcluster/android-cache-armeabi-v7a-android-24.yml b/taskcluster/android-cache-armeabi-v7a-android-24.yml index 3536ab3f..4c686739 100644 --- a/taskcluster/android-cache-armeabi-v7a-android-24.yml +++ b/taskcluster/android-cache-armeabi-v7a-android-24.yml @@ -7,7 +7,7 @@ build: artifact_url: ${system.android_cache.armeabi_v7a.android_24.url} artifact_namespace: ${system.android_cache.armeabi_v7a.android_24.namespace} scripts: - build: "taskcluster/android_cache-build.sh armeabi-v7a android-24" + build: "taskcluster/android_cache-build.sh armeabi-v7a android-24 default" package: "taskcluster/android_cache-package.sh" metadata: name: "Builds Android cache armeabi-v7a / android-24" diff --git a/taskcluster/android_cache-build.sh b/taskcluster/android_cache-build.sh index c22ece47..b1e1b843 100755 --- a/taskcluster/android_cache-build.sh +++ b/taskcluster/android_cache-build.sh @@ -6,6 +6,7 @@ source $(dirname "$0")/tc-tests-utils.sh arm_flavor=$1 api_level=$2 +api_kind=$3 export ANDROID_HOME=${ANDROID_SDK_HOME} @@ -17,5 +18,5 @@ android_install_sdk android_install_sdk_platform "android-27" if [ "${arm_flavor}" != "sdk" ]; then - android_setup_emulator "${arm_flavor}" "${api_level}" + android_setup_emulator "${arm_flavor}" "${api_level}" "${api_kind}" fi; diff --git a/taskcluster/tc-android-utils.sh b/taskcluster/tc-android-utils.sh index a71edfb3..3bf66927 100755 --- a/taskcluster/tc-android-utils.sh +++ b/taskcluster/tc-android-utils.sh @@ -112,6 +112,11 @@ android_setup_emulator() local _flavor=$1 local _api_level=${2:-android-25} + local _api_kind=${3:-google_apis} + + if [ -z "${_api_kind}" ]; then + _api_kind="google_apis" + fi export PATH=${ANDROID_SDK_HOME}/tools/bin/:${ANDROID_SDK_HOME}/platform-tools/:$PATH export DS_BINARY_PREFIX="adb shell LD_LIBRARY_PATH=${ANDROID_TMP_DIR}/ds/ ${ANDROID_TMP_DIR}/ds/" @@ -123,11 +128,11 @@ android_setup_emulator() android_install_sdk_platform "${_api_level}" # Same, yes in case of license - yes | sdkmanager --install "system-images;${_api_level};google_apis;${_flavor}" + yes | sdkmanager --install "system-images;${_api_level};${_api_kind};${_flavor}" android_sdk_accept_licenses - avdmanager create avd --name "${_flavor}-ds-pixel-${_api_level}" --device 17 --package "system-images;${_api_level};google_apis;${_flavor}" + avdmanager create avd --name "${_flavor}-ds-pixel-${_api_level}" --device 17 --package "system-images;${_api_level};${_api_kind};${_flavor}" } android_start_emulator()