From bb1ad0019497492f925c43721a4ccef373b7be66 Mon Sep 17 00:00:00 2001 From: Reuben Morais Date: Thu, 17 Dec 2020 19:03:44 +0000 Subject: [PATCH] Convert to .taskcluster.yml v1 forward TASK_ID, add created and deadline more fixes typo try without TASK_ID fix task templates add missing env vars to tc decision dry runs avoid repetition in .taskcluster and manually forward varibles to tc-decision.py url -> clone_url simulate GITHUB_EVENT separate ref an sha correct pull request actions correct pull request policy --- .taskcluster.yml | 153 +++++++++++++---------- taskcluster/build-python-wheel.tyml | 4 - taskcluster/darwin-opt-base.tyml | 4 - taskcluster/docker-build-base.tyml | 4 - taskcluster/docs.tyml | 4 - taskcluster/examples-base.tyml | 4 - taskcluster/github-events.cyml | 15 --- taskcluster/linux-opt-base.tyml | 4 - taskcluster/node-package-opt-base.tyml | 4 - taskcluster/simple-task.tyml | 6 - taskcluster/tc-schedule.sh | 41 +++++- taskcluster/test-android-opt-base.tyml | 4 - taskcluster/test-armbian-opt-base.tyml | 4 - taskcluster/test-darwin-opt-base.tyml | 4 - taskcluster/test-linux-opt-base.tyml | 4 - taskcluster/test-linux-opt-tag-base.tyml | 4 - taskcluster/test-raspbian-opt-base.tyml | 4 - taskcluster/test-win-cuda-opt-base.tyml | 4 - taskcluster/test-win-opt-base.tyml | 4 - taskcluster/win-opt-base.tyml | 4 - 20 files changed, 126 insertions(+), 153 deletions(-) delete mode 100644 taskcluster/github-events.cyml diff --git a/.taskcluster.yml b/.taskcluster.yml index ab816850..8ebc5cb8 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -1,65 +1,92 @@ -# The version is always required -version: 0 -# Top level metadata is always required -metadata: - name: "DeepSpeech" - description: "DeepSpeech builds" - owner: "{{ event.head.user.email }}" # the user who sent the pr/push e-mail will be inserted here - source: "{{ event.head.repo.url }}" # the repo where the pr came from will be inserted here +version: 1 +policy: + pullRequests: collaborators_quiet tasks: - - provisionerId: "proj-deepspeech" - workerType: "ci-decision-task" - extra: - github: - env: true - events: - - pull_request.opened - - pull_request.synchronize - - pull_request.reopened - - push - - tag - branches: - - master - - scopes: [ - "queue:create-task:highest:proj-deepspeech/*", - "queue:route:index.project.deepspeech.*", - "index:insert-task:project.deepspeech.*", - "queue:scheduler-id:taskcluster-github", - "generic-worker:cache:deepspeech-macos-pyenv", - "docker-worker:capability:device:kvm" - ] - - payload: - maxRunTime: 600 - image: "ubuntu:16.04" - - features: - taskclusterProxy: true - - env: - TC_DECISION_SHA: ef67832e6657f43e139a10f37eb326a7d9d96dad - - command: - - "/bin/bash" - - "--login" - - "-cxe" - - > - echo "deb http://archive.ubuntu.com/ubuntu/ xenial-updates main" > /etc/apt/sources.list.d/xenial-updates.list && - apt-get -qq update && apt-get -qq -y install git python3-pip curl sudo && - adduser --system --home /home/build-user build-user && - cd /home/build-user/ && - echo -e "#!/bin/bash\nset -xe\nenv && id && mkdir ~/DeepSpeech/ && git clone --quiet {{event.head.repo.url}} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet {{event.head.sha}}" > /tmp/clone.sh && chmod +x /tmp/clone.sh && - sudo -H -u build-user /bin/bash /tmp/clone.sh && - sudo -H -u build-user --preserve-env /bin/bash /home/build-user/DeepSpeech/ds/taskcluster/tc-schedule.sh - artifacts: - "public": - type: "directory" - path: "/tmp/artifacts/" - expires: "{{ '7 days' | $fromNow }}" - # Each task also requires explicit metadata + $let: metadata: - name: "DeepSpeech Decision Task" - description: "DeepSpeech Decision Task: triggers everything." - owner: "{{ event.head.user.email }}" - source: "{{ event.head.repo.url }}" + task_id: {$eval: as_slugid("decision_task")} + github: + $if: 'tasks_for == "github-pull-request"' + then: + action: "pull_request.${event.action}" + login: ${event.pull_request.user.login} + ref: ${event.pull_request.head.ref} + sha: ${event.pull_request.head.sha} + clone_url: ${event.pull_request.head.repo.clone_url} + else: + action: + $if: 'event.ref[:10] == "refs/tags/"' + then: "tag" + else: "push" + login: ${event.pusher.name} + ref: ${event.ref} + sha: ${event.after} + clone_url: ${event.repository.clone_url} + in: + $let: + decision_task: + taskId: ${metadata.task_id} + created: {$fromNow: ''} + deadline: {$fromNow: '60 minutes'} + + provisionerId: "proj-deepspeech" + workerType: "ci-decision-task" + + scopes: [ + "queue:create-task:highest:proj-deepspeech/*", + "queue:route:index.project.deepspeech.*", + "index:insert-task:project.deepspeech.*", + "queue:scheduler-id:taskcluster-github", + "generic-worker:cache:deepspeech-macos-pyenv", + "docker-worker:capability:device:kvm" + ] + + payload: + maxRunTime: 600 + image: "ubuntu:16.04" + + features: + taskclusterProxy: true + + env: + TC_DECISION_SHA: ef67832e6657f43e139a10f37eb326a7d9d96dad + + command: + - "/bin/bash" + - "--login" + - "-cxe" + - > + echo "deb http://archive.ubuntu.com/ubuntu/ xenial-updates main" > /etc/apt/sources.list.d/xenial-updates.list && + apt-get -qq update && apt-get -qq -y install git python3-pip curl sudo && + adduser --system --home /home/build-user build-user && + cd /home/build-user/ && + echo -e "#!/bin/bash\nset -xe\nenv && id && mkdir ~/DeepSpeech/ && git clone --quiet ${metadata.github.clone_url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${metadata.github.ref}" > /tmp/clone.sh && chmod +x /tmp/clone.sh && + sudo -H -u build-user /bin/bash /tmp/clone.sh && + sudo -H -u build-user --preserve-env /bin/bash /home/build-user/DeepSpeech/ds/taskcluster/tc-schedule.sh \ + TASK_ID=${metadata.task_id} \ + GITHUB_HEAD_USER_LOGIN=${metadata.github.login} \ + GITHUB_HEAD_USER_EMAIL=${metadata.github.login}@users.noreply.github.com \ + GITHUB_EVENT=${metadata.github.action} \ + GITHUB_HEAD_REPO_URL=${metadata.github.clone_url} \ + GITHUB_HEAD_BRANCH=${metadata.github.ref} \ + GITHUB_HEAD_REF=${metadata.github.ref} \ + GITHUB_HEAD_SHA=${metadata.github.sha} + artifacts: + "public": + type: "directory" + path: "/tmp/artifacts/" + expires: {$fromNow: '7 days'} + + metadata: + name: "DeepSpeech decision task" + description: "DeepSpeech decision task" + owner: "${metadata.github.login}@users.noreply.github.com" + source: "${metadata.github.clone_url}" + in: + $flattenDeep: + - $if: 'tasks_for == "github-pull-request" && event["action"] in ["opened", "reopened", "synchronize"]' + then: {$eval: decision_task} + - $if: 'tasks_for == "github-push" && event.ref == "refs/heads/master"' + then: {$eval: decision_task} + - $if: 'tasks_for == "github-push" && event.ref[:10] == "refs/tags/"' + then: {$eval: decision_task} diff --git a/taskcluster/build-python-wheel.tyml b/taskcluster/build-python-wheel.tyml index e85e60f6..0407285f 100644 --- a/taskcluster/build-python-wheel.tyml +++ b/taskcluster/build-python-wheel.tyml @@ -13,10 +13,6 @@ then: deadline: { $fromNow: '1 day' } expires: { $fromNow: '7 days' } - extra: - github: - { $eval: taskcluster.github_events.pull_request } - payload: maxRunTime: 14400 image: "arm64v8/debian:buster-20190812" diff --git a/taskcluster/darwin-opt-base.tyml b/taskcluster/darwin-opt-base.tyml index bb35cec0..dbccbb34 100644 --- a/taskcluster/darwin-opt-base.tyml +++ b/taskcluster/darwin-opt-base.tyml @@ -16,10 +16,6 @@ expires: extra: nc_asset_name: { $eval: build.nc_asset_name } - github: - $if: '(event.event == "push") || (event.event == "tag")' - then: { $eval: taskcluster.github_events.merge } - else: { $eval: taskcluster.github_events.pull_request } routes: $if: '(event.event == "push") || (event.event == "tag")' diff --git a/taskcluster/docker-build-base.tyml b/taskcluster/docker-build-base.tyml index d416bc0b..eb8bfade 100644 --- a/taskcluster/docker-build-base.tyml +++ b/taskcluster/docker-build-base.tyml @@ -9,10 +9,6 @@ then: deadline: { $fromNow: '1 day' } expires: { $fromNow: '7 days' } - extra: - github: - { $eval: taskcluster.github_events.pull_request } - payload: maxRunTime: { $eval: to_int(build.maxRunTime) } image: "ubuntu:14.04" diff --git a/taskcluster/docs.tyml b/taskcluster/docs.tyml index 9768de9d..2bb64ba2 100644 --- a/taskcluster/docs.tyml +++ b/taskcluster/docs.tyml @@ -18,10 +18,6 @@ then: extra: nc_asset_name: { $eval: build.nc_asset_name } - github: - $if: '(event.event == "push") || (event.event == "tag")' - then: { $eval: taskcluster.github_events.merge } - else: { $eval: taskcluster.github_events.pull_request } routes: $if: '(event.event == "push") || (event.event == "tag")' diff --git a/taskcluster/examples-base.tyml b/taskcluster/examples-base.tyml index 82f3760d..643f8d88 100644 --- a/taskcluster/examples-base.tyml +++ b/taskcluster/examples-base.tyml @@ -13,10 +13,6 @@ then: deadline: { $fromNow: '1 day' } expires: { $fromNow: '7 days' } - extra: - github: - { $eval: taskcluster.github_events.pull_request } - payload: maxRunTime: { $eval: to_int(build.maxRunTime) } image: ${build.docker_image} diff --git a/taskcluster/github-events.cyml b/taskcluster/github-events.cyml deleted file mode 100644 index ba31e2c1..00000000 --- a/taskcluster/github-events.cyml +++ /dev/null @@ -1,15 +0,0 @@ -taskcluster: - github_events: - pull_request: - env: true - events: - - pull_request.opened - - pull_request.synchronize - - pull_request.reopened - merge: - env: true - events: - - push - - tag - branches: - - master diff --git a/taskcluster/linux-opt-base.tyml b/taskcluster/linux-opt-base.tyml index f7d4f7a2..3bc6e6d0 100644 --- a/taskcluster/linux-opt-base.tyml +++ b/taskcluster/linux-opt-base.tyml @@ -18,10 +18,6 @@ then: extra: nc_asset_name: { $eval: build.nc_asset_name } - github: - $if: '(event.event == "push") || (event.event == "tag")' - then: { $eval: taskcluster.github_events.merge } - else: { $eval: taskcluster.github_events.pull_request } routes: $if: '(event.event == "push") || (event.event == "tag")' diff --git a/taskcluster/node-package-opt-base.tyml b/taskcluster/node-package-opt-base.tyml index d2682a28..5632412a 100644 --- a/taskcluster/node-package-opt-base.tyml +++ b/taskcluster/node-package-opt-base.tyml @@ -18,10 +18,6 @@ then: extra: nc_asset_name: { $eval: build.nc_asset_name } - github: - $if: '(event.event == "push") || (event.event == "tag")' - then: { $eval: taskcluster.github_events.merge } - else: { $eval: taskcluster.github_events.pull_request } routes: $if: '(event.event == "push") || (event.event == "tag")' diff --git a/taskcluster/simple-task.tyml b/taskcluster/simple-task.tyml index d4af6616..36ce9649 100644 --- a/taskcluster/simple-task.tyml +++ b/taskcluster/simple-task.tyml @@ -16,12 +16,6 @@ then: then: { $fromNow: '6 months' } else: { $fromNow: '7 days' } - extra: - github: - $if: '(event.event == "push") || (event.event == "tag")' - then: { $eval: taskcluster.github_events.merge } - else: { $eval: taskcluster.github_events.pull_request } - routes: { $eval: build.routes } diff --git a/taskcluster/tc-schedule.sh b/taskcluster/tc-schedule.sh index 5b348041..2a909bf1 100755 --- a/taskcluster/tc-schedule.sh +++ b/taskcluster/tc-schedule.sh @@ -1,7 +1,11 @@ #!/bin/bash - set -ex +# Set environment variables passed in arguments +for line in $@; do + eval "$line" +done + curdir=$(dirname "$0")/ pip3 install --quiet --user --upgrade pip @@ -14,14 +18,41 @@ pip3 install --quiet --user --upgrade -r ${curdir}/tc-decision_reqs.txt # This should help us track merge failures in advance for event in pull_request.opened pull_request.synchronize pull_request.reopened push; do - GITHUB_EVENT="${event}" TASK_ID="aa" GITHUB_HEAD_BRANCHORTAG="branchName" GITHUB_HEAD_REF="refs/heads/branchName" python3 ${curdir}/tc-decision.py --dry + GITHUB_EVENT="${event}" \ + TASK_ID="aa" \ + GITHUB_HEAD_BRANCHORTAG="branchName" \ + GITHUB_HEAD_REF="refs/heads/branchName" \ + GITHUB_HEAD_SHA="branchName" \ + GITHUB_HEAD_REPO_URL="url" \ + GITHUB_HEAD_USER_LOGIN="user" \ + GITHUB_HEAD_USER_EMAIL="email" \ + python3 ${curdir}/tc-decision.py --dry done; -GITHUB_EVENT="tag" TASK_ID="aa" GITHUB_HEAD_BRANCHORTAG="tagName" GITHUB_HEAD_REF="refs/tags/tagName" python3 ${curdir}/tc-decision.py --dry +GITHUB_EVENT="tag" \ +TASK_ID="aa" \ +GITHUB_HEAD_BRANCHORTAG="tagName" \ +GITHUB_HEAD_REF="refs/tags/tagName" \ +GITHUB_HEAD_SHA="branchName" \ +GITHUB_HEAD_REPO_URL="url" \ +GITHUB_HEAD_USER_LOGIN="user" \ +GITHUB_HEAD_USER_EMAIL="email" \ +python3 ${curdir}/tc-decision.py --dry # Create a new env variable for usage in TaskCluster .yml files -export GITHUB_HEAD_BRANCHORTAG="${GITHUB_HEAD_BRANCH}${GITHUB_HEAD_TAG}" +GITHUB_HEAD_BRANCHORTAG="${GITHUB_HEAD_BRANCH}${GITHUB_HEAD_TAG}" # Quick hack because tc-decision uses GITHUB_HEAD_BRANCH -export GITHUB_HEAD_BRANCH="${GITHUB_HEAD_BRANCH}${GITHUB_HEAD_TAG}" +GITHUB_HEAD_BRANCH="${GITHUB_HEAD_BRANCH}${GITHUB_HEAD_TAG}" + +# Forward all variables to tc-decision.py +GITHUB_EVENT="$GITHUB_EVENT" \ +TASK_ID="$TASK_ID" \ +GITHUB_HEAD_BRANCHORTAG="${GITHUB_HEAD_BRANCHORTAAG}" \ +GITHUB_HEAD_BRANCH="${GITHUB_HEAD_BRANCH}" \ +GITHUB_HEAD_REF="$GITHUB_HEAD_REF" \ +GITHUB_HEAD_SHA="$GITHUB_HEAD_SHA" \ +GITHUB_HEAD_REPO_URL="$GITHUB_HEAD_REPO_URL" \ +GITHUB_HEAD_USER_LOGIN="$GITHUB_HEAD_USER_LOGIN" \ +GITHUB_HEAD_USER_EMAIL="$GITHUB_HEAD_USER_EMAIL" \ python3 ${curdir}/tc-decision.py diff --git a/taskcluster/test-android-opt-base.tyml b/taskcluster/test-android-opt-base.tyml index 4389165f..28a1f509 100644 --- a/taskcluster/test-android-opt-base.tyml +++ b/taskcluster/test-android-opt-base.tyml @@ -15,10 +15,6 @@ then: scopes: - "docker-worker:capability:device:kvm" - extra: - github: - { $eval: taskcluster.github_events.pull_request } - payload: maxRunTime: { $eval: to_int(build.maxRunTime) } image: "ubuntu:18.04" diff --git a/taskcluster/test-armbian-opt-base.tyml b/taskcluster/test-armbian-opt-base.tyml index f9aec02a..ec61447b 100644 --- a/taskcluster/test-armbian-opt-base.tyml +++ b/taskcluster/test-armbian-opt-base.tyml @@ -13,10 +13,6 @@ then: deadline: { $fromNow: '1 day' } expires: { $fromNow: '7 days' } - extra: - github: - { $eval: taskcluster.github_events.pull_request } - payload: maxRunTime: { $eval: to_int(build.maxRunTime) } image: "arm64v8/debian:buster-20200422" diff --git a/taskcluster/test-darwin-opt-base.tyml b/taskcluster/test-darwin-opt-base.tyml index 87755b02..e9ddd08f 100644 --- a/taskcluster/test-darwin-opt-base.tyml +++ b/taskcluster/test-darwin-opt-base.tyml @@ -13,10 +13,6 @@ then: deadline: { $fromNow: '1 day' } expires: { $fromNow: '7 days' } - extra: - github: - { $eval: taskcluster.github_events.pull_request } - scopes: [ "generic-worker:cache:deepspeech-macos-pyenv", ] diff --git a/taskcluster/test-linux-opt-base.tyml b/taskcluster/test-linux-opt-base.tyml index 21a0fefa..cab02e75 100644 --- a/taskcluster/test-linux-opt-base.tyml +++ b/taskcluster/test-linux-opt-base.tyml @@ -13,10 +13,6 @@ then: deadline: { $fromNow: '1 day' } expires: { $fromNow: '7 days' } - extra: - github: - { $eval: taskcluster.github_events.pull_request } - payload: maxRunTime: { $eval: to_int(build.maxRunTime) } image: ${build.docker_image} diff --git a/taskcluster/test-linux-opt-tag-base.tyml b/taskcluster/test-linux-opt-tag-base.tyml index b3f084f8..ce69782e 100644 --- a/taskcluster/test-linux-opt-tag-base.tyml +++ b/taskcluster/test-linux-opt-tag-base.tyml @@ -13,10 +13,6 @@ then: deadline: { $fromNow: '1 day' } expires: { $fromNow: '7 days' } - extra: - github: - { $eval: taskcluster.github_events.pull_request } - payload: maxRunTime: { $eval: to_int(build.maxRunTime) } image: ${build.docker_image} diff --git a/taskcluster/test-raspbian-opt-base.tyml b/taskcluster/test-raspbian-opt-base.tyml index 2a37f9ca..607c2e1d 100644 --- a/taskcluster/test-raspbian-opt-base.tyml +++ b/taskcluster/test-raspbian-opt-base.tyml @@ -13,10 +13,6 @@ then: deadline: { $fromNow: '1 day' } expires: { $fromNow: '7 days' } - extra: - github: - { $eval: taskcluster.github_events.pull_request } - payload: maxRunTime: { $eval: to_int(build.maxRunTime) } image: "balenalib/rpi-raspbian:buster-20200429" diff --git a/taskcluster/test-win-cuda-opt-base.tyml b/taskcluster/test-win-cuda-opt-base.tyml index 1813dced..7e779c14 100644 --- a/taskcluster/test-win-cuda-opt-base.tyml +++ b/taskcluster/test-win-cuda-opt-base.tyml @@ -13,10 +13,6 @@ then: deadline: { $fromNow: '1 day' } expires: { $fromNow: '7 days' } - extra: - github: - { $eval: taskcluster.github_events.pull_request } - payload: maxRunTime: { $eval: to_int(build.maxRunTime) } diff --git a/taskcluster/test-win-opt-base.tyml b/taskcluster/test-win-opt-base.tyml index 9cfdfa43..e8293d2a 100644 --- a/taskcluster/test-win-opt-base.tyml +++ b/taskcluster/test-win-opt-base.tyml @@ -13,10 +13,6 @@ then: deadline: { $fromNow: '1 day' } expires: { $fromNow: '7 days' } - extra: - github: - { $eval: taskcluster.github_events.pull_request } - payload: maxRunTime: { $eval: to_int(build.maxRunTime) } diff --git a/taskcluster/win-opt-base.tyml b/taskcluster/win-opt-base.tyml index 9d0a01c8..0247f140 100644 --- a/taskcluster/win-opt-base.tyml +++ b/taskcluster/win-opt-base.tyml @@ -16,10 +16,6 @@ expires: extra: nc_asset_name: { $eval: build.nc_asset_name } - github: - $if: '(event.event == "push") || (event.event == "tag")' - then: { $eval: taskcluster.github_events.merge } - else: { $eval: taskcluster.github_events.pull_request } routes: $if: '(event.event == "push") || (event.event == "tag")'