Convert to .taskcluster.yml v1

forward TASK_ID, add created and deadline more fixes typo try without TASK_ID fix task templates add missing env vars to tc decision dry runs avoid repetition in .taskcluster and manually forward varibles to tc-decision.py url -> clone_url simulate GITHUB_EVENT separate ref an sha correct pull request actions correct pull request policy
2020-12-17 19:03:44 +00:00 · 2020-12-17 19:03:44 +00:00 · bb1ad00194
commit bb1ad00194
parent 07d0e93083
20 changed files with 126 additions and 153 deletions
--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@ -1,65 +1,92 @@
-# The version is always required
-version: 0
-# Top level metadata is always required
-metadata:
-  name: "DeepSpeech"
-  description: "DeepSpeech builds"
-  owner: "{{ event.head.user.email }}" # the user who sent the pr/push e-mail will be inserted here
-  source: "{{ event.head.repo.url }}"  # the repo where the pr came from will be inserted here
+version: 1
+policy:
+  pullRequests: collaborators_quiet
 tasks:
-  - provisionerId: "proj-deepspeech"
-    workerType: "ci-decision-task"
-    extra:
-      github:
-        env: true
-        events:
-          - pull_request.opened
-          - pull_request.synchronize
-          - pull_request.reopened
-          - push
-          - tag
-        branches:
-          - master
-
-    scopes: [
-      "queue:create-task:highest:proj-deepspeech/*",
-      "queue:route:index.project.deepspeech.*",
-      "index:insert-task:project.deepspeech.*",
-      "queue:scheduler-id:taskcluster-github",
-      "generic-worker:cache:deepspeech-macos-pyenv",
-      "docker-worker:capability:device:kvm"
-    ]
-
-    payload:
-      maxRunTime: 600
-      image: "ubuntu:16.04"
-
-      features:
-        taskclusterProxy: true
-
-      env:
-        TC_DECISION_SHA: ef67832e6657f43e139a10f37eb326a7d9d96dad
-
-      command:
-        - "/bin/bash"
-        - "--login"
-        - "-cxe"
-        - >
-          echo "deb http://archive.ubuntu.com/ubuntu/ xenial-updates main" > /etc/apt/sources.list.d/xenial-updates.list &&
-          apt-get -qq update && apt-get -qq -y install git python3-pip curl sudo &&
-          adduser --system --home /home/build-user build-user &&
-          cd /home/build-user/ &&
-          echo -e "#!/bin/bash\nset -xe\nenv && id && mkdir ~/DeepSpeech/ && git clone --quiet {{event.head.repo.url}} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet {{event.head.sha}}" > /tmp/clone.sh && chmod +x /tmp/clone.sh &&
-          sudo -H -u build-user /bin/bash /tmp/clone.sh &&
-          sudo -H -u build-user --preserve-env /bin/bash /home/build-user/DeepSpeech/ds/taskcluster/tc-schedule.sh
-      artifacts:
-        "public":
-          type: "directory"
-          path: "/tmp/artifacts/"
-          expires: "{{ '7 days' | $fromNow }}"
-    # Each task also requires explicit metadata
+  $let:
    metadata:
-      name: "DeepSpeech Decision Task"
-      description: "DeepSpeech Decision Task: triggers everything."
-      owner: "{{ event.head.user.email }}"
-      source: "{{ event.head.repo.url }}"
+      task_id: {$eval: as_slugid("decision_task")}
+      github:
+        $if: 'tasks_for == "github-pull-request"'
+        then:
+          action: "pull_request.${event.action}"
+          login: ${event.pull_request.user.login}
+          ref: ${event.pull_request.head.ref}
+          sha: ${event.pull_request.head.sha}
+          clone_url: ${event.pull_request.head.repo.clone_url}
+        else:
+          action:
+            $if: 'event.ref[:10] == "refs/tags/"'
+            then: "tag"
+            else: "push"
+          login: ${event.pusher.name}
+          ref: ${event.ref}
+          sha: ${event.after}
+          clone_url: ${event.repository.clone_url}
+  in:
+    $let:
+      decision_task:
+        taskId: ${metadata.task_id}
+        created: {$fromNow: ''}
+        deadline: {$fromNow: '60 minutes'}
+
+        provisionerId: "proj-deepspeech"
+        workerType: "ci-decision-task"
+
+        scopes: [
+          "queue:create-task:highest:proj-deepspeech/*",
+          "queue:route:index.project.deepspeech.*",
+          "index:insert-task:project.deepspeech.*",
+          "queue:scheduler-id:taskcluster-github",
+          "generic-worker:cache:deepspeech-macos-pyenv",
+          "docker-worker:capability:device:kvm"
+        ]
+
+        payload:
+          maxRunTime: 600
+          image: "ubuntu:16.04"
+
+          features:
+            taskclusterProxy: true
+
+          env:
+            TC_DECISION_SHA: ef67832e6657f43e139a10f37eb326a7d9d96dad
+
+          command:
+            - "/bin/bash"
+            - "--login"
+            - "-cxe"
+            - >
+              echo "deb http://archive.ubuntu.com/ubuntu/ xenial-updates main" > /etc/apt/sources.list.d/xenial-updates.list &&
+              apt-get -qq update && apt-get -qq -y install git python3-pip curl sudo &&
+              adduser --system --home /home/build-user build-user &&
+              cd /home/build-user/ &&
+              echo -e "#!/bin/bash\nset -xe\nenv && id && mkdir ~/DeepSpeech/ && git clone --quiet ${metadata.github.clone_url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${metadata.github.ref}" > /tmp/clone.sh && chmod +x /tmp/clone.sh &&
+              sudo -H -u build-user /bin/bash /tmp/clone.sh &&
+              sudo -H -u build-user --preserve-env /bin/bash /home/build-user/DeepSpeech/ds/taskcluster/tc-schedule.sh \
+                TASK_ID=${metadata.task_id} \
+                GITHUB_HEAD_USER_LOGIN=${metadata.github.login} \
+                GITHUB_HEAD_USER_EMAIL=${metadata.github.login}@users.noreply.github.com \
+                GITHUB_EVENT=${metadata.github.action} \
+                GITHUB_HEAD_REPO_URL=${metadata.github.clone_url} \
+                GITHUB_HEAD_BRANCH=${metadata.github.ref} \
+                GITHUB_HEAD_REF=${metadata.github.ref} \
+                GITHUB_HEAD_SHA=${metadata.github.sha}
+          artifacts:
+            "public":
+              type: "directory"
+              path: "/tmp/artifacts/"
+              expires: {$fromNow: '7 days'}
+
+        metadata:
+          name: "DeepSpeech decision task"
+          description: "DeepSpeech decision task"
+          owner: "${metadata.github.login}@users.noreply.github.com"
+          source: "${metadata.github.clone_url}"
+    in:
+      $flattenDeep:
+      - $if: 'tasks_for == "github-pull-request" && event["action"] in ["opened", "reopened", "synchronize"]'
+        then: {$eval: decision_task}
+      - $if: 'tasks_for == "github-push" && event.ref == "refs/heads/master"'
+        then: {$eval: decision_task}
+      - $if: 'tasks_for == "github-push" && event.ref[:10] == "refs/tags/"'
+        then: {$eval: decision_task}
--- a/taskcluster/build-python-wheel.tyml
+++ b/taskcluster/build-python-wheel.tyml
@ -13,10 +13,6 @@ then:
  deadline: { $fromNow: '1 day' }
  expires: { $fromNow: '7 days' }

-  extra:
-    github:
-      { $eval: taskcluster.github_events.pull_request }
-
  payload:
    maxRunTime: 14400
    image: "arm64v8/debian:buster-20190812"
--- a/taskcluster/darwin-opt-base.tyml
+++ b/taskcluster/darwin-opt-base.tyml
@ -16,10 +16,6 @@ expires:

 extra:
  nc_asset_name: { $eval: build.nc_asset_name }
-  github:
-    $if: '(event.event == "push") || (event.event == "tag")'
-    then: { $eval: taskcluster.github_events.merge }
-    else: { $eval: taskcluster.github_events.pull_request }

 routes:
  $if: '(event.event == "push") || (event.event == "tag")'
--- a/taskcluster/docker-build-base.tyml
+++ b/taskcluster/docker-build-base.tyml
@ -9,10 +9,6 @@ then:
  deadline: { $fromNow: '1 day' }
  expires: { $fromNow: '7 days' }

-  extra:
-    github:
-      { $eval: taskcluster.github_events.pull_request }
-
  payload:
    maxRunTime: { $eval: to_int(build.maxRunTime) }
    image: "ubuntu:14.04"
--- a/taskcluster/docs.tyml
+++ b/taskcluster/docs.tyml
@ -18,10 +18,6 @@ then:

  extra:
    nc_asset_name: { $eval: build.nc_asset_name }
-    github:
-      $if: '(event.event == "push") || (event.event == "tag")'
-      then: { $eval: taskcluster.github_events.merge }
-      else: { $eval: taskcluster.github_events.pull_request }

  routes:
    $if: '(event.event == "push") || (event.event == "tag")'
--- a/taskcluster/examples-base.tyml
+++ b/taskcluster/examples-base.tyml
@ -13,10 +13,6 @@ then:
  deadline: { $fromNow: '1 day' }
  expires: { $fromNow: '7 days' }

-  extra:
-    github:
-      { $eval: taskcluster.github_events.pull_request }
-
  payload:
    maxRunTime: { $eval: to_int(build.maxRunTime) }
    image: ${build.docker_image}
--- a/taskcluster/github-events.cyml
+++ b/taskcluster/github-events.cyml
@ -1,15 +0,0 @@
-taskcluster:
-  github_events:
-    pull_request:
-      env: true
-      events:
-        - pull_request.opened
-        - pull_request.synchronize
-        - pull_request.reopened
-    merge:
-      env: true
-      events:
-        - push
-        - tag
-      branches:
-        - master
--- a/taskcluster/linux-opt-base.tyml
+++ b/taskcluster/linux-opt-base.tyml
@ -18,10 +18,6 @@ then:

  extra:
    nc_asset_name: { $eval: build.nc_asset_name }
-    github:
-      $if: '(event.event == "push") || (event.event == "tag")'
-      then: { $eval: taskcluster.github_events.merge }
-      else: { $eval: taskcluster.github_events.pull_request }

  routes:
    $if: '(event.event == "push") || (event.event == "tag")'
--- a/taskcluster/node-package-opt-base.tyml
+++ b/taskcluster/node-package-opt-base.tyml
@ -18,10 +18,6 @@ then:

  extra:
    nc_asset_name: { $eval: build.nc_asset_name }
-    github:
-      $if: '(event.event == "push") || (event.event == "tag")'
-      then: { $eval: taskcluster.github_events.merge }
-      else: { $eval: taskcluster.github_events.pull_request }

  routes:
    $if: '(event.event == "push") || (event.event == "tag")'
--- a/taskcluster/simple-task.tyml
+++ b/taskcluster/simple-task.tyml
@ -16,12 +16,6 @@ then:
    then: { $fromNow: '6 months' }
    else: { $fromNow: '7 days' }

-  extra:
-    github:
-      $if: '(event.event == "push") || (event.event == "tag")'
-      then: { $eval: taskcluster.github_events.merge }
-      else: { $eval: taskcluster.github_events.pull_request }
-
  routes:
    { $eval: build.routes }

--- a/taskcluster/tc-schedule.sh
+++ b/taskcluster/tc-schedule.sh
@ -1,7 +1,11 @@
 #!/bin/bash
-
 set -ex

+# Set environment variables passed in arguments
+for line in $@; do
+  eval "$line"
+done
+
 curdir=$(dirname "$0")/

 pip3 install --quiet --user --upgrade pip
@ -14,14 +18,41 @@ pip3 install --quiet --user --upgrade -r ${curdir}/tc-decision_reqs.txt
 # This should help us track merge failures in advance
 for event in pull_request.opened pull_request.synchronize pull_request.reopened push;
 do
-    GITHUB_EVENT="${event}" TASK_ID="aa" GITHUB_HEAD_BRANCHORTAG="branchName" GITHUB_HEAD_REF="refs/heads/branchName" python3 ${curdir}/tc-decision.py --dry
+    GITHUB_EVENT="${event}" \
+    TASK_ID="aa" \
+    GITHUB_HEAD_BRANCHORTAG="branchName" \
+    GITHUB_HEAD_REF="refs/heads/branchName" \
+    GITHUB_HEAD_SHA="branchName" \
+    GITHUB_HEAD_REPO_URL="url" \
+    GITHUB_HEAD_USER_LOGIN="user" \
+    GITHUB_HEAD_USER_EMAIL="email" \
+    python3 ${curdir}/tc-decision.py --dry
 done;

-GITHUB_EVENT="tag" TASK_ID="aa" GITHUB_HEAD_BRANCHORTAG="tagName" GITHUB_HEAD_REF="refs/tags/tagName" python3 ${curdir}/tc-decision.py --dry
+GITHUB_EVENT="tag" \
+TASK_ID="aa" \
+GITHUB_HEAD_BRANCHORTAG="tagName" \
+GITHUB_HEAD_REF="refs/tags/tagName" \
+GITHUB_HEAD_SHA="branchName" \
+GITHUB_HEAD_REPO_URL="url" \
+GITHUB_HEAD_USER_LOGIN="user" \
+GITHUB_HEAD_USER_EMAIL="email" \
+python3 ${curdir}/tc-decision.py --dry

 # Create a new env variable for usage in TaskCluster .yml files
-export GITHUB_HEAD_BRANCHORTAG="${GITHUB_HEAD_BRANCH}${GITHUB_HEAD_TAG}"
+GITHUB_HEAD_BRANCHORTAG="${GITHUB_HEAD_BRANCH}${GITHUB_HEAD_TAG}"

 # Quick hack because tc-decision uses GITHUB_HEAD_BRANCH
-export GITHUB_HEAD_BRANCH="${GITHUB_HEAD_BRANCH}${GITHUB_HEAD_TAG}"
+GITHUB_HEAD_BRANCH="${GITHUB_HEAD_BRANCH}${GITHUB_HEAD_TAG}"
+
+# Forward all variables to tc-decision.py
+GITHUB_EVENT="$GITHUB_EVENT" \
+TASK_ID="$TASK_ID" \
+GITHUB_HEAD_BRANCHORTAG="${GITHUB_HEAD_BRANCHORTAAG}" \
+GITHUB_HEAD_BRANCH="${GITHUB_HEAD_BRANCH}" \
+GITHUB_HEAD_REF="$GITHUB_HEAD_REF" \
+GITHUB_HEAD_SHA="$GITHUB_HEAD_SHA" \
+GITHUB_HEAD_REPO_URL="$GITHUB_HEAD_REPO_URL" \
+GITHUB_HEAD_USER_LOGIN="$GITHUB_HEAD_USER_LOGIN" \
+GITHUB_HEAD_USER_EMAIL="$GITHUB_HEAD_USER_EMAIL" \
 python3 ${curdir}/tc-decision.py
--- a/taskcluster/test-android-opt-base.tyml
+++ b/taskcluster/test-android-opt-base.tyml
@ -15,10 +15,6 @@ then:
  scopes:
    - "docker-worker:capability:device:kvm"

-  extra:
-    github:
-      { $eval: taskcluster.github_events.pull_request }
-
  payload:
    maxRunTime: { $eval: to_int(build.maxRunTime) }
    image: "ubuntu:18.04"
--- a/taskcluster/test-armbian-opt-base.tyml
+++ b/taskcluster/test-armbian-opt-base.tyml
@ -13,10 +13,6 @@ then:
  deadline: { $fromNow: '1 day' }
  expires: { $fromNow: '7 days' }

-  extra:
-    github:
-      { $eval: taskcluster.github_events.pull_request }
-
  payload:
    maxRunTime: { $eval: to_int(build.maxRunTime) }
    image: "arm64v8/debian:buster-20200422"
--- a/taskcluster/test-darwin-opt-base.tyml
+++ b/taskcluster/test-darwin-opt-base.tyml
@ -13,10 +13,6 @@ then:
  deadline: { $fromNow: '1 day' }
  expires: { $fromNow: '7 days' }

-  extra:
-    github:
-      { $eval: taskcluster.github_events.pull_request }
-
  scopes: [
    "generic-worker:cache:deepspeech-macos-pyenv",
  ]
--- a/taskcluster/test-linux-opt-base.tyml
+++ b/taskcluster/test-linux-opt-base.tyml
@ -13,10 +13,6 @@ then:
  deadline: { $fromNow: '1 day' }
  expires: { $fromNow: '7 days' }

-  extra:
-    github:
-      { $eval: taskcluster.github_events.pull_request }
-
  payload:
    maxRunTime: { $eval: to_int(build.maxRunTime) }
    image: ${build.docker_image}
--- a/taskcluster/test-linux-opt-tag-base.tyml
+++ b/taskcluster/test-linux-opt-tag-base.tyml
@ -13,10 +13,6 @@ then:
  deadline: { $fromNow: '1 day' }
  expires: { $fromNow: '7 days' }

-  extra:
-    github:
-      { $eval: taskcluster.github_events.pull_request }
-
  payload:
    maxRunTime: { $eval: to_int(build.maxRunTime) }
    image: ${build.docker_image}
--- a/taskcluster/test-raspbian-opt-base.tyml
+++ b/taskcluster/test-raspbian-opt-base.tyml
@ -13,10 +13,6 @@ then:
  deadline: { $fromNow: '1 day' }
  expires: { $fromNow: '7 days' }

-  extra:
-    github:
-      { $eval: taskcluster.github_events.pull_request }
-
  payload:
    maxRunTime: { $eval: to_int(build.maxRunTime) }
    image: "balenalib/rpi-raspbian:buster-20200429"
--- a/taskcluster/test-win-cuda-opt-base.tyml
+++ b/taskcluster/test-win-cuda-opt-base.tyml
@ -13,10 +13,6 @@ then:
  deadline: { $fromNow: '1 day' }
  expires: { $fromNow: '7 days' }

-  extra:
-    github:
-      { $eval: taskcluster.github_events.pull_request }
-
  payload:
    maxRunTime: { $eval: to_int(build.maxRunTime) }

--- a/taskcluster/test-win-opt-base.tyml
+++ b/taskcluster/test-win-opt-base.tyml
@ -13,10 +13,6 @@ then:
  deadline: { $fromNow: '1 day' }
  expires: { $fromNow: '7 days' }

-  extra:
-    github:
-      { $eval: taskcluster.github_events.pull_request }
-
  payload:
    maxRunTime: { $eval: to_int(build.maxRunTime) }

--- a/taskcluster/win-opt-base.tyml
+++ b/taskcluster/win-opt-base.tyml
@ -16,10 +16,6 @@ expires:

 extra:
  nc_asset_name: { $eval: build.nc_asset_name }
-  github:
-    $if: '(event.event == "push") || (event.event == "tag")'
-    then: { $eval: taskcluster.github_events.merge }
-    else: { $eval: taskcluster.github_events.pull_request }

 routes:
  $if: '(event.event == "push") || (event.event == "tag")'