diff --git a/.github/actions/upload-cache-asset/action.yml b/.github/actions/upload-cache-asset/action.yml
deleted file mode 100644
index 40b35eb9..00000000
--- a/.github/actions/upload-cache-asset/action.yml
+++ /dev/null
@@ -1,58 +0,0 @@
-name: "Upload cache asset to release"
-description: "Upload a build cache asset to a release"
-inputs:
-  name:
-    description: "Artifact name"
-    required: true
-  path:
-    description: "Path of file to upload"
-    required: true
-  token:
-    description: "GitHub token"
-    required: false
-    default: ${{ github.token }}
-  repo:
-    description: "Repository name with owner (like actions/checkout)"
-    required: false
-    default: ${{ github.repository }}
-  release-tag:
-    description: "Tag of release to check artifacts under"
-    required: false
-    default: "v0.10.0-alpha.7"
-runs:
-  using: "composite"
-  steps:
-    - run: |
-        set -xe
-
-        asset_name="${{ inputs.name }}"
-        filename="${{ inputs.path }}"
-
-        # Check input
-        if [[ ! -f "${filename}" ]]; then
-          echo "Error: Input file (${filename}) missing"
-          exit 1;
-        fi
-
-        AUTH="Authorization: token ${{ inputs.token }}"
-
-        owner=$(echo "${{inputs.repo}}" | cut -f1 -d/)
-        repo=$(echo "${{inputs.repo}}" | cut -f2 -d/)
-        tag="${{ inputs.release-tag }}"
-
-        GH_REPO="https://api.github.com/repos/${owner}/${repo}"
-
-        # Check token
-        curl -o /dev/null -sH "$AUTH" $GH_REPO || { echo "Error: Invalid repo, token or network issue!";  exit 1; }
-
-        # Get ID of the release based on given tag name
-        GH_TAGS="${GH_REPO}/releases/tags/${tag}"
-        response=$(curl -sH "$AUTH" $GH_TAGS)
-        eval $(echo "$response" | grep -m 1 "id.:" | grep -w id | tr : = | tr -cd '[[:alnum:]]=')
-        [ "$id" ] || { echo "Error: Failed to get release id for tag: $tag"; echo "$response" | awk 'length($0)<100' >&2; exit 1; }
-
-        # Upload asset
-        echo "Uploading asset..."
-        GH_ASSET="https://uploads.github.com/repos/${owner}/${repo}/releases/${id}/assets?name=${asset_name}"
-        curl -T "${filename}" -X POST -H "${AUTH}" -H "Content-Type: application/octet-stream" $GH_ASSET
-      shell: bash
diff --git a/.github/actions/upload-release-asset/action.yml b/.github/actions/upload-release-asset/action.yml
new file mode 100644
index 00000000..93327473
--- /dev/null
+++ b/.github/actions/upload-release-asset/action.yml
@@ -0,0 +1,107 @@
+name: "Upload cache asset to release"
+description: "Upload a build cache asset to a release"
+inputs:
+  name:
+    description: "Artifact name"
+    required: true
+  path:
+    description: "Path of file to upload"
+    required: true
+  token:
+    description: "GitHub token"
+    required: false
+    default: ${{ github.token }}
+  repo:
+    description: "Repository name with owner (like actions/checkout)"
+    required: false
+    default: ${{ github.repository }}
+  release-tag:
+    description: "Tag of release to check artifacts under"
+    required: false
+    default: "v0.10.0-alpha.7"
+  should-create-release:
+    description: "Whether this action should automatically create a release for the given tag if one doesn't already exist"
+    required: false
+    default: false
+runs:
+  using: "composite"
+  steps:
+    - run: |
+        set -xe
+
+        asset_name="${{ inputs.name }}"
+        filenames="${{ inputs.path }}"
+
+        if [ $(compgen -G "$filenames" | wc -l) -gt 1 -a -n "$asset_name" ]; then
+          echo "Error: multiple input files specified, but also specified an asset_name."
+          echo "When uploading multiple files leave asset_name empty to use the file names as asset names."
+          exit 1
+        fi
+
+        # Check input
+        for file in $filenames; do
+          if [[ ! -f $file ]]; then
+            echo "Error: Input file (${filename}) missing"
+            exit 1;
+          fi
+        done
+
+        # If no asset name is specified, use filename
+        [ "$asset_name" ] || asset_name=$(basename "$filename")
+
+        AUTH="Authorization: token ${{ inputs.token }}"
+
+        owner=$(echo "${{inputs.repo}}" | cut -f1 -d/)
+        repo=$(echo "${{inputs.repo}}" | cut -f2 -d/)
+        tag="${{ inputs.release-tag }}"
+        should_create="${{ inputs.should-create-release }}"
+
+        GH_REPO="https://api.github.com/repos/${owner}/${repo}"
+
+        # Check token
+        curl -o /dev/null -sH "$AUTH" $GH_REPO || {
+          echo "Error: Invalid repo, token or network issue!"
+          exit 1
+        }
+
+        # Check if tag exists
+        response=$(curl -sH "$AUTH" "${GH_REPO}/git/refs/tags/${tag}")
+        eval $(echo "$response" | grep -m 1 "sha.:" | grep -w sha | tr : = | tr -cd '[[:alnum:]]=')
+        [ "$sha" ] || {
+          echo "Error: Tag does not exist: $tag"
+          echo "$response" | awk 'length($0)<100' >&2
+          exit 1
+        }
+
+        # Get ID of the release based on given tag name
+        GH_TAGS="${GH_REPO}/releases/tags/${tag}"
+        response=$(curl -sH "$AUTH" $GH_TAGS)
+        eval $(echo "$response" | grep -m 1 "id.:" | grep -w id | tr : = | tr -cd '[[:alnum:]]=')
+        [ "$id" ] || {
+          # If release does not exist, create it
+          if [[ "$should_create" == "true" ]]; then
+            echo "Tag does not have corresponding release, creating release for tag: $tag..."
+            response=$(curl -X POST -sH "$AUTH" -H "Content-Type: application/json" "${GH_REPO}/releases" -d '{"tag_name":"$tag","name":"Coqui STT $tag","prerelease":true}')
+            eval $(echo "$response" | grep -m 1 "id.:" | grep -w id | tr : = | tr -cd '[[:alnum:]]=')
+            [ "$id" ] || {
+              echo "Error: Could not create release for tag: $tag"
+              echo "$response" | awk 'length($0)<100' >&2
+              exit 1
+            }
+          else
+            echo "Error: Could not find release for tag: $tag"
+            echo "$response" | awk 'length($0)<100' >&2
+            exit 1
+          fi
+        }
+
+        # Upload assets
+        for $file in $filenames; do
+          if [ -z $asset_name ]; then
+            asset_name=$(basename $file)
+          fi
+          echo "Uploading asset with name: $asset_name from file: $file"
+          GH_ASSET="https://uploads.github.com/repos/${owner}/${repo}/releases/${id}/assets?name=${asset_name}"
+          curl -T $filename -X POST -H "${AUTH}" -H "Content-Type: application/octet-stream" $GH_ASSET
+        done
+      shell: bash
diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 71576a1c..09b2af3a 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -320,7 +320,7 @@ jobs:
         if: needs.tensorflow_opt-Linux.outputs.status == 'missing'
       - uses: ./.github/actions/package-tensorflow
         if: needs.tensorflow_opt-Linux.outputs.status == 'missing'
-      - uses: ./.github/actions/upload-cache-asset
+      - uses: ./.github/actions/upload-release-asset
         with:
           name: ${{ needs.tensorflow_opt-Linux.outputs.cache_key }}.tar.xz
           path: ${{ github.workspace }}/artifacts/home.tar.xz
@@ -365,7 +365,7 @@ jobs:
           path: ${{ github.workspace }}/artifacts/native_client.tar.xz
       - uses: actions/upload-artifact@v2
         with:
-          name: "libstt.tflite.zip"
+          name: "libstt.tflite.Linux.zip"
           path: ${{ github.workspace }}/artifacts/libstt.zip
   build-python-Linux:
     name: "Lin|Build Python bindings"
@@ -834,14 +834,6 @@ jobs:
       - uses: actions/download-artifact@v2
         with:
           name: stt-tflite-3.9-Linux.whl
-      # We need proper manylinux2014 builds before we can publish these wheels.
-      # https://github.com/coqui-ai/STT/issues/1904
-      # - uses: actions/download-artifact@v2
-      #   with:
-      #     name: stt-tflite-3.7-armv7.whl
-      # - uses: actions/download-artifact@v2
-      #   with:
-      #     name: stt-tflite-3.7-aarch64.whl
       - name: Setup PyPI config
         run: |
           cat << EOF > ~/.pypirc
@@ -851,6 +843,68 @@ jobs:
           EOF
       - run: |
           twine upload --repository pypi *.whl
+      # PyPI only supports ARM wheels built on manylinux images, but those aren't
+      # ready for use yet, so we upload our wheels to the corresponding release
+      # for this tag.
+      - uses: actions/download-artifact@v2
+        with:
+          name: stt-tflite-3.7-armv7.whl
+      - uses: actions/download-artifact@v2
+        with:
+          name: stt-tflite-3.7-aarch64.whl
+      - name: Get tag name
+        id: get-tag
+        run: |
+          tag=$(echo "${{ github.ref }}" | sed -e 's|^refs/tags/||)'
+          echo "::set-output name=tag::$tag"
+      - name: Upload artifacts to GitHub release
+        uses: ./.github/actions/upload-release-asset
+        with:
+          name: '' # use filename
+          path: "*.whl"
+          release-tag: ${{ steps.get-tag.outputs.tag }}
+          should-create-release: true
+  upload-nc-release-assets:
+    name: "Upload native client artifacts to release assets"
+    runs-on: ubuntu-20.04
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
+    needs: [build-lib_Windows, build-lib_Linux, build-lib_macOS, build-lib_LinuxAarch64, build-lib_LinuxArmv7]
+    steps:
+      - uses: actions/download-artifact@v2
+        with:
+          name: libstt.tflite.Linux.zip
+      - uses: actions/download-artifact@v2
+        with:
+          name: native_client.tflite.Linux.tar.xz
+      - uses: actions/download-artifact@v2
+        with:
+          name: libstt.tflite.macOS.zip
+      - uses: actions/download-artifact@v2
+        with:
+          name: native_client.tflite.macOS.tar.xz
+      - uses: actions/download-artifact@v2
+        with:
+          name: libstt.tflite.Windows.zip
+      - uses: actions/download-artifact@v2
+        with:
+          name: native_client.tflite.Windows.tar.xz
+      - uses: actions/download-artifact@v2
+        with:
+          name: native_client.tflite.linux.armv7.tar.xz
+      - uses: actions/download-artifact@v2
+        with:
+          name: native_client.tflite.linux.aarch64.tar.xz
+      - name: Get tag name
+        id: get-tag
+        run: |
+          tag=$(echo "${{ github.ref }}" | sed -e 's|^refs/tags/||)'
+          echo "::set-output name=tag::$tag"
+      - uses: ./.github/actions/upload-release-asset
+        with:
+          name: '' # use filename
+          path: "*.{tar.gz,zip}"
+          release-tag: ${{ steps.get-tag.outputs.tag }}
+          should-create-release: true
   docker-build:
     name: "Build Dockerfile.build image"
     runs-on: ubuntu-20.04
@@ -964,6 +1018,18 @@ jobs:
           EOF
       - run: |
           twine upload --repository pypi *.whl
+      - name: Get tag name
+        id: get-tag
+        run: |
+          tag=$(echo "${{ github.ref }}" | sed -e 's|^refs/tags/||)'
+          echo "::set-output name=tag::$tag"
+      - name: Upload artifacts to GitHub release
+        uses: ./.github/actions/upload-release-asset
+        with:
+          name: '' # use filename
+          path: "*.whl"
+          release-tag: ${{ steps.get-tag.outputs.tag }}
+          should-create-release: true
   npmjs-publish:
     name: "Upload STT packages to npmjs.com"
     runs-on: ubuntu-20.04
@@ -998,6 +1064,18 @@ jobs:
           npm publish --access=public --verbose ${{ github.workspace }}/stt-*.tgz --tag ${{ steps.compute-npm-tag.outputs.npm-tag }}
         env:
           NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+      - name: Get tag name
+        id: get-tag
+        run: |
+          tag=$(echo "${{ github.ref }}" | sed -e 's|^refs/tags/||)'
+          echo "::set-output name=tag::$tag"
+      - name: Upload artifacts to GitHub release
+        uses: ./.github/actions/upload-release-asset
+        with:
+          name: '' # use filename
+          path: "*.tgz"
+          release-tag: ${{ steps.get-tag.outputs.tag }}
+          should-create-release: true
   # macOS jobs
   swig_macOS:
     name: "Mac|Build SWIG"
@@ -1191,7 +1269,7 @@ jobs:
         if: needs.tensorflow_opt-macOS.outputs.status == 'missing'
       - uses: ./.github/actions/package-tensorflow
         if: needs.tensorflow_opt-macOS.outputs.status == 'missing'
-      - uses: ./.github/actions/upload-cache-asset
+      - uses: ./.github/actions/upload-release-asset
         with:
           name: ${{ needs.tensorflow_opt-macOS.outputs.cache_key }}.tar.xz
           path: ${{ github.workspace }}/artifacts/home.tar.xz
@@ -1229,7 +1307,7 @@ jobs:
           path: ${{ github.workspace }}/artifacts/native_client.tar.xz
       - uses: actions/upload-artifact@v2
         with:
-          name: "libstt.tflite.zip"
+          name: "libstt.tflite.macOS.zip"
           path: ${{ github.workspace }}/artifacts/libstt.zip
   build-python-macOS:
     name: "Mac|Build Python bindings"
@@ -1630,7 +1708,7 @@ jobs:
         if: needs.tensorflow_opt-Windows.outputs.status == 'missing'
       - run: ./ci_scripts/tf-package.sh
         if: needs.tensorflow_opt-Windows.outputs.status == 'missing'
-      - uses: ./.github/actions/upload-cache-asset
+      - uses: ./.github/actions/upload-release-asset
         with:
           name: ${{ needs.tensorflow_opt-Windows.outputs.cache_key }}.tar.xz
           path: ${{ github.workspace }}/artifacts/home.tar.xz
@@ -1676,7 +1754,7 @@ jobs:
           path: ${{ github.workspace }}/artifacts/native_client.tar.xz
       - uses: actions/upload-artifact@v2
         with:
-          name: "libstt.tflite.zip"
+          name: "libstt.tflite.Windows.zip"
           path: ${{ github.workspace }}/artifacts/libstt.zip
   build-python-Windows:
     name: "Win|Build Python bindings"
@@ -2491,7 +2569,7 @@ jobs:
         if: needs.tensorflow_opt-LinuxArmv7.outputs.status == 'missing'
       - uses: ./.github/actions/package-tensorflow
         if: needs.tensorflow_opt-LinuxArmv7.outputs.status == 'missing'
-      - uses: ./.github/actions/upload-cache-asset
+      - uses: ./.github/actions/upload-release-asset
         with:
           name: ${{ needs.tensorflow_opt-LinuxArmv7.outputs.cache_key }}.tar.xz
           path: ${{ github.workspace }}/artifacts/home.tar.xz
@@ -2516,7 +2594,7 @@ jobs:
         if: needs.tensorflow_opt-LinuxAarch64.outputs.status == 'missing'
       - uses: ./.github/actions/package-tensorflow
         if: needs.tensorflow_opt-LinuxAarch64.outputs.status == 'missing'
-      - uses: ./.github/actions/upload-cache-asset
+      - uses: ./.github/actions/upload-release-asset
         with:
           name: ${{ needs.tensorflow_opt-LinuxAarch64.outputs.cache_key }}.tar.xz
           path: ${{ github.workspace }}/artifacts/home.tar.xz