Explicitly name TF build cache destination file

GitHub's API has stopped sending the artifact name as the file name, so we ended up with a file matching the artifact ID.
Name the full file path explicitly so there's no room for changes.
This commit is contained in:
Reuben Morais 2021-11-11 12:42:16 +01:00
parent a5c981bb48
commit 2af6f8da89
4 changed files with 56 additions and 50 deletions

View File

@ -7,6 +7,10 @@ const fs = require('fs');
const { throttling } = require('@octokit/plugin-throttling');
const { GitHub } = require('@actions/github/lib/utils');
const Download = require('download');
const Util = require('util');
const Stream = require('stream');
const Pipeline = Util.promisify(Stream.pipeline);
async function getGoodArtifacts(client, owner, repo, releaseId, name) {
console.log(`==> GET /repos/${owner}/${repo}/releases/${releaseId}/assets`);
@ -94,22 +98,24 @@ async function main() {
console.log("==> # artifacts:", goodArtifacts.length);
const artifact = goodArtifacts[0];
console.log("==> Artifact:", artifact.id)
const size = filesize(artifact.size, { base: 10 })
console.log(`==> Downloading: ${artifact.name} (${size}) to path: ${path}`)
console.log("==> Downloading:", artifact.name, `(${size})`)
const dir = name ? path : pathname.join(path, artifact.name)
const dir = pathname.dirname(path)
console.log(`==> Creating containing dir if needed: ${dir}`)
fs.mkdirSync(dir, { recursive: true })
await Download(artifact.url, dir, {
headers: {
"Accept": "application/octet-stream",
"Authorization": `token ${token}`,
},
});
await Pipeline(
Download(artifact.url, {
headers: {
"Accept": "application/octet-stream",
"Authorization": `token ${token}`,
},
}),
fs.createWriteStream(path)
)
}
if (artifactStatus === "missing" && download == "true") {

View File

@ -394,15 +394,15 @@ jobs:
- uses: ./.github/actions/check_artifact_exists
with:
name: ${{ needs.tensorflow_opt-Linux.outputs.cache_key }}.tar.xz
path: ${{ github.workspace }}/
path: ${{ github.workspace }}/tf-cache.tar.xz
download: true
- name: Install dependencies
run: |
apt-get update
apt-get install -y --no-install-recommends xz-utils zip
- run: |
tar --skip-old-files -xf ${{ needs.tensorflow_opt-linux.outputs.cache_key }}.tar.xz
rm ${{ needs.tensorflow_opt-Linux.outputs.cache_key }}.tar.xz
tar --skip-old-files -xf tf-cache.tar.xz
rm tf-cache.tar.xz
- name: Setup venv
run: |
/opt/python/cp37-cp37m/bin/python -m venv /tmp/venv
@ -1353,11 +1353,11 @@ jobs:
- uses: ./.github/actions/check_artifact_exists
with:
name: ${{ needs.tensorflow_opt-macOS.outputs.cache_key }}.tar.xz
path: ${{ github.workspace }}/
path: ${{ github.workspace }}/tf-cache.tar.xz
download: true
- run: |
tar xkf ${{ needs.tensorflow_opt-macOS.outputs.cache_key }}.tar.xz
rm ${{ needs.tensorflow_opt-macOS.outputs.cache_key }}.tar.xz
tar xkf tf-cache.tar.xz
rm tf-cache.tar.xz
- run: |
git status
- uses: ./.github/actions/select-xcode
@ -1818,11 +1818,11 @@ jobs:
- uses: ./.github/actions/check_artifact_exists
with:
name: ${{ needs.tensorflow_opt-Windows.outputs.cache_key }}.tar.xz
path: ${{ github.workspace }}/
path: ${{ github.workspace }}/tf-cache.tar.xz
download: true
- run: |
"C:/Program Files/7-Zip/7z.exe" x ${{ needs.tensorflow_opt-Windows.outputs.cache_key }}.tar.xz -so | "C:/Program Files/7-Zip/7z.exe" x -aos -si -ttar -o`pwd`
rm ${{ needs.tensorflow_opt-Windows.outputs.cache_key }}.tar.xz
"C:/Program Files/7-Zip/7z.exe" x tf-cache.tar.xz -so | "C:/Program Files/7-Zip/7z.exe" x -aos -si -ttar -o`pwd`
rm tf-cache.tar.xz
- name: Workaround bazel bug when LLVM is installed https://github.com/bazelbuild/bazel/issues/12144
run: |
rm -f /c/msys64/mingw64/clang-cl*
@ -2696,11 +2696,11 @@ jobs:
- uses: ./.github/actions/check_artifact_exists
with:
name: ${{ needs.tensorflow_opt-LinuxArmv7.outputs.cache_key }}.tar.xz
path: ${{ github.workspace }}/
path: ${{ github.workspace }}/tf-cache.tar.xz
download: true
- run: |
tar --skip-old-files -xf ${{ needs.tensorflow_opt-linuxarmv7.outputs.cache_key }}.tar.xz
rm ${{ needs.tensorflow_opt-LinuxArmv7.outputs.cache_key }}.tar.xz
tar --skip-old-files -xf tf-cache.tar.xz
rm tf-cache.tar.xz
- run: |
git status
- name: "Install chroot"
@ -2733,11 +2733,11 @@ jobs:
- uses: ./.github/actions/check_artifact_exists
with:
name: ${{ needs.tensorflow_opt-LinuxAarch64.outputs.cache_key }}.tar.xz
path: ${{ github.workspace }}/
path: ${{ github.workspace }}/tf-cache.tar.xz
download: true
- run: |
tar --skip-old-files -xf ${{ needs.tensorflow_opt-linuxaarch64.outputs.cache_key }}.tar.xz
rm ${{ needs.tensorflow_opt-LinuxAarch64.outputs.cache_key }}.tar.xz
tar --skip-old-files -xf tf-cache.tar.xz
rm tf-cache.tar.xz
- run: |
git status
- name: "Install chroot"
@ -2794,11 +2794,11 @@ jobs:
- uses: ./.github/actions/check_artifact_exists
with:
name: ${{ needs.tensorflow_opt-LinuxArmv7.outputs.cache_key }}.tar.xz
path: ${{ github.workspace }}/
path: ${{ github.workspace }}/tf-cache.tar.xz
download: true
- run: |
tar --skip-old-files -xf ${{ needs.tensorflow_opt-linuxarmv7.outputs.cache_key }}.tar.xz
rm ${{ needs.tensorflow_opt-LinuxArmv7.outputs.cache_key }}.tar.xz
tar --skip-old-files -xf tf-cache.tar.xz
rm tf-cache.tar.xz
- uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
@ -2860,11 +2860,11 @@ jobs:
- uses: ./.github/actions/check_artifact_exists
with:
name: ${{ needs.tensorflow_opt-LinuxArmv7.outputs.cache_key }}.tar.xz
path: ${{ github.workspace }}/
path: ${{ github.workspace }}/tf-cache.tar.xz
download: true
- run: |
tar --skip-old-files -xf ${{ needs.tensorflow_opt-linuxarmv7.outputs.cache_key }}.tar.xz
rm ${{ needs.tensorflow_opt-LinuxArmv7.outputs.cache_key }}.tar.xz
tar --skip-old-files -xf tf-cache.tar.xz
rm tf-cache.tar.xz
- uses: ./.github/actions/install-xldd
with:
target: ${{ env.SYSTEM_TARGET }}
@ -2939,11 +2939,11 @@ jobs:
- uses: ./.github/actions/check_artifact_exists
with:
name: ${{ needs.tensorflow_opt-LinuxAarch64.outputs.cache_key }}.tar.xz
path: ${{ github.workspace }}/
path: ${{ github.workspace }}/tf-cache.tar.xz
download: true
- run: |
tar --skip-old-files -xf ${{ needs.tensorflow_opt-linuxaarch64.outputs.cache_key }}.tar.xz
rm ${{ needs.tensorflow_opt-LinuxAarch64.outputs.cache_key }}.tar.xz
tar --skip-old-files -xf tf-cache.tar.xz
rm tf-cache.tar.xz
- uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
@ -3005,11 +3005,11 @@ jobs:
- uses: ./.github/actions/check_artifact_exists
with:
name: ${{ needs.tensorflow_opt-LinuxAarch64.outputs.cache_key }}.tar.xz
path: ${{ github.workspace }}/
path: ${{ github.workspace }}/tf-cache.tar.xz
download: true
- run: |
tar --skip-old-files -xf ${{ needs.tensorflow_opt-linuxaarch64.outputs.cache_key }}.tar.xz
rm ${{ needs.tensorflow_opt-LinuxAarch64.outputs.cache_key }}.tar.xz
tar --skip-old-files -xf tf-cache.tar.xz
rm tf-cache.tar.xz
- uses: ./.github/actions/install-xldd
with:
target: ${{ env.SYSTEM_TARGET }}
@ -3397,11 +3397,11 @@ jobs:
- uses: ./.github/actions/check_artifact_exists
with:
name: ${{ needs.tensorflow_opt-AndroidArmv7.outputs.cache_key }}.tar.xz
path: ${{ github.workspace }}/
path: ${{ github.workspace }}/tf-cache.tar.xz
download: true
- run: |
tar --skip-old-files -xf ${{ needs.tensorflow_opt-AndroidArmv7.outputs.cache_key }}.tar.xz
rm ${{ needs.tensorflow_opt-AndroidArmv7.outputs.cache_key }}.tar.xz
tar --skip-old-files -xf tf-cache.tar.xz
rm tf-cache.tar.xz
- uses: ./.github/actions/libstt-build
with:
arch: android-armv7
@ -3469,11 +3469,11 @@ jobs:
- uses: ./.github/actions/check_artifact_exists
with:
name: ${{ needs.tensorflow_opt-AndroidArm64.outputs.cache_key }}.tar.xz
path: ${{ github.workspace }}/
path: ${{ github.workspace }}/tf-cache.tar.xz
download: true
- run: |
tar --skip-old-files -xf ${{ needs.tensorflow_opt-AndroidArm64.outputs.cache_key }}.tar.xz
rm ${{ needs.tensorflow_opt-AndroidArm64.outputs.cache_key }}.tar.xz
tar --skip-old-files -xf tf-cache.tar.xz
rm tf-cache.tar.xz
- uses: ./.github/actions/libstt-build
with:
arch: android-arm64
@ -3493,11 +3493,11 @@ jobs:
- uses: ./.github/actions/check_artifact_exists
with:
name: ${{ needs.tensorflow_opt-AndroidArm64.outputs.cache_key }}.tar.xz
path: ${{ github.workspace }}/
path: ${{ github.workspace }}/tf-cache.tar.xz
download: true
- run: |
tar --skip-old-files -xf ${{ needs.tensorflow_opt-AndroidArm64.outputs.cache_key }}.tar.xz
rm ${{ needs.tensorflow_opt-AndroidArm64.outputs.cache_key }}.tar.xz
tar --skip-old-files -xf tf-cache.tar.xz
rm tf-cache.tar.xz
- uses: ./.github/actions/libstt-build
with:
arch: android-x86_64

View File

@ -64,7 +64,7 @@ def get_importers_parser(description):
parser = argparse.ArgumentParser(description=description)
parser.add_argument(
"--validate_label_locale",
help="Path to a Python file defining a |validate_label| function for your locale. WARNING: THIS WILL ADD THIS FILE's DIRECTORY INTO PYTHONPATH.",
help="Path to a Python file defining a |validate_label| function for your locale.",
)
return parser
@ -81,15 +81,15 @@ def get_validate_label(args):
:return: The user-supplied validate_label function
:type: function
"""
# Python 3.5 does not support passing a pathlib.Path to os.path.* methods
if "validate_label_locale" not in args or (args.validate_label_locale is None):
print(
"WARNING: No --validate_label_locale specified, your might end with inconsistent dataset."
"WARNING: No --validate_label_locale specified, you might end with inconsistent dataset."
)
return validate_label_eng
# Python 3.5 does not support passing a pathlib.Path to os.path.* methods
validate_label_locale = str(args.validate_label_locale)
if not os.path.exists(os.path.abspath(validate_label_locale)):
print("ERROR: Inexistent --validate_label_locale specified. Please check.")
print("ERROR: Path specified in --validate_label_locale is not a file.")
return None
module_dir = os.path.abspath(os.path.dirname(validate_label_locale))
sys.path.insert(1, module_dir)