experiments/STT
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #1651 from lissyx/stable-tc-workdir

Browse Source 
Stable tc workdir
This commit is contained in:
lissyx 2018-10-16 17:14:40 +02:00 committed by GitHub
commit 9d52750595
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 45 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
taskcluster/darwin-amd64-cpu-opt.yml
View File

@ -6,8 +6,8 @@ build:
- "index.project.deepspeech.deepspeech.native_client.osx.${event.head.sha}"
- "notify.irc-channel.${notifications.irc}.on-exception"
- "notify.irc-channel.${notifications.irc}.on-failed"
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.689b0daf0bac618551e7dd94bd2045b7eded1458.osx/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.689b0daf0bac618551e7dd94bd2045b7eded1458.osx/artifacts/public/summarize_graph"
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.osx/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.osx/artifacts/public/summarize_graph"
scripts:
build: "taskcluster/host-build.sh"
package: "taskcluster/package.sh"

53
taskcluster/darwin-opt-base.tyml
View File

@ -79,54 +79,16 @@ payload:
- "-cxe"
- >
export TASKCLUSTER_ARTIFACTS="$(pwd)/public/" &&
export TASKCLUSTER_ORIG_TASKDIR="$(pwd)" &&
(mkdir ../tc-workdir/ || rm -fr ../tc-workdir/*) && cd ../tc-workdir/ &&
(mv $TASKCLUSTER_ORIG_TASKDIR/homebrew/ homebrew/ || true) &&
(mv $TASKCLUSTER_ORIG_TASKDIR/homebrew.cache/ homebrew.cache/ || true) &&
export TASKCLUSTER_TASK_DIR="$(pwd)" &&
export LC_ALL=C &&
export TASKCLUSTER_ORIGIN_PATH="${system.homedir.osx}/TaskCluster/.*/task_[[:digit:]]\{10\}" &&
export TASKCLUSTER_REWRITE_PATH="$TASKCLUSTER_TASK_DIR" &&
export PKG_CONFIG_PATH="$TASKCLUSTER_TASK_DIR/homebrew/lib/pkgconfig" &&
export MACOSX_DEPLOYMENT_TARGET=10.10 &&
env &&
(
(
for link in $(find -L "$TASKCLUSTER_TASK_DIR/homebrew/" -type l);
do
newloc=$(readlink "$link" | sed -e "s|$TASKCLUSTER_ORIGIN_PATH|$TASKCLUSTER_REWRITE_PATH|g");
ln -hfs "$newloc" "$link";
done) || true) &&
(
(
for lib in $(find "$TASKCLUSTER_TASK_DIR/homebrew/" -type f -name "*.dylib");
do
chmod +w "$lib";
for id in $(otool -D "$lib" | grep "$TASKCLUSTER_ORIGIN_PATH");
do
newid=$(echo $id | sed -e "s|$TASKCLUSTER_ORIGIN_PATH|$TASKCLUSTER_REWRITE_PATH|g");
install_name_tool -id "$newid" "$lib";
done;
for dep in $(otool -L "$lib" | awk '{ print $1 }' | grep "$TASKCLUSTER_ORIGIN_PATH");
do
newdep=$(echo $dep | sed -e "s|$TASKCLUSTER_ORIGIN_PATH|$TASKCLUSTER_REWRITE_PATH|g");
install_name_tool -change "$dep" "$newdep" "$lib";
done;
chmod -w "$lib";
otool -L "$lib";
done) || true) &&
(
(
for bin in $(find "$TASKCLUSTER_TASK_DIR/homebrew/" -perm +111 -type f);
do
chmod +w "$bin";
for dep in $(otool -L "$bin" | awk '{ print $1 }' | grep "$TASKCLUSTER_ORIGIN_PATH");
do
newdep=$(echo $dep | sed -e "s|$TASKCLUSTER_ORIGIN_PATH|$TASKCLUSTER_REWRITE_PATH|g");
install_name_tool -change "$dep" "$newdep" "$bin";
done;
chmod -w "$bin";
otool -L "$bin";
done) || true) &&
(wget -O - $TENSORFLOW_BUILD_ARTIFACT | pixz -d | gtar -C $TASKCLUSTER_TASK_DIR --transform "s|$TASKCLUSTER_ORIGIN_PATH|$TASKCLUSTER_REWRITE_PATH|g" -xf - ) &&
((grep -R -I --files-with-matches "$TASKCLUSTER_ORIGIN_PATH" $TASKCLUSTER_TASK_DIR | grep -v "$TASKCLUSTER_TASK_DIR/generic-worker/" | xargs -n 1 -P 16 sed -i '' -e "s|$TASKCLUSTER_ORIGIN_PATH|$TASKCLUSTER_REWRITE_PATH|g") || true) &&
(wget -O - $TENSORFLOW_BUILD_ARTIFACT | pixz -d | gtar -C $TASKCLUSTER_TASK_DIR -xf - ) &&
git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ &&
cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git checkout --quiet ${event.head.sha} &&
ln -s $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/native_client/ $TASKCLUSTER_TASK_DIR/DeepSpeech/tf/native_client &&
@ -134,7 +96,10 @@ payload:
$TASKCLUSTER_TASK_DIR/DeepSpeech/tf/tc-brew.sh &&
${swig.patch_nodejs.osx} &&
$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.build} &&
$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.package}
$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.package} ;
mv $TASKCLUSTER_TASK_DIR/homebrew/ $TASKCLUSTER_ORIG_TASKDIR/homebrew/ &&
mv $TASKCLUSTER_TASK_DIR/homebrew.cache/ $TASKCLUSTER_ORIG_TASKDIR/homebrew.cache/ &&
cd $TASKCLUSTER_TASK_DIR/../ && rm -fr tc-workdir/
artifacts:
- type: "directory"

4
taskcluster/linux-amd64-cpu-opt.yml
View File

@ -14,8 +14,8 @@ build:
system_config:
>
${swig.patch_nodejs.linux}
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.689b0daf0bac618551e7dd94bd2045b7eded1458.cpu/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.689b0daf0bac618551e7dd94bd2045b7eded1458.cpu/artifacts/public/summarize_graph"
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.cpu/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.cpu/artifacts/public/summarize_graph"
scripts:
build: "taskcluster/host-build.sh"
package: "taskcluster/package.sh"

4
taskcluster/linux-amd64-ctc-opt.yml
View File

@ -4,8 +4,8 @@ build:
- "pull_request.synchronize"
- "pull_request.reopened"
template_file: linux-opt-base.tyml
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.689b0daf0bac618551e7dd94bd2045b7eded1458.cpu/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.689b0daf0bac618551e7dd94bd2045b7eded1458.cpu/artifacts/public/summarize_graph"
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.cpu/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.cpu/artifacts/public/summarize_graph"
scripts:
build: 'taskcluster/decoder-build.sh'
package: 'taskcluster/decoder-package.sh'

4
taskcluster/linux-amd64-gpu-opt.yml
View File

@ -12,8 +12,8 @@ build:
system_config:
>
${swig.patch_nodejs.linux}
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.689b0daf0bac618551e7dd94bd2045b7eded1458.gpu/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.689b0daf0bac618551e7dd94bd2045b7eded1458.gpu/artifacts/public/summarize_graph"
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.gpu/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.gpu/artifacts/public/summarize_graph"
maxRunTime: 14400
scripts:
build: "taskcluster/cuda-build.sh"

4
taskcluster/linux-arm64-cpu-opt.yml
View File

@ -4,8 +4,8 @@ build:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.arm64"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.arm64"
- "index.project.deepspeech.deepspeech.native_client.arm64.${event.head.sha}"
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.689b0daf0bac618551e7dd94bd2045b7eded1458.arm64/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.689b0daf0bac618551e7dd94bd2045b7eded1458.cpu/artifacts/public/summarize_graph"
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.arm64/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.cpu/artifacts/public/summarize_graph"
## multistrap 2.2.0-ubuntu1 is broken in 14.04: https://bugs.launchpad.net/ubuntu/+source/multistrap/+bug/1313787
system_setup:
>

4
taskcluster/linux-rpi3-cpu-opt.yml
View File

@ -4,8 +4,8 @@ build:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.arm"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.arm"
- "index.project.deepspeech.deepspeech.native_client.arm.${event.head.sha}"
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.689b0daf0bac618551e7dd94bd2045b7eded1458.arm/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.689b0daf0bac618551e7dd94bd2045b7eded1458.cpu/artifacts/public/summarize_graph"
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.arm/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.cpu/artifacts/public/summarize_graph"
## multistrap 2.2.0-ubuntu1 is broken in 14.04: https://bugs.launchpad.net/ubuntu/+source/multistrap/+bug/1313787
system_setup:
>

4
taskcluster/node-package.yml
View File

@ -16,8 +16,8 @@ build:
system_config:
>
${swig.patch_nodejs.linux}
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.689b0daf0bac618551e7dd94bd2045b7eded1458.cpu/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.689b0daf0bac618551e7dd94bd2045b7eded1458.cpu/artifacts/public/summarize_graph"
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.cpu/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.cpu/artifacts/public/summarize_graph"
scripts:
build: "taskcluster/node-build.sh"
package: "taskcluster/node-package.sh"

2
taskcluster/test-armbian-opt-base.tyml
View File

@ -44,7 +44,7 @@ then:
PIP_DEFAULT_TIMEOUT: "60"
PIP_EXTRA_INDEX_URL: "https://lissyx.github.io/deepspeech-python-wheels/"
EXTRA_PYTHON_CONFIGURE_OPTS: "--with-fpectl" # Required by Debian Stretch
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.11.0-7-g689b0da"
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.11.0-9-g97d851f"
command:
- "/bin/bash"

19
taskcluster/test-darwin-opt-base.tyml
View File

@ -23,6 +23,8 @@ then:
scopes: [
"queue:route:notify.irc-channel.*",
"generic-worker:cache:deepspeech-homebrew-bin",
"generic-worker:cache:deepspeech-homebrew-cache",
]
payload:
@ -39,7 +41,7 @@ then:
DEEPSPEECH_TEST_MODEL: https://queue.taskcluster.net/v1/task/${training}/artifacts/public/output_graph.pb
DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.2.0-prod/output_graph.pb
DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.2.0-prod/output_graph.pbmm
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.11.0-7-g689b0da"
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.11.0-9-g97d851f"
command:
- - "/bin/bash"
@ -49,6 +51,10 @@ then:
extraSystemSetup: { $eval: strip(str(build.system_setup)) }
in: >
export TASKCLUSTER_ARTIFACTS="$(pwd)/public/" &&
export TASKCLUSTER_ORIG_TASKDIR="$(pwd)" &&
(mkdir ../tc-workdir/ || rm -fr ../tc-workdir/*) && cd ../tc-workdir/ &&
(mv $TASKCLUSTER_ORIG_TASKDIR/homebrew/ homebrew/ || true) &&
(mv $TASKCLUSTER_ORIG_TASKDIR/homebrew.cache/ homebrew.cache/ || true) &&
export TASKCLUSTER_TASK_DIR="$(pwd)" &&
export TASKCLUSTER_TMP_DIR="$TASKCLUSTER_TASK_DIR/tmp" &&
export LC_ALL=C &&
@ -59,7 +65,16 @@ then:
cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git checkout --quiet ${event.head.sha} &&
cd $TASKCLUSTER_TASK_DIR &&
source $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/tc-brew-tests.sh && ${extraSystemSetup} &&
/bin/bash ${build.args.tests_cmdline}
/bin/bash ${build.args.tests_cmdline} ;
mv $TASKCLUSTER_TASK_DIR/homebrew/ $TASKCLUSTER_ORIG_TASKDIR/homebrew/ &&
mv $TASKCLUSTER_TASK_DIR/homebrew.cache/ $TASKCLUSTER_ORIG_TASKDIR/homebrew.cache/ &&
cd $TASKCLUSTER_TASK_DIR/../ && rm -fr tc-workdir/
mounts:
- cacheName: deepspeech-homebrew-bin
directory: homebrew/
- cacheName: deepspeech-homebrew-cache
directory: homebrew.cache/
metadata:
name: ${build.metadata.name}

2
taskcluster/test-linux-opt-base.tyml
View File

@ -44,7 +44,7 @@ then:
DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.2.0-prod/output_graph.pb
DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.2.0-prod/output_graph.pbmm
PIP_DEFAULT_TIMEOUT: "60"
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.11.0-7-g689b0da"
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.11.0-9-g97d851f"
command:
- "/bin/bash"

2
taskcluster/test-raspbian-opt-base.tyml
View File

@ -44,7 +44,7 @@ then:
PIP_DEFAULT_TIMEOUT: "60"
PIP_EXTRA_INDEX_URL: "https://www.piwheels.org/simple"
EXTRA_PYTHON_CONFIGURE_OPTS: "--with-fpectl" # Required by Raspbian Stretch / PiWheels
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.11.0-7-g689b0da"
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.11.0-9-g97d851f"
command:
- "/bin/bash"

2
taskcluster/test-training_upstream-linux-amd64-py27mu-opt.yml
View File

@ -7,7 +7,7 @@ build:
apt-get -qq -y install ${python.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-train-tests.sh 2.7.14:mu"
convert_graphdef: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.689b0daf0bac618551e7dd94bd2045b7eded1458.cpu/artifacts/public/convert_graphdef_memmapped_format"
convert_graphdef: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.cpu/artifacts/public/convert_graphdef_memmapped_format"
metadata:
name: "DeepSpeech Linux AMD64 CPU upstream training Py2.7 mu"
description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 using upstream TensorFlow Python 2.7 mu, CPU only, optimized version"

2
tc-brew-tests.sh
View File

@ -37,7 +37,7 @@ install_local_homebrew()
mkdir -p "${LOCAL_HOMEBREW_DIRECTORY}"
mkdir -p "${HOMEBREW_CACHE}"
curl -L https://github.com/Homebrew/brew/tarball/master | tar xz --strip 1 -C "${LOCAL_HOMEBREW_DIRECTORY}"
curl -L https://github.com/Homebrew/brew/tarball/1.7.7 | tar xz --strip 1 -C "${LOCAL_HOMEBREW_DIRECTORY}"
export PATH=${LOCAL_HOMEBREW_DIRECTORY}/bin:$PATH
if [ ! -x "${LOCAL_HOMEBREW_DIRECTORY}/bin/brew" ]; then