Merge pull request #1697 from lissyx/add-benchmark_model-tasks

Add benchmark model tasks
2018-11-06 07:54:07 +01:00 · 2018-11-06 07:54:07 +01:00 · 1c71263790
commit 1c71263790
parent 4a520e8b57 d2438397ce
16 changed files with 54 additions and 12 deletions
--- a/taskcluster/.build.yml
+++ b/taskcluster/.build.yml
@ -21,3 +21,4 @@ build:
  args:
    tests_cmdline: ''
  convert_graphdef: ''
+  benchmark_model_bin: ''
--- a/taskcluster/darwin-amd64-cpu-opt.yml
+++ b/taskcluster/darwin-amd64-cpu-opt.yml
@ -6,7 +6,7 @@ build:
    - "index.project.deepspeech.deepspeech.native_client.osx.${event.head.sha}"
    - "notify.irc-channel.${notifications.irc}.on-exception"
    - "notify.irc-channel.${notifications.irc}.on-failed"
-  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.12.1c93ca24c99d7011ad639eea4cd96e4fe45e1a95.osx/artifacts/public/home.tar.xz"
+  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.12.bea86c1e884730cf7f8615eb24d31872c198c766.osx/artifacts/public/home.tar.xz"
  scripts:
    build: "taskcluster/host-build.sh"
    package: "taskcluster/package.sh"
--- a/taskcluster/linux-amd64-cpu-opt.yml
+++ b/taskcluster/linux-amd64-cpu-opt.yml
@ -14,7 +14,7 @@ build:
  system_config:
    >
      ${swig.patch_nodejs.linux}
-  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.12.1c93ca24c99d7011ad639eea4cd96e4fe45e1a95.cpu/artifacts/public/home.tar.xz"
+  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.12.bea86c1e884730cf7f8615eb24d31872c198c766.cpu/artifacts/public/home.tar.xz"
  scripts:
    build: "taskcluster/host-build.sh"
    package: "taskcluster/package.sh"
--- a/taskcluster/linux-amd64-ctc-opt.yml
+++ b/taskcluster/linux-amd64-ctc-opt.yml
@ -4,7 +4,7 @@ build:
    - "pull_request.synchronize"
    - "pull_request.reopened"
  template_file: linux-opt-base.tyml
-  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.12.1c93ca24c99d7011ad639eea4cd96e4fe45e1a95.cpu/artifacts/public/home.tar.xz"
+  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.12.bea86c1e884730cf7f8615eb24d31872c198c766.cpu/artifacts/public/home.tar.xz"
  scripts:
    build: 'taskcluster/decoder-build.sh'
    package: 'taskcluster/decoder-package.sh'
--- a/taskcluster/linux-amd64-gpu-opt.yml
+++ b/taskcluster/linux-amd64-gpu-opt.yml
@ -12,7 +12,7 @@ build:
  system_config:
    >
      ${swig.patch_nodejs.linux}
-  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.12.1c93ca24c99d7011ad639eea4cd96e4fe45e1a95.gpu/artifacts/public/home.tar.xz"
+  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.12.bea86c1e884730cf7f8615eb24d31872c198c766.gpu/artifacts/public/home.tar.xz"
  maxRunTime: 14400
  scripts:
    build: "taskcluster/cuda-build.sh"
--- a/taskcluster/linux-arm64-cpu-opt.yml
+++ b/taskcluster/linux-arm64-cpu-opt.yml
@ -4,7 +4,7 @@ build:
    - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.arm64"
    - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.arm64"
    - "index.project.deepspeech.deepspeech.native_client.arm64.${event.head.sha}"
-  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.12.1c93ca24c99d7011ad639eea4cd96e4fe45e1a95.arm64/artifacts/public/home.tar.xz"
+  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.12.bea86c1e884730cf7f8615eb24d31872c198c766.arm64/artifacts/public/home.tar.xz"
  ## multistrap 2.2.0-ubuntu1 is broken in 14.04: https://bugs.launchpad.net/ubuntu/+source/multistrap/+bug/1313787
  system_setup:
    >
--- a/taskcluster/linux-rpi3-cpu-opt.yml
+++ b/taskcluster/linux-rpi3-cpu-opt.yml
@ -4,7 +4,7 @@ build:
    - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.arm"
    - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.arm"
    - "index.project.deepspeech.deepspeech.native_client.arm.${event.head.sha}"
-  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.12.1c93ca24c99d7011ad639eea4cd96e4fe45e1a95.arm/artifacts/public/home.tar.xz"
+  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.12.bea86c1e884730cf7f8615eb24d31872c198c766.arm/artifacts/public/home.tar.xz"
  ## multistrap 2.2.0-ubuntu1 is broken in 14.04: https://bugs.launchpad.net/ubuntu/+source/multistrap/+bug/1313787
  system_setup:
    >
--- a/taskcluster/node-package.yml
+++ b/taskcluster/node-package.yml
@ -16,7 +16,7 @@ build:
  system_config:
    >
      ${swig.patch_nodejs.linux}
-  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.12.1c93ca24c99d7011ad639eea4cd96e4fe45e1a95.cpu/artifacts/public/home.tar.xz"
+  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.12.bea86c1e884730cf7f8615eb24d31872c198c766.cpu/artifacts/public/home.tar.xz"
  scripts:
    build: "taskcluster/node-build.sh"
    package: "taskcluster/node-package.sh"
--- a/taskcluster/test-armbian-opt-base.tyml
+++ b/taskcluster/test-armbian-opt-base.tyml
@ -43,7 +43,7 @@ then:
        PIP_DEFAULT_TIMEOUT: "60"
        PIP_EXTRA_INDEX_URL: "https://lissyx.github.io/deepspeech-python-wheels/"
        EXTRA_PYTHON_CONFIGURE_OPTS: "--with-fpectl" # Required by Debian Stretch
-        EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.12.0-rc2-5-g1c93ca2"
+        EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.12.0-rc2-11-gbea86c1"

    command:
      - "/bin/bash"
--- a/taskcluster/test-darwin-opt-base.tyml
+++ b/taskcluster/test-darwin-opt-base.tyml
@ -41,7 +41,7 @@ then:
        DEEPSPEECH_TEST_MODEL: https://queue.taskcluster.net/v1/task/${training}/artifacts/public/output_graph.pb
        DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.2.0-prod-ctcdecode/output_graph.pb
        DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.2.0-prod-ctcdecode/output_graph.pbmm
-        EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.12.0-rc2-5-g1c93ca2"
+        EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.12.0-rc2-11-gbea86c1"

    command:
      - - "/bin/bash"
--- a/taskcluster/test-linux-opt-base.tyml
+++ b/taskcluster/test-linux-opt-base.tyml
@ -37,6 +37,7 @@ then:
        node_package: { $eval: as_slugid("node-package") }
      in:
        CONVERT_GRAPHDEF_MEMMAPPED: ${build.convert_graphdef}
+        BENCHMARK_MODEL_BIN: ${build.benchmark_model_bin}
        DEEPSPEECH_ARTIFACTS_ROOT: https://queue.taskcluster.net/v1/task/${linux_amd64_build}/artifacts/public
        DEEPSPEECH_NODEJS: https://queue.taskcluster.net/v1/task/${node_package}/artifacts/public
        DEEPSPEECH_LIBCTC: https://queue.taskcluster.net/v1/task/${linux_amd64_ctc}/artifacts/public/decoder.tar.xz
@ -44,7 +45,7 @@ then:
        DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.2.0-prod-ctcdecode/output_graph.pb
        DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.2.0-prod-ctcdecode/output_graph.pbmm
        PIP_DEFAULT_TIMEOUT: "60"
-        EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.12.0-rc2-5-g1c93ca2"
+        EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.12.0-rc2-11-gbea86c1"

    command:
      - "/bin/bash"
--- a/taskcluster/test-lite_benchmark_model-linux-amd64-opt.yml
+++ b/taskcluster/test-lite_benchmark_model-linux-amd64-opt.yml
@ -0,0 +1,10 @@
+build:
+  template_file: test-linux-opt-base.tyml
+  dependencies:
+    - "test-training_upstream-linux-amd64-py27mu-opt"
+  args:
+    tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-lite_benchmark_model-ds-tests.sh"
+  benchmark_model_bin: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.12.bea86c1e884730cf7f8615eb24d31872c198c766.cpu/artifacts/public/lite_benchmark_model"
+  metadata:
+    name: "DeepSpeech Linux AMD64 CPU TF Lite benchmark_model"
+    description: "Testing DeepSpeech TF Lite benchmark_model for Linux/AMD64, CPU only, optimized version"
--- a/taskcluster/test-raspbian-opt-base.tyml
+++ b/taskcluster/test-raspbian-opt-base.tyml
@ -43,7 +43,7 @@ then:
        PIP_DEFAULT_TIMEOUT: "60"
        PIP_EXTRA_INDEX_URL: "https://www.piwheels.org/simple"
        EXTRA_PYTHON_CONFIGURE_OPTS: "--with-fpectl" # Required by Raspbian Stretch / PiWheels
-        EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.12.0-rc2-5-g1c93ca2"
+        EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.12.0-rc2-11-gbea86c1"

    command:
      - "/bin/bash"
--- a/taskcluster/test-training_upstream-linux-amd64-py27mu-opt.yml
+++ b/taskcluster/test-training_upstream-linux-amd64-py27mu-opt.yml
@ -7,7 +7,7 @@ build:
      apt-get -qq -y install ${python.packages_trusty.apt}
  args:
    tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-train-tests.sh 2.7.14:mu"
-  convert_graphdef: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.12.1c93ca24c99d7011ad639eea4cd96e4fe45e1a95.cpu/artifacts/public/convert_graphdef_memmapped_format"
+  convert_graphdef: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.12.bea86c1e884730cf7f8615eb24d31872c198c766.cpu/artifacts/public/convert_graphdef_memmapped_format"
  metadata:
    name: "DeepSpeech Linux AMD64 CPU upstream training Py2.7 mu"
    description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 using upstream TensorFlow Python 2.7 mu, CPU only, optimized version"
--- a/tc-lite_benchmark_model-ds-tests.sh
+++ b/tc-lite_benchmark_model-ds-tests.sh
@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -xe
+
+source $(dirname "$0")/tc-tests-utils.sh
+
+model_source=${DEEPSPEECH_TEST_MODEL//.pb/.tflite}
+model_name=$(basename "${model_source}")
+
+download_benchmark_model "${TASKCLUSTER_TMP_DIR}/ds"
+
+export PATH=${TASKCLUSTER_TMP_DIR}/ds/:$PATH
+
+lite_benchmark_model \
+	--graph=${TASKCLUSTER_TMP_DIR}/ds/${model_name} \
+	--show_flops \
+	--input_layer=input_node,previous_state_c,previous_state_h \
+	--input_layer_type=float,float,float \
+	--input_layer_shape=1,16,19,26:1:1,494:1,494 \
+	--output_layer=logits,new_state_c,new_state_h
--- a/tc-tests-utils.sh
+++ b/tc-tests-utils.sh
@ -275,6 +275,16 @@ download_material()
  ls -hal ${TASKCLUSTER_TMP_DIR}/${model_name} ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} ${TASKCLUSTER_TMP_DIR}/LDC93S1*.wav ${TASKCLUSTER_TMP_DIR}/alphabet.txt
 }

+download_benchmark_model()
+{
+  target_dir=$1
+
+  mkdir -p ${target_dir} || true
+
+  wget -P "${target_dir}" "${model_source}"
+  wget -P "${target_dir}" "${BENCHMARK_MODEL_BIN}" && chmod +x ${target_dir}/*benchmark_model
+}
+
 install_pyenv()
 {
  if [ -z "${PYENV_ROOT}" ]; then