Merge pull request #2645 from reuben/tf-1.15

Update to TensorFlow 1.15
2020-01-13 10:07:03 +01:00 · 2020-01-13 10:07:03 +01:00 · 7225336cea
commit 7225336cea
parent faed282cfc 17597f4526
27 changed files with 38 additions and 29 deletions
--- a/4
+++ b/4
@ -68,7 +68,7 @@ RUN wget https://bootstrap.pypa.io/get-pip.py && \
 # Clone TensoFlow from Mozilla repo
 RUN git clone https://github.com/mozilla/tensorflow/
 WORKDIR /tensorflow
-RUN git checkout r1.14
+RUN git checkout r1.15


 # GPU Environment Setup
@ -192,7 +192,7 @@ RUN cp /tensorflow/bazel-bin/native_client/generate_trie /DeepSpeech/native_clie

 # Install TensorFlow
 WORKDIR /DeepSpeech/
-RUN pip3 install tensorflow-gpu==1.14.0
+RUN pip3 install tensorflow-gpu==1.15.0


 # Make DeepSpeech and install Python bindings
--- a/requirements.txt
+++ b/requirements.txt
@ -1,6 +1,6 @@
 # Main training requirements
-tensorflow == 1.14.0
-numpy == 1.15.4
+tensorflow == 1.15.0
+numpy == 1.18.1
 progressbar2
 pandas
 six
--- a/taskcluster/.build.yml
+++ b/taskcluster/.build.yml
@ -23,5 +23,5 @@ build:
    tests_cmdline: ''
  convert_graphdef: ''
  benchmark_model_bin: ''
-  tensorflow_git_desc: 'TensorFlow: v1.14.0-21-ge77504a'
+  tensorflow_git_desc: 'TensorFlow: v1.15.0-22-gbd115ee'
  test_model_task: ''
--- a/taskcluster/android-arm64-cpu-opt.yml
+++ b/taskcluster/android-arm64-cpu-opt.yml
@ -12,7 +12,7 @@ build:
  system_config:
    >
      ${swig.patch_nodejs.linux}
-  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.14.e77504ac6b941fd2af495b49a0bcd56f25ca0e1e.android-arm64/artifacts/public/home.tar.xz"
+  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.bd115ee104da3d6bedea16e3bede341d1d7d7a4b.android-arm64/artifacts/public/home.tar.xz"
  scripts:
    build: "taskcluster/android-build.sh arm64-v8a"
    package: "taskcluster/android-package.sh arm64-v8a"
--- a/taskcluster/android-armv7-cpu-opt.yml
+++ b/taskcluster/android-armv7-cpu-opt.yml
@ -12,7 +12,7 @@ build:
  system_config:
    >
      ${swig.patch_nodejs.linux}
-  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.14.e77504ac6b941fd2af495b49a0bcd56f25ca0e1e.android-armv7/artifacts/public/home.tar.xz"
+  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.bd115ee104da3d6bedea16e3bede341d1d7d7a4b.android-armv7/artifacts/public/home.tar.xz"
  scripts:
    build: "taskcluster/android-build.sh armeabi-v7a"
    package: "taskcluster/android-package.sh armeabi-v7a"
--- a/taskcluster/android-java-opt.yml
+++ b/taskcluster/android-java-opt.yml
@ -13,7 +13,7 @@ build:
  system_setup:
    >
      apt-get -qq -y install curl && ${swig.packages.install_script}
-  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.14.e77504ac6b941fd2af495b49a0bcd56f25ca0e1e.android-armv7/artifacts/public/home.tar.xz"
+  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.bd115ee104da3d6bedea16e3bede341d1d7d7a4b.android-armv7/artifacts/public/home.tar.xz"
  scripts:
    build: "taskcluster/android-apk-build.sh"
    package: "taskcluster/android-apk-package.sh"
--- a/taskcluster/android-x86_64-cpu-opt.yml
+++ b/taskcluster/android-x86_64-cpu-opt.yml
@ -12,7 +12,7 @@ build:
  system_config:
    >
      ${swig.patch_nodejs.linux}
-  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.14.e77504ac6b941fd2af495b49a0bcd56f25ca0e1e.android-arm64/artifacts/public/home.tar.xz"
+  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.bd115ee104da3d6bedea16e3bede341d1d7d7a4b.android-arm64/artifacts/public/home.tar.xz"
  scripts:
    build: "taskcluster/android-build.sh x86_64"
    package: "taskcluster/android-package.sh x86_64"
--- a/taskcluster/darwin-amd64-cpu-opt.yml
+++ b/taskcluster/darwin-amd64-cpu-opt.yml
@ -6,7 +6,7 @@ build:
    - "index.project.deepspeech.deepspeech.native_client.osx.${event.head.sha}"
    - "notify.irc-channel.${notifications.irc}.on-exception"
    - "notify.irc-channel.${notifications.irc}.on-failed"
-  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.14.e77504ac6b941fd2af495b49a0bcd56f25ca0e1e.osx/artifacts/public/home.tar.xz"
+  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.bd115ee104da3d6bedea16e3bede341d1d7d7a4b.osx/artifacts/public/home.tar.xz"
  scripts:
    build: "taskcluster/host-build.sh"
    package: "taskcluster/package.sh"
--- a/taskcluster/darwin-amd64-ctc-opt.yml
+++ b/taskcluster/darwin-amd64-ctc-opt.yml
@ -6,7 +6,7 @@ build:
    - "index.project.deepspeech.deepspeech.native_client.osx-ctc.${event.head.sha}"
    - "notify.irc-channel.${notifications.irc}.on-exception"
    - "notify.irc-channel.${notifications.irc}.on-failed"
-  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.14.e77504ac6b941fd2af495b49a0bcd56f25ca0e1e.osx/artifacts/public/home.tar.xz"
+  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.bd115ee104da3d6bedea16e3bede341d1d7d7a4b.osx/artifacts/public/home.tar.xz"
  maxRunTime: 14400
  scripts:
    build: 'taskcluster/decoder-build.sh'
--- a/taskcluster/darwin-amd64-tflite-opt.yml
+++ b/taskcluster/darwin-amd64-tflite-opt.yml
@ -6,7 +6,7 @@ build:
    - "index.project.deepspeech.deepspeech.native_client.osx-tflite.${event.head.sha}"
    - "notify.irc-channel.${notifications.irc}.on-exception"
    - "notify.irc-channel.${notifications.irc}.on-failed"
-  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.14.e77504ac6b941fd2af495b49a0bcd56f25ca0e1e.osx/artifacts/public/home.tar.xz"
+  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.bd115ee104da3d6bedea16e3bede341d1d7d7a4b.osx/artifacts/public/home.tar.xz"
  scripts:
    build: "taskcluster/host-build.sh tflite"
    package: "taskcluster/package.sh"
--- a/taskcluster/linux-amd64-cpu-opt.yml
+++ b/taskcluster/linux-amd64-cpu-opt.yml
@ -14,7 +14,7 @@ build:
  system_config:
    >
      ${swig.patch_nodejs.linux}
-  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.14.e77504ac6b941fd2af495b49a0bcd56f25ca0e1e.cpu/artifacts/public/home.tar.xz"
+  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.bd115ee104da3d6bedea16e3bede341d1d7d7a4b.cpu/artifacts/public/home.tar.xz"
  scripts:
    build: "taskcluster/host-build.sh"
    package: "taskcluster/package.sh"
--- a/taskcluster/linux-amd64-ctc-opt.yml
+++ b/taskcluster/linux-amd64-ctc-opt.yml
@ -14,7 +14,7 @@ build:
  system_config:
    >
      ${swig.patch_nodejs.linux}
-  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.14.e77504ac6b941fd2af495b49a0bcd56f25ca0e1e.cpu/artifacts/public/home.tar.xz"
+  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.bd115ee104da3d6bedea16e3bede341d1d7d7a4b.cpu/artifacts/public/home.tar.xz"
  scripts:
    build: 'taskcluster/decoder-build.sh'
    package: 'taskcluster/decoder-package.sh'
--- a/taskcluster/linux-amd64-gpu-opt.yml
+++ b/taskcluster/linux-amd64-gpu-opt.yml
@ -12,7 +12,7 @@ build:
  system_config:
    >
      ${swig.patch_nodejs.linux}
-  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.14.e77504ac6b941fd2af495b49a0bcd56f25ca0e1e.gpu/artifacts/public/home.tar.xz"
+  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.bd115ee104da3d6bedea16e3bede341d1d7d7a4b.gpu/artifacts/public/home.tar.xz"
  maxRunTime: 14400
  scripts:
    build: "taskcluster/cuda-build.sh"
--- a/taskcluster/linux-amd64-tflite-opt.yml
+++ b/taskcluster/linux-amd64-tflite-opt.yml
@ -14,7 +14,7 @@ build:
  system_config:
    >
      ${swig.patch_nodejs.linux}
-  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.14.e77504ac6b941fd2af495b49a0bcd56f25ca0e1e.cpu/artifacts/public/home.tar.xz"
+  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.bd115ee104da3d6bedea16e3bede341d1d7d7a4b.cpu/artifacts/public/home.tar.xz"
  scripts:
    build: "taskcluster/host-build.sh tflite"
    package: "taskcluster/package.sh"
--- a/taskcluster/linux-arm64-cpu-opt.yml
+++ b/taskcluster/linux-arm64-cpu-opt.yml
@ -4,7 +4,7 @@ build:
    - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.arm64"
    - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.arm64"
    - "index.project.deepspeech.deepspeech.native_client.arm64.${event.head.sha}"
-  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.14.e77504ac6b941fd2af495b49a0bcd56f25ca0e1e.arm64/artifacts/public/home.tar.xz"
+  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.bd115ee104da3d6bedea16e3bede341d1d7d7a4b.arm64/artifacts/public/home.tar.xz"
  ## multistrap 2.2.0-ubuntu1 is broken in 14.04: https://bugs.launchpad.net/ubuntu/+source/multistrap/+bug/1313787
  system_setup:
    >
--- a/taskcluster/linux-rpi3-cpu-opt.yml
+++ b/taskcluster/linux-rpi3-cpu-opt.yml
@ -4,7 +4,7 @@ build:
    - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.arm"
    - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.arm"
    - "index.project.deepspeech.deepspeech.native_client.arm.${event.head.sha}"
-  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.14.e77504ac6b941fd2af495b49a0bcd56f25ca0e1e.arm/artifacts/public/home.tar.xz"
+  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.bd115ee104da3d6bedea16e3bede341d1d7d7a4b.arm/artifacts/public/home.tar.xz"
  ## multistrap 2.2.0-ubuntu1 is broken in 14.04: https://bugs.launchpad.net/ubuntu/+source/multistrap/+bug/1313787
  system_setup:
    >
--- a/taskcluster/node-package-cpu.yml
+++ b/taskcluster/node-package-cpu.yml
@ -17,7 +17,7 @@ build:
  system_config:
    >
      ${swig.patch_nodejs.linux}
-  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.14.e77504ac6b941fd2af495b49a0bcd56f25ca0e1e.cpu/artifacts/public/home.tar.xz"
+  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.bd115ee104da3d6bedea16e3bede341d1d7d7a4b.cpu/artifacts/public/home.tar.xz"
  scripts:
    build: "taskcluster/node-build.sh"
    package: "taskcluster/node-package.sh"
--- a/taskcluster/node-package-gpu.yml
+++ b/taskcluster/node-package-gpu.yml
@ -14,7 +14,7 @@ build:
  system_config:
    >
      ${swig.patch_nodejs.linux}
-  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.14.e77504ac6b941fd2af495b49a0bcd56f25ca0e1e.cpu/artifacts/public/home.tar.xz"
+  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.bd115ee104da3d6bedea16e3bede341d1d7d7a4b.cpu/artifacts/public/home.tar.xz"
  scripts:
    build: "taskcluster/node-build.sh --cuda"
    package: "taskcluster/node-package.sh"
--- a/taskcluster/tc-benchmark-tests.sh
+++ b/taskcluster/tc-benchmark-tests.sh
@ -93,7 +93,10 @@ pyenv install ${pyver}
 pyenv virtualenv ${pyver} ${PYENV_NAME}
 source ${PYENV_ROOT}/versions/${pyver}/envs/${PYENV_NAME}/bin/activate

+set -o pipefail
+pip install --upgrade pip setuptools wheel | cat
 pip install -r ${DS_ROOT_TASK}/DeepSpeech/ds/requirements.txt | cat
+set +o pipefail

 exec_benchmark "/tmp/test.frozen.e75.lstm494.ldc93s1.pb" "single-model"
 exec_benchmark "/tmp/test.frozen.e75.lstm100-900.ldc93s1.zip" "zipfile-model"
--- a/taskcluster/tc-lite_benchmark_model-ds-tests.sh
+++ b/taskcluster/tc-lite_benchmark_model-ds-tests.sh
@ -14,7 +14,7 @@ export PATH=${TASKCLUSTER_TMP_DIR}/ds/:$PATH
 lite_benchmark_model \
 	--graph=${TASKCLUSTER_TMP_DIR}/ds/${model_name} \
 	--show_flops \
-	--input_layer=input_node,previous_state_c,previous_state_h \
-	--input_layer_type=float,float,float \
-	--input_layer_shape=1,16,19,26:1:1,494:1,494 \
-	--output_layer=logits,new_state_c,new_state_h
+	--input_layer=input_node,previous_state_c,previous_state_h,input_samples \
+	--input_layer_type=float,float,float,float \
+	--input_layer_shape=1,16,19,26:1,100:1,100:512 \
+	--output_layer=logits,new_state_c,new_state_h,mfccs
--- a/taskcluster/tc-single-shot-inference.sh
+++ b/taskcluster/tc-single-shot-inference.sh
@ -28,7 +28,10 @@ PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf}" pyenv_install ${pyver}
 setup_pyenv_virtualenv "${pyver}" "${PYENV_NAME}"
 virtualenv_activate "${pyver}" "${PYENV_NAME}"

+set -o pipefail
+pip install --upgrade pip setuptools wheel | cat
 pip install --upgrade -r ${HOME}/DeepSpeech/ds/requirements.txt | cat
+set +o pipefail

 decoder_pkg_url=$(get_python_pkg_url ${pyver_pkg} ${py_unicode_type} "ds_ctcdecoder" "${DECODER_ARTIFACTS_ROOT}")
 LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH pip install --verbose --only-binary :all: ${PY37_SOURCE_PACKAGE} --upgrade ${decoder_pkg_url} | cat
--- a/taskcluster/tc-train-tests.sh
+++ b/taskcluster/tc-train-tests.sh
@ -42,7 +42,10 @@ PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf}" pyenv install ${pyver}
 pyenv virtualenv ${pyver} ${PYENV_NAME}
 source ${PYENV_ROOT}/versions/${pyver}/envs/${PYENV_NAME}/bin/activate

+set -o pipefail
+pip install --upgrade pip setuptools wheel | cat
 pip install --upgrade -r ${HOME}/DeepSpeech/ds/requirements.txt | cat
+set +o pipefail

 pushd ${HOME}/DeepSpeech/ds/
    verify_ctcdecoder_url
--- a/taskcluster/test-lite_benchmark_model-linux-amd64-opt.yml
+++ b/taskcluster/test-lite_benchmark_model-linux-amd64-opt.yml
@ -5,7 +5,7 @@ build:
  test_model_task: "test-training_16k-linux-amd64-py36m-opt"
  args:
    tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-lite_benchmark_model-ds-tests.sh"
-  benchmark_model_bin: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.14.e77504ac6b941fd2af495b49a0bcd56f25ca0e1e.cpu/artifacts/public/lite_benchmark_model"
+  benchmark_model_bin: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.bd115ee104da3d6bedea16e3bede341d1d7d7a4b.cpu/artifacts/public/lite_benchmark_model"
  metadata:
    name: "DeepSpeech Linux AMD64 CPU TF Lite benchmark_model"
    description: "Testing DeepSpeech TF Lite benchmark_model for Linux/AMD64, CPU only, optimized version"
--- a/taskcluster/test-training_16k-linux-amd64-py35m-opt.yml
+++ b/taskcluster/test-training_16k-linux-amd64-py35m-opt.yml
@ -7,7 +7,7 @@ build:
      apt-get -qq -y install ${python.packages_trusty.apt}
  args:
    tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-tests.sh 3.5.5:m 16k"
-  convert_graphdef: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.14.e77504ac6b941fd2af495b49a0bcd56f25ca0e1e.cpu/artifacts/public/convert_graphdef_memmapped_format"
+  convert_graphdef: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.bd115ee104da3d6bedea16e3bede341d1d7d7a4b.cpu/artifacts/public/convert_graphdef_memmapped_format"
  metadata:
    name: "DeepSpeech Linux AMD64 CPU 16kHz training Py3.5"
    description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 16kHz Python 3.5, CPU only, optimized version"
--- a/taskcluster/win-amd64-cpu-opt.yml
+++ b/taskcluster/win-amd64-cpu-opt.yml
@ -6,7 +6,7 @@ build:
    - "index.project.deepspeech.deepspeech.native_client.win.${event.head.sha}"
    - "notify.irc-channel.${notifications.irc}.on-exception"
    - "notify.irc-channel.${notifications.irc}.on-failed"
-  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.14.e77504ac6b941fd2af495b49a0bcd56f25ca0e1e.win/artifacts/public/home.tar.xz"
+  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.bd115ee104da3d6bedea16e3bede341d1d7d7a4b.win/artifacts/public/home.tar.xz"
  scripts:
    build: "taskcluster/win-build.sh"
    package: "taskcluster/win-package.sh"
--- a/taskcluster/win-amd64-gpu-opt.yml
+++ b/taskcluster/win-amd64-gpu-opt.yml
@ -6,7 +6,7 @@ build:
    - "index.project.deepspeech.deepspeech.native_client.win-cuda.${event.head.sha}"
    - "notify.irc-channel.${notifications.irc}.on-exception"
    - "notify.irc-channel.${notifications.irc}.on-failed"
-  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.14.e77504ac6b941fd2af495b49a0bcd56f25ca0e1e.win-cuda/artifacts/public/home.tar.xz"
+  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.bd115ee104da3d6bedea16e3bede341d1d7d7a4b.win-cuda/artifacts/public/home.tar.xz"
  scripts:
    build: "taskcluster/win-build.sh --cuda"
    package: "taskcluster/win-package.sh"
--- a/taskcluster/win-amd64-tflite-opt.yml
+++ b/taskcluster/win-amd64-tflite-opt.yml
@ -6,7 +6,7 @@ build:
    - "index.project.deepspeech.deepspeech.native_client.win-tflite.${event.head.sha}"
    - "notify.irc-channel.${notifications.irc}.on-exception"
    - "notify.irc-channel.${notifications.irc}.on-failed"
-  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.14.e77504ac6b941fd2af495b49a0bcd56f25ca0e1e.win/artifacts/public/home.tar.xz"
+  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.bd115ee104da3d6bedea16e3bede341d1d7d7a4b.win/artifacts/public/home.tar.xz"
  scripts:
    build: "taskcluster/win-build.sh tflite"
    package: "taskcluster/win-package.sh"