Merge pull request #3046 from mozilla/setup-decoder-pypi

Use decoder package from PyPI (Fixes #3044)

setup.py

@@ -7,20 +7,8 @@ from pkg_resources import parse_version
 from setuptools import find_packages, setup
 
 
-def get_decoder_pkg_url(version, artifacts_root=None):
-    is_arm = 'arm' in platform.machine()
-    is_mac = 'darwin' in sys.platform
-    is_win = 'win32' in sys.platform
-    is_64bit = sys.maxsize > (2**31 - 1)
-
-    if is_arm:
-        tc_arch = 'arm64-ctc' if is_64bit else 'arm-ctc'
-    elif is_mac:
-        tc_arch = 'osx-ctc'
-    elif is_win:
-        tc_arch = 'win-ctc'
-    else:
-        tc_arch = 'cpu-ctc'
+def get_tc_decoder_pkg_url(version, artifacts_root):
+    assert artifacts_root
 
     ds_version = parse_version(version)
     branch = "v{}".format(version)
@@ -37,20 +25,15 @@ def get_decoder_pkg_url(version, artifacts_root=None):
     if plat == 'windows':
         plat = 'win'
 
-    is_ucs2 = sys.maxunicode < 0x10ffff
-    m_or_mu = 'mu' if is_ucs2 else 'm'
-
     # ABI does not contain m / mu anymore after Python 3.8
     if sys.version_info.major == 3 and sys.version_info.minor >= 8:
         m_or_mu = ''
+    else:
+        is_ucs2 = sys.maxunicode < 0x10ffff
+        m_or_mu = 'mu' if is_ucs2 else 'm'
 
     pyver = ''.join(str(i) for i in sys.version_info[0:2])
 
-    if not artifacts_root:
-        artifacts_root = 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.deepspeech.native_client.{branch_name}.{tc_arch_string}/artifacts/public'.format(
-            branch_name=branch,
-            tc_arch_string=tc_arch)
-
     return 'ds_ctcdecoder @ {artifacts_root}/ds_ctcdecoder-{ds_version}-cp{pyver}-cp{pyver}{m_or_mu}-{platform}_{arch}.whl'.format(
         artifacts_root=artifacts_root,
         ds_version=ds_version,
@@ -66,8 +49,6 @@ def main():
     with open(str(version_file)) as fin:
         version = fin.read().strip()
 
-    decoder_pkg_url = get_decoder_pkg_url(version)
-
     install_requires_base = [
         'tensorflow == 1.15.2',
         'numpy',
@@ -89,18 +70,22 @@ def main():
         'soundfile',
     ]
 
+    decoder_pypi_dep = [
+        'ds_ctcdecoder == {}'.format(version)
+    ]
+
     # Due to pip craziness environment variables are the only consistent way to
     # get options into this script when doing `pip install`.
     tc_decoder_artifacts_root = os.environ.get('DECODER_ARTIFACTS_ROOT', '')
     if tc_decoder_artifacts_root:
         # We're running inside the TaskCluster environment, override the decoder
         # package URL with the one we just built.
-        decoder_pkg_url = get_decoder_pkg_url(version, tc_decoder_artifacts_root)
+        decoder_pkg_url = get_tc_decoder_pkg_url(version, tc_decoder_artifacts_root)
         install_requires = install_requires_base + [decoder_pkg_url]
     elif os.environ.get('DS_NODECODER', ''):
         install_requires = install_requires_base
     else:
-        install_requires = install_requires_base + [decoder_pkg_url]
+        install_requires = install_requires_base + decoder_pypi_dep
 
     setup(
         name='deepspeech_training',

taskcluster/scriptworker-task-pypi.yml

@@ -5,13 +5,16 @@ build:
     # - "linux-arm64-cpu-opt" Aarch64 packages are refused by upload.pypi.org
     - "darwin-amd64-cpu-opt"
     - "darwin-amd64-tflite-opt"
+    - "darwin-amd64-ctc-opt"
     - "linux-amd64-cpu-opt"
     - "linux-amd64-tflite-opt"
     - "linux-amd64-gpu-opt"
+    - "linux-amd64-ctc-opt"
     - "linux-rpi3-cpu-opt"
     - "win-amd64-cpu-opt"
     - "win-amd64-tflite-opt"
     - "win-amd64-gpu-opt"
+    - "win-amd64-ctc-opt"
   allowed:
     - "tag"
   ref_match: "refs/tags/"
@@ -24,13 +27,16 @@ build:
     python:
       - "darwin-amd64-cpu-opt"
       - "darwin-amd64-tflite-opt"
+      - "darwin-amd64-ctc-opt"
       - "linux-amd64-cpu-opt"
       - "linux-amd64-tflite-opt"
       - "linux-amd64-gpu-opt"
+      - "linux-amd64-ctc-opt"
       - "linux-rpi3-cpu-opt"
       - "win-amd64-cpu-opt"
       - "win-amd64-tflite-opt"
       - "win-amd64-gpu-opt"
+      - "win-amd64-ctc-opt"
       # - "linux-arm64-cpu-opt" Aarch64 packages are refused by upload.pypi.org
     nuget: []
   metadata:

taskcluster/tc-train-tests.sh

@@ -8,6 +8,13 @@ extract_python_versions "$1" "pyver" "pyver_pkg" "py_unicode_type" "pyconf" "pya
 
 bitrate=$2
 
+decoder_src=$3
+
+if [ "$decoder_src" = "--pypi" ]; then
+    # Disable automatically picking up decoder package built in this CI group
+    export DECODER_ARTIFACTS_ROOT=""
+fi
+
 mkdir -p ${TASKCLUSTER_ARTIFACTS} || true
 mkdir -p /tmp/train || true
 mkdir -p /tmp/train_tflite || true

taskcluster/test-linux-opt-tag-base.tyml

@@ -0,0 +1,65 @@
+$if: '(event.event in build.allowed) && ((event.event != "tag") || (build.ref_match in event.head.ref))'
+then:
+  taskId: ${taskcluster.taskId}
+  provisionerId: ${taskcluster.docker.provisionerId}
+  workerType: ${taskcluster.docker.workerType}
+  taskGroupId: ${taskcluster.taskGroupId}
+  schedulerId: ${taskcluster.schedulerId}
+  dependencies:
+    $map: { $eval: build.dependencies }
+    each(b):
+      $eval: as_slugid(b)
+  created: { $fromNow: '0 sec' }
+  deadline: { $fromNow: '1 day' }
+  expires: { $fromNow: '7 days' }
+
+  extra:
+    github:
+      { $eval: taskcluster.github_events.pull_request }
+
+  payload:
+    maxRunTime: { $eval: to_int(build.maxRunTime) }
+    image: ${build.docker_image}
+
+    env:
+      $let:
+        training: { $eval: as_slugid(build.test_model_task) }
+        linux_amd64_build: { $eval: as_slugid("linux-amd64-cpu-opt") }
+        linux_amd64_tflite: { $eval: as_slugid("linux-amd64-tflite-opt") }
+        linux_amd64_ctc: { $eval: as_slugid("linux-amd64-ctc-opt") }
+      in:
+        DEEPSPEECH_ARTIFACTS_ROOT: https://community-tc.services.mozilla.com/api/queue/v1/task/${linux_amd64_build}/artifacts/public
+        DEEPSPEECH_ARTIFACTS_TFLITE_ROOT: https://community-tc.services.mozilla.com/api/queue/v1/task/${linux_amd64_tflite}/artifacts/public
+        DEEPSPEECH_TEST_MODEL: https://community-tc.services.mozilla.com/api/queue/v1/task/${training}/artifacts/public/output_graph.pb
+        DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pb
+        DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pbmm
+        DECODER_ARTIFACTS_ROOT: https://community-tc.services.mozilla.com/api/queue/v1/task/${linux_amd64_ctc}/artifacts/public
+        PIP_DEFAULT_TIMEOUT: "60"
+        EXPECTED_TENSORFLOW_VERSION: "${build.tensorflow_git_desc}"
+
+    command:
+      - "/bin/bash"
+      - "--login"
+      - "-cxe"
+      - $let:
+          extraSystemSetup: { $eval: strip(str(build.system_setup)) }
+        in: >
+          ${aptEc2Mirrors} &&
+          apt-get -qq update && apt-get -qq -y install curl python-simplejson git pixz sox sudo wget && ${extraSystemSetup} &&
+          adduser --system --home ${system.homedir.linux} ${system.username} &&
+          cd ${system.homedir.linux} &&
+          echo -e "#!/bin/bash\nset -xe\n env && id && mkdir ~/DeepSpeech/ && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha}&& mkdir -p ${system.homedir.linux}/pyenv-root/ && wget -O - ${system.pyenv.linux.url} | tar -C ${system.homedir.linux}/pyenv-root/ -xzf -" > /tmp/clone.sh && chmod +x /tmp/clone.sh &&
+          sudo -H -u ${system.username} /bin/bash /tmp/clone.sh &&
+          sudo -H -u ${system.username} --preserve-env /bin/bash ${build.args.tests_cmdline}
+
+    artifacts:
+      "public":
+        type: "directory"
+        path: "/tmp/artifacts/"
+        expires: { $fromNow: '7 days' }
+
+  metadata:
+    name: ${build.metadata.name}
+    description: ${build.metadata.description}
+    owner: ${event.head.user.email}
+    source: ${event.head.repo.url}

taskcluster/test-training-pypi_16k-linux-amd64-py35m-opt.yml

@@ -0,0 +1,15 @@
+build:
+  template_file: test-linux-opt-tag-base.tyml
+  dependencies:
+    - "scriptworker-task-pypi"
+  allowed:
+    - "tag"
+  ref_match: "refs/tags/"
+  system_setup:
+    >
+      apt-get -qq update && apt-get -qq -y install ${training.packages_trusty.apt}
+  args:
+    tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-tests.sh 3.5.8:m 16k --pypi"
+  metadata:
+    name: "DeepSpeech Linux AMD64 CPU 16kHz PyPI training Py3.5"
+    description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 16kHz Python 3.5, CPU only, optimized version, decoder package from PyPI"

taskcluster/test-training-pypi_16k-linux-amd64-py36m-opt.yml

@@ -0,0 +1,15 @@
+build:
+  template_file: test-linux-opt-tag-base.tyml
+  dependencies:
+    - "scriptworker-task-pypi"
+  allowed:
+    - "tag"
+  ref_match: "refs/tags/"
+  system_setup:
+    >
+      apt-get -qq update && apt-get -qq -y install ${training.packages_trusty.apt}
+  args:
+    tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-tests.sh 3.6.10:m 16k --pypi"
+  metadata:
+    name: "DeepSpeech Linux AMD64 CPU 16kHz PyPI training Py3.6"
+    description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 16kHz Python 3.6, CPU only, optimized version, decoder package from PyPI"

taskcluster/test-training-pypi_16k-linux-amd64-py37m-opt.yml

@@ -0,0 +1,15 @@
+build:
+  template_file: test-linux-opt-tag-base.tyml
+  dependencies:
+    - "scriptworker-task-pypi"
+  allowed:
+    - "tag"
+  ref_match: "refs/tags/"
+  system_setup:
+    >
+      apt-get -qq update && apt-get -qq -y install ${training.packages_trusty.apt}
+  args:
+    tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-tests.sh 3.7.6:m 16k --pypi"
+  metadata:
+    name: "DeepSpeech Linux AMD64 CPU 16kHz PyPI training Py3.7"
+    description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 16kHz Python 3.7, CPU only, optimized version, decoder package from PyPI"

taskcluster/test-training-pypi_8k-linux-amd64-py36m-opt.yml

@@ -0,0 +1,15 @@
+build:
+  template_file: test-linux-opt-tag-base.tyml
+  dependencies:
+    - "scriptworker-task-pypi"
+  allowed:
+    - "tag"
+  ref_match: "refs/tags/"
+  system_setup:
+    >
+      apt-get -qq update && apt-get -qq -y install ${training.packages_trusty.apt}
+  args:
+    tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-tests.sh 3.6.10:m 8k --pypi"
+  metadata:
+    name: "DeepSpeech Linux AMD64 CPU 8kHz PyPI training Py3.6"
+    description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 8kHz Python 3.6, CPU only, optimized version, decoder package from PyPI"

taskcluster/test-training-pypi_8k-linux-amd64-py37m-opt.yml

@@ -0,0 +1,15 @@
+build:
+  template_file: test-linux-opt-tag-base.tyml
+  dependencies:
+    - "scriptworker-task-pypi"
+  allowed:
+    - "tag"
+  ref_match: "refs/tags/"
+  system_setup:
+    >
+      apt-get -qq update && apt-get -qq -y install ${training.packages_trusty.apt}
+  args:
+    tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-tests.sh 3.7.6:m 8k  --pypi"
+  metadata:
+    name: "DeepSpeech Linux AMD64 CPU 8kHz PyPI training Py3.7"
+    description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 8kHz Python 3.7, CPU only, optimized version, decoder package from PyPI"