Move Raspbian support to Buster

Fixes #2272

native_client/definitions.mk

@@ -45,7 +45,7 @@ endif
 
 ifeq ($(TARGET),rpi3)
 TOOLCHAIN   ?= ${TFDIR}/bazel-$(shell basename "${TFDIR}")/external/LinaroArmGcc72/bin/arm-linux-gnueabihf-
-RASPBIAN    ?= $(abspath $(NC_DIR)/../multistrap-raspbian-stretch)
+RASPBIAN    ?= $(abspath $(NC_DIR)/../multistrap-raspbian-buster)
 CFLAGS      := -march=armv7-a -mtune=cortex-a53 -mfpu=neon-fp-armv8 -mfloat-abi=hard -D_GLIBCXX_USE_CXX11_ABI=0 --sysroot $(RASPBIAN)
 CXXFLAGS    := $(CXXFLAGS)
 LDFLAGS     := -Wl,-rpath-link,$(RASPBIAN)/lib/arm-linux-gnueabihf/ -Wl,-rpath-link,$(RASPBIAN)/usr/lib/arm-linux-gnueabihf/
@@ -55,8 +55,9 @@ SOX_LDFLAGS := $(RASPBIAN)/usr/lib/arm-linux-gnueabihf/libsox.so
 
 PYVER := $(shell python -c "import platform; maj, min, _ = platform.python_version_tuple(); print(maj+'.'+min);")
 PYTHON_PACKAGES      :=
-PYTHON_PATH          := PYTHONPATH=$(RASPBIAN)/usr/lib/python$(PYVER)/:$(RASPBIAN)/usr/lib/python$(PYVER)/plat-arm-linux-gnueabihf/:$(RASPBIAN)/usr/lib/python3/dist-packages/
+PYTHON_PATH          := PYTHONPATH=$(RASPBIAN)/usr/lib/python$(PYVER)/:$(RASPBIAN)/usr/lib/python3/dist-packages/
-NUMPY_INCLUDE        := NUMPY_INCLUDE=$(RASPBIAN)/usr/include/python3.5/
+NUMPY_INCLUDE        := NUMPY_INCLUDE=$(RASPBIAN)/usr/include/python3.7m/
+PYTHON_SYSCONFIGDATA := _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_m_linux_arm-linux-gnueabihf
 PYTHON_PLATFORM_NAME := --plat-name linux_armv7l
 NODE_PLATFORM_TARGET := --target_arch=arm --target_platform=linux
 TOOLCHAIN_LDD_OPTS   := --root $(RASPBIAN)/

native_client/multistrap_raspbian_buster.conf

@@ -0,0 +1,14 @@
+[General]
+arch=armhf
+noauth=true
+unpack=true
+debootstrap=Raspbian
+aptsources=Raspbian
+cleanup=true
+
+[Raspbian]
+packages=libc6 libc6-dev libffi-dev libstdc++-6-dev linux-libc-dev libpython3.7-dev libsox-dev python3-numpy python3-setuptools
+source=http://raspbian.raspberrypi.org/raspbian/
+keyring=raspbian-archive-keyring
+components=main
+suite=buster

native_client/python/Makefile

@@ -8,7 +8,7 @@ bindings-clean:
 
 bindings-build:
 	pip install --quiet $(PYTHON_PACKAGES) wheel==0.31.0 setuptools==39.1.0
-	PATH=$(TOOLCHAIN):$$PATH AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED) $(RPATH_PYTHON)" MODEL_LDFLAGS="$(LDFLAGS_DIRS)" MODEL_LIBS="$(LIBS)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext $(PYTHON_PLATFORM_NAME)
+	PATH=$(TOOLCHAIN):$$PATH AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED) $(RPATH_PYTHON)" MODEL_LDFLAGS="$(LDFLAGS_DIRS)" MODEL_LIBS="$(LIBS)" $(PYTHON_PATH) $(PYTHON_SYSCONFIGDATA) $(NUMPY_INCLUDE) python ./setup.py build_ext $(PYTHON_PLATFORM_NAME)
 
 MANIFEST.in: bindings-build
 	> $@
@@ -19,6 +19,6 @@ MANIFEST.in: bindings-build
 bindings-package: MANIFEST.in
 	cat MANIFEST.in
 	rm -f temp_build/*_wrap.o temp_build/Release/*_wrap.obj
-	AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED) $(RPATH_PYTHON)" MODEL_LDFLAGS="$(LDFLAGS_DIRS)" MODEL_LIBS="$(LIBS)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
+	AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED) $(RPATH_PYTHON)" MODEL_LDFLAGS="$(LDFLAGS_DIRS)" MODEL_LIBS="$(LIBS)" $(PYTHON_PATH) $(PYTHON_SYSCONFIGDATA) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
 
 bindings: bindings-build bindings-package

taskcluster/.shared.yml

@@ -3,6 +3,8 @@ python:
     apt: 'make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev liblzma-dev curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev'
   packages_stretch:
     apt: 'make build-essential libssl1.0-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev'
+  packages_buster:
+    apt: 'make build-essential libssl1.0-dev libffi-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev'
   brew:
     setup: 'install_local_homebrew "python-ds-test" && install_pkg_local_homebrew "sox" && install_pkg_local_homebrew "readline" && install_pkg_local_homebrew "openssl" && install_pkg_local_homebrew "pkg-config"'
     env: 'export EXTRA_ENV="PATH=$TASKCLUSTER_TASK_DIR/python-ds-test.brew/bin/:$PATH"'

taskcluster/linux-rpi3-cpu-opt.yml

@@ -16,7 +16,7 @@ build:
       apt-get -qq -y install ${python.packages_trusty.apt} && ${swig.packages.install_script}
   system_config:
     >
-      multistrap -d /tmp/multistrap-raspbian-stretch/ -f ${system.homedir.linux}/DeepSpeech/ds/native_client/multistrap_raspbian_stretch.conf &&
+      multistrap -d /tmp/multistrap-raspbian-buster/ -f ${system.homedir.linux}/DeepSpeech/ds/native_client/multistrap_raspbian_buster.conf &&
       ${swig.patch_nodejs.linux}
   scripts:
     build: "taskcluster/rpi3-build.sh"

taskcluster/rpi3-build.sh

@@ -14,7 +14,7 @@ BAZEL_TARGETS="
 BAZEL_BUILD_FLAGS="${BAZEL_ARM_FLAGS} ${BAZEL_EXTRA_FLAGS}"
 BAZEL_ENV_FLAGS="TF_NEED_CUDA=0"
 SYSTEM_TARGET=rpi3
-SYSTEM_RASPBIAN=/tmp/multistrap-raspbian-stretch
+SYSTEM_RASPBIAN=/tmp/multistrap-raspbian-buster
 
 maybe_install_xldd
 
@@ -22,7 +22,7 @@ do_bazel_build
 
 do_deepspeech_binary_build
 
-export SUPPORTED_PYTHON_VERSIONS="3.4.8:ucs4 3.5.3:ucs4"
+export SUPPORTED_PYTHON_VERSIONS="3.7.3:ucs4"
 do_deepspeech_python_build
 
 do_deepspeech_nodejs_build

taskcluster/tc-tests-utils.sh

@@ -960,9 +960,11 @@ maybe_ssl102_py37()
     unset PY37_LDPATH
     unset PY37_SOURCE_PACKAGE
 
+    ARCH=$(uname -m)
+
     case "${pyver}" in
         3.7*)
-            if [ "${OS}" = "Linux" ]; then
+            if [ "${OS}" = "Linux" -a "${ARCH}" = "x86_64" ]; then
                 PY37_OPENSSL_DIR=${DS_ROOT_TASK}/ssl-xenial
 
                 if [ -d "${PY37_OPENSSL_DIR}" ]; then

taskcluster/test-python_35m-raspbian-rpi3-opt.yml

@@ -1,13 +0,0 @@
-build:
-  template_file: test-raspbian-opt-base.tyml
-  dependencies:
-    - "linux-rpi3-cpu-opt"
-    - "test-training_upstream-linux-amd64-py35m-opt"
-  system_setup:
-    >
-      apt-get -qq -y install ${python.packages_stretch.apt}
-  args:
-    tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests.sh 3.5.5:m"
-  metadata:
-    name: "DeepSpeech Raspbian RPi3/ARMv7 CPU Python v3.5 tests"
-    description: "Testing DeepSpeech for Raspbian RPi3/ARMv7 on Python v3.5, CPU only, optimized version"

taskcluster/test-python_37m-raspbian-rpi3-opt.yml

@@ -5,9 +5,9 @@ build:
     - "test-training_upstream-linux-amd64-py35m-opt"
   system_setup:
     >
-      apt-get -qq -y install ${python.packages_stretch.apt}
+      apt-get -qq -y install ${python.packages_buster.apt}
   args:
-    tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests.sh 3.4.8:m"
+    tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests.sh 3.7.3:m"
   metadata:
-    name: "DeepSpeech Raspbian RPi3/ARMv7 CPU Python v3.4 tests"
+    name: "DeepSpeech Raspbian RPi3/ARMv7 CPU Python v3.7 tests"
-    description: "Testing DeepSpeech for Raspbian RPi3/ARMv7 on Python v3.4, CPU only, optimized version"
+    description: "Testing DeepSpeech for Raspbian RPi3/ARMv7 on Python v3.7, CPU only, optimized version"

taskcluster/test-raspbian-opt-base.tyml

@@ -27,7 +27,7 @@ then:
 
   payload:
     maxRunTime: { $eval: to_int(build.maxRunTime) }
-    image: "resin/rpi-raspbian:stretch-20180228"
+    image: "resin/rpi-raspbian:buster-20190121"
 
     env:
       $let: