Move Raspbian support to Buster

Fixes #2272
2019-08-20 10:43:20 +02:00 · 2019-08-20 10:43:20 +02:00 · e06fce51ac
commit e06fce51ac
parent 3e60413f27
10 changed files with 33 additions and 27 deletions
--- a/native_client/definitions.mk
+++ b/native_client/definitions.mk
@ -45,7 +45,7 @@ endif

 ifeq ($(TARGET),rpi3)
 TOOLCHAIN   ?= ${TFDIR}/bazel-$(shell basename "${TFDIR}")/external/LinaroArmGcc72/bin/arm-linux-gnueabihf-
-RASPBIAN    ?= $(abspath $(NC_DIR)/../multistrap-raspbian-stretch)
+RASPBIAN    ?= $(abspath $(NC_DIR)/../multistrap-raspbian-buster)
 CFLAGS      := -march=armv7-a -mtune=cortex-a53 -mfpu=neon-fp-armv8 -mfloat-abi=hard -D_GLIBCXX_USE_CXX11_ABI=0 --sysroot $(RASPBIAN)
 CXXFLAGS    := $(CXXFLAGS)
 LDFLAGS     := -Wl,-rpath-link,$(RASPBIAN)/lib/arm-linux-gnueabihf/ -Wl,-rpath-link,$(RASPBIAN)/usr/lib/arm-linux-gnueabihf/
@ -55,8 +55,9 @@ SOX_LDFLAGS := $(RASPBIAN)/usr/lib/arm-linux-gnueabihf/libsox.so

 PYVER := $(shell python -c "import platform; maj, min, _ = platform.python_version_tuple(); print(maj+'.'+min);")
 PYTHON_PACKAGES      :=
-PYTHON_PATH          := PYTHONPATH=$(RASPBIAN)/usr/lib/python$(PYVER)/:$(RASPBIAN)/usr/lib/python$(PYVER)/plat-arm-linux-gnueabihf/:$(RASPBIAN)/usr/lib/python3/dist-packages/
-NUMPY_INCLUDE        := NUMPY_INCLUDE=$(RASPBIAN)/usr/include/python3.5/
+PYTHON_PATH          := PYTHONPATH=$(RASPBIAN)/usr/lib/python$(PYVER)/:$(RASPBIAN)/usr/lib/python3/dist-packages/
+NUMPY_INCLUDE        := NUMPY_INCLUDE=$(RASPBIAN)/usr/include/python3.7m/
+PYTHON_SYSCONFIGDATA := _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_m_linux_arm-linux-gnueabihf
 PYTHON_PLATFORM_NAME := --plat-name linux_armv7l
 NODE_PLATFORM_TARGET := --target_arch=arm --target_platform=linux
 TOOLCHAIN_LDD_OPTS   := --root $(RASPBIAN)/
--- a/native_client/multistrap_raspbian_buster.conf
+++ b/native_client/multistrap_raspbian_buster.conf
@ -0,0 +1,14 @@
+[General]
+arch=armhf
+noauth=true
+unpack=true
+debootstrap=Raspbian
+aptsources=Raspbian
+cleanup=true
+
+[Raspbian]
+packages=libc6 libc6-dev libffi-dev libstdc++-6-dev linux-libc-dev libpython3.7-dev libsox-dev python3-numpy python3-setuptools
+source=http://raspbian.raspberrypi.org/raspbian/
+keyring=raspbian-archive-keyring
+components=main
+suite=buster
--- a/native_client/python/Makefile
+++ b/native_client/python/Makefile
@ -8,7 +8,7 @@ bindings-clean:

 bindings-build:
 	pip install --quiet $(PYTHON_PACKAGES) wheel==0.31.0 setuptools==39.1.0
-	PATH=$(TOOLCHAIN):$$PATH AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED) $(RPATH_PYTHON)" MODEL_LDFLAGS="$(LDFLAGS_DIRS)" MODEL_LIBS="$(LIBS)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext $(PYTHON_PLATFORM_NAME)
+	PATH=$(TOOLCHAIN):$$PATH AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED) $(RPATH_PYTHON)" MODEL_LDFLAGS="$(LDFLAGS_DIRS)" MODEL_LIBS="$(LIBS)" $(PYTHON_PATH) $(PYTHON_SYSCONFIGDATA) $(NUMPY_INCLUDE) python ./setup.py build_ext $(PYTHON_PLATFORM_NAME)

 MANIFEST.in: bindings-build
 	> $@
@ -19,6 +19,6 @@ MANIFEST.in: bindings-build
 bindings-package: MANIFEST.in
 	cat MANIFEST.in
 	rm -f temp_build/*_wrap.o temp_build/Release/*_wrap.obj
-	AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED) $(RPATH_PYTHON)" MODEL_LDFLAGS="$(LDFLAGS_DIRS)" MODEL_LIBS="$(LIBS)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
+	AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED) $(RPATH_PYTHON)" MODEL_LDFLAGS="$(LDFLAGS_DIRS)" MODEL_LIBS="$(LIBS)" $(PYTHON_PATH) $(PYTHON_SYSCONFIGDATA) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)

 bindings: bindings-build bindings-package
--- a/taskcluster/.shared.yml
+++ b/taskcluster/.shared.yml
@ -3,6 +3,8 @@ python:
    apt: 'make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev liblzma-dev curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev'
  packages_stretch:
    apt: 'make build-essential libssl1.0-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev'
+  packages_buster:
+    apt: 'make build-essential libssl1.0-dev libffi-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev'
  brew:
    setup: 'install_local_homebrew "python-ds-test" && install_pkg_local_homebrew "sox" && install_pkg_local_homebrew "readline" && install_pkg_local_homebrew "openssl" && install_pkg_local_homebrew "pkg-config"'
    env: 'export EXTRA_ENV="PATH=$TASKCLUSTER_TASK_DIR/python-ds-test.brew/bin/:$PATH"'
--- a/taskcluster/linux-rpi3-cpu-opt.yml
+++ b/taskcluster/linux-rpi3-cpu-opt.yml
@ -16,7 +16,7 @@ build:
      apt-get -qq -y install ${python.packages_trusty.apt} && ${swig.packages.install_script}
  system_config:
    >
-      multistrap -d /tmp/multistrap-raspbian-stretch/ -f ${system.homedir.linux}/DeepSpeech/ds/native_client/multistrap_raspbian_stretch.conf &&
+      multistrap -d /tmp/multistrap-raspbian-buster/ -f ${system.homedir.linux}/DeepSpeech/ds/native_client/multistrap_raspbian_buster.conf &&
      ${swig.patch_nodejs.linux}
  scripts:
    build: "taskcluster/rpi3-build.sh"
--- a/taskcluster/rpi3-build.sh
+++ b/taskcluster/rpi3-build.sh
@ -14,7 +14,7 @@ BAZEL_TARGETS="
 BAZEL_BUILD_FLAGS="${BAZEL_ARM_FLAGS} ${BAZEL_EXTRA_FLAGS}"
 BAZEL_ENV_FLAGS="TF_NEED_CUDA=0"
 SYSTEM_TARGET=rpi3
-SYSTEM_RASPBIAN=/tmp/multistrap-raspbian-stretch
+SYSTEM_RASPBIAN=/tmp/multistrap-raspbian-buster

 maybe_install_xldd

@ -22,7 +22,7 @@ do_bazel_build

 do_deepspeech_binary_build

-export SUPPORTED_PYTHON_VERSIONS="3.4.8:ucs4 3.5.3:ucs4"
+export SUPPORTED_PYTHON_VERSIONS="3.7.3:ucs4"
 do_deepspeech_python_build

 do_deepspeech_nodejs_build
--- a/taskcluster/tc-tests-utils.sh
+++ b/taskcluster/tc-tests-utils.sh
@ -960,9 +960,11 @@ maybe_ssl102_py37()
    unset PY37_LDPATH
    unset PY37_SOURCE_PACKAGE

+    ARCH=$(uname -m)
+
    case "${pyver}" in
        3.7*)
-            if [ "${OS}" = "Linux" ]; then
+            if [ "${OS}" = "Linux" -a "${ARCH}" = "x86_64" ]; then
                PY37_OPENSSL_DIR=${DS_ROOT_TASK}/ssl-xenial

                if [ -d "${PY37_OPENSSL_DIR}" ]; then
--- a/taskcluster/test-python_35m-raspbian-rpi3-opt.yml
+++ b/taskcluster/test-python_35m-raspbian-rpi3-opt.yml
@ -1,13 +0,0 @@
-build:
-  template_file: test-raspbian-opt-base.tyml
-  dependencies:
-    - "linux-rpi3-cpu-opt"
-    - "test-training_upstream-linux-amd64-py35m-opt"
-  system_setup:
-    >
-      apt-get -qq -y install ${python.packages_stretch.apt}
-  args:
-    tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests.sh 3.5.5:m"
-  metadata:
-    name: "DeepSpeech Raspbian RPi3/ARMv7 CPU Python v3.5 tests"
-    description: "Testing DeepSpeech for Raspbian RPi3/ARMv7 on Python v3.5, CPU only, optimized version"
--- a/taskcluster/test-python_37m-raspbian-rpi3-opt.yml
+++ b/taskcluster/test-python_37m-raspbian-rpi3-opt.yml
@ -5,9 +5,9 @@ build:
    - "test-training_upstream-linux-amd64-py35m-opt"
  system_setup:
    >
-      apt-get -qq -y install ${python.packages_stretch.apt}
+      apt-get -qq -y install ${python.packages_buster.apt}
  args:
-    tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests.sh 3.4.8:m"
+    tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests.sh 3.7.3:m"
  metadata:
-    name: "DeepSpeech Raspbian RPi3/ARMv7 CPU Python v3.4 tests"
-    description: "Testing DeepSpeech for Raspbian RPi3/ARMv7 on Python v3.4, CPU only, optimized version"
+    name: "DeepSpeech Raspbian RPi3/ARMv7 CPU Python v3.7 tests"
+    description: "Testing DeepSpeech for Raspbian RPi3/ARMv7 on Python v3.7, CPU only, optimized version"
--- a/taskcluster/test-raspbian-opt-base.tyml
+++ b/taskcluster/test-raspbian-opt-base.tyml
@ -27,7 +27,7 @@ then:

  payload:
    maxRunTime: { $eval: to_int(build.maxRunTime) }
-    image: "resin/rpi-raspbian:stretch-20180228"
+    image: "resin/rpi-raspbian:buster-20190121"

    env:
      $let: