From e06fce51ac0b47dcbdef7588050bac0d808d25f0 Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Tue, 20 Aug 2019 10:43:20 +0200 Subject: [PATCH] Move Raspbian support to Buster Fixes #2272 --- native_client/definitions.mk | 7 ++++--- native_client/multistrap_raspbian_buster.conf | 14 ++++++++++++++ native_client/python/Makefile | 4 ++-- taskcluster/.shared.yml | 2 ++ taskcluster/linux-rpi3-cpu-opt.yml | 2 +- taskcluster/rpi3-build.sh | 4 ++-- taskcluster/tc-tests-utils.sh | 4 +++- taskcluster/test-python_35m-raspbian-rpi3-opt.yml | 13 ------------- ...t.yml => test-python_37m-raspbian-rpi3-opt.yml} | 8 ++++---- taskcluster/test-raspbian-opt-base.tyml | 2 +- 10 files changed, 33 insertions(+), 27 deletions(-) create mode 100644 native_client/multistrap_raspbian_buster.conf delete mode 100644 taskcluster/test-python_35m-raspbian-rpi3-opt.yml rename taskcluster/{test-python_34m-raspbian-rpi3-opt.yml => test-python_37m-raspbian-rpi3-opt.yml} (61%) diff --git a/native_client/definitions.mk b/native_client/definitions.mk index da404a69..941e4a49 100644 --- a/native_client/definitions.mk +++ b/native_client/definitions.mk @@ -45,7 +45,7 @@ endif ifeq ($(TARGET),rpi3) TOOLCHAIN ?= ${TFDIR}/bazel-$(shell basename "${TFDIR}")/external/LinaroArmGcc72/bin/arm-linux-gnueabihf- -RASPBIAN ?= $(abspath $(NC_DIR)/../multistrap-raspbian-stretch) +RASPBIAN ?= $(abspath $(NC_DIR)/../multistrap-raspbian-buster) CFLAGS := -march=armv7-a -mtune=cortex-a53 -mfpu=neon-fp-armv8 -mfloat-abi=hard -D_GLIBCXX_USE_CXX11_ABI=0 --sysroot $(RASPBIAN) CXXFLAGS := $(CXXFLAGS) LDFLAGS := -Wl,-rpath-link,$(RASPBIAN)/lib/arm-linux-gnueabihf/ -Wl,-rpath-link,$(RASPBIAN)/usr/lib/arm-linux-gnueabihf/ @@ -55,8 +55,9 @@ SOX_LDFLAGS := $(RASPBIAN)/usr/lib/arm-linux-gnueabihf/libsox.so PYVER := $(shell python -c "import platform; maj, min, _ = platform.python_version_tuple(); print(maj+'.'+min);") PYTHON_PACKAGES := -PYTHON_PATH := PYTHONPATH=$(RASPBIAN)/usr/lib/python$(PYVER)/:$(RASPBIAN)/usr/lib/python$(PYVER)/plat-arm-linux-gnueabihf/:$(RASPBIAN)/usr/lib/python3/dist-packages/ -NUMPY_INCLUDE := NUMPY_INCLUDE=$(RASPBIAN)/usr/include/python3.5/ +PYTHON_PATH := PYTHONPATH=$(RASPBIAN)/usr/lib/python$(PYVER)/:$(RASPBIAN)/usr/lib/python3/dist-packages/ +NUMPY_INCLUDE := NUMPY_INCLUDE=$(RASPBIAN)/usr/include/python3.7m/ +PYTHON_SYSCONFIGDATA := _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_m_linux_arm-linux-gnueabihf PYTHON_PLATFORM_NAME := --plat-name linux_armv7l NODE_PLATFORM_TARGET := --target_arch=arm --target_platform=linux TOOLCHAIN_LDD_OPTS := --root $(RASPBIAN)/ diff --git a/native_client/multistrap_raspbian_buster.conf b/native_client/multistrap_raspbian_buster.conf new file mode 100644 index 00000000..b5d5ee05 --- /dev/null +++ b/native_client/multistrap_raspbian_buster.conf @@ -0,0 +1,14 @@ +[General] +arch=armhf +noauth=true +unpack=true +debootstrap=Raspbian +aptsources=Raspbian +cleanup=true + +[Raspbian] +packages=libc6 libc6-dev libffi-dev libstdc++-6-dev linux-libc-dev libpython3.7-dev libsox-dev python3-numpy python3-setuptools +source=http://raspbian.raspberrypi.org/raspbian/ +keyring=raspbian-archive-keyring +components=main +suite=buster diff --git a/native_client/python/Makefile b/native_client/python/Makefile index d03de4c0..085ce8b0 100644 --- a/native_client/python/Makefile +++ b/native_client/python/Makefile @@ -8,7 +8,7 @@ bindings-clean: bindings-build: pip install --quiet $(PYTHON_PACKAGES) wheel==0.31.0 setuptools==39.1.0 - PATH=$(TOOLCHAIN):$$PATH AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED) $(RPATH_PYTHON)" MODEL_LDFLAGS="$(LDFLAGS_DIRS)" MODEL_LIBS="$(LIBS)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext $(PYTHON_PLATFORM_NAME) + PATH=$(TOOLCHAIN):$$PATH AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED) $(RPATH_PYTHON)" MODEL_LDFLAGS="$(LDFLAGS_DIRS)" MODEL_LIBS="$(LIBS)" $(PYTHON_PATH) $(PYTHON_SYSCONFIGDATA) $(NUMPY_INCLUDE) python ./setup.py build_ext $(PYTHON_PLATFORM_NAME) MANIFEST.in: bindings-build > $@ @@ -19,6 +19,6 @@ MANIFEST.in: bindings-build bindings-package: MANIFEST.in cat MANIFEST.in rm -f temp_build/*_wrap.o temp_build/Release/*_wrap.obj - AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED) $(RPATH_PYTHON)" MODEL_LDFLAGS="$(LDFLAGS_DIRS)" MODEL_LIBS="$(LIBS)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) + AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED) $(RPATH_PYTHON)" MODEL_LDFLAGS="$(LDFLAGS_DIRS)" MODEL_LIBS="$(LIBS)" $(PYTHON_PATH) $(PYTHON_SYSCONFIGDATA) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) bindings: bindings-build bindings-package diff --git a/taskcluster/.shared.yml b/taskcluster/.shared.yml index f0139bdf..e85124ba 100644 --- a/taskcluster/.shared.yml +++ b/taskcluster/.shared.yml @@ -3,6 +3,8 @@ python: apt: 'make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev liblzma-dev curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev' packages_stretch: apt: 'make build-essential libssl1.0-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev' + packages_buster: + apt: 'make build-essential libssl1.0-dev libffi-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev' brew: setup: 'install_local_homebrew "python-ds-test" && install_pkg_local_homebrew "sox" && install_pkg_local_homebrew "readline" && install_pkg_local_homebrew "openssl" && install_pkg_local_homebrew "pkg-config"' env: 'export EXTRA_ENV="PATH=$TASKCLUSTER_TASK_DIR/python-ds-test.brew/bin/:$PATH"' diff --git a/taskcluster/linux-rpi3-cpu-opt.yml b/taskcluster/linux-rpi3-cpu-opt.yml index 86e6244a..ca2e30d1 100644 --- a/taskcluster/linux-rpi3-cpu-opt.yml +++ b/taskcluster/linux-rpi3-cpu-opt.yml @@ -16,7 +16,7 @@ build: apt-get -qq -y install ${python.packages_trusty.apt} && ${swig.packages.install_script} system_config: > - multistrap -d /tmp/multistrap-raspbian-stretch/ -f ${system.homedir.linux}/DeepSpeech/ds/native_client/multistrap_raspbian_stretch.conf && + multistrap -d /tmp/multistrap-raspbian-buster/ -f ${system.homedir.linux}/DeepSpeech/ds/native_client/multistrap_raspbian_buster.conf && ${swig.patch_nodejs.linux} scripts: build: "taskcluster/rpi3-build.sh" diff --git a/taskcluster/rpi3-build.sh b/taskcluster/rpi3-build.sh index f9f330da..1ba95a72 100755 --- a/taskcluster/rpi3-build.sh +++ b/taskcluster/rpi3-build.sh @@ -14,7 +14,7 @@ BAZEL_TARGETS=" BAZEL_BUILD_FLAGS="${BAZEL_ARM_FLAGS} ${BAZEL_EXTRA_FLAGS}" BAZEL_ENV_FLAGS="TF_NEED_CUDA=0" SYSTEM_TARGET=rpi3 -SYSTEM_RASPBIAN=/tmp/multistrap-raspbian-stretch +SYSTEM_RASPBIAN=/tmp/multistrap-raspbian-buster maybe_install_xldd @@ -22,7 +22,7 @@ do_bazel_build do_deepspeech_binary_build -export SUPPORTED_PYTHON_VERSIONS="3.4.8:ucs4 3.5.3:ucs4" +export SUPPORTED_PYTHON_VERSIONS="3.7.3:ucs4" do_deepspeech_python_build do_deepspeech_nodejs_build diff --git a/taskcluster/tc-tests-utils.sh b/taskcluster/tc-tests-utils.sh index a2ede408..f6987e4a 100755 --- a/taskcluster/tc-tests-utils.sh +++ b/taskcluster/tc-tests-utils.sh @@ -960,9 +960,11 @@ maybe_ssl102_py37() unset PY37_LDPATH unset PY37_SOURCE_PACKAGE + ARCH=$(uname -m) + case "${pyver}" in 3.7*) - if [ "${OS}" = "Linux" ]; then + if [ "${OS}" = "Linux" -a "${ARCH}" = "x86_64" ]; then PY37_OPENSSL_DIR=${DS_ROOT_TASK}/ssl-xenial if [ -d "${PY37_OPENSSL_DIR}" ]; then diff --git a/taskcluster/test-python_35m-raspbian-rpi3-opt.yml b/taskcluster/test-python_35m-raspbian-rpi3-opt.yml deleted file mode 100644 index 722204e2..00000000 --- a/taskcluster/test-python_35m-raspbian-rpi3-opt.yml +++ /dev/null @@ -1,13 +0,0 @@ -build: - template_file: test-raspbian-opt-base.tyml - dependencies: - - "linux-rpi3-cpu-opt" - - "test-training_upstream-linux-amd64-py35m-opt" - system_setup: - > - apt-get -qq -y install ${python.packages_stretch.apt} - args: - tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests.sh 3.5.5:m" - metadata: - name: "DeepSpeech Raspbian RPi3/ARMv7 CPU Python v3.5 tests" - description: "Testing DeepSpeech for Raspbian RPi3/ARMv7 on Python v3.5, CPU only, optimized version" diff --git a/taskcluster/test-python_34m-raspbian-rpi3-opt.yml b/taskcluster/test-python_37m-raspbian-rpi3-opt.yml similarity index 61% rename from taskcluster/test-python_34m-raspbian-rpi3-opt.yml rename to taskcluster/test-python_37m-raspbian-rpi3-opt.yml index 086e2cbb..7d0674ff 100644 --- a/taskcluster/test-python_34m-raspbian-rpi3-opt.yml +++ b/taskcluster/test-python_37m-raspbian-rpi3-opt.yml @@ -5,9 +5,9 @@ build: - "test-training_upstream-linux-amd64-py35m-opt" system_setup: > - apt-get -qq -y install ${python.packages_stretch.apt} + apt-get -qq -y install ${python.packages_buster.apt} args: - tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests.sh 3.4.8:m" + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests.sh 3.7.3:m" metadata: - name: "DeepSpeech Raspbian RPi3/ARMv7 CPU Python v3.4 tests" - description: "Testing DeepSpeech for Raspbian RPi3/ARMv7 on Python v3.4, CPU only, optimized version" + name: "DeepSpeech Raspbian RPi3/ARMv7 CPU Python v3.7 tests" + description: "Testing DeepSpeech for Raspbian RPi3/ARMv7 on Python v3.7, CPU only, optimized version" diff --git a/taskcluster/test-raspbian-opt-base.tyml b/taskcluster/test-raspbian-opt-base.tyml index 444cec05..5abdda6e 100644 --- a/taskcluster/test-raspbian-opt-base.tyml +++ b/taskcluster/test-raspbian-opt-base.tyml @@ -27,7 +27,7 @@ then: payload: maxRunTime: { $eval: to_int(build.maxRunTime) } - image: "resin/rpi-raspbian:stretch-20180228" + image: "resin/rpi-raspbian:buster-20190121" env: $let: