From f8dbea7767012363e64ee9bbb29eccdf27c15e81 Mon Sep 17 00:00:00 2001 From: Advait Jain Date: Mon, 4 Nov 2019 09:59:12 -0800 Subject: [PATCH] Improve micro continuous integration. PiperOrigin-RevId: 278403820 Change-Id: Ia26583b04c534cddd4b00d3d783213005d9e288b --- .../micro/tools/ci_build/helper_functions.sh | 35 +++++++++++++++++++ .../micro/tools/ci_build/test_all.sh | 12 +++++++ .../micro/tools/ci_build/test_arduino.sh | 16 ++++----- .../micro/tools/ci_build/test_sparkfun.sh | 11 +++--- .../micro/tools/ci_build/test_x86.sh | 11 +++--- 5 files changed, 64 insertions(+), 21 deletions(-) create mode 100644 tensorflow/lite/experimental/micro/tools/ci_build/helper_functions.sh diff --git a/tensorflow/lite/experimental/micro/tools/ci_build/helper_functions.sh b/tensorflow/lite/experimental/micro/tools/ci_build/helper_functions.sh new file mode 100644 index 00000000000..0195b0c435b --- /dev/null +++ b/tensorflow/lite/experimental/micro/tools/ci_build/helper_functions.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + + +# Collection of helper functions that can be used in the different continuous +# integration scripts. + +function die() { + echo "$@" 1>&2 ; exit 1; +} + +# A small utility to run the command and only print logs if the command fails. +# On success, all logs are hidden. This helps to keep the log output clean and +# makes debugging easier. +function readable_run { + # Disable debug mode to avoid printing of variables here. + set +x + result=$("$@" 2>&1) || die "$result" + echo "$@" + echo "Command completed successfully at $(date)" + set -x +} diff --git a/tensorflow/lite/experimental/micro/tools/ci_build/test_all.sh b/tensorflow/lite/experimental/micro/tools/ci_build/test_all.sh index 1a526150ac5..4e855b0e4a9 100755 --- a/tensorflow/lite/experimental/micro/tools/ci_build/test_all.sh +++ b/tensorflow/lite/experimental/micro/tools/ci_build/test_all.sh @@ -24,11 +24,23 @@ ROOT_DIR=${SCRIPT_DIR}/../../../../../.. cd ${ROOT_DIR} pwd +make -f tensorflow/lite/experimental/micro/tools/make/Makefile \ + clean clean_downloads + # Add all the test scripts for the various supported platforms here. This # emables running all the tests together has part of the continuous integration # pipeline and reduces duplication associated with setting up the docker # environment. + +echo "Starting to run micro tests at `date`" + +echo "Running Arduino tests at `date`" tensorflow/lite/experimental/micro/tools/ci_build/test_arduino.sh + +echo "Running Sparkfun tests at `date`" tensorflow/lite/experimental/micro/tools/ci_build/test_sparkfun.sh + +echo "Running x86 tests at `date`" tensorflow/lite/experimental/micro/tools/ci_build/test_x86.sh +echo "Finished all micro tests at `date`" diff --git a/tensorflow/lite/experimental/micro/tools/ci_build/test_arduino.sh b/tensorflow/lite/experimental/micro/tools/ci_build/test_arduino.sh index 3a5fb6ee04d..db8b0097dd4 100755 --- a/tensorflow/lite/experimental/micro/tools/ci_build/test_arduino.sh +++ b/tensorflow/lite/experimental/micro/tools/ci_build/test_arduino.sh @@ -22,20 +22,18 @@ set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT_DIR=${SCRIPT_DIR}/../../../../../.. cd ${ROOT_DIR} -pwd -make -f tensorflow/lite/experimental/micro/tools/make/Makefile \ - clean clean_downloads +source tensorflow/lite/experimental/micro/tools/ci_build/helper_functions.sh -make -f tensorflow/lite/experimental/micro/tools/make/Makefile \ +readable_run make -f tensorflow/lite/experimental/micro/tools/make/Makefile clean + +# TODO(b/143715361): parallel builds do not work with generated files right now. +readable_run make -f tensorflow/lite/experimental/micro/tools/make/Makefile \ TARGET="arduino" \ TAGS="portable_optimized" \ generate_arduino_zip -tensorflow/lite/experimental/micro/tools/ci_build/install_arduino_cli.sh +readable_run tensorflow/lite/experimental/micro/tools/ci_build/install_arduino_cli.sh -tensorflow/lite/experimental/micro/tools/ci_build/test_arduino_library.sh \ +readable_run tensorflow/lite/experimental/micro/tools/ci_build/test_arduino_library.sh \ tensorflow/lite/experimental/micro/tools/make/gen/arduino_x86_64/prj/tensorflow_lite.zip - -# Needed to solve CI build bug triggered by files added to source tree. -make -f tensorflow/lite/experimental/micro/tools/make/Makefile clean_downloads diff --git a/tensorflow/lite/experimental/micro/tools/ci_build/test_sparkfun.sh b/tensorflow/lite/experimental/micro/tools/ci_build/test_sparkfun.sh index 194391c1c9c..23540d260e0 100755 --- a/tensorflow/lite/experimental/micro/tools/ci_build/test_sparkfun.sh +++ b/tensorflow/lite/experimental/micro/tools/ci_build/test_sparkfun.sh @@ -21,12 +21,11 @@ set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT_DIR=${SCRIPT_DIR}/../../../../../.. cd ${ROOT_DIR} -pwd -make -f tensorflow/lite/experimental/micro/tools/make/Makefile \ - clean clean_downloads +source tensorflow/lite/experimental/micro/tools/ci_build/helper_functions.sh -make -f tensorflow/lite/experimental/micro/tools/make/Makefile TARGET=sparkfun_edge micro_speech_bin +readable_run make -f tensorflow/lite/experimental/micro/tools/make/Makefile clean -# Needed to solve CI build bug triggered by files added to source tree. -make -f tensorflow/lite/experimental/micro/tools/make/Makefile clean_downloads +# TODO(b/143715361): downloading first to allow for parallel builds. +readable_run make -j8 -f tensorflow/lite/experimental/micro/tools/make/Makefile TARGET=sparkfun_edge third_party_downloads +readable_run make -j8 -f tensorflow/lite/experimental/micro/tools/make/Makefile TARGET=sparkfun_edge micro_speech_bin diff --git a/tensorflow/lite/experimental/micro/tools/ci_build/test_x86.sh b/tensorflow/lite/experimental/micro/tools/ci_build/test_x86.sh index f5ba225a240..b2ce96bd0bd 100755 --- a/tensorflow/lite/experimental/micro/tools/ci_build/test_x86.sh +++ b/tensorflow/lite/experimental/micro/tools/ci_build/test_x86.sh @@ -21,12 +21,11 @@ set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT_DIR=${SCRIPT_DIR}/../../../../../.. cd ${ROOT_DIR} -pwd -make -f tensorflow/lite/experimental/micro/tools/make/Makefile \ - clean clean_downloads +source tensorflow/lite/experimental/micro/tools/ci_build/helper_functions.sh -make -f tensorflow/lite/experimental/micro/tools/make/Makefile test +readable_run make -f tensorflow/lite/experimental/micro/tools/make/Makefile clean -# Needed to solve CI build bug triggered by files added to source tree. -make -f tensorflow/lite/experimental/micro/tools/make/Makefile clean_downloads +# TODO(b/143715361): downloading first to allow for parallel builds. +readable_run make -j8 -f tensorflow/lite/experimental/micro/tools/make/Makefile third_party_downloads +readable_run make -j8 -f tensorflow/lite/experimental/micro/tools/make/Makefile test