From 0c96f9772dc91e19740cd3affdb3d98796d94f77 Mon Sep 17 00:00:00 2001 From: Revan Sopher Date: Wed, 6 May 2020 14:33:30 -0700 Subject: [PATCH] Add shell function to create a Bazel test summary XML file. Bazel normally generates one of these per target, which causes signficant overhead on upload. Instead we create a single XML file for the status of each target. PiperOrigin-RevId: 310229087 Change-Id: I77f614e0d32222eeacf62c34f1cbeec07ed3d73a --- tensorflow/tools/ci_build/release/common.sh | 43 +++++++++++++++++++ .../release/macos/cpu_py2_full/nonpip.sh | 2 + .../release/macos/cpu_py35_full/nonpip.sh | 2 + .../release/macos/cpu_py36_full/nonpip.sh | 3 +- .../release/macos/cpu_py37_full/nonpip.sh | 2 + .../release/macos/cpu_py38_full/nonpip.sh | 2 + .../release/ubuntu_16/cpu_py2_full/nonpip.sh | 2 + .../release/ubuntu_16/cpu_py35_full/nonpip.sh | 2 + .../release/ubuntu_16/cpu_py36_full/nonpip.sh | 2 + .../release/ubuntu_16/cpu_py37_full/nonpip.sh | 2 + .../release/ubuntu_16/cpu_py38_full/nonpip.sh | 2 + .../release/ubuntu_16/gpu_py2_full/nonpip.sh | 2 + .../release/ubuntu_16/gpu_py35_full/nonpip.sh | 2 + .../release/ubuntu_16/gpu_py36_full/nonpip.sh | 2 + .../release/ubuntu_16/gpu_py37_full/nonpip.sh | 2 + .../release/ubuntu_16/gpu_py38_full/nonpip.sh | 2 + .../release/ubuntu_16/tpu_py37_full/nonpip.sh | 2 + 17 files changed, 75 insertions(+), 1 deletion(-) diff --git a/tensorflow/tools/ci_build/release/common.sh b/tensorflow/tools/ci_build/release/common.sh index 62d13a7e66d..a6ef52b8bea 100644 --- a/tensorflow/tools/ci_build/release/common.sh +++ b/tensorflow/tools/ci_build/release/common.sh @@ -268,3 +268,46 @@ function copy_to_new_project_name { popd rm -rf "${TMP_DIR}" } + +# Create minimalist test XML for web view. It includes the pass/fail status +# of each target, without including errors or stacktraces. +# Remember to "set +e" before calling bazel or we'll only generate the XML for +# passing runs. +function test_xml_summary { + set +x + set +e + mkdir -p "${KOKORO_ARTIFACTS_DIR}/${KOKORO_JOB_NAME}/summary" + # First build the repeated inner XML blocks, since the header block needs to + # report the number of test cases / failures / errors. + # TODO(rsopher): handle build breakages + # TODO(rsopher): extract per-test times as well + TESTCASE_XML="$(sed -n '/INFO:\ Build\ completed/,/INFO:\ Build\ completed/p' \ + /tmpfs/kokoro_build.log \ + | grep -E '(PASSED|FAILED|TIMEOUT)\ in' \ + | while read -r line; \ + do echo ''"$( \ + case "$(echo "${line}" | tr -s ' ' | cut -d ' ' -f 2)" in \ + FAILED) echo '' ;; \ + TIMEOUT) echo '' ;; \ + esac; \ + )"''; done; \ + )" + NUMBER_OF_TESTS="$(echo "${TESTCASE_XML}" | wc -l)" + NUMBER_OF_FAILURES="$(echo "${TESTCASE_XML}" | grep -c ''\ + ''\ + ''\ + "${TESTCASE_XML}"''\ + > "${KOKORO_ARTIFACTS_DIR}/${KOKORO_JOB_NAME}/summary/sponge_log.xml" +} + +# Create minimalist test XML for web view, then exit. +# Ends script with value of previous command, meant to be called immediately +# after bazel as the last call in the build script. +function test_xml_summary_exit { + RETVAL=$? + test_xml_summary + exit "${RETVAL}" +} diff --git a/tensorflow/tools/ci_build/release/macos/cpu_py2_full/nonpip.sh b/tensorflow/tools/ci_build/release/macos/cpu_py2_full/nonpip.sh index 1a0cdd26d55..02e9e2eb9f8 100644 --- a/tensorflow/tools/ci_build/release/macos/cpu_py2_full/nonpip.sh +++ b/tensorflow/tools/ci_build/release/macos/cpu_py2_full/nonpip.sh @@ -41,9 +41,11 @@ source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh tag_filters="-no_oss,-oss_serial,-nomac,-no_mac,-no_oss_py2,-v1only,-gpu,-tpu,-benchmark-test" # Run tests +set +e bazel test --test_output=errors --config=opt \ --action_env=TF2_BEHAVIOR="${TF2_BEHAVIOR}" \ --build_tag_filters="${tag_filters}" \ --test_tag_filters="${tag_filters}" -- \ ${DEFAULT_BAZEL_TARGETS} \ -//tensorflow/lite/... +test_xml_summary_exit \ No newline at end of file diff --git a/tensorflow/tools/ci_build/release/macos/cpu_py35_full/nonpip.sh b/tensorflow/tools/ci_build/release/macos/cpu_py35_full/nonpip.sh index 63b614dd687..06fabd7b1c7 100644 --- a/tensorflow/tools/ci_build/release/macos/cpu_py35_full/nonpip.sh +++ b/tensorflow/tools/ci_build/release/macos/cpu_py35_full/nonpip.sh @@ -41,9 +41,11 @@ tag_filters="-no_oss,-oss_serial,-nomac,-no_mac,-no_oss_py35,-v1only,-gpu,-tpu,- source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh # Run tests +set +e bazel test --test_output=errors --config=opt \ --action_env=TF2_BEHAVIOR="${TF2_BEHAVIOR}" \ --build_tag_filters="${tag_filters}" \ --test_tag_filters="${tag_filters}" -- \ ${DEFAULT_BAZEL_TARGETS} \ -//tensorflow/lite/... +test_xml_summary_exit diff --git a/tensorflow/tools/ci_build/release/macos/cpu_py36_full/nonpip.sh b/tensorflow/tools/ci_build/release/macos/cpu_py36_full/nonpip.sh index a80cdd88ddc..51cc3da62d6 100644 --- a/tensorflow/tools/ci_build/release/macos/cpu_py36_full/nonpip.sh +++ b/tensorflow/tools/ci_build/release/macos/cpu_py36_full/nonpip.sh @@ -41,10 +41,11 @@ tag_filters="-no_oss,-oss_serial,-nomac,-no_mac,-no_oss_py36,-v1only,-gpu,-tpu,- source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh # Run tests +set +e bazel test --test_output=errors --config=opt \ --action_env=TF2_BEHAVIOR="${TF2_BEHAVIOR}" \ --build_tag_filters="${tag_filters}" \ --test_tag_filters="${tag_filters}" -- \ ${DEFAULT_BAZEL_TARGETS} \ -//tensorflow/lite/... - +test_xml_summary_exit diff --git a/tensorflow/tools/ci_build/release/macos/cpu_py37_full/nonpip.sh b/tensorflow/tools/ci_build/release/macos/cpu_py37_full/nonpip.sh index e7234024ca5..e0f2968b45a 100644 --- a/tensorflow/tools/ci_build/release/macos/cpu_py37_full/nonpip.sh +++ b/tensorflow/tools/ci_build/release/macos/cpu_py37_full/nonpip.sh @@ -41,9 +41,11 @@ tag_filters="-no_oss,-oss_serial,-nomac,-no_mac$(maybe_skip_v1),-gpu,-tpu,-bench source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh # Run tests +set +e bazel test --test_output=errors --config=opt \ --action_env=TF2_BEHAVIOR="${TF2_BEHAVIOR}" \ --build_tag_filters="${tag_filters}" \ --test_tag_filters="${tag_filters}" -- \ ${DEFAULT_BAZEL_TARGETS} \ -//tensorflow/lite/... +test_xml_summary_exit diff --git a/tensorflow/tools/ci_build/release/macos/cpu_py38_full/nonpip.sh b/tensorflow/tools/ci_build/release/macos/cpu_py38_full/nonpip.sh index b9a4157577d..22475f35491 100644 --- a/tensorflow/tools/ci_build/release/macos/cpu_py38_full/nonpip.sh +++ b/tensorflow/tools/ci_build/release/macos/cpu_py38_full/nonpip.sh @@ -41,9 +41,11 @@ tag_filters="-no_oss,-oss_serial,-nomac,-no_mac$(maybe_skip_v1),-gpu,-tpu,-bench source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh # Run tests +set +e bazel test --test_output=errors --config=opt \ --action_env=TF2_BEHAVIOR="${TF2_BEHAVIOR}" \ --build_tag_filters="${tag_filters}" \ --test_tag_filters="${tag_filters}" -- \ ${DEFAULT_BAZEL_TARGETS} \ -//tensorflow/lite/... +test_xml_summary_exit diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/cpu_py2_full/nonpip.sh b/tensorflow/tools/ci_build/release/ubuntu_16/cpu_py2_full/nonpip.sh index 8634b9bfb97..5bdb5794e95 100644 --- a/tensorflow/tools/ci_build/release/ubuntu_16/cpu_py2_full/nonpip.sh +++ b/tensorflow/tools/ci_build/release/ubuntu_16/cpu_py2_full/nonpip.sh @@ -38,6 +38,7 @@ tag_filters="-no_oss,-oss_serial,-gpu,-tpu,-benchmark-test,-no_oss_py2,-v1only" source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh # Run tests +set +e bazel test --test_output=errors --config=opt --test_lang_filters=py \ --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1:toolchain \ --linkopt=-lrt \ @@ -45,3 +46,4 @@ bazel test --test_output=errors --config=opt --test_lang_filters=py \ --build_tag_filters="${tag_filters}" \ --test_tag_filters="${tag_filters}" -- \ ${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... +test_xml_summary_exit diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/cpu_py35_full/nonpip.sh b/tensorflow/tools/ci_build/release/ubuntu_16/cpu_py35_full/nonpip.sh index 5b161b2f53b..5339671cce3 100644 --- a/tensorflow/tools/ci_build/release/ubuntu_16/cpu_py35_full/nonpip.sh +++ b/tensorflow/tools/ci_build/release/ubuntu_16/cpu_py35_full/nonpip.sh @@ -37,6 +37,7 @@ tag_filters="-no_oss,-oss_serial,-gpu,-tpu,-benchmark-test,-no_oss_py35,-v1only" source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh # Run tests +set +e bazel test --test_output=errors --config=opt --test_lang_filters=py \ --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1:toolchain \ --linkopt=-lrt \ @@ -44,3 +45,4 @@ bazel test --test_output=errors --config=opt --test_lang_filters=py \ --build_tag_filters="${tag_filters}" \ --test_tag_filters="${tag_filters}" -- \ ${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... +test_xml_summary_exit diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/cpu_py36_full/nonpip.sh b/tensorflow/tools/ci_build/release/ubuntu_16/cpu_py36_full/nonpip.sh index 2b621a5d8ca..c2790420afc 100644 --- a/tensorflow/tools/ci_build/release/ubuntu_16/cpu_py36_full/nonpip.sh +++ b/tensorflow/tools/ci_build/release/ubuntu_16/cpu_py36_full/nonpip.sh @@ -37,6 +37,7 @@ tag_filters="-no_oss,-oss_serial,-gpu,-tpu,-benchmark-test,-no_oss_py36,-v1only" source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh # Run tests +set +e bazel test --test_output=errors --config=opt --test_lang_filters=py \ --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1:toolchain \ --linkopt=-lrt \ @@ -44,3 +45,4 @@ bazel test --test_output=errors --config=opt --test_lang_filters=py \ --build_tag_filters="${tag_filters}" \ --test_tag_filters="${tag_filters}" -- \ ${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... +test_xml_summary_exit diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/cpu_py37_full/nonpip.sh b/tensorflow/tools/ci_build/release/ubuntu_16/cpu_py37_full/nonpip.sh index 3fd9fd66afd..f6415a7c9ad 100644 --- a/tensorflow/tools/ci_build/release/ubuntu_16/cpu_py37_full/nonpip.sh +++ b/tensorflow/tools/ci_build/release/ubuntu_16/cpu_py37_full/nonpip.sh @@ -37,6 +37,7 @@ tag_filters="-no_oss,-oss_serial,-gpu,-tpu,-benchmark-test,-no_oss_py37,-v1only" source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh # Run tests +set +e bazel test --test_output=errors --config=opt --test_lang_filters=py \ --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1:toolchain \ --linkopt=-lrt \ @@ -44,3 +45,4 @@ bazel test --test_output=errors --config=opt --test_lang_filters=py \ --build_tag_filters="${tag_filters}" \ --test_tag_filters="${tag_filters}" -- \ ${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... +test_xml_summary_exit diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/cpu_py38_full/nonpip.sh b/tensorflow/tools/ci_build/release/ubuntu_16/cpu_py38_full/nonpip.sh index 205c488847d..ff7a9f3baef 100644 --- a/tensorflow/tools/ci_build/release/ubuntu_16/cpu_py38_full/nonpip.sh +++ b/tensorflow/tools/ci_build/release/ubuntu_16/cpu_py38_full/nonpip.sh @@ -37,6 +37,7 @@ tag_filters="-no_oss,-oss_serial,-gpu,-tpu,-benchmark-test,-no_oss_py38,-v1only" source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh # Run tests +set +e bazel test --test_output=errors --config=opt --test_lang_filters=py \ --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1:toolchain \ --linkopt=-lrt \ @@ -44,3 +45,4 @@ bazel test --test_output=errors --config=opt --test_lang_filters=py \ --build_tag_filters="${tag_filters}" \ --test_tag_filters="${tag_filters}" -- \ ${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... +test_xml_summary_exit diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py2_full/nonpip.sh b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py2_full/nonpip.sh index bfacea08173..a886b42daa4 100644 --- a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py2_full/nonpip.sh +++ b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py2_full/nonpip.sh @@ -45,6 +45,7 @@ source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh tag_filters="gpu,requires-gpu,-no_gpu,-no_oss,-oss_serial,-no_oss_py2" +set +e bazel test --config=cuda --config=opt \ --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1:toolchain \ --linkopt=-lrt \ @@ -56,3 +57,4 @@ bazel test --config=cuda --config=opt \ --test_output=errors --verbose_failures=true --keep_going \ --run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute \ -- ${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... +test_xml_summary_exit diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py35_full/nonpip.sh b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py35_full/nonpip.sh index c576be51bd7..bdff1f654f8 100644 --- a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py35_full/nonpip.sh +++ b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py35_full/nonpip.sh @@ -45,6 +45,7 @@ source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh tag_filters="gpu,requires-gpu,-no_gpu,-no_oss,-oss_serial,-no_oss_py35" +set +e bazel test --config=cuda --config=opt \ --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1:toolchain \ --linkopt=-lrt \ @@ -56,3 +57,4 @@ bazel test --config=cuda --config=opt \ --test_output=errors --verbose_failures=true --keep_going \ --run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute \ -- ${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... +test_xml_summary_exit diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py36_full/nonpip.sh b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py36_full/nonpip.sh index deb0dcafbb0..3fecf9abd29 100644 --- a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py36_full/nonpip.sh +++ b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py36_full/nonpip.sh @@ -45,6 +45,7 @@ source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh tag_filters="gpu,requires-gpu,-no_gpu,-no_oss,-oss_serial,-no_oss_py36" +set +e bazel test --config=cuda --config=opt \ --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1:toolchain \ --linkopt=-lrt \ @@ -56,3 +57,4 @@ bazel test --config=cuda --config=opt \ --test_output=errors --verbose_failures=true --keep_going \ --run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute \ -- ${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... +test_xml_summary_exit diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py37_full/nonpip.sh b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py37_full/nonpip.sh index 7c85556b722..ff11f954c67 100644 --- a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py37_full/nonpip.sh +++ b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py37_full/nonpip.sh @@ -45,6 +45,7 @@ source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh tag_filters="gpu,requires-gpu,-no_gpu,-no_oss,-oss_serial,-no_oss_py37" +set +e bazel test --config=cuda --config=opt \ --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1:toolchain \ --linkopt=-lrt \ @@ -56,3 +57,4 @@ bazel test --config=cuda --config=opt \ --test_output=errors --verbose_failures=true --keep_going \ --run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute \ -- ${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... +test_xml_summary_exit diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py38_full/nonpip.sh b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py38_full/nonpip.sh index 639ba9edb5a..917fbce7563 100644 --- a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py38_full/nonpip.sh +++ b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py38_full/nonpip.sh @@ -45,6 +45,7 @@ source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh tag_filters="gpu,requires-gpu,-no_gpu,-no_oss,-oss_serial,-no_oss_py38" +test +e bazel test --config=cuda --config=opt \ --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1:toolchain \ --linkopt=-lrt \ @@ -56,3 +57,4 @@ bazel test --config=cuda --config=opt \ --test_output=errors --verbose_failures=true --keep_going \ --run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute \ -- ${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... +test_xml_summary_exit diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/tpu_py37_full/nonpip.sh b/tensorflow/tools/ci_build/release/ubuntu_16/tpu_py37_full/nonpip.sh index fd803569d33..1667316d214 100644 --- a/tensorflow/tools/ci_build/release/ubuntu_16/tpu_py37_full/nonpip.sh +++ b/tensorflow/tools/ci_build/release/ubuntu_16/tpu_py37_full/nonpip.sh @@ -61,4 +61,6 @@ test_args=( --local_test_jobs=1 ) +set +e bazel test "${bazel_args[@]}" "${test_args[@]}" -- "${test_patterns[@]}" +test_xml_summary_exit