From 9dd3efb5aa3bacba8c66042ff975a3b9d4d30f95 Mon Sep 17 00:00:00 2001 From: Thomas Joerg Date: Thu, 14 May 2020 10:06:52 -0700 Subject: [PATCH] Do not silently ignore ptxas compilation failures. Change the xla_gpu_unsafe_fallback_to_driver_on_ptxas_error default to false. PiperOrigin-RevId: 311554370 Change-Id: I9a7f9ff114957998a84136e16333addf4a2cd354 --- tensorflow/compiler/xla/debug_options_flags.cc | 2 +- tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tensorflow/compiler/xla/debug_options_flags.cc b/tensorflow/compiler/xla/debug_options_flags.cc index 216fb0a7422..60a563ee956 100644 --- a/tensorflow/compiler/xla/debug_options_flags.cc +++ b/tensorflow/compiler/xla/debug_options_flags.cc @@ -66,7 +66,7 @@ DebugOptions DefaultDebugOptionsIgnoringFlags() { opts.set_xla_cpu_enable_xprof_traceme(true); // TODO(b/155295372): disable ptxas fallback by default. opts.set_xla_gpu_unsafe_fallback_to_driver_on_ptxas_not_found(true); - opts.set_xla_gpu_unsafe_fallback_to_driver_on_ptxas_error(true); + opts.set_xla_gpu_unsafe_fallback_to_driver_on_ptxas_error(false); return opts; } diff --git a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc index 0196267d904..7ff8d40b440 100644 --- a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc +++ b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc @@ -416,11 +416,12 @@ std::vector NVPTXCompiler::CompileGpuAsmOrGetCachedResult( CHECK(hlo_module_config.debug_options() .xla_gpu_unsafe_fallback_to_driver_on_ptxas_error()) << "There was an error when trying to compile ptx into sass " - "code. If you want to try falling back to the GPU driver to " - "jit compile ptx, you can use the flag " - "--xla_gpu_unsafe_fallback_to_driver_on_ptxas_error." - " Use at your own risk though, it has known drawbacks like " - "increased memory consumption."; + "code. Up until May 14 2020, XLA silently ignored such " + "errors and fell back to the GPU driver. This is likely to " + "trigger subtle runtime issues and is hence discouraged. " + "If you want to temporarily restore this behavior use the " + "flag --xla_gpu_unsafe_fallback_to_driver_on_ptxas_error " + "and file a bug in b/components/366096."; } // We're going to use the driver to JIT our PTX->SASS, so warn if