diff --git a/third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_rocm.tpl b/third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_rocm.tpl index c1595312632..7c30e921bc9 100755 --- a/third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_rocm.tpl +++ b/third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_rocm.tpl @@ -31,10 +31,13 @@ GCC_HOST_COMPILER_PATH = ('%{gcc_host_compiler_path}') HIPCC_PATH = '%{hipcc_path}' PREFIX_DIR = os.path.dirname(GCC_HOST_COMPILER_PATH) HIPCC_ENV = '%{hipcc_env}' +HIPCC_IS_HIPCLANG = '%{hipcc_is_hipclang}'=="True" HIP_RUNTIME_PATH = '%{hip_runtime_path}' HIP_RUNTIME_LIBRARY = '%{hip_runtime_library}' HCC_RUNTIME_PATH = '%{hcc_runtime_path}' HCC_RUNTIME_LIBRARY = '%{hcc_runtime_library}' +ROCR_RUNTIME_PATH = '%{rocr_runtime_path}' +ROCR_RUNTIME_LIBRARY = '%{rocr_runtime_library}' VERBOSE = '%{crosstool_verbose}'=='1' def Log(s): @@ -200,8 +203,11 @@ def InvokeHipcc(argv, log=False): # TODO(zhengxq): for some reason, 'gcc' needs this help to find 'as'. # Need to investigate and fix. - cmd = 'PATH=' + PREFIX_DIR + ':$PATH ' + cmd + cmd = 'PATH=' + PREFIX_DIR + ':$PATH '\ + + HIPCC_ENV.replace(';', ' ') + ' '\ + + cmd if log: Log(cmd) + if VERBOSE: print(cmd) return os.system(cmd) @@ -215,6 +221,9 @@ def main(): parser.add_argument('-pass-exit-codes', action='store_true') args, leftover = parser.parse_known_args(sys.argv[1:]) + if VERBOSE: print('PWD=' + os.getcwd()) + if VERBOSE: print('HIPCC_ENV=' + HIPCC_ENV) + if args.x and args.x[0] == 'rocm': # compilation for GPU objects if args.rocm_log: Log('-x rocm') @@ -231,9 +240,14 @@ def main(): gpu_linker_flags = [flag for flag in sys.argv[1:] if not flag.startswith(('--rocm_log'))] - gpu_linker_flags.append('-L' + HCC_RUNTIME_PATH) - gpu_linker_flags.append('-Wl,-rpath=' + HCC_RUNTIME_PATH) - gpu_linker_flags.append('-l' + HCC_RUNTIME_LIBRARY) + gpu_linker_flags.append('-L' + ROCR_RUNTIME_PATH) + gpu_linker_flags.append('-Wl,-rpath=' + ROCR_RUNTIME_PATH) + gpu_linker_flags.append('-l' + ROCR_RUNTIME_LIBRARY) + # do not link with HCC runtime library in case hip-clang toolchain is used + if not HIPCC_IS_HIPCLANG: + gpu_linker_flags.append('-L' + HCC_RUNTIME_PATH) + gpu_linker_flags.append('-Wl,-rpath=' + HCC_RUNTIME_PATH) + gpu_linker_flags.append('-l' + HCC_RUNTIME_LIBRARY) gpu_linker_flags.append('-L' + HIP_RUNTIME_PATH) gpu_linker_flags.append('-Wl,-rpath=' + HIP_RUNTIME_PATH) gpu_linker_flags.append('-l' + HIP_RUNTIME_LIBRARY) diff --git a/third_party/gpus/rocm_configure.bzl b/third_party/gpus/rocm_configure.bzl index 985e791216c..576a648a459 100644 --- a/third_party/gpus/rocm_configure.bzl +++ b/third_party/gpus/rocm_configure.bzl @@ -181,6 +181,11 @@ def _rocm_include_path(repository_ctx, rocm_config): # Add HIP headers inc_dirs.append("/opt/rocm/include/hip") inc_dirs.append("/opt/rocm/include/hip/hcc_detail") + inc_dirs.append("/opt/rocm/hip/include") + + # Add HIP-Clang headers + inc_dirs.append("/opt/rocm/llvm/lib/clang/8.0/include") + inc_dirs.append("/opt/rocm/llvm/lib/clang/9.0.0/include") # Add rocrand and hiprand headers inc_dirs.append("/opt/rocm/rocrand/include") @@ -259,12 +264,60 @@ def _hipcc_env(repository_ctx): "HIP_VDI_HOME", "HIPCC_VERBOSE", "HIPCC_COMPILE_FLAGS_APPEND", + "HIPPCC_LINK_FLAGS_APPEND", + "HCC_AMDGPU_TARGET", + "HIP_PLATFORM", ]: if name in repository_ctx.os.environ: hipcc_env = (hipcc_env + " " + name + "=\"" + repository_ctx.os.environ[name].strip() + "\";") return hipcc_env.strip() +def _hipcc_is_hipclang(repository_ctx): + """Returns if hipcc is based on hip-clang toolchain. + + Args: + repository_ctx: The repository context. + + Returns: + A string "True" if hipcc is based on hip-clang toolchain. + The functions returns "False" if not (ie: based on HIP/HCC toolchain). + """ + + # check user-defined hip-clang environment variables + for name in ["HIP_CLANG_PATH", "HIP_VDI_HOME"]: + if name in repository_ctx.os.environ: + return "True" + + # grep for "HIP_COMPILER=clang" in /opt/rocm/hip/lib/.hipInfo + grep_result = _execute( + repository_ctx, + ["grep", "HIP_COMPILER=clang", "/opt/rocm/hip/lib/.hipInfo"], + empty_stdout_fine = True, + ) + result = grep_result.stdout + if result == "HIP_COMPILER=clang": + return "True" + return "False" + +def _if_hipcc_is_hipclang(repository_ctx, if_true, if_false = []): + """ + Returns either the if_true or if_false arg based on whether hipcc + is based on the hip-clang toolchain + + Args : + repository_ctx: The repository context. + if_true : value to return if hipcc is hip-clang based + if_false : value to return if hipcc is not hip-clang based + (optional, defaults to empty list) + + Returns : + either the if_true arg or the of_False arg + """ + if _hipcc_is_hipclang(repository_ctx) == "True": + return if_true + return if_false + def _crosstool_verbose(repository_ctx): """Returns the environment variable value CROSSTOOL_VERBOSE. @@ -703,7 +756,21 @@ def _create_local_rocm_repository(repository_ctx): # bazel's header check failing. rocm_defines["%{extra_no_canonical_prefixes_flags}"] = "\"-fno-canonical-system-headers\"" - rocm_defines["%{unfiltered_compile_flags}"] = to_list_of_strings(["-DTENSORFLOW_USE_ROCM", "-D__HIP_PLATFORM_HCC__", "-DEIGEN_USE_HIP"]) + rocm_defines["%{unfiltered_compile_flags}"] = to_list_of_strings([ + "-DTENSORFLOW_USE_ROCM=1", + "-D__HIP_PLATFORM_HCC__", + "-DEIGEN_USE_HIP", + ] + _if_hipcc_is_hipclang(repository_ctx, [ + # + # define "TENSORFLOW_COMPILER_IS_HIP_CLANG" when we are using clang + # based hipcc to compile/build tensorflow + # + # Note that this #define should not be used to check whether or not + # tensorflow is being built with ROCm support + # (only TENSORFLOW_USE_ROCM should be used for that purpose) + # + "-DTENSORFLOW_COMPILER_IS_HIP_CLANG=1", + ])) rocm_defines["%{host_compiler_path}"] = "clang/bin/crosstool_wrapper_driver_rocm" @@ -745,6 +812,9 @@ def _create_local_rocm_repository(repository_ctx): "%{cpu_compiler}": str(cc), "%{hipcc_path}": "/opt/rocm/bin/hipcc", "%{hipcc_env}": _hipcc_env(repository_ctx), + "%{hipcc_is_hipclang}": _hipcc_is_hipclang(repository_ctx), + "%{rocr_runtime_path}": "/opt/rocm/lib", + "%{rocr_runtime_library}": "hsa-runtime64", "%{hip_runtime_path}": "/opt/rocm/hip/lib", "%{hip_runtime_library}": "hip_hcc", "%{hcc_runtime_path}": "/opt/rocm/hcc/lib",