experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Update gcc7_manylinux2010-nvcc-cuda10 config, primarily for TF nightly and releases, but also Windows presubmits.

Browse Source 
I think the preconfig hasn't been updated since cl/308598480. Most users have moved off of preconfigs, but some remain.

Manually, because the update.sh script is broken for me.

PiperOrigin-RevId: 314310108
Change-Id: I746ac0ac62f90bef6624eedf01c7be7a346a2dfd
This commit is contained in:
Christian Sigg 2020-06-02 05:00:35 -07:00 committed by TensorFlower Gardener
parent 424a3072f9
commit 83eb4048ba
2 changed files with 73 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.0/windows/msvc_wrapper_for_nvcc.py vendored
View File

@ -37,13 +37,6 @@ GCC_HOST_COMPILER_PATH = ('/dt7/usr/bin/gcc')
NVCC_PATH = '/usr/local/cuda-10.0/bin/nvcc'
NVCC_VERSION = '10.0'
NVCC_TEMP_DIR = "C:\\Windows\\Temp\\nvcc_inter_files_tmp_dir"
DEFAULT_CUDA_COMPUTE_CAPABILITIES = '3.5,6.0'
# Taken from environment variable for supported TF CUDA Compute Capabilities
# eg. export TF_CUDA_COMPUTE_CAPABILITIES=3.5,3.7,5.2,6.0,6.1,7.0
supported_cuda_compute_capabilities = os.environ.get(
'TF_CUDA_COMPUTE_CAPABILITIES',
DEFAULT_CUDA_COMPUTE_CAPABILITIES).split(',')
def Log(s):
print('gpus/crosstool: {0}'.format(s))
@ -53,7 +46,7 @@ def GetOptionValue(argv, option):
"""Extract the list of values for option from options.
Args:
option: The option whose value to extract, without the leading '/'.
option: The option whose value to extract.
Returns:
1. A list of values, either directly following the option,
@ -62,8 +55,9 @@ def GetOptionValue(argv, option):
2. The leftover options.
"""
parser = ArgumentParser(prefix_chars='/')
parser.add_argument('/' + option, nargs='*', action='append')
parser = ArgumentParser(prefix_chars='-/')
parser.add_argument(option, nargs='*', action='append')
option = option.lstrip('-/').replace('-', '_')
args, leftover = parser.parse_known_args(argv)
if args and vars(args)[option]:
return (sum(vars(args)[option], []), leftover)
@ -122,18 +116,18 @@ def InvokeNvcc(argv, log=False):
nvcc_compiler_options, argv = GetNvccOptions(argv)
opt_option, argv = GetOptionValue(argv, 'O')
opt = ['-g', '-G']
opt_option, argv = GetOptionValue(argv, '/O')
opt = ['-g']
if (len(opt_option) > 0 and opt_option[0] != 'd'):
opt = ['-O2']
include_options, argv = GetOptionValue(argv, 'I')
include_options, argv = GetOptionValue(argv, '/I')
includes = ["-I " + include for include in include_options]
defines, argv = GetOptionValue(argv, 'D')
defines, argv = GetOptionValue(argv, '/D')
defines = ['-D' + define for define in defines]
undefines, argv = GetOptionValue(argv, 'U')
undefines, argv = GetOptionValue(argv, '/U')
undefines = ['-U' + define for define in undefines]
# The rest of the unrecognized options should be passed to host compiler
@ -142,10 +136,20 @@ def InvokeNvcc(argv, log=False):
m_options = ["-m64"]
nvccopts = ['-D_FORCE_INLINES']
for capability in supported_cuda_compute_capabilities:
capability = capability.replace('.', '')
nvccopts += [r'-gencode=arch=compute_%s,"code=sm_%s,compute_%s"' % (
capability, capability, capability)]
compute_capabilities, argv = GetOptionValue(argv, "--cuda-gpu-arch")
for capability in compute_capabilities:
capability = capability[len('sm_'):]
nvccopts += [
r'-gencode=arch=compute_%s,"code=sm_%s"' % (capability, capability)
]
compute_capabilities, argv = GetOptionValue(argv, '--cuda-include-ptx')
for capability in compute_capabilities:
capability = capability[len('sm_'):]
nvccopts += [
r'-gencode=arch=compute_%s,"code=compute_%s"' % (capability, capability)
]
_, argv = GetOptionValue(argv, '--no-cuda-include-ptx')
nvccopts += nvcc_compiler_options
nvccopts += undefines
nvccopts += defines
@ -163,10 +167,15 @@ def InvokeNvcc(argv, log=False):
# Provide a unique dir for each compiling action to avoid conflicts.
tempdir = tempfile.mkdtemp(dir = NVCC_TEMP_DIR)
nvccopts += ['--keep', '--keep-dir', tempdir]
cmd = [NVCC_PATH] + nvccopts
if log:
Log(cmd)
proc = subprocess.Popen(cmd,
Log([NVCC_PATH] + nvccopts)
# Store command line options in a file to avoid hitting the character limit.
optsfile = tempfile.NamedTemporaryFile(mode='w', dir=tempdir, delete=False)
optsfile.write("\n".join(nvccopts))
optsfile.close()
proc = subprocess.Popen([NVCC_PATH, "--options-file", optsfile.name],
stdout=sys.stdout,
stderr=sys.stderr,
env=os.environ.copy(),

70
third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1/clang/bin/crosstool_wrapper_driver_is_not_gcc vendored
View File

@ -53,11 +53,6 @@ NVCC_PATH = '/usr/local/cuda-10.1/bin/nvcc'
PREFIX_DIR = os.path.dirname(GCC_HOST_COMPILER_PATH)
NVCC_VERSION = '10.1'
# Environment variable for supported TF CUDA Compute Capabilities
# eg. export TF_CUDA_COMPUTE_CAPABILITIES=3.5,3.7,5.2,6.0,6.1,7.0
CUDA_COMPUTE_ENV_VAR = 'TF_CUDA_COMPUTE_CAPABILITIES'
DEFAULT_CUDA_COMPUTE_CAPABILITIES = '3.5,6.0'
def Log(s):
print('gpus/crosstool: {0}'.format(s))
@ -76,7 +71,8 @@ def GetOptionValue(argv, option):
"""
parser = ArgumentParser()
parser.add_argument('-' + option, nargs='*', action='append')
parser.add_argument(option, nargs='*', action='append')
option = option.lstrip('-').replace('-', '_')
args, _ = parser.parse_known_args(argv)
if not args or not vars(args)[option]:
return []
@ -149,6 +145,21 @@ def GetNvccOptions(argv):
return ' '.join(['--'+a for a in options])
return ''
def system(cmd):
"""Invokes cmd with os.system().
Args:
cmd: The command.
Returns:
The exit code if the process exited with exit() or -signal
if the process was terminated by a signal.
"""
retv = os.system(cmd)
if os.WIFEXITED(retv):
return os.WEXITSTATUS(retv)
else:
return -os.WTERMSIG(retv)
def InvokeNvcc(argv, log=False):
"""Call nvcc with arguments assembled from argv.
@ -158,30 +169,30 @@ def InvokeNvcc(argv, log=False):
log: True if logging is requested.
Returns:
The return value of calling os.system('nvcc ' + args)
The return value of calling system('nvcc ' + args)
"""
host_compiler_options = GetHostCompilerOptions(argv)
nvcc_compiler_options = GetNvccOptions(argv)
opt_option = GetOptionValue(argv, 'O')
m_options = GetOptionValue(argv, 'm')
opt_option = GetOptionValue(argv, '-O')
m_options = GetOptionValue(argv, '-m')
m_options = ''.join([' -m' + m for m in m_options if m in ['32', '64']])
include_options = GetOptionValue(argv, 'I')
out_file = GetOptionValue(argv, 'o')
depfiles = GetOptionValue(argv, 'MF')
defines = GetOptionValue(argv, 'D')
include_options = GetOptionValue(argv, '-I')
out_file = GetOptionValue(argv, '-o')
depfiles = GetOptionValue(argv, '-MF')
defines = GetOptionValue(argv, '-D')
defines = ''.join([' -D' + define for define in defines])
undefines = GetOptionValue(argv, 'U')
undefines = GetOptionValue(argv, '-U')
undefines = ''.join([' -U' + define for define in undefines])
std_options = GetOptionValue(argv, 'std')
# currently only c++11 is supported by Cuda 7.0 std argument
nvcc_allowed_std_options = ["c++11"]
std_options = GetOptionValue(argv, '-std')
# Supported -std flags as of CUDA 9.0. Only keep last to mimic gcc/clang.
nvcc_allowed_std_options = ["c++03", "c++11", "c++14"]
std_options = ''.join([' -std=' + define
for define in std_options if define in nvcc_allowed_std_options])
for define in std_options if define in nvcc_allowed_std_options][-1:])
# The list of source files get passed after the -c option. I don't know of
# any other reliable way to just get the list of source files to be compiled.
src_files = GetOptionValue(argv, 'c')
src_files = GetOptionValue(argv, '-c')
# Pass -w through from host to nvcc, but don't do anything fancier with
# warnings-related flags, since they're not necessarily the same across
@ -194,7 +205,7 @@ def InvokeNvcc(argv, log=False):
return 1
opt = (' -O2' if (len(opt_option) > 0 and int(opt_option[0]) > 0)
else ' -g -G')
else ' -g')
includes = (' -I ' + ' -I '.join(include_options)
if len(include_options) > 0
@ -207,13 +218,16 @@ def InvokeNvcc(argv, log=False):
srcs = ' '.join(src_files)
out = ' -o ' + out_file[0]
supported_cuda_compute_capabilities = os.environ.get(CUDA_COMPUTE_ENV_VAR, DEFAULT_CUDA_COMPUTE_CAPABILITIES).split(',')
nvccopts = '-D_FORCE_INLINES '
for capability in supported_cuda_compute_capabilities:
capability = capability.replace('.', '')
nvccopts += r'-gencode=arch=compute_%s,\"code=sm_%s,compute_%s\" ' % (
capability, capability, capability)
nvccopts += ' ' + nvcc_compiler_options
for capability in GetOptionValue(argv, "--cuda-gpu-arch"):
capability = capability[len('sm_'):]
nvccopts += r'-gencode=arch=compute_%s,\"code=sm_%s\" ' % (capability,
capability)
for capability in GetOptionValue(argv, '--cuda-include-ptx'):
capability = capability[len('sm_'):]
nvccopts += r'-gencode=arch=compute_%s,\"code=compute_%s\" ' % (capability,
capability)
nvccopts += nvcc_compiler_options
nvccopts += undefines
nvccopts += defines
nvccopts += std_options
@ -229,7 +243,7 @@ def InvokeNvcc(argv, log=False):
' -I .' +
' -x cu ' + opt + includes + ' ' + srcs + ' -M -o ' + depfile)
if log: Log(cmd)
exit_status = os.system(cmd)
exit_status = system(cmd)
if exit_status != 0:
return exit_status
@ -243,7 +257,7 @@ def InvokeNvcc(argv, log=False):
# Need to investigate and fix.
cmd = 'PATH=' + PREFIX_DIR + ':$PATH ' + cmd
if log: Log(cmd)
return os.system(cmd)
return system(cmd)
def main():