experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Implement a workaround for TF http archives being downloaded multiple times.

Browse Source 
While on linux this only saves a minute, on windows this saves 40+ minutes.

PiperOrigin-RevId: 287977569
Change-Id: I795dfa97caee3cde8fea307b56a7b7ebf9e637d7
This commit is contained in:
Gunhan Gulsoy 2020-01-03 05:12:10 -08:00 committed by TensorFlower Gardener
parent b379e6012a
commit c27d06e49c
1 changed files with 20 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
third_party/repo.bzl vendored
View File

@ -87,6 +87,15 @@ def _tf_http_archive(ctx):
"someone will come along shortly thereafter and mirror the file.") "someone will come along shortly thereafter and mirror the file.")
use_syslib = _use_system_lib(ctx, ctx.attr.name) use_syslib = _use_system_lib(ctx, ctx.attr.name)
# Work around the bazel bug that redownloads the whole library.
# Remove this after https://github.com/bazelbuild/bazel/issues/10515 is fixed.
if ctx.attr.additional_build_files:
for internal_src in ctx.attr.additional_build_files:
_ = ctx.path(Label(internal_src))
# End of workaround.
if not use_syslib: if not use_syslib:
ctx.download_and_extract( ctx.download_and_extract(
ctx.attr.urls, ctx.attr.urls,
@ -123,10 +132,12 @@ def _tf_http_archive(ctx):
ctx.symlink(Label(internal_src), ctx.path(external_dest)) ctx.symlink(Label(internal_src), ctx.path(external_dest))
tf_http_archive = repository_rule( tf_http_archive = repository_rule(
implementation = _tf_http_archive,
attrs = { attrs = {
"sha256": attr.string(mandatory = True), "sha256": attr.string(mandatory = True),
"urls": attr.string_list(mandatory = True, allow_empty = False), "urls": attr.string_list(
mandatory = True,
allow_empty = False,
),
"strip_prefix": attr.string(), "strip_prefix": attr.string(),
"type": attr.string(), "type": attr.string(),
"delete": attr.string_list(), "delete": attr.string_list(),
@ -139,7 +150,9 @@ tf_http_archive = repository_rule(
environ = [ environ = [
"TF_SYSTEM_LIBS", "TF_SYSTEM_LIBS",
], ],
implementation = _tf_http_archive,
) )
"""Downloads and creates Bazel repos for dependencies. """Downloads and creates Bazel repos for dependencies.
This is a swappable replacement for both http_archive() and This is a swappable replacement for both http_archive() and
@ -204,10 +217,12 @@ def _third_party_http_archive(ctx):
# For link_files, specify each dict entry as: # For link_files, specify each dict entry as:
# "//path/to/source:file": "localfile" # "//path/to/source:file": "localfile"
third_party_http_archive = repository_rule( third_party_http_archive = repository_rule(
implementation = _third_party_http_archive,
attrs = { attrs = {
"sha256": attr.string(mandatory = True), "sha256": attr.string(mandatory = True),
"urls": attr.string_list(mandatory = True, allow_empty = False), "urls": attr.string_list(
mandatory = True,
allow_empty = False,
),
"strip_prefix": attr.string(), "strip_prefix": attr.string(),
"type": attr.string(), "type": attr.string(),
"delete": attr.string_list(), "delete": attr.string_list(),
@ -220,4 +235,5 @@ third_party_http_archive = repository_rule(
environ = [ environ = [
"TF_SYSTEM_LIBS", "TF_SYSTEM_LIBS",
], ],
implementation = _third_party_http_archive,
) )