experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add Clutz to TensorBoard build

Browse Source 
This is so we can get JavaScript protobufs. This CL also improves the
web_aspect and makes some peculiar Closure Compiler errors go away
relating to externs.

PiperOrigin-RevId: 158061198
This commit is contained in:
Justine Tunney 2017-06-05 14:21:25 -07:00 committed by TensorFlower Gardener
parent 0df6760fe9
commit 515b3ac677
10 changed files with 390 additions and 171 deletions
WORKSPACE
tensorflow
tensorboard
BUILDclutz.d.tsdefs.bzl
java/org/tensorflow/tensorboard/vulcanize
Vulcanize.java
vulcanize.bzlweb.bzl
workspace.bzl
third_party
clutz.BUILDclutz.bzl

8
WORKSPACE
View File

@ -2,11 +2,11 @@ workspace(name = "org_tensorflow")
http_archive(
name = "io_bazel_rules_closure",
sha256 = "edc91f556b762fc5212d1050d00b12e40dd0b0b1c1d5d96886b59e9a30a6cae4",
strip_prefix = "rules_closure-3f07fb6a58870afbb36051bd5d54da4479561cc6",
sha256 = "bc41b80486413aaa551860fc37471dbc0666e1dbb5236fb6177cb83b0c105846",
strip_prefix = "rules_closure-dec425a4ff3faf09a56c85d082e4eed05d8ce38f",
urls = [
"http://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/3f07fb6a58870afbb36051bd5d54da4479561cc6.tar.gz", # 2017-05-31
"https://github.com/bazelbuild/rules_closure/archive/3f07fb6a58870afbb36051bd5d54da4479561cc6.tar.gz",
"http://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/dec425a4ff3faf09a56c85d082e4eed05d8ce38f.tar.gz", # 2017-06-02
"https://github.com/bazelbuild/rules_closure/archive/dec425a4ff3faf09a56c85d082e4eed05d8ce38f.tar.gz",
],
)

11
tensorflow/tensorboard/BUILD
View File

@ -33,6 +33,17 @@ filegroup(
],
)
filegroup(
name = "ts_web_library_default_typings",
srcs = [
# Ordering probably matters.
"@com_microsoft_typescript//:lib.es6.d.ts",
"@io_angular_clutz//:src/resources/closure.lib.d.ts",
"clutz.d.ts",
],
visibility = ["//visibility:public"],
)
filegroup(
name = "all_files",
srcs = glob(

19
tensorflow/tensorboard/clutz.d.ts vendored Normal file
View File

@ -0,0 +1,19 @@
// Copyright 2017 The TensorFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// tslint:disable
declare namespace ಠ_ಠ.clutz {
interface IteratorIterable<T> extends Iterator<T>, Iterable<T> {}
interface IIterableResult<T> extends IteratorResult<T> {}
}

7
tensorflow/tensorboard/defs.bzl
View File

@ -15,3 +15,10 @@
def tensorboard_webcomponent_library(**kwargs):
"""Rules referencing this will be deleted from the codebase soon."""
pass
def _legacy_js_impl(target, ctx):
return struct()
legacy_js = aspect(
implementation=_legacy_js_impl,
attr_aspects=["exports"])

73
tensorflow/tensorboard/java/org/tensorflow/tensorboard/vulcanize/Vulcanize.java
View File

@ -24,7 +24,6 @@ import com.google.common.base.Joiner;
import com.google.common.base.Optional;
import com.google.common.base.Splitter;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMultimap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
@ -38,6 +37,7 @@ import com.google.javascript.jscomp.DiagnosticGroup;
import com.google.javascript.jscomp.DiagnosticGroups;
import com.google.javascript.jscomp.DiagnosticType;
import com.google.javascript.jscomp.JSError;
import com.google.javascript.jscomp.ModuleIdentifier;
import com.google.javascript.jscomp.PropertyRenamingPolicy;
import com.google.javascript.jscomp.Result;
import com.google.javascript.jscomp.SourceFile;
@ -53,8 +53,10 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Deque;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
@ -63,6 +65,7 @@ import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Attribute;
import org.jsoup.nodes.Comment;
@ -92,6 +95,7 @@ public final class Vulcanize {
private static final Set<String> legalese = new HashSet<>();
private static final List<String> licenses = new ArrayList<>();
private static final List<Webpath> stack = new ArrayList<>();
private static final List<SourceFile> externs = new ArrayList<>();
private static final List<SourceFile> sourcesFromJsLibraries = new ArrayList<>();
private static final Map<Webpath, String> sourcesFromScriptTags = new LinkedHashMap<>();
private static final Map<Webpath, Node> sourceTags = new LinkedHashMap<>();
@ -111,7 +115,13 @@ public final class Vulcanize {
Path output = Paths.get(args[4]);
for (int i = 5; i < args.length; i++) {
if (args[i].endsWith(".js")) {
sourcesFromJsLibraries.add(SourceFile.fromFile(args[i]));
String code = new String(Files.readAllBytes(Paths.get(args[i])), UTF_8);
SourceFile sourceFile = SourceFile.fromCode(args[i], code);
if (code.contains("@externs")) {
externs.add(sourceFile);
} else {
sourcesFromJsLibraries.add(sourceFile);
}
continue;
}
if (!args[i].endsWith(".pbtxt")) {
@ -339,12 +349,20 @@ public final class Vulcanize {
options.setRemoveUnusedPrototypePropertiesInExterns(false);
options.setRemoveUnusedClassProperties(false);
// Closure pass.
// Dependency management.
options.setClosurePass(true);
options.setManageClosureDependencies(true);
options.getDependencyOptions().setDependencyPruning(true);
options.getDependencyOptions().setDependencySorting(false);
options.getDependencyOptions().setDependencySorting(true);
options.getDependencyOptions().setMoocherDropping(false);
options.getDependencyOptions()
.setEntryPoints(
sourceTags
.keySet()
.stream()
.map(Webpath::toString)
.map(ModuleIdentifier::forFile)
.collect(Collectors.toList()));
// Polymer pass.
options.setPolymerVersion(1);
@ -362,6 +380,16 @@ public final class Vulcanize {
new WarningsGuard() {
@Override
public CheckLevel level(JSError error) {
if (error.sourceName == null) {
return null;
}
if (error.sourceName.startsWith("javascript/externs")
|| error.sourceName.contains("com_google_javascript_closure_compiler_externs")) {
// TODO(jart): Figure out why these "mismatch of the removeEventListener property on
// type" warnings are showing up.
// https://github.com/google/closure-compiler/pull/1959
return CheckLevel.OFF;
}
if (IGNORE_PATHS_PATTERN.matcher(error.sourceName).matches()) {
return CheckLevel.OFF;
}
@ -395,42 +423,39 @@ public final class Vulcanize {
// Compile everything into a single script.
Compiler compiler = new Compiler();
compiler.disableThreads();
Result result = compiler.compile(ImmutableList.<SourceFile>of(), sauce, options);
Result result = compiler.compile(externs, sauce, options);
if (!result.success) {
System.exit(1);
}
String jsBlob = compiler.toSource();
// Split apart the JS blob and put it back in the original <script> locations.
Deque<Map.Entry<Webpath, Node>> tags = new ArrayDeque<>();
tags.addAll(sourceTags.entrySet());
Matcher matcher = WEBPATH_PATTERN.matcher(jsBlob);
Webpath path = null;
String pureJsDeps = "";
int start = -1;
verify(matcher.find(), "Nothing found in compiled JS blob!");
Webpath path = Webpath.get(matcher.group(1));
int start = 0;
while (matcher.find()) {
if (!sourceTags.containsKey(Webpath.get(matcher.group(1)))) {
continue; // Skip over js_library dependencies, which must group at beginning of args.
}
if (path != null) {
swapScript(path, pureJsDeps + jsBlob.substring(start, matcher.start()));
pureJsDeps = "";
} else {
pureJsDeps = jsBlob.substring(0, matcher.start());
if (sourceTags.containsKey(path)) {
swapScript(tags, path, jsBlob.substring(start, matcher.start()));
start = matcher.start();
}
path = Webpath.get(matcher.group(1));
start = matcher.start();
}
swapScript(path, pureJsDeps + jsBlob.substring(start));
if (!sourceTags.isEmpty()) {
throw new RuntimeException("Couldn't pull out: " + ImmutableSet.copyOf(sourceTags.keySet()));
}
swapScript(tags, path, jsBlob.substring(start));
verify(tags.isEmpty(), "<script> wasn't compiled: %s", tags);
}
private static void swapScript(Webpath path, String script) {
Node tag = sourceTags.get(path);
private static void swapScript(
Deque<Map.Entry<Webpath, Node>> tags, Webpath path, String script) {
verify(!tags.isEmpty(), "jscomp compiled %s after last <script>?!", path);
Webpath want = tags.getFirst().getKey();
verify(path.equals(want), "<script> tag for %s should come before %s", path, want);
Node tag = tags.removeFirst().getValue();
tag.replaceWith(
new Element(Tag.valueOf("script"), tag.baseUri())
.appendChild(new DataNode(script, tag.baseUri())));
sourceTags.remove(path);
}
private static void handleLicense(String text) {

24
tensorflow/tensorboard/vulcanize.bzl
View File

@ -12,26 +12,25 @@
# See the License for the specific language governing permissions and
# limitations under the License.
load("@io_bazel_rules_closure//closure/private:defs.bzl", "unfurl", "long_path")
load("//tensorflow/tensorboard:defs.bzl", "legacy_js")
load("@io_bazel_rules_closure//closure/private:defs.bzl", "collect_js", "unfurl", "long_path")
load("//tensorflow/tensorboard:web.bzl", "web_aspect")
def _tensorboard_html_binary(ctx):
deps = unfurl(ctx.attr.deps, provider="webfiles")
manifests = depset(order="topological")
files = depset()
jslibs = depset(ctx.files._jslibs)
webpaths = depset()
manifests = set(order="topological")
files = set()
webpaths = set()
for dep in deps:
manifests += dep.webfiles.manifests
webpaths += dep.webfiles.webpaths
files += dep.data_runfiles.files
if hasattr(dep.webfiles, "jslibs"):
jslibs += dep.webfiles.jslibs
if hasattr(dep, "closure_js_library"):
jslibs += getattr(dep.closure_js_library, "srcs", [])
webpaths += [ctx.attr.output_path]
closure_js_library=collect_js(
ctx, unfurl(ctx.attr.deps, provider="closure_js_library"))
# vulcanize
jslibs = depset(ctx.files._jslibs) + closure_js_library.srcs
ctx.action(
inputs=list(manifests | files | jslibs),
outputs=[ctx.outputs.html],
@ -101,7 +100,12 @@ tensorboard_html_binary = rule(
"input_path": attr.string(mandatory=True),
"output_path": attr.string(mandatory=True),
"data": attr.label_list(cfg="data", allow_files=True),
"deps": attr.label_list(aspects=[web_aspect], mandatory=True),
"deps": attr.label_list(
aspects=[
web_aspect,
legacy_js,
],
mandatory=True),
"external_assets": attr.string_dict(default={"/_/runfiles": "."}),
"_jslibs": attr.label(
default=Label("//tensorflow/tensorboard/java/org/tensorflow/tensorboard/vulcanize:jslibs"),

273
tensorflow/tensorboard/web.bzl
View File

@ -14,7 +14,18 @@
"""Same as web_library but supports TypeScript."""
load("//tensorflow/tensorboard:defs.bzl", "legacy_js")
load("//third_party:clutz.bzl",
"CLUTZ_ATTRIBUTES",
"CLUTZ_OUTPUTS",
"clutz_aspect",
"extract_dts_from_closure_libraries")
load("@io_bazel_rules_closure//closure/private:defs.bzl",
"CLOSURE_LIBRARY_BASE_ATTR",
"CLOSURE_LIBRARY_DEPS_ATTR",
"collect_js",
"collect_runfiles",
"convert_path_to_es6_module_name",
"create_argfile",
@ -22,6 +33,14 @@ load("@io_bazel_rules_closure//closure/private:defs.bzl",
"long_path",
"unfurl")
_ASPECT_SLURP_FILE_TYPE = FileType([
".html", ".js", ".css", ".gss", ".png", ".jpg", ".gif", ".ico", ".svg"])
_CLOSURE_WORKER = attr.label(
default=Label("@io_bazel_rules_closure//java/io/bazel/rules/closure:ClosureWorker"),
executable=True,
cfg="host")
def _ts_web_library(ctx):
if not ctx.attr.srcs:
if ctx.attr.deps:
@ -43,36 +62,33 @@ def _ts_web_library(ctx):
# process what came before
deps = unfurl(ctx.attr.deps, provider="webfiles")
webpaths = depset()
manifests = depset(order="topological")
jslibs = depset(order="postorder")
ts_typings = depset(ctx.files._es6dts)
ts_typings_paths = depset()
ts_typings = depset(ctx.files._default_typings)
ts_typings_paths = depset(
[long_path(ctx, f) for f in ctx.files._default_typings])
ts_typings_execroots = depset()
aspect_runfiles = depset()
for dep in deps:
webpaths += dep.webfiles.webpaths
manifests += dep.webfiles.manifests
if hasattr(dep.webfiles, "ts_typings"):
ts_typings += dep.webfiles.ts_typings
if hasattr(dep.webfiles, "ts_typings_paths"):
ts_typings_paths += dep.webfiles.ts_typings_paths
if hasattr(dep.webfiles, "ts_typings_execroots"):
ts_typings_execroots += dep.webfiles.ts_typings_execroots
if hasattr(dep.webfiles, "jslibs"):
jslibs += dep.webfiles.jslibs
if hasattr(dep, "closure_js_library"):
jslibs += getattr(dep.closure_js_library, "srcs", [])
if hasattr(dep.webfiles, "aspect_runfiles"):
aspect_runfiles += dep.webfiles.aspect_runfiles
# process what comes now
manifest_srcs = []
new_webpaths = []
ts_inputs = depset()
ts_outputs = []
ts_files = ["lib.es6.d.ts"] + list(ts_typings_paths)
ts_files = list(ts_typings_paths)
new_typings = []
new_typings_paths = []
new_typings_execroot = struct(inputs=[])
execroot = struct(
inputs=[("lib.es6.d.ts", ctx.files._es6dts[0].path)],
inputs=[(long_path(ctx, f), f.path) for f in ctx.files._default_typings],
outputs=[],
program=[ctx.executable._tsc.path, "-p"])
web_srcs = []
@ -118,23 +134,20 @@ def _ts_web_library(ctx):
else:
web_srcs.append(src)
# create webfiles manifest
manifest = ctx.new_file(ctx.configuration.bin_dir,
"%s.pbtxt" % ctx.label.name)
ctx.file_action(
output=manifest,
content=struct(
label=str(ctx.label),
src=manifest_srcs).to_proto())
manifests += [manifest]
webpaths += new_webpaths
# get typings for closure code
clutz_dts = extract_dts_from_closure_libraries(ctx)
if clutz_dts:
entry = (long_path(ctx, clutz_dts), clutz_dts.path)
ts_inputs += [clutz_dts]
ts_files.append(entry[0])
execroot.inputs.append(entry)
# compile typescript
workspace = ""
if ctx.label.workspace_root:
workspace = "/" + ctx.label.workspace_root
if execroot.outputs:
ts_config = ctx.new_file(ctx.bin_dir, "%s-tsc.json" % ctx.label.name)
ts_config = _new_file(ctx, "-tsc.json")
execroot.inputs.append(("tsconfig.json", ts_config.path))
ctx.file_action(
output=ts_config,
@ -149,10 +162,9 @@ def _ts_web_library(ctx):
noResolve=True,
target="es5",
),
files=list(ts_files),
files=ts_files,
).to_json())
er_config = ctx.new_file(ctx.bin_dir,
"%s-tsc-execroot.json" % ctx.label.name)
er_config = _new_file(ctx, "-tsc-execroot.json")
ctx.file_action(output=er_config, content=execroot.to_json())
ts_inputs += collect_runfiles([ctx.attr._tsc])
ts_inputs += ctx.files._tsc
@ -164,41 +176,14 @@ def _ts_web_library(ctx):
outputs=ts_outputs,
executable=ctx.executable._execrooter,
arguments=[er_config.path] + [f.path for f in ts_typings_execroots],
progress_message="Compiling %d TypeScript files" % len(ts_files))
progress_message="Compiling %d TypeScript files %s" % (
len(ts_files), ctx.label))
# perform strict dependency checking
inputs = [manifest]
direct_manifests = depset([manifest])
args = ["WebfilesValidator",
"--dummy", ctx.outputs.dummy.path,
"--target", manifest.path]
for category in ctx.attr.suppress:
args.append("--suppress")
args.append(category)
inputs.extend(web_srcs)
for dep in deps:
inputs.append(dep.webfiles.dummy)
for f in dep.files:
inputs.append(f)
direct_manifests += [dep.webfiles.manifest]
inputs.append(dep.webfiles.manifest)
args.append("--direct_dep")
args.append(dep.webfiles.manifest.path)
for man in difference(manifests, direct_manifests):
inputs.append(man)
args.append("--transitive_dep")
args.append(man.path)
argfile = create_argfile(ctx, args)
inputs.append(argfile)
ctx.action(
inputs=inputs,
outputs=[ctx.outputs.dummy],
executable=ctx.executable._ClosureWorker,
arguments=["@@" + argfile.path],
mnemonic="Closure",
execution_requirements={"supports-workers": "1"},
progress_message="Checking webfiles in %s" % ctx.label)
web_srcs.append(ctx.outputs.dummy)
manifest = _make_manifest(ctx, manifest_srcs)
webpaths += new_webpaths
dummy, manifests = _run_webfiles_validator(ctx, web_srcs, deps, manifest)
web_srcs.append(dummy)
# define development web server that only applies to this transitive closure
params = struct(
@ -207,7 +192,7 @@ def _ts_web_library(ctx):
manifest=[long_path(ctx, man) for man in manifests],
external_asset=[struct(webpath=k, path=v)
for k, v in ctx.attr.external_assets.items()])
params_file = ctx.new_file(ctx.bin_dir, "%s_params.pbtxt" % ctx.label.name)
params_file = _new_file(ctx, "-params.pbtxt")
ctx.file_action(output=params_file, content=params.to_proto())
ctx.file_action(
executable=True,
@ -217,8 +202,7 @@ def _ts_web_library(ctx):
long_path(ctx, params_file)))
if new_typings:
er_config = ctx.new_file(ctx.bin_dir,
"%s-typings-execroot.json" % ctx.label.name)
er_config = _new_file(ctx, "-typings-execroot.json")
ctx.file_action(output=er_config, content=new_typings_execroot.to_json())
ts_typings += new_typings
ts_typings_paths += new_typings_paths
@ -230,83 +214,120 @@ def _ts_web_library(ctx):
# export data to parent rules
return struct(
files=depset(web_srcs),
files=depset(web_srcs + [dummy]),
exports=unfurl(ctx.attr.exports),
webfiles=struct(
manifest=manifest,
manifests=manifests,
webpaths=webpaths,
dummy=ctx.outputs.dummy,
jslibs=jslibs,
dummy=dummy,
ts_typings=ts_typings,
ts_typings_paths=ts_typings_paths,
ts_typings_execroots=ts_typings_execroots),
closure_js_library=collect_js(
ctx, unfurl(ctx.attr.deps, provider="closure_js_library")),
runfiles=ctx.runfiles(
files=ctx.files.srcs + ctx.files.data + ts_outputs + [
manifest,
params_file,
ctx.outputs.executable,
ctx.outputs.dummy],
dummy],
transitive_files=(collect_runfiles([ctx.attr._WebfilesServer]) |
collect_runfiles(deps) |
collect_runfiles(ctx.attr.data))))
collect_runfiles(ctx.attr.data) |
aspect_runfiles)))
def _web_aspect_impl(target, ctx):
if ctx.rule.kind in ("js_library", "pinto_library"):
return _web_aspect_js_library(target, ctx, [], depset())
if hasattr(target, "js"):
return _web_aspect_js_library(
target,
ctx,
target.files,
target.js.full_tc(True))
return struct()
def _web_aspect_js_library(target, ctx, extra_srcs, extra_transitive):
deps = unfurl((ctx.rule.attr.deps +
getattr(ctx.rule.attr, 'sticky_deps', [])),
provider="webfiles")
# process what came before
if hasattr(target, "webfiles"):
return struct()
srcs = []
deps = []
if hasattr(ctx.rule.files, "srcs"):
srcs.extend(_ASPECT_SLURP_FILE_TYPE.filter(ctx.rule.files.srcs))
for attr in ("deps", "sticky_deps", "module_deps"):
value = getattr(ctx.rule.attr, attr, None)
if value:
deps.extend(value)
deps = unfurl(deps, provider="webfiles")
webpaths = depset()
manifests = depset(order="topological")
jslibs = depset(order="postorder")
aspect_runfiles = depset(srcs)
for dep in deps:
webpaths += dep.webfiles.webpaths
manifests += dep.webfiles.manifests
if hasattr(dep.webfiles, "jslibs"):
jslibs += dep.webfiles.jslibs
# process what comes now
srcs = ctx.rule.files.srcs + extra_srcs
jslibs += [src for src in srcs if src.path.endswith(".js")]
if hasattr(dep.webfiles, "aspect_runfiles"):
aspect_runfiles += dep.webfiles.aspect_runfiles
manifest_srcs = []
new_webpaths = []
web_srcs = []
for src in srcs:
webpath = "/" + long_path(ctx, src)
_add_webpath(ctx, src, webpath, webpaths, new_webpaths, manifest_srcs)
web_srcs.append(src)
# create webfiles manifest
manifest = ctx.new_file(ctx.configuration.bin_dir,
"%s-webfiles.pbtxt" % ctx.label.name)
ctx.file_action(
output=manifest,
content=struct(
label=str(ctx.label),
src=manifest_srcs).to_proto())
manifests += [manifest]
webpaths += new_webpaths
manifest = _make_manifest(ctx, manifest_srcs)
dummy, manifests = _run_webfiles_validator(ctx, srcs, deps, manifest)
aspect_runfiles += [dummy, manifest]
return struct(
exports=[] if srcs else deps,
webfiles=struct(
manifest=manifest,
manifests=manifests,
webpaths=webpaths,
dummy=manifest,
jslibs=jslibs),
closure_legacy_js_runfiles=(depset(srcs + ctx.rule.files.data) |
extra_transitive |
collect_runfiles(deps) |
collect_runfiles(ctx.rule.files.data)))
dummy=dummy,
aspect_runfiles=aspect_runfiles))
def _make_manifest(ctx, src_list):
manifest = _new_file(ctx, "-webfiles.pbtxt")
ctx.file_action(
output=manifest,
content=struct(
label=str(ctx.label),
src=src_list).to_proto())
return manifest
def _run_webfiles_validator(ctx, srcs, deps, manifest):
dummy = _new_file(ctx, "-webfiles.ignoreme")
manifests = depset(order="topological")
for dep in deps:
manifests += dep.webfiles.manifests
if srcs:
args = ["WebfilesValidator",
"--dummy", dummy.path,
"--target", manifest.path]
if hasattr(ctx, "attr") and hasattr(ctx.attr, "suppress"):
for category in ctx.attr.suppress:
args.append("--suppress")
args.append(category)
inputs = [manifest]
inputs.extend(srcs)
direct_manifests = depset()
for dep in deps:
inputs.append(dep.webfiles.dummy)
for f in dep.files:
inputs.append(f)
direct_manifests += [dep.webfiles.manifest]
inputs.append(dep.webfiles.manifest)
args.append("--direct_dep")
args.append(dep.webfiles.manifest.path)
for man in difference(manifests, direct_manifests):
inputs.append(man)
args.append("--transitive_dep")
args.append(man.path)
argfile = _new_file(ctx, "-webfiles-checker-args.txt")
ctx.file_action(output=argfile, content="\n".join(args))
inputs.append(argfile)
ctx.action(
inputs=inputs,
outputs=[dummy],
executable=(getattr(ctx.executable, "_ClosureWorker", None) or
getattr(ctx.executable, "_ClosureWorkerAspect", None)),
arguments=["@@" + argfile.path],
mnemonic="Closure",
execution_requirements={"supports-workers": "1"},
progress_message="Checking webfiles %s" % ctx.label)
else:
ctx.file_action(output=dummy, content="BOO!")
manifests += [manifest]
return dummy, manifests
def _new_file(ctx, suffix):
return ctx.new_file(ctx.bin_dir, "%s%s" % (ctx.label.name, suffix))
def _add_webpath(ctx, src, webpath, webpaths, new_webpaths, manifest_srcs):
if webpath in new_webpaths:
@ -354,45 +375,45 @@ def _get_strip(ctx):
web_aspect = aspect(
implementation=_web_aspect_impl,
attr_aspects=["deps"])
attr_aspects=["deps", "sticky_deps", "module_deps"],
attrs={"_ClosureWorkerAspect": _CLOSURE_WORKER})
ts_web_library = rule(
implementation=_ts_web_library,
executable=True,
attrs={
attrs=CLUTZ_ATTRIBUTES + {
"path": attr.string(),
"srcs": attr.label_list(allow_files=True),
"deps": attr.label_list(aspects=[web_aspect]),
"deps": attr.label_list(
aspects=[
web_aspect,
clutz_aspect,
legacy_js,
]),
"exports": attr.label_list(),
"data": attr.label_list(cfg="data", allow_files=True),
"suppress": attr.string_list(),
"strip_prefix": attr.string(),
"external_assets": attr.string_dict(default={"/_/runfiles": "."}),
"clutz_entry_points": attr.string_list(),
"_execrooter": attr.label(
default=Label(
"//tensorflow/tensorboard/scripts:execrooter"),
default=Label("//tensorflow/tensorboard/scripts:execrooter"),
executable=True,
cfg="host"),
"_tsc": attr.label(
default=Label(
"@com_microsoft_typescript//:tsc"),
default=Label("@com_microsoft_typescript//:tsc"),
allow_files=True,
executable=True,
cfg="host"),
"_es6dts": attr.label(
default=Label(
"@com_microsoft_typescript//:lib.es6.d.ts"),
"_default_typings": attr.label(
default=Label("//tensorflow/tensorboard:ts_web_library_default_typings"),
allow_files=True),
"_ClosureWorker": attr.label(
default=Label("@io_bazel_rules_closure//java/io/bazel/rules/closure:ClosureWorker"),
executable=True,
cfg="host"),
"_WebfilesServer": attr.label(
default=Label(
"@io_bazel_rules_closure//java/io/bazel/rules/closure/webfiles/server:WebfilesServer"),
default=Label("@io_bazel_rules_closure//java/io/bazel/rules/closure/webfiles/server:WebfilesServer"),
executable=True,
cfg="host"),
"_ClosureWorker": _CLOSURE_WORKER,
"_closure_library_base": CLOSURE_LIBRARY_BASE_ATTR,
"_closure_library_deps": CLOSURE_LIBRARY_DEPS_ATTR,
},
outputs={
"dummy": "%{name}.ignoreme",
})
outputs=CLUTZ_OUTPUTS)

25
tensorflow/workspace.bzl
View File

@ -741,19 +741,27 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
]),
)
##############################################################################
# TensorBoard JavaScript Production Dependencies
native.new_http_archive(
name = "io_angular_clutz",
build_file = "//third_party:clutz.BUILD",
sha256 = "2981de41d1ff4774b544423da9a2cd8beb3be649e95aef2ef2fd83957300b3fe",
strip_prefix = "clutz-b0db5ade9bb535d387f05292316c422790c9848e",
urls = [
"http://mirror.bazel.build/github.com/angular/clutz/archive/b0db5ade9bb535d387f05292316c422790c9848e.tar.gz", # 2017-05-22
"https://github.com/angular/clutz/archive/b0db5ade9bb535d387f05292316c422790c9848e.tar.gz",
],
)
filegroup_external(
name = "com_google_javascript_closure_compiler_externs",
licenses = ["notice"], # Apache 2.0
sha256_urls_extract = {
"0ee7b88ed2955b622eaa038bece283e28d0fb5abebfbb80871fc3d0353f0000b": [
"http://mirror.bazel.build/github.com/google/closure-compiler/archive/v20170423.tar.gz",
"https://github.com/google/closure-compiler/archive/v20170423.tar.gz",
"0f515a6ebfa138490b3c5ea9f3591ea1a7e4a930d3074f18b3eca86084ad9b66": [
"http://mirror.bazel.build/github.com/google/closure-compiler/archive/b37e6000001b0a6bf4c0be49024ebda14a8711d9.tar.gz", # 2017-06-02
"https://github.com/google/closure-compiler/archive/b37e6000001b0a6bf4c0be49024ebda14a8711d9.tar.gz",
],
},
strip_prefix = {"v20170423.tar.gz": "closure-compiler-20170423/externs"},
strip_prefix = {"b37e6000001b0a6bf4c0be49024ebda14a8711d9.tar.gz": "closure-compiler-b37e6000001b0a6bf4c0be49024ebda14a8711d9/externs"},
)
filegroup_external(
@ -761,12 +769,15 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
licenses = ["notice"], # Apache 2.0
sha256_urls = {
"23baad9a200a717a821c6df504c84d3a893d7ea9102b14876eb80097e3b94292": [
"http://mirror.bazel.build/raw.githubusercontent.com/google/closure-compiler/0e8dc5597a295ee259e3fecd98d6535dc621232f/contrib/externs/polymer-1.0.js",
"http://mirror.bazel.build/raw.githubusercontent.com/google/closure-compiler/0e8dc5597a295ee259e3fecd98d6535dc621232f/contrib/externs/polymer-1.0.js", # 2017-05-27
"https://raw.githubusercontent.com/google/closure-compiler/0e8dc5597a295ee259e3fecd98d6535dc621232f/contrib/externs/polymer-1.0.js",
],
},
)
##############################################################################
# TensorBoard JavaScript Production Dependencies
web_library_external(
name = "com_lodash",
licenses = ["notice"], # MIT

44
third_party/clutz.BUILD vendored Normal file
View File

@ -0,0 +1,44 @@
# Description:
# Build tool for making TypeScript .d.ts files from Closure JavaScript.
package(default_visibility = ["//visibility:public"])
licenses(["notice"]) # MIT
exports_files([
"LICENSE",
"src/resources/closure.lib.d.ts",
])
JVM_FLAGS = [
"-Xss20m", # JSCompiler needs big stacks for recursive parsing
"-XX:+UseParallelGC", # Best GC when app isn't latency sensitive
]
java_binary(
name = "clutz",
srcs = glob(["src/main/java/com/google/javascript/clutz/**/*.java"]),
jvm_flags = JVM_FLAGS,
main_class = "com.google.javascript.clutz.DeclarationGenerator",
deps = [
"@args4j",
"@com_google_code_findbugs_jsr305",
"@com_google_code_gson",
"@com_google_guava",
"@com_google_javascript_closure_compiler",
],
)
java_binary(
name = "gents",
srcs = glob(["src/main/java/com/google/javascript/gents/**/*.java"]),
jvm_flags = JVM_FLAGS,
main_class = "com.google.javascript.gents.TypeScriptGenerator",
deps = [
"@args4j",
"@com_google_code_findbugs_jsr305",
"@com_google_code_gson",
"@com_google_guava",
"@com_google_javascript_closure_compiler",
],
)

77
third_party/clutz.bzl vendored Normal file
View File

@ -0,0 +1,77 @@
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Build definitions for TypeScript from Closure JavaScript libraries."""
load("@io_bazel_rules_closure//closure/private:defs.bzl",
"JS_FILE_TYPE",
"collect_js",
"unfurl")
CLUTZ_ATTRIBUTES = {
"_clutz": attr.label(
default=Label("@io_angular_clutz//:clutz"),
executable=True,
cfg="host"),
"_clutz_externs": attr.label(
default=Label("@com_google_javascript_closure_compiler_externs"),
allow_files=True),
}
def extract_dts_from_closure_libraries(ctx):
"""Extracts type definitions from closure dependencies.
This just generates one big .d.ts file for all transitive Closure sources,
and does not pass it down. That means each rule has to duplicate the effort,
but on the other hand allows transitive dependencies on shared rules without
causing duplicate definition errors.
Args:
ctx: A Skylark context.
Returns:
The generated Clutz typings file, or None if there were no JS deps.
"""
deps = unfurl(ctx.attr.deps, provider="closure_js_library")
js = collect_js(ctx, deps)
if not js.srcs:
return None
js_typings = ctx.new_file(ctx.bin_dir, "%s-js-typings.d.ts" % ctx.label.name)
srcs = depset(JS_FILE_TYPE.filter(ctx.files._clutz_externs)) + js.srcs
args = ["-o", js_typings.path]
for src in srcs:
args.append(src.path)
if getattr(ctx.attr, "clutz_entry_points", None):
args.append("--closure_entry_points")
args.extend(ctx.attr.clutz_entry_points)
ctx.action(
inputs=list(srcs),
outputs=[js_typings],
executable=ctx.executable._clutz,
arguments=args,
mnemonic="Clutz",
progress_message="Running Clutz on %d JS files %s" % (
len(srcs), ctx.label))
return js_typings
################################################################################
# The following definitions are for API compatibility with internal clutz.bzl
CLUTZ_OUTPUTS = {}
def _clutz_aspect_impl(target, ctx):
return struct()
clutz_aspect = aspect(
implementation=_clutz_aspect_impl,
attr_aspects=["exports"])