Proper re-use of Bazel cache

Fixes #1189

bazel.patch

@@ -0,0 +1,264 @@
+diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/FileWriteAction.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/FileWriteAction.java
+index c7aa4cb63..e084bc27c 100644
+--- a/src/main/java/com/google/devtools/build/lib/analysis/actions/FileWriteAction.java
++++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/FileWriteAction.java
+@@ -28,6 +28,7 @@ import java.io.ByteArrayInputStream;
+ import java.io.ByteArrayOutputStream;
+ import java.io.IOException;
+ import java.io.OutputStream;
++import java.io.PrintWriter;
+ import java.util.zip.GZIPInputStream;
+ import java.util.zip.GZIPOutputStream;
+ 
+@@ -73,6 +74,8 @@ public final class FileWriteAction extends AbstractFileWriteAction {
+    */
+   private final CharSequence fileContents;
+ 
++  private final Artifact output;
++
+   /** Minimum length (in chars) for content to be eligible for compression. */
+   private static final int COMPRESS_CHARS_THRESHOLD = 256;
+ 
+@@ -90,6 +93,7 @@ public final class FileWriteAction extends AbstractFileWriteAction {
+       fileContents = new CompressedString((String) fileContents);
+     }
+     this.fileContents = fileContents;
++    this.output = output;
+   }
+ 
+   /**
+@@ -230,11 +234,32 @@ public final class FileWriteAction extends AbstractFileWriteAction {
+    */
+   @Override
+   protected String computeKey() {
++    // System.err.println("src/main/java/com/google/devtools/build/lib/analysis/actions/FileWriteAction.java => output: " + output.getExecPath());
++    // ".ckd" Compute Key Debug
++    PrintWriter computeKeyDebugWriter = null;
++    String computeKeyDebugFile = output.getExecPath() + ".FileWriteAction.ckd";
++    try {
++      computeKeyDebugWriter = new PrintWriter(computeKeyDebugFile, "UTF-8");
++    } catch (java.io.FileNotFoundException ex) {
++      System.err.println("Unable to create " + computeKeyDebugFile);
++    } catch (java.io.UnsupportedEncodingException ex) {
++      System.err.println("Unsupported encoding");
++    }
++
+     Fingerprint f = new Fingerprint();
+     f.addString(GUID);
++    computeKeyDebugWriter.println("GUID: " + GUID);
++
+     f.addString(String.valueOf(makeExecutable));
++    computeKeyDebugWriter.println("MAKEEXECUTABLE: " + String.valueOf(makeExecutable));
++
+     f.addString(getFileContents());
+-    return f.hexDigestAndReset();
++    computeKeyDebugWriter.println("FILECONTENTS: " + getFileContents());
++
++    String rv = f.hexDigestAndReset();
++    computeKeyDebugWriter.println("KEY: " + rv);
++    computeKeyDebugWriter.close();
++    return rv;
+   }
+ 
+   /**
+diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/SpawnAction.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/SpawnAction.java
+index 580788160..26883eb92 100644
+--- a/src/main/java/com/google/devtools/build/lib/analysis/actions/SpawnAction.java
++++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/SpawnAction.java
+@@ -60,6 +60,7 @@ import com.google.devtools.build.lib.util.ShellEscaper;
+ import com.google.devtools.build.lib.vfs.PathFragment;
+ import com.google.protobuf.GeneratedMessage.GeneratedExtension;
+ import java.nio.charset.Charset;
++import java.io.PrintWriter;
+ import java.util.ArrayList;
+ import java.util.Collections;
+ import java.util.LinkedHashMap;
+@@ -91,6 +92,9 @@ public class SpawnAction extends AbstractAction implements ExecutionInfoSpecifie
+ 
+   private final CommandLine argv;
+ 
++  private final Iterable<Artifact> inputs;
++  private final Iterable<Artifact> outputs;
++
+   private final boolean executeUnconditionally;
+   private final boolean isShellCommand;
+   private final String progressMessage;
+@@ -197,6 +201,9 @@ public class SpawnAction extends AbstractAction implements ExecutionInfoSpecifie
+     this.mnemonic = mnemonic;
+     this.executeUnconditionally = executeUnconditionally;
+     this.extraActionInfoSupplier = extraActionInfoSupplier;
++
++    this.inputs = inputs;
++    this.outputs = outputs;
+   }
+ 
+   @Override
+@@ -312,23 +319,89 @@ public class SpawnAction extends AbstractAction implements ExecutionInfoSpecifie
+ 
+   @Override
+   protected String computeKey() {
++    boolean genruleSetup = String.valueOf(Iterables.get(inputs, 0).getExecPath()).contains("genrule/genrule-setup.sh");
++    boolean validGenrule = genruleSetup && (Iterables.size(inputs) > 1);
++
++    String genruleScript = null;
++    if (validGenrule) {
++      genruleScript = String.valueOf(Iterables.get(inputs, 1).getExecPath());
++    }
++
++    // ".ckd" Compute Key Debug
++    PrintWriter computeKeyDebugWriter = null;
++    if (validGenrule) {
++      String computeKeyDebugFile = genruleScript + ".SpawnAction.ckd";
++      try {
++        computeKeyDebugWriter = new PrintWriter(computeKeyDebugFile, "UTF-8");
++      } catch (java.io.FileNotFoundException ex) {
++        System.err.println("Unable to create " + computeKeyDebugFile);
++      } catch (java.io.UnsupportedEncodingException ex) {
++        System.err.println("Unsupported encoding");
++      }
++    }
++
++    validGenrule = validGenrule && (computeKeyDebugWriter != null);
++
+     Fingerprint f = new Fingerprint();
+     f.addString(GUID);
++    if (validGenrule) { computeKeyDebugWriter.println("GUID: " + GUID); }
++
+     f.addStrings(argv.arguments());
++    if (validGenrule) {
++      for (String input : argv.arguments()) {
++        computeKeyDebugWriter.println("ARGUMENTS: " + input);
++      }
++    }
++
+     f.addString(getMnemonic());
++    if (validGenrule) { computeKeyDebugWriter.println("MNEMONIC: " + getMnemonic()); }
++
+     // We don't need the toolManifests here, because they are a subset of the inputManifests by
+     // definition and the output of an action shouldn't change whether something is considered a
+     // tool or not.
+     f.addPaths(getRunfilesSupplier().getRunfilesDirs());
++    if (validGenrule) {
++      for (PathFragment path : getRunfilesSupplier().getRunfilesDirs()) {
++        computeKeyDebugWriter.println("RUNFILESDIRS: " + path.getPathString());
++      }
++    }
++
+     ImmutableList<Artifact> runfilesManifests = getRunfilesSupplier().getManifests();
+     f.addInt(runfilesManifests.size());
++    if (validGenrule) { computeKeyDebugWriter.println("RUNFILESMANIFESTSSIZE: " + runfilesManifests.size()); }
++
+     for (Artifact runfilesManifest : runfilesManifests) {
+       f.addPath(runfilesManifest.getExecPath());
++      if (validGenrule) { computeKeyDebugWriter.println("RUNFILESMANIFEST: " + runfilesManifest.getExecPath().getPathString()); }
+     }
++
+     f.addStringMap(getEnvironment());
++    if (validGenrule) {
++      for (Map.Entry<String, String> entry : getEnvironment().entrySet()) {
++        computeKeyDebugWriter.println("ENV: " + entry.getKey() + "=" + entry.getValue());
++      }
++    }
++
+     f.addStrings(getClientEnvironmentVariables());
++    if (validGenrule) {
++      for (String input : argv.arguments()) {
++        computeKeyDebugWriter.println("CLIENTENV: " + input);
++      }
++    }
++
+     f.addStringMap(getExecutionInfo());
+-    return f.hexDigestAndReset();
++    if (validGenrule) {
++      for (Map.Entry<String, String> entry : executionInfo.entrySet()) {
++        computeKeyDebugWriter.println("EXECINFO: " + entry.getKey() + "=" + entry.getValue());
++      }
++    }
++
++    String rv = f.hexDigestAndReset();
++    if (validGenrule) {
++      computeKeyDebugWriter.println("KEY: " + rv);
++      computeKeyDebugWriter.close();
++    }
++    return rv;
+   }
+ 
+   @Override
+diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileAction.java
+index 3559fffde..3ba39617c 100644
+--- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileAction.java
++++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileAction.java
+@@ -1111,10 +1111,30 @@ public class CppCompileAction extends AbstractAction
+ 
+   @Override
+   public String computeKey() {
++    // ".ckd" Compute Key Debug
++    PrintWriter computeKeyDebugWriter = null;
++    String computeKeyDebugFile = getInternalOutputFile() + ".CppCompileAction.ckd";
++    try {
++      computeKeyDebugWriter = new PrintWriter(computeKeyDebugFile, "UTF-8");
++    } catch (java.io.FileNotFoundException ex) {
++      System.err.println("Unable to create " + computeKeyDebugFile);
++    } catch (java.io.UnsupportedEncodingException ex) {
++      System.err.println("Unsupported encoding");
++    }
++
+     Fingerprint f = new Fingerprint();
+     f.addUUID(actionClassId);
++    computeKeyDebugWriter.println("UUID: " + actionClassId);
++
+     f.addStringMap(getEnvironment());
++    for (Map.Entry<String, String> entry : getEnvironment().entrySet()) {
++      computeKeyDebugWriter.println("ENV: " + entry.getKey() + "=" + entry.getValue());
++    }
++
+     f.addStringMap(executionInfo);
++    for (Map.Entry<String, String> entry : executionInfo.entrySet()) {
++      computeKeyDebugWriter.println("EXECINFO: " + entry.getKey() + "=" + entry.getValue());
++    }
+ 
+     // For the argv part of the cache key, ignore all compiler flags that explicitly denote module
+     // file (.pcm) inputs. Depending on input discovery, some of the unused ones are removed from
+@@ -1124,6 +1144,9 @@ public class CppCompileAction extends AbstractAction
+     // A better long-term solution would be to make the compiler to find them automatically and
+     // never hand in the .pcm files explicitly on the command line in the first place.
+     f.addStrings(compileCommandLine.getArgv(getInternalOutputFile(), null));
++    for (String input : compileCommandLine.getArgv(getInternalOutputFile(), null)) {
++      computeKeyDebugWriter.println("COMMAND: " + input);
++    }
+ 
+     /*
+      * getArgv() above captures all changes which affect the compilation
+@@ -1133,19 +1156,31 @@ public class CppCompileAction extends AbstractAction
+      * have changed, otherwise we might miss some errors.
+      */
+     f.addPaths(context.getDeclaredIncludeDirs());
++    for (PathFragment path : context.getDeclaredIncludeDirs()) {
++      computeKeyDebugWriter.println("DECLAREDINCLUDEDIRS: " + path.getPathString());
++    }
+     f.addPaths(context.getDeclaredIncludeWarnDirs());
++    for (PathFragment path : context.getDeclaredIncludeWarnDirs()) {
++      computeKeyDebugWriter.println("DECLAREDINCLUDEWARNDIRS: " + path.getPathString());
++    }
+     for (Artifact declaredIncludeSrc : context.getDeclaredIncludeSrcs()) {
+       f.addPath(declaredIncludeSrc.getExecPath());
++      computeKeyDebugWriter.println("DECLAREDINCLUDESRCS: " + declaredIncludeSrc.getExecPath().getPathString());
+     }
+     f.addInt(0);  // mark the boundary between input types
+     for (Artifact input : getMandatoryInputs()) {
+       f.addPath(input.getExecPath());
++      computeKeyDebugWriter.println("MANDATORYINPUTS: " + input.getExecPath().getPathString());
+     }
+     f.addInt(0);
+     for (Artifact input : prunableInputs) {
+       f.addPath(input.getExecPath());
++      computeKeyDebugWriter.println("PRUNABLEINPUTS: " + input.getExecPath().getPathString());
+     }
+-    return f.hexDigestAndReset();
++    String rv = f.hexDigestAndReset();
++    computeKeyDebugWriter.println("KEY: " + rv);
++    computeKeyDebugWriter.close();
++    return rv;
+   }
+ 
+   @Override

native_client/BUILD

@@ -26,7 +26,7 @@ tf_library(
     config = "tfcompile.config.pbtxt",
     # This depends on //tensorflow:rpi3 condition defined in mozilla/tensorflow
     tfcompile_flags = select({
-        "//tensorflow:rpi3": str('--target_triple="armv6-linux-gnueabihf" --target_cpu="cortex-a53" --target_features="+neon-fp-armv8"'),
+        "//tensorflow:rpi3": str('--target_cpu="cortex-a53"'),
         "//conditions:default": str('')
     }),
 )

taskcluster/cuda-build.sh

@@ -13,7 +13,7 @@ BAZEL_TARGETS="
 "
 
 BAZEL_ENV_FLAGS="TF_NEED_CUDA=1 ${TF_CUDA_FLAGS}"
-BAZEL_BUILD_FLAGS="${BAZEL_CUDA_FLAGS} ${BAZEL_OPT_FLAGS} ${BAZEL_EXTRA_FLAGS}"
+BAZEL_BUILD_FLAGS="${BAZEL_CUDA_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BAZEL_OPT_FLAGS}"
 SYSTEM_TARGET=host
 EXTRA_LOCAL_CFLAGS=""
 EXTRA_LOCAL_LDFLAGS="-L${DS_ROOT_TASK}/DeepSpeech/CUDA/lib64/ -L${DS_ROOT_TASK}/DeepSpeech/CUDA/lib64/stubs/ -lcudart -lcuda"

taskcluster/darwin-amd64-cpu-aot_prod-opt.yml

@@ -4,8 +4,8 @@ build:
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branch}.osx_aot"
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branch}.${event.head.sha}.osx_aot"
     - "index.project.deepspeech.deepspeech.native_client.osx_aot.${event.head.sha}"
-  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.osx/artifacts/public/home.tar.xz"
+  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.995194cbb18744f48347ce3c003e999b4d5187f1.osx/artifacts/public/home.tar.xz"
-  summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.osx/artifacts/public/summarize_graph"
+  summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.995194cbb18744f48347ce3c003e999b4d5187f1.osx/artifacts/public/summarize_graph"
   scripts:
     build: "taskcluster/host-build.sh --aot"
     package: "taskcluster/package.sh"

taskcluster/darwin-amd64-cpu-opt.yml

@@ -6,8 +6,8 @@ build:
     - "index.project.deepspeech.deepspeech.native_client.osx.${event.head.sha}"
     - "notify.irc-channel.${notifications.irc}.on-exception"
     - "notify.irc-channel.${notifications.irc}.on-failed"
-  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.osx/artifacts/public/home.tar.xz"
+  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.995194cbb18744f48347ce3c003e999b4d5187f1.osx/artifacts/public/home.tar.xz"
-  summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.osx/artifacts/public/summarize_graph"
+  summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.995194cbb18744f48347ce3c003e999b4d5187f1.osx/artifacts/public/summarize_graph"
   scripts:
     build: "taskcluster/host-build.sh"
     package: "taskcluster/package.sh"

taskcluster/linux-amd64-cpu-aot_prod-opt.yml

@@ -4,8 +4,8 @@ build:
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branch}.cpu_aot"
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branch}.${event.head.sha}.cpu_aot"
     - "index.project.deepspeech.deepspeech.native_client.cpu_aot.${event.head.sha}"
-  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.cpu/artifacts/public/home.tar.xz"
+  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.995194cbb18744f48347ce3c003e999b4d5187f1.cpu/artifacts/public/home.tar.xz"
-  summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.cpu/artifacts/public/summarize_graph"
+  summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.995194cbb18744f48347ce3c003e999b4d5187f1.cpu/artifacts/public/summarize_graph"
   system_setup:
     >
       ${nodejs.packages.prep_6} && apt-get -qq update && apt-get -qq -y install nodejs python-yaml &&

taskcluster/linux-amd64-cpu-aot_test-opt.yml

@@ -6,8 +6,8 @@ build:
   template_file: linux-opt-base.tyml
   dependencies:
     - "test-training_upstream-linux-amd64-py27-opt"
-  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.cpu/artifacts/public/home.tar.xz"
+  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.995194cbb18744f48347ce3c003e999b4d5187f1.cpu/artifacts/public/home.tar.xz"
-  summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.cpu/artifacts/public/summarize_graph"
+  summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.995194cbb18744f48347ce3c003e999b4d5187f1.cpu/artifacts/public/summarize_graph"
   system_setup:
     >
       ${nodejs.packages.prep_6} && apt-get -qq update && apt-get -qq -y install nodejs python-yaml &&

taskcluster/linux-amd64-cpu-opt.yml

@@ -13,8 +13,8 @@ build:
   system_config:
     >
       ${swig.patch_nodejs.linux}
-  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.cpu/artifacts/public/home.tar.xz"
+  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.995194cbb18744f48347ce3c003e999b4d5187f1.cpu/artifacts/public/home.tar.xz"
-  summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.cpu/artifacts/public/summarize_graph"
+  summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.995194cbb18744f48347ce3c003e999b4d5187f1.cpu/artifacts/public/summarize_graph"
   scripts:
     build: "taskcluster/host-build.sh"
     package: "taskcluster/package.sh"

taskcluster/linux-amd64-ctc-opt.yml

@@ -4,8 +4,8 @@ build:
     - "pull_request.synchronize"
     - "pull_request.reopened"
   template_file: linux-opt-base.tyml
-  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.cpu/artifacts/public/home.tar.xz"
+  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.995194cbb18744f48347ce3c003e999b4d5187f1.cpu/artifacts/public/home.tar.xz"
-  summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.cpu/artifacts/public/summarize_graph"
+  summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.995194cbb18744f48347ce3c003e999b4d5187f1.cpu/artifacts/public/summarize_graph"
   scripts:
     build: 'taskcluster/decoder-build.sh'
     package: 'taskcluster/decoder-package.sh'

taskcluster/linux-amd64-gpu-opt.yml

@@ -11,8 +11,8 @@ build:
   system_config:
     >
       ${swig.patch_nodejs.linux}
-  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.gpu/artifacts/public/home.tar.xz"
+  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.995194cbb18744f48347ce3c003e999b4d5187f1.gpu/artifacts/public/home.tar.xz"
-  summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.gpu/artifacts/public/summarize_graph"
+  summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.995194cbb18744f48347ce3c003e999b4d5187f1.gpu/artifacts/public/summarize_graph"
   maxRunTime: 14400
   scripts:
     build: "taskcluster/cuda-build.sh"

taskcluster/linux-rpi3-cpu-aot_prod-opt.yml

@@ -4,8 +4,8 @@ build:
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branch}.arm_aot"
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branch}.${event.head.sha}.arm_aot"
     - "index.project.deepspeech.deepspeech.native_client.arm_aot.${event.head.sha}"
-  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.arm/artifacts/public/home.tar.xz"
+  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.995194cbb18744f48347ce3c003e999b4d5187f1.arm/artifacts/public/home.tar.xz"
-  summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.cpu/artifacts/public/summarize_graph"
+  summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.995194cbb18744f48347ce3c003e999b4d5187f1.cpu/artifacts/public/summarize_graph"
   ## multistrap 2.2.0-ubuntu1 is broken in 14.04: https://bugs.launchpad.net/ubuntu/+source/multistrap/+bug/1313787
   system_setup:
     >

taskcluster/linux-rpi3-cpu-opt.yml

@@ -4,8 +4,8 @@ build:
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branch}.arm"
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branch}.${event.head.sha}.arm"
     - "index.project.deepspeech.deepspeech.native_client.arm.${event.head.sha}"
-  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.arm/artifacts/public/home.tar.xz"
+  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.995194cbb18744f48347ce3c003e999b4d5187f1.arm/artifacts/public/home.tar.xz"
-  summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.cpu/artifacts/public/summarize_graph"
+  summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.995194cbb18744f48347ce3c003e999b4d5187f1.cpu/artifacts/public/summarize_graph"
   ## multistrap 2.2.0-ubuntu1 is broken in 14.04: https://bugs.launchpad.net/ubuntu/+source/multistrap/+bug/1313787
   system_setup:
     >

taskcluster/node-package.yml

@@ -14,8 +14,8 @@ build:
   system_config:
     >
       ${swig.patch_nodejs.linux}
-  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.cpu/artifacts/public/home.tar.xz"
+  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.995194cbb18744f48347ce3c003e999b4d5187f1.cpu/artifacts/public/home.tar.xz"
-  summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7d79b8f6422e5b7e0ba82504c500a86756c44a62.cpu/artifacts/public/summarize_graph"
+  summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.995194cbb18744f48347ce3c003e999b4d5187f1.cpu/artifacts/public/summarize_graph"
   scripts:
     build: "taskcluster/node-build.sh"
     package: "taskcluster/node-package.sh"

taskcluster/package.sh

@@ -6,6 +6,8 @@ source $(dirname "$0")/../tc-tests-utils.sh
 
 mkdir -p ${TASKCLUSTER_ARTIFACTS} || true
 
+cp ${DS_ROOT_TASK}/DeepSpeech/tf/bazel*.log ${TASKCLUSTER_ARTIFACTS}/
+
 package_native_client "native_client.tar.xz"
 
 if [ -d ${DS_ROOT_TASK}/DeepSpeech/ds/wheels ]; then

tc-tests-utils.sh

@@ -229,20 +229,77 @@ do_get_model_parameters()
   eval $__result="'--define=DS_MODEL_FRAMESIZE=${model_width} --define=DS_MODEL_FILE=${model_file}'"
 }
 
+# Checks whether we run a patched version of bazel.
+# Patching is required to dump computeKey() parameters to .ckd files
+# See bazel.patch
+# Return 0 (success exit code) on patched version, 1 on release version
+is_patched_bazel()
+{
+  bazel_version=$(bazel version | grep 'Build label:' | cut -d':' -f2)
+
+  if [ -z "${bazel_version}" ]; then
+    return 0;
+  else
+    return 1;
+  fi;
+}
+
+verify_bazel_rebuild()
+{
+  bazel_explain_file="$1"
+
+  if [ ! -f "${bazel_explain_file}" ]; then
+    echo "No such explain file: ${bazel_explain_file}"
+    exit 1
+  fi;
+
+  spurious_rebuilds=$(grep 'Executing action' "${bazel_explain_file}" | grep 'Compiling' | grep -v -E 'no entry in the cache|unconditional execution is requested' | wc -l)
+  if [ "${spurious_rebuilds}" -ne 0 ]; then
+    echo "Bazel rebuilds some file it should not, please check."
+
+    if is_patched_bazel; then
+      mkdir -p ${DS_ROOT_TASK}/DeepSpeech/ckd/ds ${DS_ROOT_TASK}/DeepSpeech/ckd/tf
+      tar xf ${DS_ROOT_TASK}/DeepSpeech/bazel-ckd-tf.tar --strip-components=4 -C ${DS_ROOT_TASK}/DeepSpeech/ckd/ds/
+      tar xf ${DS_ROOT_TASK}/DeepSpeech/bazel-ckd-ds.tar --strip-components=4 -C ${DS_ROOT_TASK}/DeepSpeech/ckd/tf/
+
+      echo "Making a diff between CKD files"
+      mkdir -p ${TASKCLUSTER_ARTIFACTS}
+      diff -urNw ${DS_ROOT_TASK}/DeepSpeech/ckd/tf/ ${DS_ROOT_TASK}/DeepSpeech/ckd/ds/ | tee ${TASKCLUSTER_ARTIFACTS}/ckd.diff
+
+      rm -fr ${DS_ROOT_TASK}/DeepSpeech/ckd/tf/ ${DS_ROOT_TASK}/DeepSpeech/ckd/ds/
+    else
+      echo "Cannot get CKD information from release, please use patched Bazel"
+    fi;
+
+    exit 1
+  fi;
+}
+
 do_bazel_build()
 {
   cd ${DS_ROOT_TASK}/DeepSpeech/tf
   eval "export ${BAZEL_ENV_FLAGS}"
-  PATH=${DS_ROOT_TASK}/bin/:$PATH bazel ${BAZEL_OUTPUT_USER_ROOT} build \
+
-    --config=monolithic -c opt ${BAZEL_BUILD_FLAGS} ${BAZEL_TARGETS}
+  if is_patched_bazel; then
+    find ${DS_ROOT_TASK}/DeepSpeech/tf/bazel-out/ -iname "*.ckd" | tar -cf ${DS_ROOT_TASK}/DeepSpeech/bazel-ckd-tf.tar -T -
+  fi;
+
+  bazel ${BAZEL_OUTPUT_USER_ROOT} build \
+    -s --explain bazel_monolithic.log --verbose_explanations --experimental_strict_action_env --config=monolithic -c opt ${BAZEL_BUILD_FLAGS} ${BAZEL_TARGETS}
+
+  if is_patched_bazel; then
+    find ${DS_ROOT_TASK}/DeepSpeech/tf/bazel-out/ -iname "*.ckd" | tar -cf ${DS_ROOT_TASK}/DeepSpeech/bazel-ckd-ds.tar -T -
+  fi;
+
+  verify_bazel_rebuild "${DS_ROOT_TASK}/DeepSpeech/tf/bazel_monolithic.log"
 }
 
 do_bazel_shared_build()
 {
   cd ${DS_ROOT_TASK}/DeepSpeech/tf
   eval "export ${BAZEL_ENV_FLAGS}"
-  PATH=${DS_ROOT_TASK}/bin/:$PATH bazel ${BAZEL_OUTPUT_USER_ROOT} build \
+  bazel ${BAZEL_OUTPUT_USER_ROOT} build \
-    -c opt ${BAZEL_BUILD_FLAGS} ${BAZEL_TARGETS}
+    -s --explain bazel_shared.log --verbose_explanations --experimental_strict_action_env -c opt ${BAZEL_BUILD_FLAGS} ${BAZEL_TARGETS}
 }
 
 do_deepspeech_binary_build()