From 09045e49d1e2d3322e8ca8cc40f0e5db10f13c58 Mon Sep 17 00:00:00 2001 From: Andrew Selle Date: Tue, 6 Sep 2016 08:19:04 -0800 Subject: [PATCH] Include sha hash in python variable __version__ It is necessary to symlink in files from .git/ in order to make bazel aware of changes to the current head. As it is this is not completely reliable when git repositories are in a dirty index state. First class support for bazel git a reported bug but not a high priority. ./configure sets up the symlinks by calling the gen_git_source.py a bazel genrule calls gen_git_source.py to generate version_info.cc Also changed cmake and make to build this properly. Change: 132328009 --- configure | 11 + tensorflow/BUILD | 1 + .../contrib/cmake/tf_core_framework.cmake | 16 ++ tensorflow/contrib/makefile/Makefile | 9 +- tensorflow/core/BUILD | 9 + tensorflow/core/public/version.h | 8 + tensorflow/python/__init__.py | 6 +- tensorflow/python/client/tf_session.i | 6 + tensorflow/python/framework/versions.py | 6 +- tensorflow/python/framework/versions_test.py | 3 + tensorflow/tensorflow.bzl | 15 ++ tensorflow/tf_exported_symbols.lds | 1 + tensorflow/tools/git/BUILD | 28 +++ tensorflow/tools/git/gen_git_source.py | 223 ++++++++++++++++++ 14 files changed, 339 insertions(+), 3 deletions(-) create mode 100644 tensorflow/tools/git/BUILD create mode 100755 tensorflow/tools/git/gen_git_source.py diff --git a/configure b/configure index f4b772c55ef..ce75bb490a7 100755 --- a/configure +++ b/configure @@ -2,6 +2,11 @@ DO_NOT_SUBMIT_WARNING="Unofficial setting. DO NOT SUBMIT!!!" +# Find out the absolute path to where ./configure resides +pushd `dirname $0` #> /dev/null +SOURCE_BASE_DIR=`pwd -P` +popd > /dev/null + ## Set up python-related environment settings while true; do fromuser="" @@ -68,6 +73,12 @@ echo "$SWIG_PATH" > tensorflow/tools/swig/swig_path # Invoke python_config and set up symlinks to python includes (./util/python/python_config.sh --setup "$PYTHON_BIN_PATH";) || exit -1 +# Run the gen_git_source to create links where bazel can track dependencies for +# git hash propagation +GEN_GIT_SOURCE=tensorflow/tools/git/gen_git_source.py +chmod a+x ${GEN_GIT_SOURCE} +${PYTHON_BIN_PATH} ${GEN_GIT_SOURCE} --configure ${SOURCE_BASE_DIR} + ## Set up Cuda-related environment settings while [ "$TF_NEED_CUDA" == "" ]; do diff --git a/tensorflow/BUILD b/tensorflow/BUILD index c5609486d59..0995c6276b4 100644 --- a/tensorflow/BUILD +++ b/tensorflow/BUILD @@ -168,6 +168,7 @@ filegroup( "//tensorflow/tools/docker:all_files", "//tensorflow/tools/docker/notebooks:all_files", "//tensorflow/tools/docs:all_files", + "//tensorflow/tools/git:all_files", "//tensorflow/tools/proto_text:all_files", "//tensorflow/tools/test:all_files", "//tensorflow/user_ops:all_files", diff --git a/tensorflow/contrib/cmake/tf_core_framework.cmake b/tensorflow/contrib/cmake/tf_core_framework.cmake index 0dbe3c194d9..cbcd11a05b9 100644 --- a/tensorflow/contrib/cmake/tf_core_framework.cmake +++ b/tensorflow/contrib/cmake/tf_core_framework.cmake @@ -180,6 +180,21 @@ add_dependencies(tf_core_lib boringssl ) +# Tricky setup to force always rebuilding +# force_rebuild always runs forcing ${VERSION_INFO_CC} target to run +# ${VERSION_INFO_CC} would cache, but it depends on a phony never produced +# target. +set(VERSION_INFO_CC ${tensorflow_source_dir}/tensorflow/core/util/version_info.cc) +add_custom_target(force_rebuild_target ALL DEPENDS ${VERSION_INFO_CC}) +add_custom_command(OUTPUT __force_rebuild COMMAND cmake -E echo) +add_custom_command(OUTPUT + ${VERSION_INFO_CC} + COMMAND ${tensorflow_source_dir}/tensorflow/tools/git/gen_git_source.py + --raw_generate ${VERSION_INFO_CC} + DEPENDS __force_rebuild) + +set(tf_version_srcs ${tensorflow_source_dir}/tensorflow/core/util/version_info.cc) + ######################################################## # tf_core_framework library @@ -211,6 +226,7 @@ list(REMOVE_ITEM tf_core_framework_srcs ${tf_core_framework_test_srcs}) add_library(tf_core_framework OBJECT ${tf_core_framework_srcs} + ${tf_version_srcs} ${PROTO_TEXT_HDRS} ${PROTO_TEXT_SRCS}) target_include_directories(tf_core_framework PUBLIC diff --git a/tensorflow/contrib/makefile/Makefile b/tensorflow/contrib/makefile/Makefile index 7f2218ada7a..5da8502845c 100644 --- a/tensorflow/contrib/makefile/Makefile +++ b/tensorflow/contrib/makefile/Makefile @@ -421,7 +421,8 @@ $(wildcard tensorflow/core/platform/*.cc) \ $(wildcard tensorflow/core/platform/*/*.cc) \ $(wildcard tensorflow/core/platform/*/*/*.cc) \ $(wildcard tensorflow/core/util/*.cc) \ -$(wildcard tensorflow/core/util/*/*.cc) +$(wildcard tensorflow/core/util/*/*.cc) \ +tensorflow/core/util/version_info.cc CORE_CC_EXCLUDE_SRCS := \ $(wildcard tensorflow/core/*/*test.cc) \ $(wildcard tensorflow/core/*/*testutil*) \ @@ -477,6 +478,11 @@ all: $(LIB_PATH) $(BENCHMARK_NAME) # Rules for target compilation. + +.phony_version_info: +tensorflow/core/util/version_info.cc: .phony_version_info + python tensorflow/tools/git/gen_git_source.py --raw_generate $@ + # Gathers together all the objects we've compiled into a single '.a' archive. $(LIB_PATH): $(LIB_OBJS) @mkdir -p $(dir $@) @@ -553,6 +559,7 @@ $(HOST_GENDIR)%.pb.cc $(HOST_GENDIR)%.pb.h: %.proto # Gets rid of all generated files. clean: rm -rf $(MAKEFILE_DIR)/gen + rm -rf tensorflow/core/util/version_info.cc # Gets rid of target files only, leaving the host alone. Also leaves the lib # directory untouched deliberately, so we can persist multiple architectures diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index 5a8b8c86e52..b100e87d6c4 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -68,6 +68,7 @@ load( ) load("//tensorflow:tensorflow.bzl", "tf_cc_test_gpu") load("//tensorflow:tensorflow.bzl", "tf_cc_tests_gpu") +load("//tensorflow:tensorflow.bzl", "tf_version_info_genrule") # For platform specific build config load( @@ -938,6 +939,13 @@ cc_library( ], ) +tf_version_info_genrule() + +cc_library( + name = "version_lib", + srcs = ["util/version_info.cc"], +) + tf_cuda_library( name = "framework_internal", srcs = glob( @@ -980,6 +988,7 @@ tf_cuda_library( ":lib_internal", ":proto_text", ":protos_all_cc", + ":version_lib", "//tensorflow/core/kernels:bounds_check", "//third_party/eigen3", ], diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h index a9b63ea1f4c..037e5795f78 100644 --- a/tensorflow/core/public/version.h +++ b/tensorflow/core/public/version.h @@ -84,4 +84,12 @@ limitations under the License. #define TF_CHECKPOINT_VERSION_MIN_CONSUMER 0 #define TF_CHECKPOINT_VERSION 1 +/// Version query functions (defined in generated version_info.cc) + +// Host compiler version (declared elsewhere to be __VERSION__) +extern const char* tf_compiler_version(); +// The git commit designator when tensorflow was built +// If no git repository, this will be "internal". +extern const char* tf_git_version(); + #endif // TENSORFLOW_CORE_PUBLIC_VERSION_H_ diff --git a/tensorflow/python/__init__.py b/tensorflow/python/__init__.py index 974d3c24573..72a149330f8 100644 --- a/tensorflow/python/__init__.py +++ b/tensorflow/python/__init__.py @@ -245,4 +245,8 @@ __all__.extend([ 'train', ]) -__all__.append('__version__') +__all__.extend([ + '__version__', + '__git_version__', + '__compiler_version__', +]) diff --git a/tensorflow/python/client/tf_session.i b/tensorflow/python/client/tf_session.i index 87391fff68d..8afd3f77f66 100644 --- a/tensorflow/python/client/tf_session.i +++ b/tensorflow/python/client/tf_session.i @@ -35,6 +35,12 @@ tensorflow::ImportNumpy(); %constant int GRAPH_DEF_VERSION_MIN_CONSUMER = TF_GRAPH_DEF_VERSION_MIN_CONSUMER; %constant int GRAPH_DEF_VERSION_MIN_PRODUCER = TF_GRAPH_DEF_VERSION_MIN_PRODUCER; +// Git version information +%constant const char* __git_version__ = tf_git_version(); + +// Compiler +%constant const char* __compiler_version__ = tf_compiler_version(); + // Release the Python GIL for the duration of most methods. %exception { Py_BEGIN_ALLOW_THREADS; diff --git a/tensorflow/python/framework/versions.py b/tensorflow/python/framework/versions.py index 369dd701cf6..63722e041cf 100644 --- a/tensorflow/python/framework/versions.py +++ b/tensorflow/python/framework/versions.py @@ -22,6 +22,9 @@ from __future__ import print_function from tensorflow.python import pywrap_tensorflow __version__ = pywrap_tensorflow.__version__ +__git_version__ = pywrap_tensorflow.__git_version__ +__compiler_version__ = pywrap_tensorflow.__compiler_version__ + GRAPH_DEF_VERSION = pywrap_tensorflow.GRAPH_DEF_VERSION GRAPH_DEF_VERSION_MIN_CONSUMER = ( pywrap_tensorflow.GRAPH_DEF_VERSION_MIN_CONSUMER) @@ -30,4 +33,5 @@ GRAPH_DEF_VERSION_MIN_PRODUCER = ( # Make sure these symbols are exported even though one starts with _. __all__ = ["__version__", "GRAPH_DEF_VERSION", "GRAPH_DEF_VERSION_MIN_CONSUMER", - "GRAPH_DEF_VERSION_MIN_PRODUCER"] + "GRAPH_DEF_VERSION_MIN_PRODUCER", "__git_version__", + "__compiler_version__"] diff --git a/tensorflow/python/framework/versions_test.py b/tensorflow/python/framework/versions_test.py index 6042048414f..025c311a2a1 100644 --- a/tensorflow/python/framework/versions_test.py +++ b/tensorflow/python/framework/versions_test.py @@ -38,6 +38,9 @@ class VersionTest(tf.test.TestCase): self.assertLessEqual(0, min_producer) self.assertLessEqual(min_producer, version) + def testGitVersion(self): + self.assertEqual(type(tf.__git_version__), str) + self.assertEqual(type(tf.__compiler_version__), str) if __name__ == "__main__": tf.test.main() diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl index 3d367d8ac64..565ecdce987 100644 --- a/tensorflow/tensorflow.bzl +++ b/tensorflow/tensorflow.bzl @@ -813,3 +813,18 @@ def tf_genrule_cmd_append_to_srcs(to_append): return ("cat $(SRCS) > $(@) && " + "echo >> $(@) && " + "echo " + to_append + " >> $(@)") + + +def tf_version_info_genrule(): + native.genrule( + name = "version_info_gen", + srcs = [ + "//tensorflow/tools/git:gen/spec.json", + "//tensorflow/tools/git:gen/head", + "//tensorflow/tools/git:gen/branch_ref", + ], + outs = ["util/version_info.cc"], + cmd = "$(location //tensorflow/tools/git:gen_git_source.py) --generate $(SRCS) \"$@\"", + local = 1, + tools = ["//tensorflow/tools/git:gen_git_source.py"], + ) diff --git a/tensorflow/tf_exported_symbols.lds b/tensorflow/tf_exported_symbols.lds index e7749ab0f70..cb81e89922c 100644 --- a/tensorflow/tf_exported_symbols.lds +++ b/tensorflow/tf_exported_symbols.lds @@ -1,2 +1,3 @@ *tensorflow* *perftools*gputools* +*tf_* diff --git a/tensorflow/tools/git/BUILD b/tensorflow/tools/git/BUILD new file mode 100644 index 00000000000..f502c8dde07 --- /dev/null +++ b/tensorflow/tools/git/BUILD @@ -0,0 +1,28 @@ +# Description: +# Contains script to generate tensorflow/core/util/version_info.cc +# Also contains information about git repository deposited by configure +# in gen/... +package(default_visibility = ["//tensorflow:internal"]) + +licenses(["notice"]) # Apache 2.0 + +exports_files( + glob(["gen/*"]) + [ + "gen_git_source.py", + ], +) + +# ----------------------------------------------------------------------------- +# Google-internal targets. These must be at the end for syncrepo. + +filegroup( + name = "all_files", + srcs = glob( + ["**/*"], + exclude = [ + "**/METADATA", + "**/OWNERS", + ], + ), + visibility = ["//tensorflow:__subpackages__"], +) diff --git a/tensorflow/tools/git/gen_git_source.py b/tensorflow/tools/git/gen_git_source.py new file mode 100755 index 00000000000..6c0770b1ffa --- /dev/null +++ b/tensorflow/tools/git/gen_git_source.py @@ -0,0 +1,223 @@ +#!/usr/bin/python +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Help include git hash in tensorflow bazel build. + +This creates symlinks from the internal git repository directory so +that the build system can see changes in the version state. We also +remember what branch git was on so when the branch changes we can +detect that the ref file is no longer correct (so we can suggest users +run ./configure again). + +NOTE: this script is only used in opensource. + +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +import argparse +import json +import os +import shutil + + +def parse_branch_ref(filename): + """Given a filename of a .git/HEAD file return ref path. + + In particular, if git is in detached head state, this will + return None. If git is in attached head, it will return + the branch reference. E.g. if on 'master', the HEAD will + contain 'ref: refs/heads/master' so 'refs/heads/master' + will be returned. + + Example: parse_branch_ref(".git/HEAD") + Args: + filename: file to treat as a git HEAD file + Returns: + None if detached head, otherwise ref subpath + Raises: + RuntimeError: if the HEAD file is unparseable. + """ + + data = open(filename).read().strip() + items = data.split(" ") + if len(items) == 1: + return None + elif len(items) == 2 and items[0] == "ref:": + return items[1].strip() + else: + raise RuntimeError("Git directory has unparseable HEAD") + + +def configure(src_base_path, debug=False): + """Configure `src_base_path` to embed git hashes if available.""" + + # TODO(aselle): No files generated or symlinked here are deleted by + # the build system. I don't know of a way to do it in bazel. It + # should only be a problem if somebody moves a sandbox directory + # without running ./configure again. + + git_path = os.path.join(src_base_path, ".git") + gen_path = os.path.join(src_base_path, "tensorflow", "tools", "git", "gen") + + # Remove and recreate the path + if os.path.exists(gen_path): + if os.path.isdir(gen_path): + shutil.rmtree(gen_path) + else: + raise RuntimeError("Cannot delete non-directory %s, inspect ", + "and remove manually" % gen_path) + os.makedirs(gen_path) + + if not os.path.isdir(gen_path): + raise RuntimeError("gen_git_source.py: Failed to create dir") + + # file that specifies what the state of the git repo is + spec = {} + + # value file names will be mapped to the keys + link_map = {"head": None, "branch_ref": None} + + if not os.path.isdir(git_path): + # No git directory + spec["git"] = False + open(os.path.join(gen_path, "head"), "w").write("") + open(os.path.join(gen_path, "branch_ref"), "w").write("") + else: + # Git directory, possibly detached or attached + spec["git"] = True + spec["path"] = src_base_path + git_head_path = os.path.join(git_path, "HEAD") + spec["branch"] = parse_branch_ref(git_head_path) + link_map["head"] = git_head_path + if spec["branch"] is not None: + # attached method + link_map["branch_ref"] = os.path.join(git_path, * + os.path.split(spec["branch"])) + # Create symlinks or dummy files + for target, src in link_map.items(): + if src is None: + open(os.path.join(gen_path, target), "w").write("") + else: + os.symlink(src, os.path.join(gen_path, target)) + + json.dump(spec, open(os.path.join(gen_path, "spec.json"), "w"), indent=2) + if debug: + print("gen_git_source.py: list %s" % gen_path) + print("gen_git_source.py: %s" + repr(os.listdir(gen_path))) + print("gen_git_source.py: spec is %r" % spec) + + +def generate(arglist): + """Generate version_info.cc as given `destination_file`. + + Args: + arglist: should be a sequence that contains + spec, head_symlink, ref_symlink, destination_file. + + `destination_file` is the filename where version_info.cc will be written + + `spec` is a filename where the file contains a JSON dictionary + 'git' bool that is true if the source is in a git repo + 'path' base path of the source code + 'branch' the name of the ref specification of the current branch/tag + + `head_symlink` is a filename to HEAD that is cross-referenced against + what is contained in the json branch designation. + + `ref_symlink` is unused in this script but passed, because the build + system uses that file to detect when commits happen. + + Raises: + RuntimeError: If ./configure needs to be run, RuntimeError will be raised. + """ + + # unused ref_symlink arg + spec, head_symlink, _, dest_file = arglist + data = json.load(open(spec)) + strs = {"tf_compiler_version": "__VERSION__"} + if not data["git"]: + strs["tf_git_version"] = "internal" + else: + old_branch = data["branch"] + new_branch = parse_branch_ref(head_symlink) + if new_branch != old_branch: + raise RuntimeError( + "Run ./configure again, branch was '%s' but is now '%s'" % + (old_branch, new_branch)) + strs["tf_git_version"] = os.popen( + "git -C \"%s\" describe --long --dirty --tags" % + (data["path"],)).read().strip() + # TODO(aselle): Check for escaping + cpp_file = "\n".join("const char* %s() {return \"%s\";}" % (x, y) + for x, y in strs.items()) + open(dest_file, "w").write(cpp_file + "\n") + + +def raw_generate(output_file): + """Simple generator used for cmake/make build systems. + + This does not create any symlinks. It requires the build system + to build unconditionally. + + Args: + output_file: Output filename for the version info cc + """ + + strs = {"tf_compiler_version": "__VERSION__"} + version = os.popen("git describe --long --dirty --tags").read().strip() + version = version if version else "unknown" + strs["tf_git_version"] = version + cpp_file = "\n".join("const char* %s() {return \"%s\";}" % (x, y) + for x, y in strs.items()) + open(output_file, "w").write(cpp_file + "\n") + + +parser = argparse.ArgumentParser(description="""Git hash injection into bazel. +If used with --configure will search for git directory and put symlinks +into source so that a bazel genrule can call --generate""") + +parser.add_argument( + "--debug", + type=bool, + help="print debugging information about paths", + default=False) + +parser.add_argument( + "--configure", type=str, + help="Path to configure as a git repo dependency tracking sentinel") + +parser.add_argument( + "--generate", + type=str, + help="Generate given spec-file, HEAD-symlink-file, ref-symlink-file", + nargs="+") + +parser.add_argument( + "--raw_generate", + type=str, + help="Generate version_info.cc (simpler version used for cmake/make)") + +args = parser.parse_args() + +if args.configure is not None: + configure(args.configure, debug=args.debug) +elif args.generate is not None: + generate(args.generate) +elif args.raw_generate is not None: + raw_generate(args.raw_generate) +else: + raise RuntimeError("--configure or --generate or --raw_generate " + "must be used")