From 6586a66b1eac161221155b8bf1c2a75677623bc3 Mon Sep 17 00:00:00 2001 From: Asim Shankar Date: Mon, 10 Oct 2016 15:24:11 -0800 Subject: [PATCH] go: Scaffolding for a package containing wrapper functions for TensorFlow ops. This includes: (1) A 'genop' command-line tool that generates Go functions for each TensorFlow op registered in the address space of the process. (2) An 'op' package that will host these generated functions and some hand-crafted functions (like Const). At this time, none of the generated files are being committed. As a result, the installation instructions in README.md have been updated to reflect how 'go generate' should be used to setup the 'op' package. Note that even after this change, packages that clients are expected to use ('tensorflow' and 'op') do not depend on protocol buffers (github.com/golang/protobuf). However, the stand-alone command-line tool 'genop' does. This change is focused on the scaffolding and introduces a dummy code generator. The real implementation of the code generator will be done in a follow up change as I wanted to separate these "mechanics" from the "implementation details". Yet another step for #10 Change: 135735176 --- tensorflow/go/README.md | 19 +++++--- tensorflow/go/genop/.gitignore | 2 + tensorflow/go/genop/generate.sh | 42 ++++++++++++++++++ tensorflow/go/genop/internal/genop.go | 64 +++++++++++++++++++++++++++ tensorflow/go/genop/internal/lib.go | 19 ++++++++ tensorflow/go/genop/main.go | 42 ++++++++++++++++++ tensorflow/go/op/.gitignore | 1 + tensorflow/go/op/generate.go | 20 +++++++++ 8 files changed, 203 insertions(+), 6 deletions(-) create mode 100644 tensorflow/go/genop/.gitignore create mode 100644 tensorflow/go/genop/generate.sh create mode 100644 tensorflow/go/genop/internal/genop.go create mode 100644 tensorflow/go/genop/internal/lib.go create mode 100644 tensorflow/go/genop/main.go create mode 100644 tensorflow/go/op/.gitignore create mode 100644 tensorflow/go/op/generate.go diff --git a/tensorflow/go/README.md b/tensorflow/go/README.md index b3cfb0bd7e8..9fa16b66d62 100644 --- a/tensorflow/go/README.md +++ b/tensorflow/go/README.md @@ -49,14 +49,21 @@ Construct and execute TensorFlow graphs in Go. a. Copying it to a system location, e.g., - ```sh - cp ${GOPATH}/src/github.com/tensorflow/tensorflow/bazel-bin/tensorflow/libtensorflow.so /usr/local/lib - ``` + ```sh + cp ${GOPATH}/src/github.com/tensorflow/tensorflow/bazel-bin/tensorflow/libtensorflow.so /usr/local/lib + ``` - OR + OR - b. Setting the `LD_LIBRARY_PATH=${GOPATH}/src/github.com/tensorflow/tensorflow/bazel-bin/tensorflow` - environment variable (`DYLD_LIBRARY_PATH` on Mac OS X). + b. Setting the + `LD_LIBRARY_PATH=${GOPATH}/src/github.com/tensorflow/tensorflow/bazel-bin/tensorflow` + environment variable (`DYLD_LIBRARY_PATH` on Mac OS X). + +4. Generate wrapper functions for TensorFlow ops: + + ```sh + go generate github.com/tensorflow/tensorflow/tensorflow/go/op + ``` After this, the `go` tool should be usable as normal. For example: diff --git a/tensorflow/go/genop/.gitignore b/tensorflow/go/genop/.gitignore new file mode 100644 index 00000000000..a84838469ba --- /dev/null +++ b/tensorflow/go/genop/.gitignore @@ -0,0 +1,2 @@ +# .pb.go files generated by generate.sh +internal/proto/* diff --git a/tensorflow/go/genop/generate.sh b/tensorflow/go/genop/generate.sh new file mode 100644 index 00000000000..ae49894b941 --- /dev/null +++ b/tensorflow/go/genop/generate.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +set -e + +go get github.com/golang/protobuf/{proto,protoc-gen-go} + +cd $(dirname $0) +TF_DIR=${GOPATH}/src/github.com/tensorflow/tensorflow +PROTOC="${TF_DIR}/bazel-out/host/bin/external/protobuf/protoc" + +if [ ! -x "${PROTOC}" ] +then + PATH_PROTOC=$(which protoc) + if [ ! -x "${PATH_PROTOC}" ] + then + echo "Protocol buffer compiler protoc not found in PATH or in ${PROTOC}" + echo "Perhaps build it using:" + echo "bazel build -c opt @protobuf//:protoc" + exit 1 + fi + PROTOC=PATH_PROTOC +fi + +mkdir -p ./internal/proto +${PROTOC} \ + -I ${TF_DIR} \ + --go_out=./internal/proto \ + ${TF_DIR}/tensorflow/core/framework/*.proto diff --git a/tensorflow/go/genop/internal/genop.go b/tensorflow/go/genop/internal/genop.go new file mode 100644 index 00000000000..be84b2322aa --- /dev/null +++ b/tensorflow/go/genop/internal/genop.go @@ -0,0 +1,64 @@ +// Copyright 2016 The TensorFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package internal generates Go source code with functions for TensorFlow operations. +// +// The generated APIs are unstable and can change without notice. +package internal + +// #include "tensorflow/c/c_api.h" +import "C" + +import ( + "fmt" + "io" + "unsafe" + + "github.com/golang/protobuf/proto" + pb "github.com/tensorflow/tensorflow/tensorflow/go/genop/internal/proto/tensorflow/core/framework" +) + +// GenerateFunctionsForRegisteredOps writes a Go source code file to w +// containing functions for each TensorFlow operation registered in the address +// space of the calling process. +func GenerateFunctionsForRegisteredOps(w io.Writer) error { + ops, err := registeredOps() + if err != nil { + return err + } + fmt.Fprintf(w, `// DO NOT EDIT +// This file was machine generated. +// +// This code generation process is a work in progress and is not ready yet. +// Eventually, the code generator will generate approximately %d wrapper +// functions for adding TensorFlow operations to a Graph. + +package op +`, len(ops.Op)) + return nil +} + +func registeredOps() (*pb.OpList, error) { + buf := C.TF_GetAllOpList() + defer C.TF_DeleteBuffer(buf) + var ( + list = new(pb.OpList) + size = int(buf.length) + // A []byte backed by C memory. + // See: https://github.com/golang/go/wiki/cgo#turning-c-arrays-into-go-slices + data = (*[1 << 30]byte)(unsafe.Pointer(buf.data))[:size:size] + err = proto.Unmarshal(data, list) + ) + return list, err +} diff --git a/tensorflow/go/genop/internal/lib.go b/tensorflow/go/genop/internal/lib.go new file mode 100644 index 00000000000..aea8037c9bb --- /dev/null +++ b/tensorflow/go/genop/internal/lib.go @@ -0,0 +1,19 @@ +// Copyright 2016 The TensorFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package internal + +// #cgo LDFLAGS: -L${SRCDIR}/../../../../bazel-bin/tensorflow -ltensorflow +// #cgo CFLAGS: -I${SRCDIR}/../../../../ +import "C" diff --git a/tensorflow/go/genop/main.go b/tensorflow/go/genop/main.go new file mode 100644 index 00000000000..a6512cce1ae --- /dev/null +++ b/tensorflow/go/genop/main.go @@ -0,0 +1,42 @@ +// Copyright 2016 The TensorFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:generate sh generate.sh + +// Command genop generates a Go source file with functions for TensorFlow ops. +package main + +import ( + "flag" + "log" + "os" + + "github.com/tensorflow/tensorflow/tensorflow/go/genop/internal" +) + +func main() { + filename := flag.String("outfile", "", "File to write generated source code to.") + flag.Parse() + if *filename == "" { + log.Fatal("--outfile must be set") + } + file, err := os.OpenFile(*filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) + if err != nil { + log.Fatalf("Failed to open %q for writing: %v", *filename, err) + } + defer file.Close() + if err = internal.GenerateFunctionsForRegisteredOps(file); err != nil { + log.Fatal(err) + } +} diff --git a/tensorflow/go/op/.gitignore b/tensorflow/go/op/.gitignore new file mode 100644 index 00000000000..61af2de400a --- /dev/null +++ b/tensorflow/go/op/.gitignore @@ -0,0 +1 @@ +wrappers.go diff --git a/tensorflow/go/op/generate.go b/tensorflow/go/op/generate.go new file mode 100644 index 00000000000..598edcf150e --- /dev/null +++ b/tensorflow/go/op/generate.go @@ -0,0 +1,20 @@ +// Copyright 2016 The TensorFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:generate go generate ../genop +//go:generate go run ../genop/main.go -outfile wrappers.go +//go:generate go fmt wrappers.go + +package op +