go: Scaffolding for a package containing wrapper functions for TensorFlow ops.

This includes:
(1) A 'genop' command-line tool that generates Go functions for each TensorFlow op registered in the address space of the process.
(2) An 'op' package that will host these generated functions and some hand-crafted
functions (like Const).

At this time, none of the generated files are being committed.
As a result, the installation instructions in README.md have been updated to
reflect how 'go generate' should be used to setup the 'op' package.

Note that even after this change, packages that clients are expected to
use ('tensorflow' and 'op') do not depend on protocol buffers (github.com/golang/protobuf).
However, the stand-alone command-line tool 'genop' does.

This change is focused on the scaffolding and introduces a dummy code generator.
The real implementation of the code generator will be done in a follow up change
as I wanted to separate these "mechanics" from the "implementation details".

Yet another step for #10
Change: 135735176

tensorflow/go/README.md

@@ -49,14 +49,21 @@ Construct and execute TensorFlow graphs in Go.
 
     a. Copying it to a system location, e.g.,
 
-       ```sh
-       cp ${GOPATH}/src/github.com/tensorflow/tensorflow/bazel-bin/tensorflow/libtensorflow.so /usr/local/lib
-       ```
+    ```sh
+    cp ${GOPATH}/src/github.com/tensorflow/tensorflow/bazel-bin/tensorflow/libtensorflow.so /usr/local/lib
+    ```
 
-       OR
+    OR
 
-    b. Setting the `LD_LIBRARY_PATH=${GOPATH}/src/github.com/tensorflow/tensorflow/bazel-bin/tensorflow`
-       environment variable (`DYLD_LIBRARY_PATH` on Mac OS X).
+    b. Setting the
+    `LD_LIBRARY_PATH=${GOPATH}/src/github.com/tensorflow/tensorflow/bazel-bin/tensorflow`
+    environment variable (`DYLD_LIBRARY_PATH` on Mac OS X).
+
+4.  Generate wrapper functions for TensorFlow ops:
+
+    ```sh
+    go generate github.com/tensorflow/tensorflow/tensorflow/go/op
+    ```
 
 After this, the `go` tool should be usable as normal. For example:
 

tensorflow/go/genop/.gitignore

@@ -0,0 +1,2 @@
+# .pb.go files generated by generate.sh
+internal/proto/*

tensorflow/go/genop/generate.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+set -e
+
+go get github.com/golang/protobuf/{proto,protoc-gen-go}
+
+cd $(dirname $0)
+TF_DIR=${GOPATH}/src/github.com/tensorflow/tensorflow
+PROTOC="${TF_DIR}/bazel-out/host/bin/external/protobuf/protoc"
+
+if [ ! -x "${PROTOC}" ]
+then
+  PATH_PROTOC=$(which protoc)
+  if [ ! -x "${PATH_PROTOC}" ]
+  then
+    echo "Protocol buffer compiler protoc not found in PATH or in ${PROTOC}"
+    echo "Perhaps build it using:"
+    echo "bazel build -c opt @protobuf//:protoc"
+    exit 1
+  fi
+  PROTOC=PATH_PROTOC
+fi
+
+mkdir -p ./internal/proto
+${PROTOC} \
+  -I ${TF_DIR} \
+  --go_out=./internal/proto \
+  ${TF_DIR}/tensorflow/core/framework/*.proto

tensorflow/go/genop/internal/genop.go

@@ -0,0 +1,64 @@
+// Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package internal generates Go source code with functions for TensorFlow operations.
+//
+// The generated APIs are unstable and can change without notice.
+package internal
+
+// #include "tensorflow/c/c_api.h"
+import "C"
+
+import (
+	"fmt"
+	"io"
+	"unsafe"
+
+	"github.com/golang/protobuf/proto"
+	pb "github.com/tensorflow/tensorflow/tensorflow/go/genop/internal/proto/tensorflow/core/framework"
+)
+
+// GenerateFunctionsForRegisteredOps writes a Go source code file to w
+// containing functions for each TensorFlow operation registered in the address
+// space of the calling process.
+func GenerateFunctionsForRegisteredOps(w io.Writer) error {
+	ops, err := registeredOps()
+	if err != nil {
+		return err
+	}
+	fmt.Fprintf(w, `// DO NOT EDIT
+// This file was machine generated.
+//
+// This code generation process is a work in progress and is not ready yet.
+// Eventually, the code generator will generate approximately %d wrapper
+// functions for adding TensorFlow operations to a Graph.
+
+package op
+`, len(ops.Op))
+	return nil
+}
+
+func registeredOps() (*pb.OpList, error) {
+	buf := C.TF_GetAllOpList()
+	defer C.TF_DeleteBuffer(buf)
+	var (
+		list = new(pb.OpList)
+		size = int(buf.length)
+		// A []byte backed by C memory.
+		// See: https://github.com/golang/go/wiki/cgo#turning-c-arrays-into-go-slices
+		data = (*[1 << 30]byte)(unsafe.Pointer(buf.data))[:size:size]
+		err  = proto.Unmarshal(data, list)
+	)
+	return list, err
+}

tensorflow/go/genop/internal/lib.go

@@ -0,0 +1,19 @@
+// Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package internal
+
+// #cgo LDFLAGS: -L${SRCDIR}/../../../../bazel-bin/tensorflow -ltensorflow
+// #cgo CFLAGS: -I${SRCDIR}/../../../../
+import "C"

tensorflow/go/genop/main.go

@@ -0,0 +1,42 @@
+// Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:generate sh generate.sh
+
+// Command genop generates a Go source file with functions for TensorFlow ops.
+package main
+
+import (
+	"flag"
+	"log"
+	"os"
+
+	"github.com/tensorflow/tensorflow/tensorflow/go/genop/internal"
+)
+
+func main() {
+	filename := flag.String("outfile", "", "File to write generated source code to.")
+	flag.Parse()
+	if *filename == "" {
+		log.Fatal("--outfile must be set")
+	}
+	file, err := os.OpenFile(*filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
+	if err != nil {
+		log.Fatalf("Failed to open %q for writing: %v", *filename, err)
+	}
+	defer file.Close()
+	if err = internal.GenerateFunctionsForRegisteredOps(file); err != nil {
+		log.Fatal(err)
+	}
+}

tensorflow/go/op/.gitignore

@@ -0,0 +1 @@
+wrappers.go

tensorflow/go/op/generate.go

@@ -0,0 +1,20 @@
+// Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:generate go generate ../genop
+//go:generate go run ../genop/main.go -outfile wrappers.go
+//go:generate go fmt wrappers.go
+
+package op
+