Add python/lib BUILD and refactor python/BUILD

PiperOrigin-RevId: 346841489
Change-Id: Id8c9e27ccf012c7b9668e7fd56d5d97d3ff09370
This commit is contained in:
Yanhua Sun 2020-12-10 12:31:18 -08:00 committed by TensorFlower Gardener
parent 62e3126273
commit 2800f688ff
19 changed files with 712 additions and 552 deletions

View File

@ -78,7 +78,7 @@ cc_library(
],
visibility = [
"//tensorflow/core:__pkg__",
"//tensorflow/python:__pkg__",
"//tensorflow/python:__subpackages__",
],
)

View File

@ -1246,7 +1246,7 @@ cc_library(
"-lpthread",
],
}),
visibility = ["//tensorflow/python:__pkg__"],
visibility = ["//tensorflow/python:__subpackages__"],
deps = tf_additional_lib_deps() + [
"@com_google_absl//absl/meta:type_traits",
"@com_google_absl//absl/strings",

File diff suppressed because it is too large Load Diff

View File

@ -69,11 +69,11 @@ cc_library(
"//tensorflow/python:ndarray_tensor_bridge",
"//tensorflow/python:numpy_lib",
"//tensorflow/python:py_exception_registry",
"//tensorflow/python:py_seq_tensor",
"//tensorflow/python:py_util",
"//tensorflow/python:safe_ptr",
"//tensorflow/python:safe_pyobject_ptr",
"//tensorflow/python:stack_trace",
"//tensorflow/python/lib/core:py_seq_tensor",
"//tensorflow/python/lib/core:py_util",
"//tensorflow/python/lib/core:safe_ptr",
"//tensorflow/python/lib/core:safe_pyobject_ptr",
"//third_party/py/numpy:headers",
"//third_party/python_runtime:headers",
"@com_google_absl//absl/container:flat_hash_map",
@ -234,8 +234,8 @@ tf_python_pybind_extension(
"//tensorflow/c/eager:custom_device_testutil",
"//tensorflow/python:cpp_python_util",
"//tensorflow/python:pybind11_lib",
"//tensorflow/python:pybind11_status",
"//tensorflow/python:safe_ptr",
"//tensorflow/python/lib/core:pybind11_status",
"//tensorflow/python/lib/core:safe_ptr",
"//third_party/python_runtime:headers",
"@pybind11",
],

View File

@ -25,8 +25,8 @@ from tensorflow.core.framework import types_pb2
# protobuf errors where a file is defined twice on MacOS.
# pylint: disable=invalid-import-order,g-bad-import-order
from tensorflow.python import pywrap_tensorflow # pylint: disable=unused-import
from tensorflow.python import _pywrap_bfloat16
from tensorflow.python import _dtypes
from tensorflow.python.lib.core import _pywrap_bfloat16
from tensorflow.python.util.tf_export import tf_export
_np_bfloat16 = _pywrap_bfloat16.TF_bfloat16_type()

View File

@ -23,10 +23,10 @@ import pickle
import warnings
from tensorflow.core.lib.core import error_codes_pb2
from tensorflow.python import _pywrap_file_io
from tensorflow.python.framework import c_api_util
from tensorflow.python.framework import errors
from tensorflow.python.framework import errors_impl
from tensorflow.python.lib.io import _pywrap_file_io
from tensorflow.python.platform import test
from tensorflow.python.util import compat

View File

@ -0,0 +1,370 @@
# python/lib/core package
# buildifier: disable=same-origin-load
load("//tensorflow:tensorflow.bzl", "tf_python_pybind_extension")
# buildifier: disable=same-origin-load
load("//tensorflow:tensorflow.bzl", "tf_py_test")
# buildifier: disable=same-origin-load
load("//tensorflow:tensorflow.bzl", "tf_external_workspace_visible")
# buildifier: disable=same-origin-load
load("//tensorflow:tensorflow.bzl", "get_compatible_with_portable")
visibility = [
"//engedu/ml/tf_from_scratch:__pkg__",
"//third_party/cloud_tpu/convergence_tools:__subpackages__",
"//third_party/mlperf:__subpackages__",
"//tensorflow:internal",
"//tensorflow/lite/toco/python:__pkg__",
"//tensorflow_models:__subpackages__",
"//tensorflow_model_optimization:__subpackages__",
"//third_party/py/cleverhans:__subpackages__",
"//third_party/py/launchpad:__subpackages__",
"//third_party/py/reverb:__subpackages__",
"//third_party/py/neural_structured_learning:__subpackages__",
"//third_party/py/tensorflow_examples:__subpackages__",
"//third_party/py/tf_agents:__subpackages__", # For benchmarks.
"//third_party/py/tf_slim:__subpackages__",
"//third_party/py/tensorflow_docs:__subpackages__",
"//third_party/py/keras:__subpackages__",
]
package(
default_visibility = visibility,
licenses = ["notice"], # Apache 2.0
)
cc_library(
name = "numpy_lib",
srcs = ["numpy.cc"],
hdrs = ["numpy.h"],
deps = [
"//third_party/py/numpy:headers",
"//third_party/python_runtime:headers",
],
)
cc_library(
name = "bfloat16_lib",
srcs = ["bfloat16.cc"],
hdrs = ["bfloat16.h"],
deps = [
":numpy_lib",
"//tensorflow/core/platform:logging",
"//third_party/eigen3",
"//third_party/python_runtime:headers",
"@com_google_absl//absl/strings",
],
)
tf_python_pybind_extension(
name = "_pywrap_bfloat16",
srcs = ["bfloat16_wrapper.cc"],
hdrs = ["bfloat16.h"],
module_name = "_pywrap_bfloat16",
deps = [
"//third_party/python_runtime:headers",
"@pybind11",
],
)
cc_library(
name = "ndarray_tensor_bridge",
srcs = ["ndarray_tensor_bridge.cc"],
hdrs = ["ndarray_tensor_bridge.h"],
visibility = tf_external_workspace_visible(
visibility + [
"//tensorflow:ndarray_tensor_allow_list",
],
),
deps = [
":bfloat16_lib",
":numpy_lib",
"//tensorflow/c:c_api_no_xla",
"//tensorflow/core:lib",
"//tensorflow/core:protos_all_cc",
],
)
cc_library(
name = "py_exception_registry",
srcs = ["py_exception_registry.cc"],
hdrs = ["py_exception_registry.h"],
deps = [
"//tensorflow/c:tf_status_headers",
"//tensorflow/core:lib",
"//tensorflow/core:protos_all_cc",
"//third_party/python_runtime:headers",
],
alwayslink = 1,
)
cc_library(
name = "pybind11_absl",
hdrs = ["pybind11_absl.h"],
features = ["-parse_headers"],
visibility = tf_external_workspace_visible(visibility),
deps = [
"//tensorflow/core/platform:stringpiece",
"@pybind11",
],
)
cc_library(
name = "pybind11_lib",
hdrs = ["pybind11_lib.h"],
compatible_with = get_compatible_with_portable(),
features = ["-parse_headers"],
visibility = tf_external_workspace_visible(visibility),
deps = [
"@pybind11",
],
)
cc_library(
name = "pybind11_status_headers",
hdrs = [
"py_exception_registry.h",
"pybind11_status.h",
"//tensorflow/c:headers",
"//tensorflow/c/eager:headers",
],
features = [
"-parse_headers",
],
visibility = tf_external_workspace_visible(visibility),
deps = [
"//tensorflow/c:tf_status_headers",
"//tensorflow/core:protos_all_cc",
"//tensorflow/core/common_runtime:core_cpu_headers_lib",
"//third_party/python_runtime:headers",
"@pybind11",
],
)
cc_library(
name = "pybind11_status",
hdrs = [
"py_exception_registry.h",
"pybind11_status.h",
"//tensorflow/c:headers",
],
features = ["-parse_headers"],
visibility = tf_external_workspace_visible(visibility),
deps = [
":pybind11_status_headers",
"//tensorflow/core:lib",
],
)
cc_library(
name = "pybind11_proto",
hdrs = ["pybind11_proto.h"],
features = ["-parse_headers"],
visibility = tf_external_workspace_visible(visibility),
deps = [
"@com_google_absl//absl/strings",
"@pybind11",
],
)
filegroup(
name = "py_exception_registry_hdr",
srcs = [
"py_exception_registry.h",
],
visibility = ["//visibility:public"],
)
filegroup(
name = "numpy_hdr",
srcs = ["numpy.h"],
)
filegroup(
name = "safe_ptr_hdr",
srcs = ["safe_ptr.h"],
)
filegroup(
name = "ndarray_tensor_hdr",
srcs = ["ndarray_tensor.h"],
)
filegroup(
name = "basic_hdrs",
srcs = [
"bfloat16.h",
"ndarray_tensor.h",
"ndarray_tensor_bridge.h",
"numpy.h",
"py_exception_registry.h",
"pybind11_status.h",
"safe_ptr.h",
"safe_pyobject_ptr.h",
],
)
cc_library(
name = "py_func_lib",
srcs = ["py_func.cc"],
hdrs = ["py_func.h"],
deps = [
":ndarray_tensor",
":ndarray_tensor_bridge",
":numpy_lib",
":py_util",
":safe_ptr",
"//tensorflow/c:tf_status_helper",
"//tensorflow/c/eager:c_api",
"//tensorflow/c/eager:tfe_context_internal",
"//tensorflow/c/eager:tfe_tensorhandle_internal",
"//tensorflow/core:framework",
"//tensorflow/core:lib",
"//tensorflow/core:protos_all_cc",
"//tensorflow/core:script_ops_op_lib",
"//tensorflow/core/common_runtime/eager:context",
"//tensorflow/core/common_runtime/eager:tensor_handle",
"//tensorflow/python/eager:pywrap_tfe_lib",
"//third_party/py/numpy:headers",
"//third_party/python_runtime:headers",
],
alwayslink = 1,
)
tf_python_pybind_extension(
name = "_pywrap_py_func",
srcs = ["py_func_wrapper.cc"],
module_name = "_pywrap_py_func",
deps = [
"//tensorflow/python:py_func_headers_lib",
"//third_party/python_runtime:headers",
"@pybind11",
],
)
cc_library(
name = "safe_pyobject_ptr",
srcs = ["safe_pyobject_ptr.cc"],
hdrs = ["safe_pyobject_ptr.h"],
deps = [
"//third_party/python_runtime:headers",
],
)
cc_library(
name = "safe_pyobject_ptr_required_hdrs",
textual_hdrs = ["safe_pyobject_ptr.h"],
)
cc_library(
name = "safe_ptr",
srcs = [
"safe_ptr.cc",
"//tensorflow/c/eager:headers",
],
hdrs = ["safe_ptr.h"],
deps = [
":safe_pyobject_ptr",
"//tensorflow/c:c_api_no_xla",
"//third_party/python_runtime:headers",
],
)
cc_library(
name = "ndarray_tensor_headers",
hdrs = [
"bfloat16.h",
"ndarray_tensor.h",
"ndarray_tensor_bridge.h",
"numpy.h",
"safe_ptr.h",
"safe_pyobject_ptr.h",
"//tensorflow/c:headers",
"//tensorflow/c/eager:headers",
],
features = [
"-parse_headers",
],
visibility = tf_external_workspace_visible(visibility + [
"//tensorflow:ndarray_tensor_allow_list",
]),
deps = [
":numpy_lib",
"//tensorflow/c:pywrap_required_hdrs",
"//tensorflow/c:tf_status_headers",
"//tensorflow/core:framework_internal_headers_lib",
"//tensorflow/core/common_runtime:core_cpu_headers_lib",
"//third_party/py/numpy:headers",
"//third_party/python_runtime:headers",
],
)
cc_library(
name = "ndarray_tensor",
srcs = ["ndarray_tensor.cc"],
hdrs = ["ndarray_tensor.h"],
visibility = tf_external_workspace_visible(visibility + [
"//tensorflow:ndarray_tensor_allow_list",
]),
deps = [
":bfloat16_lib",
":ndarray_tensor_bridge",
":numpy_lib",
":safe_ptr",
"//tensorflow/c:c_api_internal",
"//tensorflow/c:tf_status_helper",
"//tensorflow/c:tf_tensor_internal",
"//tensorflow/c/eager:tfe_context_internal",
"//tensorflow/core:framework",
"//tensorflow/core:lib",
],
)
cc_library(
name = "py_seq_tensor",
srcs = ["py_seq_tensor.cc"],
hdrs = ["py_seq_tensor.h"],
features = ["-parse_headers"],
deps = [
":ndarray_tensor",
":ndarray_tensor_bridge",
":numpy_lib",
":py_util",
":safe_ptr",
"//tensorflow/c:tensor_interface",
"//tensorflow/c:tf_tensor_internal",
"//tensorflow/c/eager:c_api_internal",
"//tensorflow/c/eager:tfe_context_internal",
"//tensorflow/c/eager:tfe_tensorhandle_internal",
"//tensorflow/core:framework",
"//tensorflow/core:lib",
"//third_party/python_runtime:headers", # build_cleaner: keep; DNR: b/35864863
],
)
cc_library(
name = "py_util",
srcs = ["py_util.cc"],
hdrs = ["py_util.h"],
deps = [
"//tensorflow/core:lib",
"//tensorflow/core:script_ops_op_lib",
"//tensorflow/core/platform:logging",
"//third_party/python_runtime:headers",
],
)
tf_py_test(
name = "bfloat16_test",
size = "small",
srcs = ["bfloat16_test.py"],
python_version = "PY3",
deps = [
"//tensorflow/python:pywrap_tensorflow",
"//tensorflow/python/lib/io:lib",
"//tensorflow/python/platform:client_testlib",
],
)

View File

@ -29,7 +29,9 @@ from absl.testing import parameterized
import numpy as np
# pylint: disable=unused-import,g-bad-import-order
from tensorflow.python import _pywrap_bfloat16
from tensorflow.python.framework import dtypes
from tensorflow.python.lib.core import _pywrap_bfloat16
from tensorflow.python.platform import test
bfloat16 = _pywrap_bfloat16.TF_bfloat16_type()

View File

@ -0,0 +1,103 @@
# python/lib/io package
# buildifier: disable=same-origin-load
load("//tensorflow:tensorflow.bzl", "tf_python_pybind_extension")
# buildifier: disable=same-origin-load
load("//tensorflow:tensorflow.bzl", "tf_py_test")
visibility = [
"//tensorflow:__subpackages__",
]
package(
default_visibility = visibility,
licenses = ["notice"], # Apache 2.0
)
cc_library(
name = "py_record_reader_lib",
srcs = ["py_record_reader.cc"],
hdrs = ["py_record_reader.h"],
deps = [
"//tensorflow/c:c_api",
"//tensorflow/c:tf_status_helper",
"//tensorflow/core:lib",
"//tensorflow/core:lib_internal",
],
)
tf_python_pybind_extension(
name = "_pywrap_file_io",
srcs = ["file_io_wrapper.cc"],
module_name = "_pywrap_file_io",
deps = [
"//tensorflow/core:framework_headers_lib",
"//tensorflow/core:protos_all_cc",
"//tensorflow/python/lib/core:pybind11_absl",
"//tensorflow/python/lib/core:pybind11_status",
"@pybind11",
],
)
py_library(
name = "lib",
srcs = [
"file_io.py",
"python_io.py",
"tf_record.py",
],
srcs_version = "PY2AND3",
deps = [
":_pywrap_file_io",
":_pywrap_record_io",
"//tensorflow/python:errors",
"//tensorflow/python:pywrap_tensorflow",
"//tensorflow/python:util",
"@six_archive//:six",
],
)
tf_python_pybind_extension(
name = "_pywrap_record_io",
srcs = ["record_io_wrapper.cc"],
module_name = "_pywrap_record_io",
deps = [
"//tensorflow/core:framework_headers_lib",
"//tensorflow/core:lib_headers_for_pybind",
"//tensorflow/core/platform:types",
"//tensorflow/python/lib/core:pybind11_absl",
"//tensorflow/python/lib/core:pybind11_status",
"@com_google_absl//absl/memory",
"@pybind11",
],
)
tf_py_test(
name = "file_io_test",
size = "small",
srcs = ["file_io_test.py"],
python_version = "PY3",
tags = [
"no_rocm",
"no_windows",
],
deps = [
":lib",
"//tensorflow/python:errors",
"//tensorflow/python/platform:client_testlib",
],
)
tf_py_test(
name = "tf_record_test",
size = "small",
srcs = ["tf_record_test.py"],
python_version = "PY3",
deps = [
":lib",
"//tensorflow/python:errors",
"//tensorflow/python:util",
"//tensorflow/python/platform:client_testlib",
],
)

View File

@ -23,8 +23,8 @@ import uuid
import six
from tensorflow.python import _pywrap_file_io
from tensorflow.python.framework import errors
from tensorflow.python.lib.io import _pywrap_file_io
from tensorflow.python.util import compat
from tensorflow.python.util import deprecation
from tensorflow.python.util.tf_export import tf_export

View File

@ -19,7 +19,7 @@ from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from tensorflow.python import _pywrap_record_io
from tensorflow.python.lib.io import _pywrap_record_io
from tensorflow.python.util import compat
from tensorflow.python.util import deprecation
from tensorflow.python.util.tf_export import tf_export

View File

@ -28,13 +28,13 @@ import weakref
import numpy as np
import six
from tensorflow.python import _pywrap_py_func
from tensorflow.python.eager import backprop
from tensorflow.python.eager import context
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import func_graph
from tensorflow.python.framework import function
from tensorflow.python.framework import ops
from tensorflow.python.lib.core import _pywrap_py_func
from tensorflow.python.ops import gen_script_ops
from tensorflow.python.ops import resource_variable_ops
from tensorflow.python.util import compat

View File

@ -47,9 +47,9 @@ py_library(
":build_info",
"//tensorflow/core:protos_all_py",
"//tensorflow/python:_pywrap_util_port",
"//tensorflow/python:lib",
"//tensorflow/python:pywrap_tfe",
"//tensorflow/python:util",
"//tensorflow/python/lib/io:lib",
"@absl_py//absl/flags",
"@rules_python//python/runfiles",
"@six_archive//:six",

View File

@ -1,7 +1,7 @@
path: "tensorflow.io.TFRecordWriter"
tf_class {
is_instance: "<class \'tensorflow.python.lib.io.tf_record.TFRecordWriter\'>"
is_instance: "<class \'tensorflow.python._pywrap_record_io.RecordWriter\'>"
is_instance: "<class \'tensorflow.python.lib.io._pywrap_record_io.RecordWriter\'>"
is_instance: "<class \'pybind11_builtins.pybind11_object\'>"
member_method {
name: "__init__"

View File

@ -1,7 +1,7 @@
path: "tensorflow.python_io.TFRecordWriter"
tf_class {
is_instance: "<class \'tensorflow.python.lib.io.tf_record.TFRecordWriter\'>"
is_instance: "<class \'tensorflow.python._pywrap_record_io.RecordWriter\'>"
is_instance: "<class \'tensorflow.python.lib.io._pywrap_record_io.RecordWriter\'>"
is_instance: "<class \'pybind11_builtins.pybind11_object\'>"
member_method {
name: "__init__"

View File

@ -1,7 +1,7 @@
path: "tensorflow.io.TFRecordWriter"
tf_class {
is_instance: "<class \'tensorflow.python.lib.io.tf_record.TFRecordWriter\'>"
is_instance: "<class \'tensorflow.python._pywrap_record_io.RecordWriter\'>"
is_instance: "<class \'tensorflow.python.lib.io._pywrap_record_io.RecordWriter\'>"
is_instance: "<class \'pybind11_builtins.pybind11_object\'>"
member_method {
name: "__init__"

View File

@ -48,7 +48,7 @@ py_test(
":test_module1",
":test_module2",
":traverse",
"//tensorflow/python:platform_test",
"//tensorflow/python/platform:test",
],
)

View File

@ -46,17 +46,17 @@ transitive_hdrs(
# Code that relies on these headers should dynamically link to
# _pywrap_tensorflow_internal.so as well.
"//tensorflow/python:model_analyzer_lib",
"//tensorflow/python:py_exception_registry",
"//tensorflow/python:pybind11_absl",
"//tensorflow/python:pybind11_lib",
"//tensorflow/python:pybind11_status",
"//tensorflow/python:pybind11_proto",
"//tensorflow/python:kernel_registry",
"//tensorflow/python:cpp_python_util",
"//tensorflow/python:py_func_lib",
"//tensorflow/python:py_seq_tensor",
"//tensorflow/python:py_util",
"//tensorflow/python:py_record_reader_lib",
"//tensorflow/python/lib/core:py_exception_registry",
"//tensorflow/python/lib/core:pybind11_proto",
"//tensorflow/python/lib/core:pybind11_absl",
"//tensorflow/python/lib/core:pybind11_lib",
"//tensorflow/python/lib/core:pybind11_status",
"//tensorflow/python/lib/core:py_func_lib",
"//tensorflow/python/lib/core:py_seq_tensor",
"//tensorflow/python/lib/core:py_util",
"//tensorflow/python/lib/io:py_record_reader_lib",
"//tensorflow/python:python_op_gen",
"//tensorflow/python:tf_session_helper",
"//third_party/eigen3",