experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #31571 from frreiss:issue-lmdb-endian

Browse Source 
PiperOrigin-RevId: 264453617
This commit is contained in:
TensorFlower Gardener 2019-08-20 14:19:50 -07:00
commit 520a0edb41
4 changed files with 18 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
tensorflow/contrib/data/python/kernel_tests/lmdb_dataset_op_test.py
View File

@ -20,6 +20,7 @@ from __future__ import print_function
import os import os
import shutil import shutil
import sys
from tensorflow.contrib.data.python.ops import readers from tensorflow.contrib.data.python.ops import readers
from tensorflow.python.data.kernel_tests import test_base from tensorflow.python.data.kernel_tests import test_base
@ -40,7 +41,10 @@ class LMDBDatasetTest(test_base.DatasetTestBase):
def setUp(self): def setUp(self):
super(LMDBDatasetTest, self).setUp() super(LMDBDatasetTest, self).setUp()
# Copy database out because we need the path to be writable to use locks. # Copy database out because we need the path to be writable to use locks.
path = os.path.join(prefix_path, "lmdb", "testdata", "data.mdb") # The on-disk format of an LMDB database is different on big-endian
# machines, because LMDB is a memory-mapped database.
db_file = "data.mdb" if sys.byteorder == "little" else "data_bigendian.mdb"
path = os.path.join(prefix_path, "lmdb", "testdata", db_file)
self.db_path = os.path.join(self.get_temp_dir(), "data.mdb") self.db_path = os.path.join(self.get_temp_dir(), "data.mdb")
shutil.copy(path, self.db_path) shutil.copy(path, self.db_path)

8
tensorflow/core/BUILD
View File

@ -5482,11 +5482,19 @@ filegroup(
testonly = 1, testonly = 1,
srcs = [ srcs = [
# A simple key-value store: # A simple key-value store:
# 0 : 'b'
# 1 : 'b'
# ...
# 9 : 'b'
# Which is then overwritten with:
# 0 : 'a' # 0 : 'a'
# 1 : 'b' # 1 : 'b'
# ... # ...
# 9 : 'j' # 9 : 'j'
"lib/lmdb/testdata/data.mdb", "lib/lmdb/testdata/data.mdb",
# LMDB, being a memory-mapped database, uses a different file format on
# big-endian systems.
"lib/lmdb/testdata/data_bigendian.mdb",
], ],
visibility = ["//visibility:public"], visibility = ["//visibility:public"],
) )

BIN
tensorflow/core/lib/lmdb/testdata/data_bigendian.mdb vendored Normal file
View File

Binary file not shown.

6
tensorflow/python/kernel_tests/reader_ops_test.py
View File

@ -22,6 +22,7 @@ import collections
import gzip import gzip
import os import os
import shutil import shutil
import sys
import threading import threading
import zlib import zlib
@ -754,7 +755,10 @@ class LMDBReaderTest(test.TestCase):
def setUp(self): def setUp(self):
super(LMDBReaderTest, self).setUp() super(LMDBReaderTest, self).setUp()
# Copy database out because we need the path to be writable to use locks. # Copy database out because we need the path to be writable to use locks.
path = os.path.join(prefix_path, "lmdb", "testdata", "data.mdb") # The on-disk format of an LMDB file is different on big-endian machines,
# because LMDB is a memory-mapped database.
db_file = "data.mdb" if sys.byteorder == "little" else "data_bigendian.mdb"
path = os.path.join(prefix_path, "lmdb", "testdata", db_file)
self.db_path = os.path.join(self.get_temp_dir(), "data.mdb") self.db_path = os.path.join(self.get_temp_dir(), "data.mdb")
shutil.copy(path, self.db_path) shutil.copy(path, self.db_path)