experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Experimental prototype of SentencePieceTokenizer for TFLite support library

Browse Source 
PiperOrigin-RevId: 300458695
Change-Id: I9d416531e16fbb4883b603b01f40c11b7533c83d
This commit is contained in:
A. Unique TensorFlower 2020-03-11 19:25:17 -07:00 committed by TensorFlower Gardener
parent 82432a5d18
commit dba1d5fa93
1 changed files with 34 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
tensorflow/lite/experimental/support/java/src/java/org/tensorflow/lite/support/common/FileUtil.java
View File

@ -22,6 +22,7 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.ByteBuffer;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.charset.Charset;
@ -67,8 +68,9 @@ public class FileUtil {
@NonNull Context context, @NonNull String filePath, Charset cs) throws IOException {
SupportPreconditions.checkNotNull(context, "Context cannot be null.");
SupportPreconditions.checkNotNull(filePath, "File path cannot be null.");
InputStream inputStream = context.getAssets().open(filePath);
return loadLabels(inputStream, cs);
try (InputStream inputStream = context.getAssets().open(filePath)) {
return loadLabels(inputStream, cs);
}
}
/**
@ -97,13 +99,13 @@ public class FileUtil {
public static List<String> loadLabels(@NonNull InputStream inputStream, Charset cs)
throws IOException {
List<String> labels = new ArrayList<>();
BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, cs));
String line;
while ((line = reader.readLine()) != null) {
labels.add(line);
try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, cs))) {
String line;
while ((line = reader.readLine()) != null) {
labels.add(line);
}
return labels;
}
reader.close();
return labels;
}
/**
@ -151,11 +153,29 @@ public class FileUtil {
throws IOException {
SupportPreconditions.checkNotNull(context, "Context should not be null.");
SupportPreconditions.checkNotNull(filePath, "File path cannot be null.");
AssetFileDescriptor fileDescriptor = context.getAssets().openFd(filePath);
FileInputStream inputStream = new FileInputStream(fileDescriptor.getFileDescriptor());
FileChannel fileChannel = inputStream.getChannel();
long startOffset = fileDescriptor.getStartOffset();
long declaredLength = fileDescriptor.getDeclaredLength();
return fileChannel.map(FileChannel.MapMode.READ_ONLY, startOffset, declaredLength);
try (AssetFileDescriptor fileDescriptor = context.getAssets().openFd(filePath);
FileInputStream inputStream = new FileInputStream(fileDescriptor.getFileDescriptor())) {
FileChannel fileChannel = inputStream.getChannel();
long startOffset = fileDescriptor.getStartOffset();
long declaredLength = fileDescriptor.getDeclaredLength();
return fileChannel.map(FileChannel.MapMode.READ_ONLY, startOffset, declaredLength);
}
}
/**
* Loads a binary file from the asset folder.
*
* @param context Application context to access assets.
* @param filePath Asset path of the file.
* @return the byte array for the binary file.
* @throws IOException if an I/O error occurs when loading file.
*/
@NonNull
public static byte[] loadByteFromFile(@NonNull Context context, @NonNull String filePath)
throws IOException {
ByteBuffer buffer = loadMappedFile(context, filePath);
byte[] byteArray = new byte[buffer.remaining()];
buffer.get(byteArray);
return byteArray;
}
}