experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Remove SplitAndParseAsInts.

Browse Source 
It is too specific and not widely used to be maintained as a part of TF's API.

PiperOrigin-RevId: 262963923
This commit is contained in:
Gunhan Gulsoy 2019-08-12 11:12:09 -07:00 committed by TensorFlower Gardener
parent e11156c229
commit d1002d97ef
10 changed files with 84 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
tensorflow/core/common_runtime/collective_param_resolver_local.cc
View File

@ -27,6 +27,7 @@ limitations under the License.
#include "tensorflow/core/lib/core/errors.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/lib/gtl/flatmap.h"
#include "tensorflow/core/lib/strings/numbers.h"
#include "tensorflow/core/lib/strings/str_util.h"
#include "tensorflow/core/lib/strings/strcat.h"
#include "tensorflow/core/platform/types.h"
@ -229,17 +230,20 @@ GlobalDeviceMap BuildDevRecs(const CollInstanceParams& ip,
}
bool ParseRingOrder(const string& gpu_ring_order_str, TaskDeviceMap* tdm) {
std::vector<int32> gpu_ring_order_vec;
if (!str_util::SplitAndParseAsInts(gpu_ring_order_str, ',',
&gpu_ring_order_vec)) {
return false;
}
if (gpu_ring_order_vec.size() != tdm->size()) return false;
std::vector<string> split_gpu_ring_order_str =
str_util::Split(gpu_ring_order_str, ',');
if (split_gpu_ring_order_str.size() != tdm->size()) return false;
// gpu id -> local rank
gtl::FlatMap<int32, int32> gpu_ranks;
for (int32 rank = 0; rank < static_cast<int32>(gpu_ring_order_vec.size());
++rank) {
gpu_ranks[gpu_ring_order_vec[rank]] = rank;
for (int32 rank = 0;
rank < static_cast<int32>(split_gpu_ring_order_str.size()); ++rank) {
int32 tmp;
if (strings::safe_strto32(split_gpu_ring_order_str[rank], &tmp)) {
gpu_ranks[tmp] = rank;
} else {
return false;
}
}
for (auto& tdm_it : *tdm) {

23
tensorflow/core/grappler/optimizers/auto_mixed_precision.cc
View File

@ -1067,16 +1067,25 @@ std::pair<int, int> GetDeviceGPUArch(
const DeviceProperties& device_properties) {
if (device_properties.type() != "GPU") return {0, 0};
string arch_str = device_properties.environment().at("architecture");
std::vector<int32> arch_pieces;
if (!str_util::SplitAndParseAsInts(arch_str, '.', &arch_pieces) ||
arch_pieces.empty()) {
std::vector<string> split_arch_str = str_util::Split(arch_str, '.');
if (split_arch_str.empty()) {
return {0, 0};
}
std::pair<int, int> arch(arch_pieces[0], 0);
if (arch_pieces.size() > 1) {
arch.second = arch_pieces[1];
int major, minor;
if (!strings::safe_strto32(split_arch_str[0], &major)) {
return {0, 0};
}
if (split_arch_str.size() > 1) {
if (strings::safe_strto32(split_arch_str[1], &minor)) {
return {major, minor};
} else {
return {0, 0};
}
} else {
return {major, 0};
}
return arch;
}
bool AutoMixedPrecisionImpl::IsOnSuitableGPUArch(const NodeDef& node) const {

7
tensorflow/core/kernels/hexagon/hexagon_rewriter_transform.cc
View File

@ -54,8 +54,13 @@ Status RewriteQuantizedStrippedModelForHexagon(
string shape_string;
TF_RETURN_IF_ERROR(context.GetOneStringParameter(
INPUT_SHAPE_PREFIX + std::to_string(i), "", &shape_string));
std::vector<string> split_shape = str_util::Split(shape_string, ',');
std::vector<int64> dims;
CHECK(str_util::SplitAndParseAsInts(shape_string, ',', &dims));
for (const string& dim : split_shape) {
int64 tmp;
CHECK(strings::safe_strto64(dim, &tmp));
dims.push_back(tmp);
}
// Get input data type
string data_type_string;

8
tensorflow/core/kernels/remote_fused_graph_rewriter_transform.cc
View File

@ -116,8 +116,14 @@ static Status PlaceShapeType(const std::vector<string>& inputs,
std::vector<std::pair<string, Tensor>> input_tensors;
for (size_t i = 0; i < inputs.size(); ++i) {
const string& name = inputs.at(i);
std::vector<string> split_input_shapes =
str_util::Split(input_shapes_strs.at(i), ',');
std::vector<int64> dims;
CHECK(str_util::SplitAndParseAsInts(input_shapes_strs.at(i), ',', &dims));
for (const string& dim : split_input_shapes) {
int64 tmp;
CHECK(strings::safe_strto64(dim, &tmp));
dims.push_back(tmp);
}
DataType data_type;
CHECK(DataTypeFromString(input_types_strs.at(i), &data_type))
<< "\"" << input_types_strs.at(i) << "\" was an invalid type";

10
tensorflow/core/lib/strings/str_util.cc
View File

@ -218,16 +218,6 @@ bool ConsumeNonWhitespace(StringPiece* s, StringPiece* val) {
}
}
bool SplitAndParseAsInts(StringPiece text, char delim,
std::vector<int32>* result) {
return SplitAndParseAsInts<int32>(text, delim, strings::safe_strto32, result);
}
bool SplitAndParseAsInts(StringPiece text, char delim,
std::vector<int64>* result) {
return SplitAndParseAsInts<int64>(text, delim, strings::safe_strto64, result);
}
size_t Strnlen(const char* str, const size_t string_max_len) {
size_t len = 0;
while (len < string_max_len && str[len] != '\0') {

4
tensorflow/core/lib/strings/str_util.h
View File

@ -150,10 +150,6 @@ std::vector<string> Split(StringPiece text, StringPiece delims, Predicate p);
// Split "text" at "delim" characters, and parse each component as
// an integer. If successful, adds the individual numbers in order
// to "*result" and returns true. Otherwise returns false.
bool SplitAndParseAsInts(StringPiece text, char delim,
std::vector<int32>* result);
bool SplitAndParseAsInts(StringPiece text, char delim,
std::vector<int64>* result);
// StartsWith()
//

47
tensorflow/core/lib/strings/str_util_test.cc
View File

@ -267,53 +267,6 @@ TEST(Split, Basic) {
"a|b|c");
}
TEST(SplitAndParseAsInts, Int32) {
std::vector<int32> nums;
EXPECT_TRUE(str_util::SplitAndParseAsInts("", ',', &nums));
EXPECT_EQ(nums.size(), 0);
EXPECT_TRUE(str_util::SplitAndParseAsInts("134", ',', &nums));
EXPECT_EQ(nums.size(), 1);
EXPECT_EQ(nums[0], 134);
EXPECT_TRUE(str_util::SplitAndParseAsInts("134,2,13,-5", ',', &nums));
EXPECT_EQ(nums.size(), 4);
EXPECT_EQ(nums[0], 134);
EXPECT_EQ(nums[1], 2);
EXPECT_EQ(nums[2], 13);
EXPECT_EQ(nums[3], -5);
EXPECT_FALSE(str_util::SplitAndParseAsInts("abc", ',', &nums));
EXPECT_FALSE(str_util::SplitAndParseAsInts("-13,abc", ',', &nums));
EXPECT_FALSE(str_util::SplitAndParseAsInts("13,abc,5", ',', &nums));
}
TEST(SplitAndParseAsInts, Int64) {
std::vector<int64> nums;
EXPECT_TRUE(str_util::SplitAndParseAsInts("", ',', &nums));
EXPECT_EQ(nums.size(), 0);
EXPECT_TRUE(str_util::SplitAndParseAsInts("134", ',', &nums));
EXPECT_EQ(nums.size(), 1);
EXPECT_EQ(nums[0], 134);
EXPECT_TRUE(
str_util::SplitAndParseAsInts("134,2,13,-4000000000", ',', &nums));
EXPECT_EQ(nums.size(), 4);
EXPECT_EQ(nums[0], 134);
EXPECT_EQ(nums[1], 2);
EXPECT_EQ(nums[2], 13);
EXPECT_EQ(nums[3], static_cast<int64>(-4000000000ull));
EXPECT_FALSE(str_util::SplitAndParseAsInts("abc", ',', &nums));
EXPECT_FALSE(str_util::SplitAndParseAsInts("-13,abc", ',', &nums));
EXPECT_FALSE(str_util::SplitAndParseAsInts("13,abc,5", ',', &nums));
}
TEST(Lowercase, Basic) {
EXPECT_EQ("", str_util::Lowercase(""));
EXPECT_EQ("hello", str_util::Lowercase("hello"));

30
tensorflow/core/platform/cloud/gcs_file_system.cc
View File

@ -624,17 +624,31 @@ class GcsWritableFile : public WritableFile {
StringPiece range_piece(received_range);
absl::ConsumePrefix(&range_piece,
"bytes="); // May or may not be present.
std::vector<int64> range_parts;
if (!str_util::SplitAndParseAsInts(range_piece, '-', &range_parts) ||
range_parts.size() != 2) {
auto return_error = [this](string error_message) {
return errors::Internal("Unexpected response from GCS when writing ",
GetGcsPath(), ": Range header '",
received_range, "' could not be parsed.");
GetGcsPath(), ": ", error_message);
};
std::vector<string> range_strs = str_util::Split(range_piece, '-');
std::vector<int64> range_parts;
for (const string& range_str : range_strs) {
int64 tmp;
if (strings::safe_strto64(range_str, &tmp)) {
range_parts.push_back(tmp);
} else {
return return_error("Range header '" + received_range +
"' could not be parsed.");
}
}
if (range_parts.size() != 2) {
return return_error("Range header '" + received_range +
"' could not be parsed.");
}
if (range_parts[0] != 0) {
return errors::Internal("Unexpected response from GCS when writing to ",
GetGcsPath(), ": the returned range '",
received_range, "' does not start at zero.");
return return_error("The returned range '" + received_range +
"' does not start at zero.");
}
// If GCS returned "Range: 0-10", this means 11 bytes were uploaded.
*uploaded = range_parts[1] + 1;

17
tensorflow/tools/benchmark/benchmark_model.cc
View File

@ -533,17 +533,20 @@ int Main(int argc, char** argv) {
InputLayerInfo input;
CHECK(DataTypeFromString(input_layer_types[n], &input.data_type))
<< input_layer_types[n] << " was an invalid type";
std::vector<int32> sizes;
CHECK(str_util::SplitAndParseAsInts(input_layer_shapes[n], ',', &sizes))
<< "Incorrect size string specified: " << input_layer_shapes[n];
for (int i = 0; i < sizes.size(); ++i) {
int32 size = sizes[i];
if (size == -1) {
std::vector<string> split_layer_shapes =
str_util::Split(input_layer_shapes[n], ',');
for (const string& layer_shape : split_layer_shapes) {
int32 tmp;
CHECK(strings::safe_strto32(layer_shape, &tmp))
<< "Incorrect size string specified: " << input_layer_shapes[n];
if (tmp == -1) {
LOG(ERROR) << "Any unknown sizes in the shapes (-1's) must be replaced"
<< " with the size you want to benchmark with.";
return -1;
} else {
input.shape.AddDim(tmp);
}
input.shape.AddDim(sizes[i]);
}
input.name = input_layers[n];
if (n < input_layer_values.size()) {

13
tensorflow/tools/graph_transforms/transform_utils.cc
View File

@ -18,6 +18,7 @@ limitations under the License.
#include "tensorflow/core/framework/node_def_util.h"
#include "tensorflow/core/framework/op.h"
#include "tensorflow/core/lib/hash/hash.h"
#include "tensorflow/core/lib/strings/numbers.h"
#include "tensorflow/core/lib/strings/str_util.h"
namespace tensorflow {
@ -596,10 +597,16 @@ Status TensorShapeFromString(const string& shape_string, TensorShape* result) {
if (shape_string.empty()) {
return errors::InvalidArgument("Specificed shape is empty.");
}
std::vector<string> dims_as_str = str_util::Split(shape_string, ",");
std::vector<int64> dims;
if (!str_util::SplitAndParseAsInts(shape_string, ',', &dims)) {
return errors::InvalidArgument("Could parse as shape: '", shape_string,
"'");
for (const string& dim : dims_as_str) {
int64 tmp;
if (strings::safe_strto64(dim, &tmp)) {
dims.push_back(tmp);
} else {
return errors::InvalidArgument("Could parse as shape: '", shape_string,
"'");
}
}
*result = TensorShape(dims);
return Status::OK();