Remove SplitAndParseAsInts.
It is too specific and not widely used to be maintained as a part of TF's API. PiperOrigin-RevId: 262963923
This commit is contained in:
parent
e11156c229
commit
d1002d97ef
@ -27,6 +27,7 @@ limitations under the License.
|
|||||||
#include "tensorflow/core/lib/core/errors.h"
|
#include "tensorflow/core/lib/core/errors.h"
|
||||||
#include "tensorflow/core/lib/core/status.h"
|
#include "tensorflow/core/lib/core/status.h"
|
||||||
#include "tensorflow/core/lib/gtl/flatmap.h"
|
#include "tensorflow/core/lib/gtl/flatmap.h"
|
||||||
|
#include "tensorflow/core/lib/strings/numbers.h"
|
||||||
#include "tensorflow/core/lib/strings/str_util.h"
|
#include "tensorflow/core/lib/strings/str_util.h"
|
||||||
#include "tensorflow/core/lib/strings/strcat.h"
|
#include "tensorflow/core/lib/strings/strcat.h"
|
||||||
#include "tensorflow/core/platform/types.h"
|
#include "tensorflow/core/platform/types.h"
|
||||||
@ -229,17 +230,20 @@ GlobalDeviceMap BuildDevRecs(const CollInstanceParams& ip,
|
|||||||
}
|
}
|
||||||
|
|
||||||
bool ParseRingOrder(const string& gpu_ring_order_str, TaskDeviceMap* tdm) {
|
bool ParseRingOrder(const string& gpu_ring_order_str, TaskDeviceMap* tdm) {
|
||||||
std::vector<int32> gpu_ring_order_vec;
|
std::vector<string> split_gpu_ring_order_str =
|
||||||
if (!str_util::SplitAndParseAsInts(gpu_ring_order_str, ',',
|
str_util::Split(gpu_ring_order_str, ',');
|
||||||
&gpu_ring_order_vec)) {
|
if (split_gpu_ring_order_str.size() != tdm->size()) return false;
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (gpu_ring_order_vec.size() != tdm->size()) return false;
|
|
||||||
// gpu id -> local rank
|
// gpu id -> local rank
|
||||||
gtl::FlatMap<int32, int32> gpu_ranks;
|
gtl::FlatMap<int32, int32> gpu_ranks;
|
||||||
for (int32 rank = 0; rank < static_cast<int32>(gpu_ring_order_vec.size());
|
for (int32 rank = 0;
|
||||||
++rank) {
|
rank < static_cast<int32>(split_gpu_ring_order_str.size()); ++rank) {
|
||||||
gpu_ranks[gpu_ring_order_vec[rank]] = rank;
|
int32 tmp;
|
||||||
|
if (strings::safe_strto32(split_gpu_ring_order_str[rank], &tmp)) {
|
||||||
|
gpu_ranks[tmp] = rank;
|
||||||
|
} else {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for (auto& tdm_it : *tdm) {
|
for (auto& tdm_it : *tdm) {
|
||||||
|
@ -1067,16 +1067,25 @@ std::pair<int, int> GetDeviceGPUArch(
|
|||||||
const DeviceProperties& device_properties) {
|
const DeviceProperties& device_properties) {
|
||||||
if (device_properties.type() != "GPU") return {0, 0};
|
if (device_properties.type() != "GPU") return {0, 0};
|
||||||
string arch_str = device_properties.environment().at("architecture");
|
string arch_str = device_properties.environment().at("architecture");
|
||||||
std::vector<int32> arch_pieces;
|
std::vector<string> split_arch_str = str_util::Split(arch_str, '.');
|
||||||
if (!str_util::SplitAndParseAsInts(arch_str, '.', &arch_pieces) ||
|
if (split_arch_str.empty()) {
|
||||||
arch_pieces.empty()) {
|
|
||||||
return {0, 0};
|
return {0, 0};
|
||||||
}
|
}
|
||||||
std::pair<int, int> arch(arch_pieces[0], 0);
|
|
||||||
if (arch_pieces.size() > 1) {
|
int major, minor;
|
||||||
arch.second = arch_pieces[1];
|
if (!strings::safe_strto32(split_arch_str[0], &major)) {
|
||||||
|
return {0, 0};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (split_arch_str.size() > 1) {
|
||||||
|
if (strings::safe_strto32(split_arch_str[1], &minor)) {
|
||||||
|
return {major, minor};
|
||||||
|
} else {
|
||||||
|
return {0, 0};
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
return {major, 0};
|
||||||
}
|
}
|
||||||
return arch;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
bool AutoMixedPrecisionImpl::IsOnSuitableGPUArch(const NodeDef& node) const {
|
bool AutoMixedPrecisionImpl::IsOnSuitableGPUArch(const NodeDef& node) const {
|
||||||
|
@ -54,8 +54,13 @@ Status RewriteQuantizedStrippedModelForHexagon(
|
|||||||
string shape_string;
|
string shape_string;
|
||||||
TF_RETURN_IF_ERROR(context.GetOneStringParameter(
|
TF_RETURN_IF_ERROR(context.GetOneStringParameter(
|
||||||
INPUT_SHAPE_PREFIX + std::to_string(i), "", &shape_string));
|
INPUT_SHAPE_PREFIX + std::to_string(i), "", &shape_string));
|
||||||
|
std::vector<string> split_shape = str_util::Split(shape_string, ',');
|
||||||
std::vector<int64> dims;
|
std::vector<int64> dims;
|
||||||
CHECK(str_util::SplitAndParseAsInts(shape_string, ',', &dims));
|
for (const string& dim : split_shape) {
|
||||||
|
int64 tmp;
|
||||||
|
CHECK(strings::safe_strto64(dim, &tmp));
|
||||||
|
dims.push_back(tmp);
|
||||||
|
}
|
||||||
|
|
||||||
// Get input data type
|
// Get input data type
|
||||||
string data_type_string;
|
string data_type_string;
|
||||||
|
@ -116,8 +116,14 @@ static Status PlaceShapeType(const std::vector<string>& inputs,
|
|||||||
std::vector<std::pair<string, Tensor>> input_tensors;
|
std::vector<std::pair<string, Tensor>> input_tensors;
|
||||||
for (size_t i = 0; i < inputs.size(); ++i) {
|
for (size_t i = 0; i < inputs.size(); ++i) {
|
||||||
const string& name = inputs.at(i);
|
const string& name = inputs.at(i);
|
||||||
|
std::vector<string> split_input_shapes =
|
||||||
|
str_util::Split(input_shapes_strs.at(i), ',');
|
||||||
std::vector<int64> dims;
|
std::vector<int64> dims;
|
||||||
CHECK(str_util::SplitAndParseAsInts(input_shapes_strs.at(i), ',', &dims));
|
for (const string& dim : split_input_shapes) {
|
||||||
|
int64 tmp;
|
||||||
|
CHECK(strings::safe_strto64(dim, &tmp));
|
||||||
|
dims.push_back(tmp);
|
||||||
|
}
|
||||||
DataType data_type;
|
DataType data_type;
|
||||||
CHECK(DataTypeFromString(input_types_strs.at(i), &data_type))
|
CHECK(DataTypeFromString(input_types_strs.at(i), &data_type))
|
||||||
<< "\"" << input_types_strs.at(i) << "\" was an invalid type";
|
<< "\"" << input_types_strs.at(i) << "\" was an invalid type";
|
||||||
|
@ -218,16 +218,6 @@ bool ConsumeNonWhitespace(StringPiece* s, StringPiece* val) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
bool SplitAndParseAsInts(StringPiece text, char delim,
|
|
||||||
std::vector<int32>* result) {
|
|
||||||
return SplitAndParseAsInts<int32>(text, delim, strings::safe_strto32, result);
|
|
||||||
}
|
|
||||||
|
|
||||||
bool SplitAndParseAsInts(StringPiece text, char delim,
|
|
||||||
std::vector<int64>* result) {
|
|
||||||
return SplitAndParseAsInts<int64>(text, delim, strings::safe_strto64, result);
|
|
||||||
}
|
|
||||||
|
|
||||||
size_t Strnlen(const char* str, const size_t string_max_len) {
|
size_t Strnlen(const char* str, const size_t string_max_len) {
|
||||||
size_t len = 0;
|
size_t len = 0;
|
||||||
while (len < string_max_len && str[len] != '\0') {
|
while (len < string_max_len && str[len] != '\0') {
|
||||||
|
@ -150,10 +150,6 @@ std::vector<string> Split(StringPiece text, StringPiece delims, Predicate p);
|
|||||||
// Split "text" at "delim" characters, and parse each component as
|
// Split "text" at "delim" characters, and parse each component as
|
||||||
// an integer. If successful, adds the individual numbers in order
|
// an integer. If successful, adds the individual numbers in order
|
||||||
// to "*result" and returns true. Otherwise returns false.
|
// to "*result" and returns true. Otherwise returns false.
|
||||||
bool SplitAndParseAsInts(StringPiece text, char delim,
|
|
||||||
std::vector<int32>* result);
|
|
||||||
bool SplitAndParseAsInts(StringPiece text, char delim,
|
|
||||||
std::vector<int64>* result);
|
|
||||||
|
|
||||||
// StartsWith()
|
// StartsWith()
|
||||||
//
|
//
|
||||||
|
@ -267,53 +267,6 @@ TEST(Split, Basic) {
|
|||||||
"a|b|c");
|
"a|b|c");
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST(SplitAndParseAsInts, Int32) {
|
|
||||||
std::vector<int32> nums;
|
|
||||||
EXPECT_TRUE(str_util::SplitAndParseAsInts("", ',', &nums));
|
|
||||||
EXPECT_EQ(nums.size(), 0);
|
|
||||||
|
|
||||||
EXPECT_TRUE(str_util::SplitAndParseAsInts("134", ',', &nums));
|
|
||||||
EXPECT_EQ(nums.size(), 1);
|
|
||||||
EXPECT_EQ(nums[0], 134);
|
|
||||||
|
|
||||||
EXPECT_TRUE(str_util::SplitAndParseAsInts("134,2,13,-5", ',', &nums));
|
|
||||||
EXPECT_EQ(nums.size(), 4);
|
|
||||||
EXPECT_EQ(nums[0], 134);
|
|
||||||
EXPECT_EQ(nums[1], 2);
|
|
||||||
EXPECT_EQ(nums[2], 13);
|
|
||||||
EXPECT_EQ(nums[3], -5);
|
|
||||||
|
|
||||||
EXPECT_FALSE(str_util::SplitAndParseAsInts("abc", ',', &nums));
|
|
||||||
|
|
||||||
EXPECT_FALSE(str_util::SplitAndParseAsInts("-13,abc", ',', &nums));
|
|
||||||
|
|
||||||
EXPECT_FALSE(str_util::SplitAndParseAsInts("13,abc,5", ',', &nums));
|
|
||||||
}
|
|
||||||
|
|
||||||
TEST(SplitAndParseAsInts, Int64) {
|
|
||||||
std::vector<int64> nums;
|
|
||||||
EXPECT_TRUE(str_util::SplitAndParseAsInts("", ',', &nums));
|
|
||||||
EXPECT_EQ(nums.size(), 0);
|
|
||||||
|
|
||||||
EXPECT_TRUE(str_util::SplitAndParseAsInts("134", ',', &nums));
|
|
||||||
EXPECT_EQ(nums.size(), 1);
|
|
||||||
EXPECT_EQ(nums[0], 134);
|
|
||||||
|
|
||||||
EXPECT_TRUE(
|
|
||||||
str_util::SplitAndParseAsInts("134,2,13,-4000000000", ',', &nums));
|
|
||||||
EXPECT_EQ(nums.size(), 4);
|
|
||||||
EXPECT_EQ(nums[0], 134);
|
|
||||||
EXPECT_EQ(nums[1], 2);
|
|
||||||
EXPECT_EQ(nums[2], 13);
|
|
||||||
EXPECT_EQ(nums[3], static_cast<int64>(-4000000000ull));
|
|
||||||
|
|
||||||
EXPECT_FALSE(str_util::SplitAndParseAsInts("abc", ',', &nums));
|
|
||||||
|
|
||||||
EXPECT_FALSE(str_util::SplitAndParseAsInts("-13,abc", ',', &nums));
|
|
||||||
|
|
||||||
EXPECT_FALSE(str_util::SplitAndParseAsInts("13,abc,5", ',', &nums));
|
|
||||||
}
|
|
||||||
|
|
||||||
TEST(Lowercase, Basic) {
|
TEST(Lowercase, Basic) {
|
||||||
EXPECT_EQ("", str_util::Lowercase(""));
|
EXPECT_EQ("", str_util::Lowercase(""));
|
||||||
EXPECT_EQ("hello", str_util::Lowercase("hello"));
|
EXPECT_EQ("hello", str_util::Lowercase("hello"));
|
||||||
|
@ -624,17 +624,31 @@ class GcsWritableFile : public WritableFile {
|
|||||||
StringPiece range_piece(received_range);
|
StringPiece range_piece(received_range);
|
||||||
absl::ConsumePrefix(&range_piece,
|
absl::ConsumePrefix(&range_piece,
|
||||||
"bytes="); // May or may not be present.
|
"bytes="); // May or may not be present.
|
||||||
std::vector<int64> range_parts;
|
|
||||||
if (!str_util::SplitAndParseAsInts(range_piece, '-', &range_parts) ||
|
auto return_error = [this](string error_message) {
|
||||||
range_parts.size() != 2) {
|
|
||||||
return errors::Internal("Unexpected response from GCS when writing ",
|
return errors::Internal("Unexpected response from GCS when writing ",
|
||||||
GetGcsPath(), ": Range header '",
|
GetGcsPath(), ": ", error_message);
|
||||||
received_range, "' could not be parsed.");
|
};
|
||||||
|
|
||||||
|
std::vector<string> range_strs = str_util::Split(range_piece, '-');
|
||||||
|
std::vector<int64> range_parts;
|
||||||
|
for (const string& range_str : range_strs) {
|
||||||
|
int64 tmp;
|
||||||
|
if (strings::safe_strto64(range_str, &tmp)) {
|
||||||
|
range_parts.push_back(tmp);
|
||||||
|
} else {
|
||||||
|
return return_error("Range header '" + received_range +
|
||||||
|
"' could not be parsed.");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
if (range_parts.size() != 2) {
|
||||||
|
return return_error("Range header '" + received_range +
|
||||||
|
"' could not be parsed.");
|
||||||
|
}
|
||||||
|
|
||||||
if (range_parts[0] != 0) {
|
if (range_parts[0] != 0) {
|
||||||
return errors::Internal("Unexpected response from GCS when writing to ",
|
return return_error("The returned range '" + received_range +
|
||||||
GetGcsPath(), ": the returned range '",
|
"' does not start at zero.");
|
||||||
received_range, "' does not start at zero.");
|
|
||||||
}
|
}
|
||||||
// If GCS returned "Range: 0-10", this means 11 bytes were uploaded.
|
// If GCS returned "Range: 0-10", this means 11 bytes were uploaded.
|
||||||
*uploaded = range_parts[1] + 1;
|
*uploaded = range_parts[1] + 1;
|
||||||
|
@ -533,17 +533,20 @@ int Main(int argc, char** argv) {
|
|||||||
InputLayerInfo input;
|
InputLayerInfo input;
|
||||||
CHECK(DataTypeFromString(input_layer_types[n], &input.data_type))
|
CHECK(DataTypeFromString(input_layer_types[n], &input.data_type))
|
||||||
<< input_layer_types[n] << " was an invalid type";
|
<< input_layer_types[n] << " was an invalid type";
|
||||||
std::vector<int32> sizes;
|
|
||||||
CHECK(str_util::SplitAndParseAsInts(input_layer_shapes[n], ',', &sizes))
|
std::vector<string> split_layer_shapes =
|
||||||
<< "Incorrect size string specified: " << input_layer_shapes[n];
|
str_util::Split(input_layer_shapes[n], ',');
|
||||||
for (int i = 0; i < sizes.size(); ++i) {
|
for (const string& layer_shape : split_layer_shapes) {
|
||||||
int32 size = sizes[i];
|
int32 tmp;
|
||||||
if (size == -1) {
|
CHECK(strings::safe_strto32(layer_shape, &tmp))
|
||||||
|
<< "Incorrect size string specified: " << input_layer_shapes[n];
|
||||||
|
if (tmp == -1) {
|
||||||
LOG(ERROR) << "Any unknown sizes in the shapes (-1's) must be replaced"
|
LOG(ERROR) << "Any unknown sizes in the shapes (-1's) must be replaced"
|
||||||
<< " with the size you want to benchmark with.";
|
<< " with the size you want to benchmark with.";
|
||||||
return -1;
|
return -1;
|
||||||
|
} else {
|
||||||
|
input.shape.AddDim(tmp);
|
||||||
}
|
}
|
||||||
input.shape.AddDim(sizes[i]);
|
|
||||||
}
|
}
|
||||||
input.name = input_layers[n];
|
input.name = input_layers[n];
|
||||||
if (n < input_layer_values.size()) {
|
if (n < input_layer_values.size()) {
|
||||||
|
@ -18,6 +18,7 @@ limitations under the License.
|
|||||||
#include "tensorflow/core/framework/node_def_util.h"
|
#include "tensorflow/core/framework/node_def_util.h"
|
||||||
#include "tensorflow/core/framework/op.h"
|
#include "tensorflow/core/framework/op.h"
|
||||||
#include "tensorflow/core/lib/hash/hash.h"
|
#include "tensorflow/core/lib/hash/hash.h"
|
||||||
|
#include "tensorflow/core/lib/strings/numbers.h"
|
||||||
#include "tensorflow/core/lib/strings/str_util.h"
|
#include "tensorflow/core/lib/strings/str_util.h"
|
||||||
|
|
||||||
namespace tensorflow {
|
namespace tensorflow {
|
||||||
@ -596,10 +597,16 @@ Status TensorShapeFromString(const string& shape_string, TensorShape* result) {
|
|||||||
if (shape_string.empty()) {
|
if (shape_string.empty()) {
|
||||||
return errors::InvalidArgument("Specificed shape is empty.");
|
return errors::InvalidArgument("Specificed shape is empty.");
|
||||||
}
|
}
|
||||||
|
std::vector<string> dims_as_str = str_util::Split(shape_string, ",");
|
||||||
std::vector<int64> dims;
|
std::vector<int64> dims;
|
||||||
if (!str_util::SplitAndParseAsInts(shape_string, ',', &dims)) {
|
for (const string& dim : dims_as_str) {
|
||||||
return errors::InvalidArgument("Could parse as shape: '", shape_string,
|
int64 tmp;
|
||||||
"'");
|
if (strings::safe_strto64(dim, &tmp)) {
|
||||||
|
dims.push_back(tmp);
|
||||||
|
} else {
|
||||||
|
return errors::InvalidArgument("Could parse as shape: '", shape_string,
|
||||||
|
"'");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
*result = TensorShape(dims);
|
*result = TensorShape(dims);
|
||||||
return Status::OK();
|
return Status::OK();
|
||||||
|
Loading…
x
Reference in New Issue
Block a user