From e6fd34c57c4ad7402c2c127a3380cfba65ecdb55 Mon Sep 17 00:00:00 2001 From: Fredrik Knutsson Date: Wed, 11 Sep 2019 09:43:52 +0200 Subject: [PATCH 01/13] Add support for offline planned tensor allocations By adding metadata to the model, it is possible to set arena offset for each tensor. Change-Id: Idd646c00a6e34e0c2603896d748cd5680a57f015 --- .../memory_planner/greedy_memory_planner.cc | 181 +++++++++++------- .../memory_planner/greedy_memory_planner.h | 19 +- tensorflow/lite/micro/micro_allocator.cc | 82 +++++++- 3 files changed, 210 insertions(+), 72 deletions(-) diff --git a/tensorflow/lite/micro/memory_planner/greedy_memory_planner.cc b/tensorflow/lite/micro/memory_planner/greedy_memory_planner.cc index faea73e9169..7763afa2075 100644 --- a/tensorflow/lite/micro/memory_planner/greedy_memory_planner.cc +++ b/tensorflow/lite/micro/memory_planner/greedy_memory_planner.cc @@ -42,8 +42,8 @@ GreedyMemoryPlanner::GreedyMemoryPlanner(unsigned char* scratch_buffer, int scratch_buffer_size) : buffer_count_(0), need_to_calculate_offsets_(true) { const int per_buffer_size = sizeof(BufferRequirements) + // requirements_ - sizeof(int) + // buffer_sizes_sorted_by_size_ - sizeof(int) + // buffer_ids_sorted_by_size_ + sizeof(int) + // buffer_sizes_sorted_ + sizeof(int) + // buffer_ids_sorted_ sizeof(ListEntry) + // buffers_sorted_by_offset_ sizeof(int); // buffer_offsets_; // Allocate the arrays we need within the scratch buffer arena. @@ -53,10 +53,10 @@ GreedyMemoryPlanner::GreedyMemoryPlanner(unsigned char* scratch_buffer, requirements_ = reinterpret_cast(next_free); next_free += sizeof(BufferRequirements) * max_buffer_count_; - buffer_sizes_sorted_by_size_ = reinterpret_cast(next_free); + buffer_sizes_sorted_ = reinterpret_cast(next_free); next_free += sizeof(int) * max_buffer_count_; - buffer_ids_sorted_by_size_ = reinterpret_cast(next_free); + buffer_ids_sorted_ = reinterpret_cast(next_free); next_free += sizeof(int) * max_buffer_count_; buffers_sorted_by_offset_ = reinterpret_cast(next_free); @@ -81,11 +81,24 @@ TfLiteStatus GreedyMemoryPlanner::AddBuffer( current->size = size; current->first_time_used = first_time_used; current->last_time_used = last_time_used; + current->offline_offset = kOnlinePlannedBuffer; ++buffer_count_; need_to_calculate_offsets_ = true; return kTfLiteOk; } +TfLiteStatus GreedyMemoryPlanner::AddBuffer( + tflite::ErrorReporter* error_reporter, int size, int first_time_used, + int last_time_used, int offline_offset) { + BufferRequirements* current = &requirements_[buffer_count_]; + if (AddBuffer(error_reporter, size, first_time_used, last_time_used) != + kTfLiteOk) { + return kTfLiteError; + } + current->offline_offset = offline_offset; + return kTfLiteOk; +} + bool GreedyMemoryPlanner::DoesEntryOverlapInTime( const GreedyMemoryPlanner::ListEntry* entry, const int first_time_used, const int last_time_used) const { @@ -107,7 +120,7 @@ GreedyMemoryPlanner::NextSimultaneouslyActiveBuffer( ListEntry* result = nullptr; ListEntry* candidate_next_entry; if (start == nullptr) { - candidate_next_entry = &buffers_sorted_by_offset_[0]; + candidate_next_entry = &buffers_sorted_by_offset_[first_entry_index_]; } else { if (start->next_entry_index == -1) { return nullptr; @@ -139,29 +152,51 @@ void GreedyMemoryPlanner::CalculateOffsetsIfNeeded() { // This helps find a more compact layout. Intuitively, you can think // about putting the large buffers in place first, and then the // smaller buffers can fit in the gaps, rather than fragmenting the - // gaps with small buffers at the beginning. + // gaps with small buffers at the beginning. Add offline planned offsets + // first in the list, since they have a predetermined offset. + int idx_from_tail = buffer_count_; + int idx_from_head = 0; for (int i = 0; i < buffer_count_; ++i) { - buffer_sizes_sorted_by_size_[i] = requirements_[i].size; - buffer_ids_sorted_by_size_[i] = i; - buffer_offsets_[i] = -1; + if (requirements_[i].offline_offset == kOnlinePlannedBuffer) { + idx_from_tail--; + buffer_sizes_sorted_[idx_from_tail] = requirements_[i].size; + buffer_ids_sorted_[idx_from_tail] = i; + buffer_offsets_[i] = -1; + } else { + buffer_sizes_sorted_[idx_from_head] = requirements_[i].size; + buffer_ids_sorted_[idx_from_head] = i; + buffer_offsets_[i] = requirements_[i].offline_offset; + idx_from_head++; + } } - // This sorting algorithm is naive, and may end up taking a very long time - // with hundreds of buffers. - ReverseSortInPlace(buffer_sizes_sorted_by_size_, buffer_ids_sorted_by_size_, - buffer_count_); - // Put the largest buffer at offset zero to start the process. - ListEntry* first_entry = &buffers_sorted_by_offset_[0]; - first_entry->offset = 0; - first_entry->requirements_index = buffer_ids_sorted_by_size_[0]; - first_entry->next_entry_index = -1; + // This sorting algorithm is naive, and may end up taking a very long time + // with hundreds of buffers. Do not sort the offline planned offsets. + ReverseSortInPlace(&buffer_sizes_sorted_[idx_from_head], + &buffer_ids_sorted_[idx_from_head], + buffer_count_ - idx_from_head); + + // Initialize the first entry to the first buffer in + // buffer_ids_sorted_. + // - If there are no offline planned offsets, the largest buffer will be + // first, and the buffers will be handled in size order. + // - If offline offsets are present, these will be handled first in order + // for the greedy algorithm to utilized gaps in the offline plan. + first_entry_index_ = 0; next_free_entry_ = 1; - buffer_offsets_[buffer_ids_sorted_by_size_[0]] = 0; + ListEntry* first_entry = &buffers_sorted_by_offset_[first_entry_index_]; + first_entry->next_entry_index = -1; // to mark the entry as end of list + int buffer_id = buffer_ids_sorted_[0]; + first_entry->requirements_index = buffer_id; + if (requirements_[buffer_id].offline_offset == kOnlinePlannedBuffer) { + buffer_offsets_[buffer_id] = 0; + } + first_entry->offset = buffer_offsets_[buffer_id]; // Work through the rest of the buffers to find a good gap to place each one. for (int i = 1; i < buffer_count_; ++i) { // The id is the order the buffer was originally added by the client. - const int buffer_id = buffer_ids_sorted_by_size_[i]; + const int buffer_id = buffer_ids_sorted_[i]; // Look at what size and time range the buffer needs to be active. BufferRequirements* wanted_requirements = &requirements_[buffer_id]; const int wanted_size = wanted_requirements->size; @@ -173,37 +208,43 @@ void GreedyMemoryPlanner::CalculateOffsetsIfNeeded() { // so that it's easy to find the next buffer in memory, and so the gap. // The candidate_entry variable holds the buffer that we're considering // placing the current buffer after. - ListEntry* prior_entry = nullptr; + int candidate_offset = 0; // Loop through the offset-ordered list of buffers, looking for gaps. - while (true) { - // Find out what the next active buffer is. - ListEntry* next_entry = NextSimultaneouslyActiveBuffer( - prior_entry, wanted_first_time_used, wanted_last_time_used); + if (wanted_requirements->offline_offset == kOnlinePlannedBuffer) { + ListEntry* prior_entry = nullptr; + while (true) { + // Find out what the next active buffer is. + ListEntry* next_entry = NextSimultaneouslyActiveBuffer( + prior_entry, wanted_first_time_used, wanted_last_time_used); - if (prior_entry) { - BufferRequirements* candidate_requirements = - &requirements_[prior_entry->requirements_index]; - const int prior_entry_offset = - prior_entry->offset + candidate_requirements->size; - if (prior_entry_offset > candidate_offset) { - candidate_offset = prior_entry_offset; + if (prior_entry) { + BufferRequirements* candidate_requirements = + &requirements_[prior_entry->requirements_index]; + const int prior_entry_offset = + prior_entry->offset + candidate_requirements->size; + if (prior_entry_offset > candidate_offset) { + candidate_offset = prior_entry_offset; + } } + if (next_entry == nullptr) { + // We're at the end of the list, so we can always append the buffer + // here. + break; + } + // Find out how much space there is between us and the next buffer. + const int gap = next_entry->offset - candidate_offset; + if (gap >= wanted_size) { + // This entry has a big enough gap between it and the next, so + // use it! + break; + } + // The gap wasn't big enough, so move on to another candidate. + prior_entry = next_entry; } - if (next_entry == nullptr) { - // We're at the end of the list, so we can always append the buffer - // here. - break; - } - // Find out how much space there is between us and the next buffer. - const int gap = next_entry->offset - candidate_offset; - if (gap >= wanted_size) { - // This entry has a big enough gap between it and the next, so - // use it! - break; - } - // The gap wasn't big enough, so move on to another candidate. - prior_entry = next_entry; + } else { + // Offline planned offset are to be considered constant + candidate_offset = wanted_requirements->offline_offset; } // At this point, we've either found a gap (possibly at the end of the // list) and want to place the buffer there, or there are no other active @@ -217,26 +258,36 @@ void GreedyMemoryPlanner::CalculateOffsetsIfNeeded() { new_entry->requirements_index = buffer_id; const int new_entry_index = next_free_entry_; ++next_free_entry_; - ListEntry* current_entry = first_entry; - // Make sure that we insert the buffer at the correct place in the ordered - // list. - while (true) { - const int next_entry_index = current_entry->next_entry_index; - if (next_entry_index == -1) { - // We're at the end of the list, so just add the new entry here. - current_entry->next_entry_index = new_entry_index; - new_entry->next_entry_index = -1; - break; + + if (first_entry->offset > candidate_offset) { + // The new entry offset is smaller than the first entry offset => + // replace the first entry + first_entry = new_entry; + first_entry->next_entry_index = first_entry_index_; + first_entry_index_ = new_entry_index; + } else { + ListEntry* current_entry = first_entry; + // Make sure that we insert the buffer at the correct place in the + // buffer-offset-ordered list + while (true) { + const int next_entry_index = current_entry->next_entry_index; + if (next_entry_index == -1) { + // We're at the end of the list, so just add the new entry here. + current_entry->next_entry_index = new_entry_index; + new_entry->next_entry_index = -1; + break; + } + // not at the end of the list -> take a look at next entry + ListEntry* next_entry = &buffers_sorted_by_offset_[next_entry_index]; + if (next_entry->offset > candidate_offset) { + // We're at the right spot to do an insertion and retain the sorting + // order, so place the new entry here. + new_entry->next_entry_index = current_entry->next_entry_index; + current_entry->next_entry_index = new_entry_index; + break; + } + current_entry = next_entry; } - ListEntry* next_entry = &buffers_sorted_by_offset_[next_entry_index]; - if (next_entry->offset > candidate_offset) { - // We're at the right spot to do an insertion and retain the sorting - // order, so place the new entry here. - new_entry->next_entry_index = current_entry->next_entry_index; - current_entry->next_entry_index = new_entry_index; - break; - } - current_entry = next_entry; } } } diff --git a/tensorflow/lite/micro/memory_planner/greedy_memory_planner.h b/tensorflow/lite/micro/memory_planner/greedy_memory_planner.h index f2c77ed94f3..d874b70e732 100644 --- a/tensorflow/lite/micro/memory_planner/greedy_memory_planner.h +++ b/tensorflow/lite/micro/memory_planner/greedy_memory_planner.h @@ -21,6 +21,8 @@ limitations under the License. namespace tflite { +constexpr int kOnlinePlannedBuffer = -1; + // A memory planner that uses a greedy algorithm to arrange buffers in memory // to minimize the overall arena size needed. // @@ -59,6 +61,12 @@ class GreedyMemoryPlanner : public MemoryPlanner { TfLiteStatus AddBuffer(ErrorReporter* error_reporter, int size, int first_time_used, int last_time_used) override; + // Record details of an offline planned buffer offset we want to place. + // offline_offset is the buffer offset from the start of the arena. + TfLiteStatus AddBuffer(ErrorReporter* error_reporter, int size, + int first_time_used, int last_time_used, + int offline_offset); + // Returns the high-water mark of used memory. This is the minimum size of a // memory arena you'd need to allocate to hold these buffers. size_t GetMaximumMemorySize() override; @@ -110,16 +118,23 @@ class GreedyMemoryPlanner : public MemoryPlanner { // Records the client-provided information about each buffer. struct BufferRequirements { int size; + int offline_offset; int first_time_used; int last_time_used; }; // Working arrays used during the layout algorithm. BufferRequirements* requirements_; - int* buffer_sizes_sorted_by_size_; - int* buffer_ids_sorted_by_size_; + // buffer_sizes_sorted_ and buffer_ids_sorted_ are sorted according to: + // { + // offline planned buffers, + // online planned buffers sorted by size + // } + int* buffer_sizes_sorted_; + int* buffer_ids_sorted_; ListEntry* buffers_sorted_by_offset_; int next_free_entry_; + int first_entry_index_; // Stores the outcome of the plan, the location of each buffer in the arena. int* buffer_offsets_; diff --git a/tensorflow/lite/micro/micro_allocator.cc b/tensorflow/lite/micro/micro_allocator.cc index c3044a0351f..46edec2bf43 100644 --- a/tensorflow/lite/micro/micro_allocator.cc +++ b/tensorflow/lite/micro/micro_allocator.cc @@ -37,6 +37,7 @@ struct AllocationInfo { int last_used; bool needs_allocating; void** output_ptr; + int offline_offset; }; // We align tensor buffers to 16-byte boundaries, since this is a common @@ -112,9 +113,17 @@ class AllocationInfoBuilder { return Allocate(); } + // Check if model contains offline planned buffer offsets. + // - If there's no metadata available, offline_planner_offsets is not set + // - If there's metadata available, offline_planner_offsets will point to the + // first offset in the metadata buffer list. + TfLiteStatus GetOfflinePlannedOffsets(const Model* model, + int** offline_planner_offsets); + // Add allocaiton information for the tensors. - TfLiteStatus AddTensors(const SubGraph* subgraph, + TfLiteStatus AddTensors(const SubGraph* subgraph, int* offline_offsets, TfLiteTensor* runtime_tensors); + // Add allocation information for the scratch buffers. TfLiteStatus AddScratchBuffers(internal::ScratchBufferHandle* buffer_handles); @@ -148,6 +157,7 @@ TfLiteStatus AllocationInfoBuilder::Allocate() { } TfLiteStatus AllocationInfoBuilder::AddTensors(const SubGraph* subgraph, + int* offline_offsets, TfLiteTensor* runtime_tensors) { // Set up allocation info for all tensors. for (size_t i = 0; i < tensor_count_; ++i) { @@ -159,6 +169,11 @@ TfLiteStatus AllocationInfoBuilder::AddTensors(const SubGraph* subgraph, current->last_used = -1; current->needs_allocating = (runtime_tensors[i].data.raw == nullptr) && (!subgraph->tensors()->Get(i)->is_variable()); + if (offline_offsets) { + current->offline_offset = offline_offsets[i]; + } else { + current->offline_offset = kOnlinePlannedBuffer; + } } for (size_t i = 0; i < subgraph->inputs()->size(); ++i) { @@ -216,6 +231,51 @@ TfLiteStatus AllocationInfoBuilder::AddTensors(const SubGraph* subgraph, return kTfLiteOk; } +// The tensor offsets will be encoded in the metadata:[Metadata] field of the +// Model. The following encoding applies: +// +// | Metadata component | Value | +// | name:string | “OfflineMemoryAllocation” | +// | buffer:unit | Index of buffer containing memory allocation data | +// +// The buffer contents for the memory allocation is a list of 32-bit integers of +// the following format: +// +// | Offset | Value | +// | 0 | Offline allocation format version – set to 0 | +// | 1 | Subgraph index to which this allocation applies | +// | 2 | Number offsets following: n | +// | 3 | Arena byte offset of tensor #0 or -1 to allocate at runtime | +// | 4 | Arena byte offset of tensor #1 or -1 to allocate at runtime | +// | 3+(n-1) | Arena byte offset of tensor #(n-1) or -1 to allocate at runtime | +TfLiteStatus AllocationInfoBuilder::GetOfflinePlannedOffsets( + const Model* model, int** offline_planner_offsets) { + if (model->metadata()) { + for (int i = 0; i < model->metadata()->size(); ++i) { + auto metadata = model->metadata()->Get(i); + if (strncmp(metadata->name()->c_str(), "OfflineMemoryAllocation", + strlen("OfflineMemoryAllocation")) == 0) { + const flatbuffers::Vector>* buffers = + model->buffers(); + auto* buffer = (*buffers)[metadata->buffer()]; + auto* array = buffer->data(); + const uint32_t* metadata_buffer = (uint32_t*)array->data(); + const int32_t nbr_tensors = metadata_buffer[2]; + *offline_planner_offsets = (int32_t*)&metadata_buffer[3]; + + if (tensor_count_ != nbr_tensors) { + TF_LITE_REPORT_ERROR(reporter_, + "Nbr of offline buffer offsets (%d) in metadata " + "not equal nbr tensors (%d)\n", + nbr_tensors, tensor_count_); + return kTfLiteError; + } + } + } + } + return kTfLiteOk; +} + TfLiteStatus AllocationInfoBuilder::AddScratchBuffers( internal::ScratchBufferHandle* buffer_handles) { // Set up allocation info for buffers. @@ -241,9 +301,17 @@ TfLiteStatus CreatePlan(ErrorReporter* error_reporter, MemoryPlanner* planner, if (current->needs_allocating) { size_t aligned_bytes_required = AlignSizeUp(current->bytes, kBufferAlignment); - TF_LITE_ENSURE_STATUS( - planner->AddBuffer(error_reporter, aligned_bytes_required, - current->first_created, current->last_used)); + if (current->offline_offset == kOnlinePlannedBuffer) { + TF_LITE_ENSURE_STATUS( + planner->AddBuffer(error_reporter, aligned_bytes_required, + current->first_created, current->last_used)); + } else { + TF_LITE_ENSURE_STATUS( + (static_cast(planner)) + ->AddBuffer(error_reporter, aligned_bytes_required, + current->first_created, current->last_used, + current->offline_offset)); + } } } return kTfLiteOk; @@ -546,7 +614,11 @@ TfLiteStatus MicroAllocator::FinishTensorAllocation() { AllocationInfoBuilder builder(error_reporter_, &tmp_allocator); TF_LITE_ENSURE_STATUS( builder.Init(tensors_->size(), scratch_buffer_count_)); - TF_LITE_ENSURE_STATUS(builder.AddTensors(subgraph_, context_->tensors)); + int* offline_planner_offsets = nullptr; + TF_LITE_ENSURE_STATUS( + builder.GetOfflinePlannedOffsets(model_, &offline_planner_offsets)); + TF_LITE_ENSURE_STATUS(builder.AddTensors(subgraph_, offline_planner_offsets, + context_->tensors)); TF_LITE_ENSURE_STATUS(builder.AddScratchBuffers(scratch_buffer_handles_)); const AllocationInfo* allocation_info = builder.Finish(); From 2ac92c48cd31721ec882b110e2af95edc505dc66 Mon Sep 17 00:00:00 2001 From: Fredrik Knutsson Date: Wed, 25 Mar 2020 12:30:25 +0100 Subject: [PATCH 02/13] Add helper functions in micro allocator One function to check metadata correctness and another to print model data. Change-Id: I2500fbbac25b376d068e3d9a1d190249da461eef --- tensorflow/lite/micro/micro_allocator.cc | 88 ++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/tensorflow/lite/micro/micro_allocator.cc b/tensorflow/lite/micro/micro_allocator.cc index 46edec2bf43..a9324eb35cd 100644 --- a/tensorflow/lite/micro/micro_allocator.cc +++ b/tensorflow/lite/micro/micro_allocator.cc @@ -96,6 +96,94 @@ TfLiteStatus AllocateVariables( return kTfLiteOk; } +// Helper function to print model flatbuffer data. This function is not called +// by default. Hence it's not linked in to the final binary code. +void PrintModelData(const Model* model, ErrorReporter* error_reporter) { + auto* subgraphs = model->subgraphs(); + const SubGraph* subgraph = (*subgraphs)[0]; + const flatbuffers::Vector>* tensors = + subgraph->tensors(); + const flatbuffers::Vector>* buffers = + model->buffers(); + TF_LITE_REPORT_ERROR(error_reporter, "==== Model info: ====="); + for (int i = 0; i < tensors->size(); ++i) { + const tflite::Tensor& flatbuffer_tensor = *tensors->Get(i); + auto* quantization = flatbuffer_tensor.quantization(); + size_t type_size, tensor_size; + auto* buffer = (*buffers)[flatbuffer_tensor.buffer()]; + auto* array = buffer->data(); + int array_size = 0; + if (array) { + array_size = array->size(); + } + BytesRequiredForTensor(flatbuffer_tensor, &tensor_size, &type_size, + error_reporter); + TF_LITE_REPORT_ERROR( + error_reporter, "Tensor index: %d arena tensor %d size %d", i, + !array_size && !flatbuffer_tensor.is_variable(), tensor_size); + } +} + +// Helper function to check flatbuffer metadata correctness. This function is +// not called by default. Hence it's not linked in to the final binary code. +TfLiteStatus CheckOfflinePlannedOffsets(const Model* model, + ErrorReporter* error_reporter) { + if (model->metadata()) { + for (int i = 0; i < model->metadata()->size(); ++i) { + auto metadata = model->metadata()->Get(i); + if (strncmp(metadata->name()->c_str(), "OfflineMemoryAllocation", + strlen("OfflineMemoryAllocation")) == 0) { + auto* subgraphs = model->subgraphs(); + const SubGraph* subgraph = (*subgraphs)[0]; + const flatbuffers::Vector>* tensors = + subgraph->tensors(); + const flatbuffers::Vector>* buffers = + model->buffers(); + int nbr_tflite_tensors = tensors->size(); + auto* buffer = (*buffers)[metadata->buffer()]; + auto* array = buffer->data(); + const uint32_t* metadata_buffer = (uint32_t*)array->data(); + int version = metadata_buffer[0]; + int subgraph_idx = metadata_buffer[1]; + const int nbr_offline_offsets = metadata_buffer[2]; + int* offline_planner_offsets = (int*)&metadata_buffer[3]; + + TF_LITE_REPORT_ERROR(error_reporter, "==== Model metadata info: ====="); + TF_LITE_REPORT_ERROR(error_reporter, + "Offline planner metadata found, version %d, " + "subgraph %d, nbr offline offsets %d", + version, subgraph_idx, nbr_offline_offsets); + for (int i = 0; i < nbr_offline_offsets; ++i) { + TF_LITE_REPORT_ERROR( + error_reporter, + "Offline planner tensor index %d, offline offset: %d", i, + offline_planner_offsets[i]); + } + + if (version != 1) { + TF_LITE_REPORT_ERROR(error_reporter, "Version not supported! (%d)\n", + version); + return kTfLiteError; + } + if (subgraph_idx != 0) { + TF_LITE_REPORT_ERROR(error_reporter, + "Only 1 subgraph supported! Subgraph idx (%d)\n", + subgraph_idx); + return kTfLiteError; + } + if (nbr_tflite_tensors != nbr_offline_offsets) { + TF_LITE_REPORT_ERROR(error_reporter, + "Nbr of offline buffer offsets (%d) in metadata " + "not equal nbr tensors (%d)\n", + nbr_offline_offsets, nbr_tflite_tensors); + return kTfLiteError; + } + } + } + } + return kTfLiteOk; +} + // A helper class to construct AllocationInfo array. This array contains the // lifetime of tensors / scratch_buffer and will be used to calculate the memory // plan. Methods need to be called in order from `Init`, `Add*`, to `Finish`. From dd673e306a0fec5b87f9c979d4c29524c575b766 Mon Sep 17 00:00:00 2001 From: Jens Elofsson Date: Fri, 6 Mar 2020 14:04:46 +0100 Subject: [PATCH 03/13] OfflinePlanner: Swap offsets in TestAllocationForModelsWithBranches The offline planner sorts the tensors in reverse order, so the testcase have to be updated accordingly. Change-Id: Ic3a1193489d6ad5f592db1c9a289b01083ad9c62 --- tensorflow/lite/micro/micro_allocator_test.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/lite/micro/micro_allocator_test.cc b/tensorflow/lite/micro/micro_allocator_test.cc index 47eefff90b5..765a447c044 100644 --- a/tensorflow/lite/micro/micro_allocator_test.cc +++ b/tensorflow/lite/micro/micro_allocator_test.cc @@ -185,10 +185,10 @@ TF_LITE_MICRO_TEST(TestAllocationForModelsWithBranches) { // bytes = 2 * 2 * 3 * sizeof(float32) = 48, same for other tensors. TF_LITE_MICRO_EXPECT_EQ(48, context.tensors[0].bytes); // t1 can't reuse any memory, as n0 requires both t0 and t1. - TF_LITE_MICRO_EXPECT_EQ(48, context.tensors[1].data.uint8 - start); + TF_LITE_MICRO_EXPECT_EQ(96, context.tensors[1].data.uint8 - start); // t2 can't reuse any memory, as n1 requires both t0 and t2. Also n2 requires // both t1 and t2. - TF_LITE_MICRO_EXPECT_EQ(96, context.tensors[2].data.uint8 - start); + TF_LITE_MICRO_EXPECT_EQ(48, context.tensors[2].data.uint8 - start); // t3 reuses the same memory from t0 as t0 is not an input to any node. TF_LITE_MICRO_EXPECT_EQ(0, context.tensors[3].data.uint8 - start); } From 7737c6222fafdb6425d0d2395a1be5c6f4205780 Mon Sep 17 00:00:00 2001 From: Jens Elofsson Date: Wed, 11 Mar 2020 12:49:23 +0100 Subject: [PATCH 04/13] Testcases for offline planned tensors Added testcases. Expanded ModelBuilder for have metadata support. Change-Id: I7e8bf3b3537d126086aef52bb3d6fe572aa8e7a0 --- tensorflow/lite/micro/micro_allocator_test.cc | 183 ++++++++++++++++++ tensorflow/lite/micro/test_helpers.cc | 106 +++++++++- tensorflow/lite/micro/test_helpers.h | 13 ++ 3 files changed, 292 insertions(+), 10 deletions(-) diff --git a/tensorflow/lite/micro/micro_allocator_test.cc b/tensorflow/lite/micro/micro_allocator_test.cc index 765a447c044..69fb82910b0 100644 --- a/tensorflow/lite/micro/micro_allocator_test.cc +++ b/tensorflow/lite/micro/micro_allocator_test.cc @@ -227,4 +227,187 @@ TF_LITE_MICRO_TEST(TestFinishComplexTensorAllocation) { tflite::testing::EnsureUniqueVariableTensorBuffer(&context, 7); } +TF_LITE_MICRO_TEST(OfflinePlannerBranchesAllOnline) { + int version = 1; + int subgraph = 0; + int nbr_tensors = 4; + const int32_t metadata_buffer[tflite::testing::kOfflinePlannerHeaderSize + + nbr_tensors] = {version, subgraph, + nbr_tensors, // header + // memory offsets: + -1, -1, -1, -1}; + + // The structure is identical to the one in + // TestAllocationForModelsWithBranches + std::vector node_list = { + { + {0}, // input + {1} // output + }, + { + {0}, // input + {2} // output + }, + { + {1, 2}, // input1, input2 + {3} // output + }}; + + const tflite::Model* model = tflite::testing::GetModelWithOfflinePlanning( + nbr_tensors, metadata_buffer, node_list); + + TfLiteContext context; + constexpr size_t arena_size = 4096; + uint8_t arena[arena_size]; + tflite::MicroAllocator allocator(&context, model, arena, arena_size, + micro_test::reporter); + TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, allocator.FinishTensorAllocation()); + + // Since all of the tensors are online planned and the model structure is + // identical to that in TestAllocationForModelsWithBranches, + // the offsets be should identical to that test. + uint8_t* start = context.tensors[0].data.uint8; + TF_LITE_MICRO_EXPECT_EQ(0, context.tensors[0].data.uint8 - start); + TF_LITE_MICRO_EXPECT_EQ(48, context.tensors[0].bytes); + TF_LITE_MICRO_EXPECT_EQ(96, context.tensors[1].data.uint8 - start); + TF_LITE_MICRO_EXPECT_EQ(48, context.tensors[2].data.uint8 - start); + TF_LITE_MICRO_EXPECT_EQ(0, context.tensors[3].data.uint8 - start); +} + +TF_LITE_MICRO_TEST(OfflinePlannerBasic) { + int nbr_tensors = 4; + const int32_t metadata_buffer[tflite::testing::kOfflinePlannerHeaderSize + + nbr_tensors] = {1, 0, nbr_tensors, + 0, // t0 + 48, // t1 + 0, // t2 + 48}; // t3 + + int t0 = 0; + int t1 = 1; + int t2 = 2; + int t3 = 3; + + std::vector node_list = {{ + {t0}, // input + {t1} // output + }, + { + {t1}, // input + {t2} // output + }, + { + {t2}, // input + {t3} // output + }}; + + const tflite::Model* model = tflite::testing::GetModelWithOfflinePlanning( + nbr_tensors, metadata_buffer, node_list); + + TfLiteContext context; + constexpr size_t arena_size = 4096; + uint8_t arena[arena_size]; + tflite::MicroAllocator allocator(&context, model, arena, arena_size, + micro_test::reporter); + TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, allocator.FinishTensorAllocation()); + + uint8_t* start = context.tensors[0].data.uint8; + TF_LITE_MICRO_EXPECT_EQ(0, context.tensors[0].data.uint8 - start); + TF_LITE_MICRO_EXPECT_EQ(48, context.tensors[1].data.uint8 - start); + TF_LITE_MICRO_EXPECT_EQ(0, context.tensors[2].data.uint8 - start); + TF_LITE_MICRO_EXPECT_EQ(48, context.tensors[3].data.uint8 - start); +} + +TF_LITE_MICRO_TEST(OfflinePlannerOverlappingAllocation) { + int nbr_tensors = 4; + const int32_t metadata_buffer[tflite::testing::kOfflinePlannerHeaderSize + + nbr_tensors] = { + 1, 0, nbr_tensors, // header: version, subgraph, nbr tensors + // memory offsets: + 0, // t0 + 0, // t1 + 48, // t2 + -1}; // t3 + + int t0 = 0; + int t1 = 1; + int t2 = 2; + int t3 = 3; + + std::vector node_list = { + { + {t0, t1}, // input, scratch + {t2} // output + }, + { + {t2}, // input + {t3} // output + }, + }; + + const tflite::Model* model = tflite::testing::GetModelWithOfflinePlanning( + nbr_tensors, metadata_buffer, node_list); + + TfLiteContext context; + constexpr size_t arena_size = 4096; + uint8_t arena[arena_size]; + tflite::MicroAllocator allocator(&context, model, arena, arena_size, + micro_test::reporter); + TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, allocator.FinishTensorAllocation()); + + uint8_t* start = context.tensors[0].data.uint8; + TF_LITE_MICRO_EXPECT_EQ(0, context.tensors[0].data.uint8 - start); + TF_LITE_MICRO_EXPECT_EQ(0, context.tensors[1].data.uint8 - start); + TF_LITE_MICRO_EXPECT_EQ(48, context.tensors[2].data.uint8 - start); + TF_LITE_MICRO_EXPECT_EQ(0, context.tensors[3].data.uint8 - start); + TF_LITE_MICRO_EXPECT_EQ(48, context.tensors[0].bytes); +} + +TF_LITE_MICRO_TEST(OfflinePlannerOfflineOnline) { + int nbr_tensors = 5; + const int32_t metadata_buffer[tflite::testing::kOfflinePlannerHeaderSize + + nbr_tensors] = { + 1, 0, nbr_tensors, // header: version, subgraph, nbr tensors + // memory offsets: + 0, // t0 + 48, // t1 + -1, // t2 + 0, // t3 + -1}; // t4 + + int t0 = 0; + int t1 = 1; + int t2 = 2; + int t3 = 3; + int t4 = 4; + + std::vector node_list = { + { + {t0, t1}, // input, scratch + {t2}, // output + }, + { + {t2}, // input + {t3, t4}, // output1, output2 + }, + }; + + const tflite::Model* model = tflite::testing::GetModelWithOfflinePlanning( + nbr_tensors, metadata_buffer, node_list); + + TfLiteContext context; + constexpr size_t arena_size = 4096; + uint8_t arena[arena_size]; + tflite::MicroAllocator allocator(&context, model, arena, arena_size, + micro_test::reporter); + TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, allocator.FinishTensorAllocation()); + + uint8_t* start = context.tensors[0].data.uint8; + TF_LITE_MICRO_EXPECT_EQ(0, context.tensors[0].data.uint8 - start); + TF_LITE_MICRO_EXPECT_EQ(48, context.tensors[1].data.uint8 - start); + TF_LITE_MICRO_EXPECT_EQ(96, context.tensors[2].data.uint8 - start); + TF_LITE_MICRO_EXPECT_EQ(48, context.tensors[4].data.uint8 - start); + TF_LITE_MICRO_EXPECT_EQ(0, context.tensors[3].data.uint8 - start); +} + TF_LITE_MICRO_TESTS_END diff --git a/tensorflow/lite/micro/test_helpers.cc b/tensorflow/lite/micro/test_helpers.cc index 77a1cc82f3b..b39d3b2916f 100644 --- a/tensorflow/lite/micro/test_helpers.cc +++ b/tensorflow/lite/micro/test_helpers.cc @@ -48,7 +48,7 @@ class StackAllocator : public flatbuffers::Allocator { return *inst; } - static constexpr size_t kStackAllocatorSize = 4096; + static constexpr size_t kStackAllocatorSize = 8192; private: uint8_t data_backing_[kStackAllocatorSize]; @@ -94,6 +94,10 @@ class ModelBuilder { Node AddNode(Operator op, std::initializer_list inputs, std::initializer_list outputs); + void AddMetadata(const char* description_string, + const int32_t* metadata_buffer_data, + size_t num_elements); + // Constructs the flatbuffer model using `builder_` and return a pointer to // it. The returned model has the same lifetime as `builder_`. const Model* BuildModel(std::initializer_list inputs, @@ -116,6 +120,16 @@ class ModelBuilder { static constexpr int kMaxTensors = 50; flatbuffers::Offset tensors_[kMaxTensors]; + + static constexpr int kMaxMetadataBuffers = 10; + + static constexpr int kMaxMetadatas = 10; + flatbuffers::Offset metadata_[kMaxMetadatas]; + + flatbuffers::Offset metadata_buffers_[kMaxMetadataBuffers]; + + int nbr_of_metadata_buffers_ = 0; + int next_tensor_id_ = 0; }; @@ -142,13 +156,34 @@ ModelBuilder::Node ModelBuilder::AddNode( return next_operator_id_ - 1; } +void ModelBuilder::AddMetadata(const char* description_string, + const int32_t* metadata_buffer_data, + size_t num_elements) { + metadata_[ModelBuilder::nbr_of_metadata_buffers_] = + CreateMetadata(*builder_, + builder_->CreateString(description_string), + 1 + ModelBuilder::nbr_of_metadata_buffers_); + + metadata_buffers_[nbr_of_metadata_buffers_] = tflite::CreateBuffer(*builder_, + builder_->CreateVector((uint8_t*)metadata_buffer_data, + sizeof(uint32_t) * num_elements)); + + ModelBuilder::nbr_of_metadata_buffers_++; +} + const Model* ModelBuilder::BuildModel( std::initializer_list inputs, std::initializer_list outputs) { // Model schema requires an empty buffer at idx 0. - constexpr size_t kBufferSize = 1; - const flatbuffers::Offset buffers[kBufferSize] = { - tflite::CreateBuffer(*builder_)}; + size_t kBufferSize = 1 + ModelBuilder::nbr_of_metadata_buffers_; + flatbuffers::Offset buffers[kBufferSize]; + buffers[0] = tflite::CreateBuffer(*builder_); + + // Place the metadata buffers first in the buffer since the indices for them + // have already been set in AddMetadata() + for (int i = 1; i < ModelBuilder::nbr_of_metadata_buffers_ + 1; ++i) { + buffers[i] = metadata_buffers_[i - 1]; + } // TFLM only supports single subgraph. constexpr size_t subgraphs_size = 1; @@ -159,12 +194,26 @@ const Model* ModelBuilder::BuildModel( builder_->CreateVector(outputs.begin(), outputs.size()), builder_->CreateVector(operators_, next_operator_id_), builder_->CreateString("test_subgraph"))}; - const flatbuffers::Offset model_offset = tflite::CreateModel( - *builder_, 0, - builder_->CreateVector(operator_codes_, next_operator_code_id_), - builder_->CreateVector(subgraphs, subgraphs_size), - builder_->CreateString("teset_model"), - builder_->CreateVector(buffers, kBufferSize)); + + flatbuffers::Offset model_offset; + if (ModelBuilder::nbr_of_metadata_buffers_ > 0) { + model_offset = tflite::CreateModel( + *builder_, 0, + builder_->CreateVector(operator_codes_, next_operator_code_id_), + builder_->CreateVector(subgraphs, subgraphs_size), + builder_->CreateString("teset_model"), + builder_->CreateVector(buffers, kBufferSize), + 0, + builder_->CreateVector(metadata_, ModelBuilder::nbr_of_metadata_buffers_)); + } else { + model_offset = tflite::CreateModel( + *builder_, 0, + builder_->CreateVector(operator_codes_, next_operator_code_id_), + builder_->CreateVector(subgraphs, subgraphs_size), + builder_->CreateString("teset_model"), + builder_->CreateVector(buffers, kBufferSize)); + } + tflite::FinishModelBuffer(*builder_, model_offset); void* model_pointer = builder_->GetBufferPointer(); const Model* model = flatbuffers::GetRoot(model_pointer); @@ -243,6 +292,35 @@ const Model* BuildSimpleModelWithBranch() { return model_builder.BuildModel({t0}, {t3}); } +const Model* BuildModelWithOfflinePlanning(int number_of_tensors, + const int32_t* metadata_buffer, + std::vector node_conn) { + using flatbuffers::Offset; + flatbuffers::FlatBufferBuilder* fb_builder = BuilderInstance(); + + ModelBuilder model_builder(fb_builder); + + const int op_id = + model_builder.RegisterOp(BuiltinOperator_CUSTOM, "mock_custom", + /* version= */ 0); + + int tensors[number_of_tensors]; + + for (int i = 0; i < number_of_tensors; ++i) { + tensors[i] = model_builder.AddTensor(TensorType_FLOAT32, {2, 2, 3}); + } + + for (int i = 0; i < node_conn.size(); i++) { + model_builder.AddNode(op_id, node_conn[i].input, node_conn[i].output); + } + + model_builder.AddMetadata("OfflineMemoryAllocation", + metadata_buffer, number_of_tensors + tflite::testing::kOfflinePlannerHeaderSize); + + return model_builder.BuildModel(node_conn[0].input, + node_conn[node_conn.size() - 1].output); +} + const Model* BuildSimpleMockModel() { using flatbuffers::Offset; flatbuffers::FlatBufferBuilder* builder = BuilderInstance(); @@ -496,6 +574,14 @@ const Model* GetSimpleModelWithBranch() { return model; } +const Model* GetModelWithOfflinePlanning(int num_tensors, + const int32_t* metadata_buffer, + std::vector node_conn) { + const Model* model = + BuildModelWithOfflinePlanning(num_tensors, metadata_buffer, node_conn); + return model; +} + const Model* GetSimpleStatefulModel() { static Model* model = nullptr; if (!model) { diff --git a/tensorflow/lite/micro/test_helpers.h b/tensorflow/lite/micro/test_helpers.h index f4e7fa8dfba..26aeeb086ef 100644 --- a/tensorflow/lite/micro/test_helpers.h +++ b/tensorflow/lite/micro/test_helpers.h @@ -27,6 +27,14 @@ limitations under the License. namespace tflite { namespace testing { +constexpr int kOfflinePlannerHeaderSize = 3; + +struct NodeConnection_ { + std::initializer_list input; + std::initializer_list output; +}; +typedef struct NodeConnection_ NodeConnection; + // Returns a simple example flatbuffer TensorFlow Lite model. Contains 1 input, // 1 layer of weights, 1 output Tensor, and 1 operator. const Model* GetSimpleMockModel(); @@ -38,6 +46,11 @@ const Model* GetComplexMockModel(); // Returns a simple flatbuffer model with two branches. const Model* GetSimpleModelWithBranch(); +// Returns a simple flatbuffer model with offline planned tensors +const Model* GetModelWithOfflinePlanning(int num_tensors, + const int32_t* metadata_buffer, + std::vector node_conn); + // Returns a flatbuffer model with `simple_stateful_op` const Model* GetSimpleStatefulModel(); From b8571e365d2e907f66693551b488ce2f72e3507b Mon Sep 17 00:00:00 2001 From: Jens Elofsson Date: Thu, 2 Apr 2020 14:31:17 +0200 Subject: [PATCH 05/13] Fix compile error when building for ARM Cortex-M4 etc. --- tensorflow/lite/micro/micro_allocator.cc | 12 ++++++------ tensorflow/lite/micro/test_helpers.h | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tensorflow/lite/micro/micro_allocator.cc b/tensorflow/lite/micro/micro_allocator.cc index a9324eb35cd..fdfeb9d409c 100644 --- a/tensorflow/lite/micro/micro_allocator.cc +++ b/tensorflow/lite/micro/micro_allocator.cc @@ -37,7 +37,7 @@ struct AllocationInfo { int last_used; bool needs_allocating; void** output_ptr; - int offline_offset; + int32_t offline_offset; }; // We align tensor buffers to 16-byte boundaries, since this is a common @@ -206,10 +206,10 @@ class AllocationInfoBuilder { // - If there's metadata available, offline_planner_offsets will point to the // first offset in the metadata buffer list. TfLiteStatus GetOfflinePlannedOffsets(const Model* model, - int** offline_planner_offsets); + int32_t** offline_planner_offsets); // Add allocaiton information for the tensors. - TfLiteStatus AddTensors(const SubGraph* subgraph, int* offline_offsets, + TfLiteStatus AddTensors(const SubGraph* subgraph, int32_t* offline_offsets, TfLiteTensor* runtime_tensors); // Add allocation information for the scratch buffers. @@ -245,7 +245,7 @@ TfLiteStatus AllocationInfoBuilder::Allocate() { } TfLiteStatus AllocationInfoBuilder::AddTensors(const SubGraph* subgraph, - int* offline_offsets, + int32_t* offline_offsets, TfLiteTensor* runtime_tensors) { // Set up allocation info for all tensors. for (size_t i = 0; i < tensor_count_; ++i) { @@ -337,7 +337,7 @@ TfLiteStatus AllocationInfoBuilder::AddTensors(const SubGraph* subgraph, // | 4 | Arena byte offset of tensor #1 or -1 to allocate at runtime | // | 3+(n-1) | Arena byte offset of tensor #(n-1) or -1 to allocate at runtime | TfLiteStatus AllocationInfoBuilder::GetOfflinePlannedOffsets( - const Model* model, int** offline_planner_offsets) { + const Model* model, int32_t** offline_planner_offsets) { if (model->metadata()) { for (int i = 0; i < model->metadata()->size(); ++i) { auto metadata = model->metadata()->Get(i); @@ -702,7 +702,7 @@ TfLiteStatus MicroAllocator::FinishTensorAllocation() { AllocationInfoBuilder builder(error_reporter_, &tmp_allocator); TF_LITE_ENSURE_STATUS( builder.Init(tensors_->size(), scratch_buffer_count_)); - int* offline_planner_offsets = nullptr; + int32_t* offline_planner_offsets = nullptr; TF_LITE_ENSURE_STATUS( builder.GetOfflinePlannedOffsets(model_, &offline_planner_offsets)); TF_LITE_ENSURE_STATUS(builder.AddTensors(subgraph_, offline_planner_offsets, diff --git a/tensorflow/lite/micro/test_helpers.h b/tensorflow/lite/micro/test_helpers.h index 26aeeb086ef..81416e06dcc 100644 --- a/tensorflow/lite/micro/test_helpers.h +++ b/tensorflow/lite/micro/test_helpers.h @@ -30,8 +30,8 @@ namespace testing { constexpr int kOfflinePlannerHeaderSize = 3; struct NodeConnection_ { - std::initializer_list input; - std::initializer_list output; + std::initializer_list input; + std::initializer_list output; }; typedef struct NodeConnection_ NodeConnection; From e9b965a29cda282bc5dd8b2bcbc7991bb623734e Mon Sep 17 00:00:00 2001 From: Fredrik Knutsson Date: Fri, 24 Apr 2020 15:19:39 +0200 Subject: [PATCH 06/13] Fix review comments, 24/4 --- .../person_detection/person_detection_test.cc | 2 + .../memory_planner/greedy_memory_planner.h | 6 ++- tensorflow/lite/micro/micro_allocator.cc | 52 +++++-------------- .../lite/micro/micro_optional_debug_tools.cc | 31 +++++++++++ .../lite/micro/micro_optional_debug_tools.h | 3 ++ 5 files changed, 54 insertions(+), 40 deletions(-) diff --git a/tensorflow/lite/micro/examples/person_detection/person_detection_test.cc b/tensorflow/lite/micro/examples/person_detection/person_detection_test.cc index 51a61881ead..f57e6ea88f1 100644 --- a/tensorflow/lite/micro/examples/person_detection/person_detection_test.cc +++ b/tensorflow/lite/micro/examples/person_detection/person_detection_test.cc @@ -22,6 +22,7 @@ limitations under the License. #include "tensorflow/lite/micro/micro_error_reporter.h" #include "tensorflow/lite/micro/micro_interpreter.h" #include "tensorflow/lite/micro/micro_mutable_op_resolver.h" +#include "tensorflow/lite/micro/micro_optional_debug_tools.h" #include "tensorflow/lite/micro/testing/micro_test.h" #include "tensorflow/lite/schema/schema_generated.h" #include "tensorflow/lite/version.h" @@ -46,6 +47,7 @@ TF_LITE_MICRO_TEST(TestInvoke) { "to supported version %d.\n", model->version(), TFLITE_SCHEMA_VERSION); } + PrintModelData(model, error_reporter); // Pull in only the operation implementations we need. // This relies on a complete list of all the ops needed by this graph. diff --git a/tensorflow/lite/micro/memory_planner/greedy_memory_planner.h b/tensorflow/lite/micro/memory_planner/greedy_memory_planner.h index c849e57645c..19a36f342fd 100644 --- a/tensorflow/lite/micro/memory_planner/greedy_memory_planner.h +++ b/tensorflow/lite/micro/memory_planner/greedy_memory_planner.h @@ -144,8 +144,10 @@ class GreedyMemoryPlanner : public MemoryPlanner { int* buffer_sizes_sorted_; int* buffer_ids_sorted_; ListEntry* buffers_sorted_by_offset_; - int next_free_entry_; - int first_entry_index_; + int next_free_entry_; // Index of the next free entry of + // buffers_sorted_by_offset_ + int first_entry_index_; // Index of the first entry (smallest offset) of + // buffers_sorted_by_offset_ // Stores the outcome of the plan, the location of each buffer in the arena. int* buffer_offsets_; diff --git a/tensorflow/lite/micro/micro_allocator.cc b/tensorflow/lite/micro/micro_allocator.cc index daa7cbcb0c9..c57294f5745 100644 --- a/tensorflow/lite/micro/micro_allocator.cc +++ b/tensorflow/lite/micro/micro_allocator.cc @@ -45,6 +45,8 @@ struct AllocationInfo { // requirement for SIMD extensions. constexpr int kBufferAlignment = 16; +constexpr char kOfflineMemAllocMetadata[] = "OfflineMemoryAllocation"; + class MicroBuiltinDataAllocator : public BuiltinDataAllocator { public: explicit MicroBuiltinDataAllocator(SimpleMemoryAllocator* memory_allocator) @@ -81,33 +83,6 @@ TfLiteStatus AllocateVariables( return kTfLiteOk; } -// Helper function to print model flatbuffer data. This function is not called -// by default. Hence it's not linked in to the final binary code. -void PrintModelData(const Model* model, ErrorReporter* error_reporter) { - auto* subgraphs = model->subgraphs(); - const SubGraph* subgraph = (*subgraphs)[0]; - const flatbuffers::Vector>* tensors = - subgraph->tensors(); - const flatbuffers::Vector>* buffers = - model->buffers(); - TF_LITE_REPORT_ERROR(error_reporter, "==== Model info: ====="); - for (int i = 0; i < tensors->size(); ++i) { - const tflite::Tensor& flatbuffer_tensor = *tensors->Get(i); - auto* quantization = flatbuffer_tensor.quantization(); - size_t type_size, tensor_size; - auto* buffer = (*buffers)[flatbuffer_tensor.buffer()]; - auto* array = buffer->data(); - int array_size = 0; - if (array) { - array_size = array->size(); - } - BytesRequiredForTensor(flatbuffer_tensor, &tensor_size, &type_size, - error_reporter); - TF_LITE_REPORT_ERROR( - error_reporter, "Tensor index: %d arena tensor %d size %d", i, - !array_size && !flatbuffer_tensor.is_variable(), tensor_size); - } -} // Helper function to check flatbuffer metadata correctness. This function is // not called by default. Hence it's not linked in to the final binary code. @@ -116,8 +91,8 @@ TfLiteStatus CheckOfflinePlannedOffsets(const Model* model, if (model->metadata()) { for (int i = 0; i < model->metadata()->size(); ++i) { auto metadata = model->metadata()->Get(i); - if (strncmp(metadata->name()->c_str(), "OfflineMemoryAllocation", - strlen("OfflineMemoryAllocation")) == 0) { + if (strncmp(metadata->name()->c_str(), kOfflineMemAllocMetadata, + strlen(kOfflineMemAllocMetadata)) == 0) { auto* subgraphs = model->subgraphs(); const SubGraph* subgraph = (*subgraphs)[0]; const flatbuffers::Vector>* tensors = @@ -311,8 +286,9 @@ TfLiteStatus AllocationInfoBuilder::AddTensors(const SubGraph* subgraph, // | name:string | “OfflineMemoryAllocation” | // | buffer:unit | Index of buffer containing memory allocation data | // -// The buffer contents for the memory allocation is a list of 32-bit integers of -// the following format: +// The buffer contents for the memory allocation is a list of 32-bit integers. +// The number of tensors, n, must be equal to the number of tensors defined in +// the model. The following encoding applies: // // | Offset | Value | // | 0 | Offline allocation format version – set to 0 | @@ -326,8 +302,8 @@ TfLiteStatus AllocationInfoBuilder::GetOfflinePlannedOffsets( if (model->metadata()) { for (int i = 0; i < model->metadata()->size(); ++i) { auto metadata = model->metadata()->Get(i); - if (strncmp(metadata->name()->c_str(), "OfflineMemoryAllocation", - strlen("OfflineMemoryAllocation")) == 0) { + if (strncmp(metadata->name()->c_str(), kOfflineMemAllocMetadata, + strlen(kOfflineMemAllocMetadata)) == 0) { const flatbuffers::Vector>* buffers = model->buffers(); auto* buffer = (*buffers)[metadata->buffer()]; @@ -365,7 +341,8 @@ TfLiteStatus AllocationInfoBuilder::AddScratchBuffers( return kTfLiteOk; } -TfLiteStatus CreatePlan(ErrorReporter* error_reporter, MemoryPlanner* planner, +TfLiteStatus CreatePlan(ErrorReporter* error_reporter, + GreedyMemoryPlanner* planner, const AllocationInfo* allocation_info, size_t allocation_info_size) { // Add the tensors to our allocation plan. @@ -380,10 +357,9 @@ TfLiteStatus CreatePlan(ErrorReporter* error_reporter, MemoryPlanner* planner, current->first_created, current->last_used)); } else { TF_LITE_ENSURE_STATUS( - (static_cast(planner)) - ->AddBuffer(error_reporter, aligned_bytes_required, - current->first_created, current->last_used, - current->offline_offset)); + planner->AddBuffer(error_reporter, aligned_bytes_required, + current->first_created, current->last_used, + current->offline_offset)); } } } diff --git a/tensorflow/lite/micro/micro_optional_debug_tools.cc b/tensorflow/lite/micro/micro_optional_debug_tools.cc index 70f16c78d79..10373d3c034 100644 --- a/tensorflow/lite/micro/micro_optional_debug_tools.cc +++ b/tensorflow/lite/micro/micro_optional_debug_tools.cc @@ -22,6 +22,8 @@ limitations under the License. #include #include "tensorflow/lite/schema/schema_generated.h" +#include "tensorflow/lite/micro/memory_helpers.h" + namespace tflite { namespace { @@ -100,6 +102,35 @@ const char* AllocTypeName(TfLiteAllocationType type) { } } // namespace +// Helper function to print model flatbuffer data. This function is not called +// by default. Hence it's not linked in to the final binary code. +void PrintModelData(const Model* model, ErrorReporter* error_reporter) { + auto* subgraphs = model->subgraphs(); + const SubGraph* subgraph = (*subgraphs)[0]; + const flatbuffers::Vector>* tensors = + subgraph->tensors(); + const flatbuffers::Vector>* buffers = + model->buffers(); + TF_LITE_REPORT_ERROR(error_reporter, "==== Model info: ====="); + for (int i = 0; i < tensors->size(); ++i) { + const tflite::Tensor& flatbuffer_tensor = *tensors->Get(i); + auto* quantization = flatbuffer_tensor.quantization(); + size_t type_size, tensor_size; + auto* buffer = (*buffers)[flatbuffer_tensor.buffer()]; + auto* array = buffer->data(); + int array_size = 0; + if (array) { + array_size = array->size(); + } + BytesRequiredForTensor(flatbuffer_tensor, &tensor_size, &type_size, + error_reporter); + TF_LITE_REPORT_ERROR( + error_reporter, + "Tensor index: %d arena tensor %d size %d ", + i, !array_size && !flatbuffer_tensor.is_variable(), tensor_size); + } +} + // Prints a dump of what tensors and what nodes are in the interpreter. void PrintInterpreterState(MicroInterpreter* interpreter) { printf("Interpreter has %zu tensors and %zu nodes\n", diff --git a/tensorflow/lite/micro/micro_optional_debug_tools.h b/tensorflow/lite/micro/micro_optional_debug_tools.h index ae96b62ab3c..cc9630e6f12 100644 --- a/tensorflow/lite/micro/micro_optional_debug_tools.h +++ b/tensorflow/lite/micro/micro_optional_debug_tools.h @@ -20,6 +20,9 @@ limitations under the License. #include "tensorflow/lite/micro/micro_interpreter.h" namespace tflite { +// Helper function to print model flatbuffer data. This function is not called +// by default. Hence it's not linked in to the final binary code. +void PrintModelData(const Model* model, ErrorReporter* error_reporter); // Prints a dump of what tensors and what nodes are in the interpreter. void PrintInterpreterState(MicroInterpreter* interpreter); } // namespace tflite From 7d3237ca0951e102dfcc04f5bd98e0bc1fa1e22c Mon Sep 17 00:00:00 2001 From: Jens Elofsson Date: Mon, 27 Apr 2020 13:16:49 +0200 Subject: [PATCH 07/13] Address reviewer comments. --- tensorflow/lite/micro/micro_allocator_test.cc | 65 ++++++++++--------- tensorflow/lite/micro/test_helpers.cc | 56 ++++++++-------- tensorflow/lite/micro/test_helpers.h | 3 +- 3 files changed, 64 insertions(+), 60 deletions(-) diff --git a/tensorflow/lite/micro/micro_allocator_test.cc b/tensorflow/lite/micro/micro_allocator_test.cc index 45bc3b06b24..b5db8fdd626 100644 --- a/tensorflow/lite/micro/micro_allocator_test.cc +++ b/tensorflow/lite/micro/micro_allocator_test.cc @@ -247,22 +247,22 @@ TF_LITE_MICRO_TEST(OfflinePlannerBranchesAllOnline) { // The structure is identical to the one in // TestAllocationForModelsWithBranches - std::vector node_list = { - { - {0}, // input - {1} // output - }, - { - {0}, // input - {2} // output - }, - { - {1, 2}, // input1, input2 - {3} // output - }}; + int num_conns = 3; + tflite::testing::NodeConnection node_list[3] = {{ + {0}, // input + {1} // output + }, + { + {0}, // input + {2} // output + }, + { + {1, 2}, // input1, input2 + {3} // output + }}; const tflite::Model* model = tflite::testing::GetModelWithOfflinePlanning( - nbr_tensors, metadata_buffer, node_list); + nbr_tensors, metadata_buffer, node_list, num_conns); TfLiteContext context; constexpr size_t arena_size = 4096; @@ -296,21 +296,22 @@ TF_LITE_MICRO_TEST(OfflinePlannerBasic) { int t2 = 2; int t3 = 3; - std::vector node_list = {{ - {t0}, // input - {t1} // output - }, - { - {t1}, // input - {t2} // output - }, - { - {t2}, // input - {t3} // output - }}; + int num_conns = 3; + tflite::testing::NodeConnection node_list[3] = {{ + {t0}, // input + {t1} // output + }, + { + {t1}, // input + {t2} // output + }, + { + {t2}, // input + {t3} // output + }}; const tflite::Model* model = tflite::testing::GetModelWithOfflinePlanning( - nbr_tensors, metadata_buffer, node_list); + nbr_tensors, metadata_buffer, node_list, num_conns); TfLiteContext context; constexpr size_t arena_size = 4096; @@ -342,7 +343,8 @@ TF_LITE_MICRO_TEST(OfflinePlannerOverlappingAllocation) { int t2 = 2; int t3 = 3; - std::vector node_list = { + int num_conns = 2; + tflite::testing::NodeConnection node_list[2] = { { {t0, t1}, // input, scratch {t2} // output @@ -354,7 +356,7 @@ TF_LITE_MICRO_TEST(OfflinePlannerOverlappingAllocation) { }; const tflite::Model* model = tflite::testing::GetModelWithOfflinePlanning( - nbr_tensors, metadata_buffer, node_list); + nbr_tensors, metadata_buffer, node_list, num_conns); TfLiteContext context; constexpr size_t arena_size = 4096; @@ -389,7 +391,8 @@ TF_LITE_MICRO_TEST(OfflinePlannerOfflineOnline) { int t3 = 3; int t4 = 4; - std::vector node_list = { + int num_conns = 2; + tflite::testing::NodeConnection node_list[2] = { { {t0, t1}, // input, scratch {t2}, // output @@ -401,7 +404,7 @@ TF_LITE_MICRO_TEST(OfflinePlannerOfflineOnline) { }; const tflite::Model* model = tflite::testing::GetModelWithOfflinePlanning( - nbr_tensors, metadata_buffer, node_list); + nbr_tensors, metadata_buffer, node_list, num_conns); TfLiteContext context; constexpr size_t arena_size = 4096; diff --git a/tensorflow/lite/micro/test_helpers.cc b/tensorflow/lite/micro/test_helpers.cc index b39d3b2916f..f52ebdc4d45 100644 --- a/tensorflow/lite/micro/test_helpers.cc +++ b/tensorflow/lite/micro/test_helpers.cc @@ -95,8 +95,7 @@ class ModelBuilder { std::initializer_list outputs); void AddMetadata(const char* description_string, - const int32_t* metadata_buffer_data, - size_t num_elements); + const int32_t* metadata_buffer_data, size_t num_elements); // Constructs the flatbuffer model using `builder_` and return a pointer to // it. The returned model has the same lifetime as `builder_`. @@ -157,16 +156,15 @@ ModelBuilder::Node ModelBuilder::AddNode( } void ModelBuilder::AddMetadata(const char* description_string, - const int32_t* metadata_buffer_data, - size_t num_elements) { + const int32_t* metadata_buffer_data, + size_t num_elements) { metadata_[ModelBuilder::nbr_of_metadata_buffers_] = - CreateMetadata(*builder_, - builder_->CreateString(description_string), - 1 + ModelBuilder::nbr_of_metadata_buffers_); + CreateMetadata(*builder_, builder_->CreateString(description_string), + 1 + ModelBuilder::nbr_of_metadata_buffers_); - metadata_buffers_[nbr_of_metadata_buffers_] = tflite::CreateBuffer(*builder_, - builder_->CreateVector((uint8_t*)metadata_buffer_data, - sizeof(uint32_t) * num_elements)); + metadata_buffers_[nbr_of_metadata_buffers_] = tflite::CreateBuffer( + *builder_, builder_->CreateVector((uint8_t*)metadata_buffer_data, + sizeof(uint32_t) * num_elements)); ModelBuilder::nbr_of_metadata_buffers_++; } @@ -175,14 +173,14 @@ const Model* ModelBuilder::BuildModel( std::initializer_list inputs, std::initializer_list outputs) { // Model schema requires an empty buffer at idx 0. - size_t kBufferSize = 1 + ModelBuilder::nbr_of_metadata_buffers_; - flatbuffers::Offset buffers[kBufferSize]; + size_t buffer_size = 1 + ModelBuilder::nbr_of_metadata_buffers_; + flatbuffers::Offset buffers[kMaxMetadataBuffers]; buffers[0] = tflite::CreateBuffer(*builder_); // Place the metadata buffers first in the buffer since the indices for them // have already been set in AddMetadata() for (int i = 1; i < ModelBuilder::nbr_of_metadata_buffers_ + 1; ++i) { - buffers[i] = metadata_buffers_[i - 1]; + buffers[i] = metadata_buffers_[i - 1]; } // TFLM only supports single subgraph. @@ -202,16 +200,16 @@ const Model* ModelBuilder::BuildModel( builder_->CreateVector(operator_codes_, next_operator_code_id_), builder_->CreateVector(subgraphs, subgraphs_size), builder_->CreateString("teset_model"), - builder_->CreateVector(buffers, kBufferSize), - 0, - builder_->CreateVector(metadata_, ModelBuilder::nbr_of_metadata_buffers_)); + builder_->CreateVector(buffers, buffer_size), 0, + builder_->CreateVector(metadata_, + ModelBuilder::nbr_of_metadata_buffers_)); } else { model_offset = tflite::CreateModel( *builder_, 0, builder_->CreateVector(operator_codes_, next_operator_code_id_), builder_->CreateVector(subgraphs, subgraphs_size), builder_->CreateString("teset_model"), - builder_->CreateVector(buffers, kBufferSize)); + builder_->CreateVector(buffers, buffer_size)); } tflite::FinishModelBuffer(*builder_, model_offset); @@ -293,8 +291,9 @@ const Model* BuildSimpleModelWithBranch() { } const Model* BuildModelWithOfflinePlanning(int number_of_tensors, - const int32_t* metadata_buffer, - std::vector node_conn) { + const int32_t* metadata_buffer, + NodeConnection* node_conn, + int num_conns) { using flatbuffers::Offset; flatbuffers::FlatBufferBuilder* fb_builder = BuilderInstance(); @@ -310,15 +309,16 @@ const Model* BuildModelWithOfflinePlanning(int number_of_tensors, tensors[i] = model_builder.AddTensor(TensorType_FLOAT32, {2, 2, 3}); } - for (int i = 0; i < node_conn.size(); i++) { + for (int i = 0; i < num_conns; ++i) { model_builder.AddNode(op_id, node_conn[i].input, node_conn[i].output); } - model_builder.AddMetadata("OfflineMemoryAllocation", - metadata_buffer, number_of_tensors + tflite::testing::kOfflinePlannerHeaderSize); + model_builder.AddMetadata( + "OfflineMemoryAllocation", metadata_buffer, + number_of_tensors + tflite::testing::kOfflinePlannerHeaderSize); return model_builder.BuildModel(node_conn[0].input, - node_conn[node_conn.size() - 1].output); + node_conn[num_conns - 1].output); } const Model* BuildSimpleMockModel() { @@ -408,8 +408,7 @@ const Model* BuildComplexMockModel() { constexpr size_t buffers_size = 7; const Offset buffers[buffers_size] = { // Op 1 buffers: - CreateBuffer(*builder), - CreateBuffer(*builder), + CreateBuffer(*builder), CreateBuffer(*builder), CreateBuffer(*builder, builder->CreateVector(buffer_data_1, buffer_data_size)), // Op 2 buffers: @@ -576,9 +575,10 @@ const Model* GetSimpleModelWithBranch() { const Model* GetModelWithOfflinePlanning(int num_tensors, const int32_t* metadata_buffer, - std::vector node_conn) { - const Model* model = - BuildModelWithOfflinePlanning(num_tensors, metadata_buffer, node_conn); + NodeConnection* node_conn, + int num_conns) { + const Model* model = BuildModelWithOfflinePlanning( + num_tensors, metadata_buffer, node_conn, num_conns); return model; } diff --git a/tensorflow/lite/micro/test_helpers.h b/tensorflow/lite/micro/test_helpers.h index e31f5061de8..647ffb92cff 100644 --- a/tensorflow/lite/micro/test_helpers.h +++ b/tensorflow/lite/micro/test_helpers.h @@ -49,7 +49,8 @@ const Model* GetSimpleModelWithBranch(); // Returns a simple flatbuffer model with offline planned tensors const Model* GetModelWithOfflinePlanning(int num_tensors, const int32_t* metadata_buffer, - std::vector node_conn); + NodeConnection* node_conn, + int num_conns); // Returns a flatbuffer model with `simple_stateful_op` const Model* GetSimpleStatefulModel(); From 28f2af10ecdde4ab8e24247a728032ea1891d730 Mon Sep 17 00:00:00 2001 From: Jens Elofsson Date: Tue, 12 May 2020 17:41:28 +0200 Subject: [PATCH 08/13] Realign AllocationInfo struct. After adding offline_offset, sizeof(AllocationInfo) = 40, which caused hello_world_test to crash. After realigning it's back to its original size (32). --- tensorflow/lite/micro/micro_allocator.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/lite/micro/micro_allocator.cc b/tensorflow/lite/micro/micro_allocator.cc index f1c1d65f1cc..5ffda9209d9 100644 --- a/tensorflow/lite/micro/micro_allocator.cc +++ b/tensorflow/lite/micro/micro_allocator.cc @@ -34,11 +34,11 @@ namespace { // Used to hold information used during allocation calculations. struct AllocationInfo { size_t bytes; + void** output_ptr; int first_created; int last_used; - bool needs_allocating; - void** output_ptr; int32_t offline_offset; + bool needs_allocating; }; // We align tensor buffers to 16-byte boundaries, since this is a common From 18aa35c75fcf64dfd99e2a3e7cdcd62bafbc030f Mon Sep 17 00:00:00 2001 From: Fredrik Knutsson Date: Wed, 27 May 2020 19:18:06 +0200 Subject: [PATCH 09/13] Fixed review comments 27/5 --- .../micro/memory_planner/greedy_memory_planner.cc | 2 +- tensorflow/lite/micro/micro_allocator.cc | 13 +++++++------ tensorflow/lite/micro/micro_optional_debug_tools.cc | 3 +-- tensorflow/lite/micro/test_helpers.cc | 4 +--- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/tensorflow/lite/micro/memory_planner/greedy_memory_planner.cc b/tensorflow/lite/micro/memory_planner/greedy_memory_planner.cc index 47bb7cfb8c0..8f21a167f67 100644 --- a/tensorflow/lite/micro/memory_planner/greedy_memory_planner.cc +++ b/tensorflow/lite/micro/memory_planner/greedy_memory_planner.cc @@ -292,7 +292,7 @@ size_t GreedyMemoryPlanner::GetMaximumMemorySize() { if (buffer_count_ == 0) { return 0; } - ListEntry* entry = &buffers_sorted_by_offset_[0]; + ListEntry* entry = &buffers_sorted_by_offset_[first_entry_index_]; size_t max_size = 0; while (entry) { BufferRequirements* requirements = diff --git a/tensorflow/lite/micro/micro_allocator.cc b/tensorflow/lite/micro/micro_allocator.cc index 9c4de6e3035..870e466b4e4 100644 --- a/tensorflow/lite/micro/micro_allocator.cc +++ b/tensorflow/lite/micro/micro_allocator.cc @@ -91,7 +91,7 @@ TfLiteStatus AllocateVariables( TfLiteStatus CheckOfflinePlannedOffsets(const Model* model, ErrorReporter* error_reporter) { if (model->metadata()) { - for (int i = 0; i < model->metadata()->size(); ++i) { + for (size_t i = 0; i < model->metadata()->size(); ++i) { auto metadata = model->metadata()->Get(i); if (strncmp(metadata->name()->c_str(), kOfflineMemAllocMetadata, strlen(kOfflineMemAllocMetadata)) == 0) { @@ -115,11 +115,11 @@ TfLiteStatus CheckOfflinePlannedOffsets(const Model* model, "Offline planner metadata found, version %d, " "subgraph %d, nbr offline offsets %d", version, subgraph_idx, nbr_offline_offsets); - for (int i = 0; i < nbr_offline_offsets; ++i) { + for (int j = 0; j < nbr_offline_offsets; ++j) { TF_LITE_REPORT_ERROR( error_reporter, - "Offline planner tensor index %d, offline offset: %d", i, - offline_planner_offsets[i]); + "Offline planner tensor index %d, offline offset: %d", j, + offline_planner_offsets[j]); } if (version != 1) { @@ -302,7 +302,7 @@ TfLiteStatus AllocationInfoBuilder::AddTensors(const SubGraph* subgraph, TfLiteStatus AllocationInfoBuilder::GetOfflinePlannedOffsets( const Model* model, int32_t** offline_planner_offsets) { if (model->metadata()) { - for (int i = 0; i < model->metadata()->size(); ++i) { + for (size_t i = 0; i < model->metadata()->size(); ++i) { auto metadata = model->metadata()->Get(i); if (strncmp(metadata->name()->c_str(), kOfflineMemAllocMetadata, strlen(kOfflineMemAllocMetadata)) == 0) { @@ -311,7 +311,7 @@ TfLiteStatus AllocationInfoBuilder::GetOfflinePlannedOffsets( auto* buffer = (*buffers)[metadata->buffer()]; auto* array = buffer->data(); const uint32_t* metadata_buffer = (uint32_t*)array->data(); - const int32_t nbr_tensors = metadata_buffer[2]; + const size_t nbr_tensors = (size_t)metadata_buffer[2]; *offline_planner_offsets = (int32_t*)&metadata_buffer[3]; if (tensor_count_ != nbr_tensors) { @@ -339,6 +339,7 @@ TfLiteStatus AllocationInfoBuilder::AddScratchBuffers( current->first_created = handle->node_idx; current->last_used = handle->node_idx; current->needs_allocating = true; + current->offline_offset = kOnlinePlannedBuffer; } return kTfLiteOk; } diff --git a/tensorflow/lite/micro/micro_optional_debug_tools.cc b/tensorflow/lite/micro/micro_optional_debug_tools.cc index 418347a5b25..22b170094d5 100644 --- a/tensorflow/lite/micro/micro_optional_debug_tools.cc +++ b/tensorflow/lite/micro/micro_optional_debug_tools.cc @@ -122,9 +122,8 @@ void PrintModelData(const Model* model, ErrorReporter* error_reporter) { const flatbuffers::Vector>* buffers = model->buffers(); TF_LITE_REPORT_ERROR(error_reporter, "==== Model info: ====="); - for (int i = 0; i < tensors->size(); ++i) { + for (size_t i = 0; i < tensors->size(); ++i) { const tflite::Tensor& flatbuffer_tensor = *tensors->Get(i); - auto* quantization = flatbuffer_tensor.quantization(); size_t type_size, tensor_size; auto* buffer = (*buffers)[flatbuffer_tensor.buffer()]; auto* array = buffer->data(); diff --git a/tensorflow/lite/micro/test_helpers.cc b/tensorflow/lite/micro/test_helpers.cc index b60d3065020..96e000b1b6d 100644 --- a/tensorflow/lite/micro/test_helpers.cc +++ b/tensorflow/lite/micro/test_helpers.cc @@ -308,10 +308,8 @@ const Model* BuildModelWithOfflinePlanning(int number_of_tensors, model_builder.RegisterOp(BuiltinOperator_CUSTOM, "mock_custom", /* version= */ 0); - int tensors[number_of_tensors]; - for (int i = 0; i < number_of_tensors; ++i) { - tensors[i] = model_builder.AddTensor(TensorType_FLOAT32, {2, 2, 3}); + model_builder.AddTensor(TensorType_FLOAT32, {2, 2, 3}); } for (int i = 0; i < num_conns; ++i) { From b1e74b227c681588a62768042816702a9518f642 Mon Sep 17 00:00:00 2001 From: Jens Elofsson Date: Thu, 4 Jun 2020 09:22:37 +0200 Subject: [PATCH 10/13] Fix compile errors. --- tensorflow/lite/micro/memory_planner/greedy_memory_planner.cc | 2 +- tensorflow/lite/micro/micro_allocator.cc | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tensorflow/lite/micro/memory_planner/greedy_memory_planner.cc b/tensorflow/lite/micro/memory_planner/greedy_memory_planner.cc index 8f21a167f67..39991ab758b 100644 --- a/tensorflow/lite/micro/memory_planner/greedy_memory_planner.cc +++ b/tensorflow/lite/micro/memory_planner/greedy_memory_planner.cc @@ -191,7 +191,7 @@ void GreedyMemoryPlanner::CalculateOffsetsIfNeeded() { // Work through the rest of the buffers to find a good gap to place each one. for (int i = 1; i < buffer_count_; ++i) { // The id is the order the buffer was originally added by the client. - const int buffer_id = buffer_ids_sorted_[i]; + buffer_id = buffer_ids_sorted_[i]; // Look at what size and time range the buffer needs to be active. BufferRequirements* wanted_requirements = &requirements_[buffer_id]; const int wanted_size = wanted_requirements->size; diff --git a/tensorflow/lite/micro/micro_allocator.cc b/tensorflow/lite/micro/micro_allocator.cc index c204f4460b4..8fac421750d 100644 --- a/tensorflow/lite/micro/micro_allocator.cc +++ b/tensorflow/lite/micro/micro_allocator.cc @@ -99,6 +99,9 @@ TfLiteStatus AllocateVariables( // not called by default. Hence it's not linked in to the final binary code. TfLiteStatus CheckOfflinePlannedOffsets(const Model* model, ErrorReporter* error_reporter) { + // Suppress compile warning for unused function + (void)CheckOfflinePlannedOffsets; + if (model->metadata()) { for (size_t i = 0; i < model->metadata()->size(); ++i) { auto metadata = model->metadata()->Get(i); From bacc5e5927360ad119cdb9967d53df4c95c53fd4 Mon Sep 17 00:00:00 2001 From: Jens Elofsson Date: Mon, 8 Jun 2020 14:38:12 +0200 Subject: [PATCH 11/13] Fix build issues. --- tensorflow/lite/micro/micro_allocator.cc | 3 ++- tensorflow/lite/micro/micro_allocator_test.cc | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/tensorflow/lite/micro/micro_allocator.cc b/tensorflow/lite/micro/micro_allocator.cc index 8fac421750d..18c82bca57d 100644 --- a/tensorflow/lite/micro/micro_allocator.cc +++ b/tensorflow/lite/micro/micro_allocator.cc @@ -94,7 +94,7 @@ TfLiteStatus AllocateVariables( return kTfLiteOk; } - +#if !defined(__clang__) // Helper function to check flatbuffer metadata correctness. This function is // not called by default. Hence it's not linked in to the final binary code. TfLiteStatus CheckOfflinePlannedOffsets(const Model* model, @@ -157,6 +157,7 @@ TfLiteStatus CheckOfflinePlannedOffsets(const Model* model, } return kTfLiteOk; } +#endif // A helper class to construct AllocationInfo array. This array contains the // lifetime of tensors / scratch_buffer and will be used to calculate the memory diff --git a/tensorflow/lite/micro/micro_allocator_test.cc b/tensorflow/lite/micro/micro_allocator_test.cc index 67052ce12d9..ac581305340 100644 --- a/tensorflow/lite/micro/micro_allocator_test.cc +++ b/tensorflow/lite/micro/micro_allocator_test.cc @@ -238,7 +238,7 @@ TF_LITE_MICRO_TEST(TestFinishComplexTensorAllocation) { TF_LITE_MICRO_TEST(OfflinePlannerBranchesAllOnline) { int version = 1; int subgraph = 0; - int nbr_tensors = 4; + constexpr int nbr_tensors = 4; const int32_t metadata_buffer[tflite::testing::kOfflinePlannerHeaderSize + nbr_tensors] = {version, subgraph, nbr_tensors, // header @@ -283,7 +283,7 @@ TF_LITE_MICRO_TEST(OfflinePlannerBranchesAllOnline) { } TF_LITE_MICRO_TEST(OfflinePlannerBasic) { - int nbr_tensors = 4; + constexpr int nbr_tensors = 4; const int32_t metadata_buffer[tflite::testing::kOfflinePlannerHeaderSize + nbr_tensors] = {1, 0, nbr_tensors, 0, // t0 @@ -328,7 +328,7 @@ TF_LITE_MICRO_TEST(OfflinePlannerBasic) { } TF_LITE_MICRO_TEST(OfflinePlannerOverlappingAllocation) { - int nbr_tensors = 4; + constexpr int nbr_tensors = 4; const int32_t metadata_buffer[tflite::testing::kOfflinePlannerHeaderSize + nbr_tensors] = { 1, 0, nbr_tensors, // header: version, subgraph, nbr tensors @@ -374,7 +374,7 @@ TF_LITE_MICRO_TEST(OfflinePlannerOverlappingAllocation) { } TF_LITE_MICRO_TEST(OfflinePlannerOfflineOnline) { - int nbr_tensors = 5; + constexpr int nbr_tensors = 5; const int32_t metadata_buffer[tflite::testing::kOfflinePlannerHeaderSize + nbr_tensors] = { 1, 0, nbr_tensors, // header: version, subgraph, nbr tensors From 77e5fb550de9aeed3a3454de06f1a7571d5e5ba3 Mon Sep 17 00:00:00 2001 From: Jens Elofsson Date: Wed, 10 Jun 2020 10:47:54 +0200 Subject: [PATCH 12/13] Fix compile errors by using the new MicroAllocator::Create() --- tensorflow/lite/micro/micro_allocator_test.cc | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tensorflow/lite/micro/micro_allocator_test.cc b/tensorflow/lite/micro/micro_allocator_test.cc index a71115dc67b..f52742ca723 100644 --- a/tensorflow/lite/micro/micro_allocator_test.cc +++ b/tensorflow/lite/micro/micro_allocator_test.cc @@ -288,9 +288,9 @@ TF_LITE_MICRO_TEST(OfflinePlannerBranchesAllOnline) { TfLiteContext context; constexpr size_t arena_size = 4096; uint8_t arena[arena_size]; - tflite::MicroAllocator allocator(&context, model, arena, arena_size, - micro_test::reporter); - TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, allocator.FinishTensorAllocation()); + tflite::MicroAllocator* allocator = tflite::MicroAllocator::Create( + &context, model, arena, arena_size, micro_test::reporter); + TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, allocator->FinishTensorAllocation()); // Since all of the tensors are online planned and the model structure is // identical to that in TestAllocationForModelsWithBranches, @@ -337,9 +337,9 @@ TF_LITE_MICRO_TEST(OfflinePlannerBasic) { TfLiteContext context; constexpr size_t arena_size = 4096; uint8_t arena[arena_size]; - tflite::MicroAllocator allocator(&context, model, arena, arena_size, - micro_test::reporter); - TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, allocator.FinishTensorAllocation()); + tflite::MicroAllocator* allocator = tflite::MicroAllocator::Create( + &context, model, arena, arena_size, micro_test::reporter); + TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, allocator->FinishTensorAllocation()); uint8_t* start = context.tensors[0].data.uint8; TF_LITE_MICRO_EXPECT_EQ(0, context.tensors[0].data.uint8 - start); @@ -382,9 +382,9 @@ TF_LITE_MICRO_TEST(OfflinePlannerOverlappingAllocation) { TfLiteContext context; constexpr size_t arena_size = 4096; uint8_t arena[arena_size]; - tflite::MicroAllocator allocator(&context, model, arena, arena_size, - micro_test::reporter); - TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, allocator.FinishTensorAllocation()); + tflite::MicroAllocator* allocator = tflite::MicroAllocator::Create( + &context, model, arena, arena_size, micro_test::reporter); + TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, allocator->FinishTensorAllocation()); uint8_t* start = context.tensors[0].data.uint8; TF_LITE_MICRO_EXPECT_EQ(0, context.tensors[0].data.uint8 - start); @@ -430,9 +430,9 @@ TF_LITE_MICRO_TEST(OfflinePlannerOfflineOnline) { TfLiteContext context; constexpr size_t arena_size = 4096; uint8_t arena[arena_size]; - tflite::MicroAllocator allocator(&context, model, arena, arena_size, - micro_test::reporter); - TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, allocator.FinishTensorAllocation()); + tflite::MicroAllocator* allocator = tflite::MicroAllocator::Create( + &context, model, arena, arena_size, micro_test::reporter); + TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, allocator->FinishTensorAllocation()); uint8_t* start = context.tensors[0].data.uint8; TF_LITE_MICRO_EXPECT_EQ(0, context.tensors[0].data.uint8 - start); From 2f9642602d9d3da721d603e583b2569c765704a1 Mon Sep 17 00:00:00 2001 From: Jens Elofsson Date: Fri, 12 Jun 2020 10:49:57 +0200 Subject: [PATCH 13/13] Adapt to changes in micro_allocator. --- tensorflow/lite/micro/micro_allocator.cc | 20 ++++--- tensorflow/lite/micro/micro_allocator.h | 3 +- tensorflow/lite/micro/micro_allocator_test.cc | 52 ++++++++++++++----- 3 files changed, 55 insertions(+), 20 deletions(-) diff --git a/tensorflow/lite/micro/micro_allocator.cc b/tensorflow/lite/micro/micro_allocator.cc index 7cd40e54435..16b3b986a52 100644 --- a/tensorflow/lite/micro/micro_allocator.cc +++ b/tensorflow/lite/micro/micro_allocator.cc @@ -375,10 +375,9 @@ TfLiteStatus CreatePlan(ErrorReporter* error_reporter, planner->AddBuffer(error_reporter, aligned_bytes_required, current->first_created, current->last_used)); } else { - TF_LITE_ENSURE_STATUS( - planner->AddBuffer(error_reporter, aligned_bytes_required, - current->first_created, current->last_used, - current->offline_offset)); + TF_LITE_ENSURE_STATUS(planner->AddBuffer( + error_reporter, aligned_bytes_required, current->first_created, + current->last_used, current->offline_offset)); } } } @@ -647,7 +646,7 @@ TfLiteStatus MicroAllocator::FinishModelAllocation(const Model* model, const SubGraph* subgraph = GetSubGraphFromModel(model); TFLITE_DCHECK(subgraph != nullptr); - TF_LITE_ENSURE_STATUS(CommitStaticMemoryPlan(subgraph, context)); + TF_LITE_ENSURE_STATUS(CommitStaticMemoryPlan(model, subgraph, context)); TF_LITE_ENSURE_STATUS(AllocateVariables(subgraph->tensors(), context->tensors, memory_allocator_)); @@ -874,7 +873,8 @@ const SubGraph* MicroAllocator::GetSubGraphFromModel(const Model* model) { return (*subgraphs)[0]; } -TfLiteStatus MicroAllocator::CommitStaticMemoryPlan(const SubGraph* subgraph, +TfLiteStatus MicroAllocator::CommitStaticMemoryPlan(const Model* model, + const SubGraph* subgraph, TfLiteContext* context) { // Create static memory plan // 1. Calculate AllocationInfo to know the lifetime of each tensor/buffer. @@ -891,7 +891,13 @@ TfLiteStatus MicroAllocator::CommitStaticMemoryPlan(const SubGraph* subgraph, AllocationInfoBuilder builder(error_reporter_, &tmp_allocator); TF_LITE_ENSURE_STATUS( builder.Init(subgraph->tensors()->size(), scratch_buffer_count_)); - TF_LITE_ENSURE_STATUS(builder.AddTensors(subgraph, context->tensors)); + + int32_t* offline_planner_offsets = nullptr; + TF_LITE_ENSURE_STATUS( + builder.GetOfflinePlannedOffsets(model, &offline_planner_offsets)); + TF_LITE_ENSURE_STATUS(builder.AddTensors(subgraph, offline_planner_offsets, + context->tensors)); + TF_LITE_ENSURE_STATUS(builder.AddScratchBuffers(scratch_buffer_handles_)); const AllocationInfo* allocation_info = builder.Finish(); diff --git a/tensorflow/lite/micro/micro_allocator.h b/tensorflow/lite/micro/micro_allocator.h index 7fc091196a5..9cfc1793fc7 100644 --- a/tensorflow/lite/micro/micro_allocator.h +++ b/tensorflow/lite/micro/micro_allocator.h @@ -185,7 +185,8 @@ class MicroAllocator { // Commits a memory plan for all non-persistent buffer allocations in the // 'head' section of the memory arena. - virtual TfLiteStatus CommitStaticMemoryPlan(const SubGraph* subgraph, + virtual TfLiteStatus CommitStaticMemoryPlan(const Model* model, + const SubGraph* subgraph, TfLiteContext* context); // A simple memory allocator that always allocate from the arena tail or head. diff --git a/tensorflow/lite/micro/micro_allocator_test.cc b/tensorflow/lite/micro/micro_allocator_test.cc index 04f4732b9d3..f3f3f32611e 100644 --- a/tensorflow/lite/micro/micro_allocator_test.cc +++ b/tensorflow/lite/micro/micro_allocator_test.cc @@ -312,6 +312,8 @@ TF_LITE_MICRO_TEST(OfflinePlannerBranchesAllOnline) { int version = 1; int subgraph = 0; constexpr int nbr_tensors = 4; + tflite::testing::MockOpResolver mock_resolver; + tflite::NodeAndRegistration* node_and_registration; const int32_t metadata_buffer[tflite::testing::kOfflinePlannerHeaderSize + nbr_tensors] = {version, subgraph, nbr_tensors, // header @@ -340,9 +342,14 @@ TF_LITE_MICRO_TEST(OfflinePlannerBranchesAllOnline) { TfLiteContext context; constexpr size_t arena_size = 4096; uint8_t arena[arena_size]; - tflite::MicroAllocator* allocator = tflite::MicroAllocator::Create( - &context, model, arena, arena_size, micro_test::reporter); - TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, allocator->FinishTensorAllocation()); + tflite::MicroAllocator* allocator = + tflite::MicroAllocator::Create(arena, arena_size, micro_test::reporter); + + TF_LITE_MICRO_EXPECT_EQ( + kTfLiteOk, allocator->StartModelAllocation(model, &context, mock_resolver, + &node_and_registration)); + TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, + allocator->FinishModelAllocation(model, &context)); // Since all of the tensors are online planned and the model structure is // identical to that in TestAllocationForModelsWithBranches, @@ -357,6 +364,8 @@ TF_LITE_MICRO_TEST(OfflinePlannerBranchesAllOnline) { TF_LITE_MICRO_TEST(OfflinePlannerBasic) { constexpr int nbr_tensors = 4; + tflite::testing::MockOpResolver mock_resolver; + tflite::NodeAndRegistration* node_and_registration; const int32_t metadata_buffer[tflite::testing::kOfflinePlannerHeaderSize + nbr_tensors] = {1, 0, nbr_tensors, 0, // t0 @@ -389,9 +398,14 @@ TF_LITE_MICRO_TEST(OfflinePlannerBasic) { TfLiteContext context; constexpr size_t arena_size = 4096; uint8_t arena[arena_size]; - tflite::MicroAllocator* allocator = tflite::MicroAllocator::Create( - &context, model, arena, arena_size, micro_test::reporter); - TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, allocator->FinishTensorAllocation()); + tflite::MicroAllocator* allocator = + tflite::MicroAllocator::Create(arena, arena_size, micro_test::reporter); + + TF_LITE_MICRO_EXPECT_EQ( + kTfLiteOk, allocator->StartModelAllocation(model, &context, mock_resolver, + &node_and_registration)); + TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, + allocator->FinishModelAllocation(model, &context)); uint8_t* start = context.tensors[0].data.uint8; TF_LITE_MICRO_EXPECT_EQ(0, context.tensors[0].data.uint8 - start); @@ -402,6 +416,8 @@ TF_LITE_MICRO_TEST(OfflinePlannerBasic) { TF_LITE_MICRO_TEST(OfflinePlannerOverlappingAllocation) { constexpr int nbr_tensors = 4; + tflite::testing::MockOpResolver mock_resolver; + tflite::NodeAndRegistration* node_and_registration; const int32_t metadata_buffer[tflite::testing::kOfflinePlannerHeaderSize + nbr_tensors] = { 1, 0, nbr_tensors, // header: version, subgraph, nbr tensors @@ -434,9 +450,14 @@ TF_LITE_MICRO_TEST(OfflinePlannerOverlappingAllocation) { TfLiteContext context; constexpr size_t arena_size = 4096; uint8_t arena[arena_size]; - tflite::MicroAllocator* allocator = tflite::MicroAllocator::Create( - &context, model, arena, arena_size, micro_test::reporter); - TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, allocator->FinishTensorAllocation()); + tflite::MicroAllocator* allocator = + tflite::MicroAllocator::Create(arena, arena_size, micro_test::reporter); + + TF_LITE_MICRO_EXPECT_EQ( + kTfLiteOk, allocator->StartModelAllocation(model, &context, mock_resolver, + &node_and_registration)); + TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, + allocator->FinishModelAllocation(model, &context)); uint8_t* start = context.tensors[0].data.uint8; TF_LITE_MICRO_EXPECT_EQ(0, context.tensors[0].data.uint8 - start); @@ -448,6 +469,8 @@ TF_LITE_MICRO_TEST(OfflinePlannerOverlappingAllocation) { TF_LITE_MICRO_TEST(OfflinePlannerOfflineOnline) { constexpr int nbr_tensors = 5; + tflite::testing::MockOpResolver mock_resolver; + tflite::NodeAndRegistration* node_and_registration; const int32_t metadata_buffer[tflite::testing::kOfflinePlannerHeaderSize + nbr_tensors] = { 1, 0, nbr_tensors, // header: version, subgraph, nbr tensors @@ -482,9 +505,14 @@ TF_LITE_MICRO_TEST(OfflinePlannerOfflineOnline) { TfLiteContext context; constexpr size_t arena_size = 4096; uint8_t arena[arena_size]; - tflite::MicroAllocator* allocator = tflite::MicroAllocator::Create( - &context, model, arena, arena_size, micro_test::reporter); - TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, allocator->FinishTensorAllocation()); + tflite::MicroAllocator* allocator = + tflite::MicroAllocator::Create(arena, arena_size, micro_test::reporter); + + TF_LITE_MICRO_EXPECT_EQ( + kTfLiteOk, allocator->StartModelAllocation(model, &context, mock_resolver, + &node_and_registration)); + TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, + allocator->FinishModelAllocation(model, &context)); uint8_t* start = context.tensors[0].data.uint8; TF_LITE_MICRO_EXPECT_EQ(0, context.tensors[0].data.uint8 - start);