Gitiles
Code Review Sign In
review.mlplatform.org / ml / armnn / 3e32a8700bf12d3b70d2824c12cdae907bde9360 / . / delegate / src / armnn_delegate.cpp
blob: 2ede23c12bc4830b865b4bc6054a9891f9d72b0d [file] [log] [blame]
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]1//
2// Copyright © 2020 Arm Ltd and Contributors. All rights reserved.
3// SPDX-License-Identifier: MIT
4//
5
6#include <armnn_delegate.hpp>
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]7
Matthew Sloyanac001ee2021-02-03 10:43:04 +0000[diff] [blame]8#include "Version.hpp"
9
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]10#include "Activation.hpp"
11#include "ArgMinMax.hpp"
12#include "BatchSpace.hpp"
13#include "Comparison.hpp"
14#include "Convolution.hpp"
15#include "Control.hpp"
16#include "ElementwiseBinary.hpp"
17#include "ElementwiseUnary.hpp"
18#include "Fill.hpp"
19#include "FullyConnected.hpp"
20#include "Gather.hpp"
Matthew Sloyanc8eb9552020-11-26 10:54:22 +0000[diff] [blame]21#include "LogicalBinary.hpp"
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]22#include "Lstm.hpp"
23#include "Normalization.hpp"
Matthew Sloyana7a12f52021-05-06 10:05:28 +0100[diff] [blame]24#include "Pack.hpp"
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]25#include "Pad.hpp"
26#include "Pooling.hpp"
James Conroy39825482021-05-27 17:44:50 +0100[diff] [blame]27#include "Prelu.hpp"
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]28#include "Quantization.hpp"
29#include "Redefine.hpp"
Sadik Armagana2747482021-02-09 10:28:54 +0000[diff] [blame]30#include "Reduce.hpp"
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]31#include "Resize.hpp"
32#include "Round.hpp"
Keith Davis0176fd82021-06-01 17:36:32 +0100[diff] [blame]33#include "Shape.hpp"
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]34#include "Slice.hpp"
35#include "Softmax.hpp"
36#include "SpaceDepth.hpp"
Sadik Armagan34fa1bd2020-11-27 12:40:52 +0000[diff] [blame]37#include "Split.hpp"
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]38#include "Transpose.hpp"
Narumol Prangnawarat7684b182021-08-12 14:48:15 +0100[diff] [blame]39#include "UnidirectionalSequenceLstm.hpp"
Kevin May8ab2d7a2021-05-07 09:32:51 +0100[diff] [blame]40#include "Unpack.hpp"
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]41
Colm Donelan3e32a872021-10-04 22:55:37 +0100[diff] [blame^]42#include <armnnUtils/Filesystem.hpp>
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]43#include <flatbuffers/flatbuffers.h>
44#include <tensorflow/lite/context_util.h>
45
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]46#include <algorithm>
Matthew Sloyanac001ee2021-02-03 10:43:04 +0000[diff] [blame]47#include <iostream>
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]48#include <sstream>
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]49
50namespace armnnDelegate
51{
52
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]53DelegateOptions TfLiteArmnnDelegateOptionsDefault()
54{
55 DelegateOptions options(armnn::Compute::CpuRef);
56 return options;
57}
58
59TfLiteDelegate* TfLiteArmnnDelegateCreate(armnnDelegate::DelegateOptions options)
60{
61 auto* armnnDelegate = new ::armnnDelegate::Delegate(options);
62 return armnnDelegate->GetDelegate();
63}
64
65void TfLiteArmnnDelegateDelete(TfLiteDelegate* tfLiteDelegate)
66{
67 if (tfLiteDelegate != nullptr)
68 {
69 delete static_cast<::armnnDelegate::Delegate*>(tfLiteDelegate->data_);
70 }
71}
72
73TfLiteStatus DoPrepare(TfLiteContext* tfLiteContext, TfLiteDelegate* tfLiteDelegate)
74{
75 TfLiteIntArray* supportedOperators =
76 static_cast<::armnnDelegate::Delegate*>(tfLiteDelegate->data_)->IdentifyOperatorsToDelegate(tfLiteContext);
77
78 // ArmNN Delegate Registration
79 static const TfLiteRegistration kArmnnSubgraphRegistration = {
80 // ArmnnSubgraph Init
81 .init = [](TfLiteContext* tfLiteContext, const char* buffer, size_t length) -> void* {
Finn Williams6f9f9902020-11-13 13:23:15 +0000[diff] [blame]82 armnn::IgnoreUnused(length);
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]83 const TfLiteDelegateParams* parameters = reinterpret_cast<const TfLiteDelegateParams*>(buffer);
84
85 return static_cast<void*>(ArmnnSubgraph::Create(
86 tfLiteContext, parameters, static_cast<::armnnDelegate::Delegate*>(parameters->delegate->data_)));
87 },
88 // ArmnnSubgraph Free
89 .free = [](TfLiteContext* tfLiteContext, void* buffer) -> void {
Finn Williams6f9f9902020-11-13 13:23:15 +0000[diff] [blame]90 armnn::IgnoreUnused(tfLiteContext);
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]91 if (buffer != nullptr)
92 {
93 delete static_cast<ArmnnSubgraph*>(buffer);
94 }
95 },
96 // ArmnnSubgraph Prepare
97 .prepare = [](TfLiteContext* tfLiteContext, TfLiteNode* tfLiteNode) -> TfLiteStatus {
98 if (tfLiteNode->user_data == nullptr)
99 {
100 return kTfLiteError;
101 }
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]102 return static_cast<ArmnnSubgraph*>(tfLiteNode->user_data)->Prepare(tfLiteContext);
103 },
104 // ArmnnSubgraph Invoke
105 .invoke = [](TfLiteContext* tfLiteContext, TfLiteNode* tfLiteNode) -> TfLiteStatus {
106 if (tfLiteNode->user_data == nullptr)
107 {
108 return kTfLiteError;
109 }
110
111 return static_cast<ArmnnSubgraph*>(tfLiteNode->user_data)->Invoke(tfLiteContext, tfLiteNode);
112 },
113
114 .profiling_string = nullptr,
115 .builtin_code = kTfLiteBuiltinDelegate,
116 .custom_name = "TfLiteArmNnDelegate",
117 .version = 1,
118 };
119
120 const TfLiteStatus status =
121 tfLiteContext->ReplaceNodeSubsetsWithDelegateKernels(
122 tfLiteContext, kArmnnSubgraphRegistration, supportedOperators, tfLiteDelegate);
123
124 TfLiteIntArrayFree(supportedOperators);
125 return status;
126
127}
128
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]129Delegate::Delegate(armnnDelegate::DelegateOptions options)
130 : m_Runtime(nullptr, nullptr),
131 m_Options(std::move(options))
132{
Jan Eilers2cd18472020-12-15 10:42:38 +0000[diff] [blame]133 // Configures logging for ARMNN
134 if (options.IsLoggingEnabled())
135 {
136 armnn::ConfigureLogging(true, true, options.GetLoggingSeverity());
137 }
138
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]139 // Create ArmNN Runtime
140 armnn::IRuntime::CreationOptions runtimeOptions;
Colm Donelan3e32a872021-10-04 22:55:37 +0100[diff] [blame^]141 runtimeOptions.m_DynamicBackendsPath = m_Options.GetDynamicBackendsPath();
142 runtimeOptions.m_EnableGpuProfiling = m_Options.GetGpuProfilingState();
143 runtimeOptions.m_ProfilingOptions = m_Options.GetExternalProfilingParams();
Sadik Armagan4189cc52020-11-11 18:01:48 +0000[diff] [blame]144
145 auto backendOptions = m_Options.GetBackendOptions();
146 if (!backendOptions.empty())
147 {
148 runtimeOptions.m_BackendOptions = backendOptions;
149 }
Narumol Prangnawarat0b51d5a2021-01-20 15:58:29 +0000[diff] [blame]150 else if (!m_Options.GetOptimizerOptions().m_ModelOptions.empty())
151 {
152 runtimeOptions.m_BackendOptions = m_Options.GetOptimizerOptions().m_ModelOptions;
153 }
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]154 m_Runtime = armnn::IRuntime::Create(runtimeOptions);
155
156 std::vector<armnn::BackendId> backends;
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]157 if (m_Runtime)
158 {
159 const armnn::BackendIdSet supportedDevices = m_Runtime->GetDeviceSpec().GetSupportedBackends();
160 for (auto& backend : m_Options.GetBackends())
161 {
162 if (std::find(supportedDevices.cbegin(), supportedDevices.cend(), backend) == supportedDevices.cend())
163 {
Sadik Armagan0534e032020-10-27 17:30:18 +0000[diff] [blame]164 TFLITE_LOG_PROD(tflite::TFLITE_LOG_INFO,
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]165 "TfLiteArmnnDelegate: Requested unknown backend %s", backend.Get().c_str());
166 }
167 else
168 {
169 backends.push_back(backend);
170 }
171 }
172 }
173
174 if (backends.empty())
175 {
176 // No known backend specified
177 throw armnn::InvalidArgumentException("TfLiteArmnnDelegate: No known backend specified.");
178 }
179 m_Options.SetBackends(backends);
180
181 TFLITE_LOG_PROD_ONCE(tflite::TFLITE_LOG_INFO, "TfLiteArmnnDelegate: Created TfLite ArmNN delegate.");
182}
183
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]184TfLiteIntArray* Delegate::IdentifyOperatorsToDelegate(TfLiteContext* tfLiteContext)
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]185{
186 TfLiteIntArray* executionPlan = nullptr;
187 if (tfLiteContext->GetExecutionPlan(tfLiteContext, &executionPlan) != kTfLiteOk)
188 {
189 TF_LITE_KERNEL_LOG(tfLiteContext, "TfLiteArmnnDelegate: Unable to get graph execution plan.");
190 return nullptr;
191 }
192
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]193 // Delegate data with null network
194 DelegateData delegateData(m_Options.GetBackends());
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]195
196 TfLiteIntArray* nodesToDelegate = TfLiteIntArrayCreate(executionPlan->size);
197 nodesToDelegate->size = 0;
198 for (int i = 0; i < executionPlan->size; ++i)
199 {
200 const int nodeIndex = executionPlan->data[i];
201
202 // If TfLite nodes can be delegated to ArmNN
203 TfLiteNode* tfLiteNode = nullptr;
204 TfLiteRegistration* tfLiteRegistration = nullptr;
205 if (tfLiteContext->GetNodeAndRegistration(
206 tfLiteContext, nodeIndex, &tfLiteNode, &tfLiteRegistration) != kTfLiteOk)
207 {
208 TF_LITE_KERNEL_LOG(tfLiteContext,
209 "TfLiteArmnnDelegate: Unable to get node and registration for node %d.",
210 nodeIndex);
211 continue;
212 }
213
214 if (ArmnnSubgraph::VisitNode(
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]215 delegateData, tfLiteContext, tfLiteRegistration, tfLiteNode, nodeIndex) != kTfLiteOk)
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]216 {
217 // node is not supported by ArmNN
218 continue;
219 }
220
221 nodesToDelegate->data[nodesToDelegate->size++] = nodeIndex;
222 }
223
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]224 std::sort(&nodesToDelegate->data[0], &nodesToDelegate->data[nodesToDelegate->size]);
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]225 return nodesToDelegate;
226}
227
228TfLiteDelegate* Delegate::GetDelegate()
229{
230 return &m_Delegate;
231}
232
Matthew Sloyanac001ee2021-02-03 10:43:04 +0000[diff] [blame]233const std::string Delegate::GetVersion()
234{
235 return DELEGATE_VERSION;
236}
237
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]238TfLiteStatus ArmnnSubgraph::AddInputLayer(DelegateData& delegateData,
239 TfLiteContext* tfLiteContext,
240 const TfLiteIntArray* inputs,
241 std::vector<armnn::BindingPointInfo>& inputBindings)
242{
Finn Williams6f9f9902020-11-13 13:23:15 +0000[diff] [blame]243 const size_t numInputs = static_cast<size_t>(inputs->size);
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]244 for (unsigned int i = 0; i < numInputs; ++i)
245 {
246 const int32_t tensorId = inputs->data[i];
247 const TfLiteTensor tensor = tfLiteContext->tensors[tensorId];
Sadik Armagan6e36a642020-11-10 21:18:41 +0000[diff] [blame]248 // Do not create bindings for constant inputs
249 if (tensor.allocation_type == kTfLiteMmapRo)
250 {
251 continue;
252 }
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]253
254 auto bindingId = static_cast<armnn::LayerBindingId>((tensorId));
255 armnn::IConnectableLayer* layer = delegateData.m_Network->AddInputLayer(bindingId);
256
257 auto tensorInfo = GetTensorInfoForTfLiteTensor(tensor);
258 armnn::IOutputSlot& outputSlot = layer->GetOutputSlot(0);
259 outputSlot.SetTensorInfo(tensorInfo);
260
261 // Store for creating connections
Finn Williams6f9f9902020-11-13 13:23:15 +0000[diff] [blame]262 delegateData.m_OutputSlotForNode[static_cast<unsigned long>(tensorId)] = &outputSlot;
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]263
Sadik Armagan6e36a642020-11-10 21:18:41 +0000[diff] [blame]264 inputBindings.push_back(std::make_pair(bindingId, tensorInfo));
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]265 }
Sadik Armagan6e36a642020-11-10 21:18:41 +0000[diff] [blame]266
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]267 return kTfLiteOk;
268}
269
270TfLiteStatus ArmnnSubgraph::AddOutputLayer(DelegateData& delegateData,
271 TfLiteContext* tfLiteContext,
272 const TfLiteIntArray* outputs,
273 std::vector<armnn::BindingPointInfo>& outputBindings)
274{
Finn Williams6f9f9902020-11-13 13:23:15 +0000[diff] [blame]275 const size_t numOutputs = static_cast<size_t>(outputs->size);
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]276 for (unsigned int i = 0; i < numOutputs; ++i)
277 {
278 const int32_t tensorId = outputs->data[i];
279 const TfLiteTensor tensor = tfLiteContext->tensors[tensorId];
280
281 auto bindingId = static_cast<armnn::LayerBindingId>((tensorId));
282 armnn::IConnectableLayer* layer = delegateData.m_Network->AddOutputLayer(bindingId);
283
284 auto tensorInfo = GetTensorInfoForTfLiteTensor(tensor);
Finn Williams6f9f9902020-11-13 13:23:15 +0000[diff] [blame]285 ARMNN_ASSERT(delegateData.m_OutputSlotForNode[static_cast<unsigned long>(tensorId)] != nullptr);
286 delegateData.m_OutputSlotForNode[static_cast<unsigned long>(tensorId)]->Connect(layer->GetInputSlot(0));
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]287 outputBindings.push_back(std::make_pair(bindingId, tensorInfo));
288 }
289
290 return kTfLiteOk;
291}
292
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]293ArmnnSubgraph* ArmnnSubgraph::Create(TfLiteContext* tfLiteContext,
294 const TfLiteDelegateParams* parameters,
295 const Delegate* delegate)
296{
297 TfLiteIntArray* executionPlan;
298 if (tfLiteContext->GetExecutionPlan(tfLiteContext, &executionPlan) != kTfLiteOk)
299 {
300 return nullptr;
301 }
302
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]303 // Initialize DelegateData holds network and output slots information
304 DelegateData delegateData(delegate->m_Options.GetBackends());
305
306 // Build ArmNN Network
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]307 armnn::NetworkOptions networkOptions = {};
308 armnn::NetworkId networkId;
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]309 delegateData.m_Network = armnn::INetwork::Create(networkOptions);
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]310
Sadik Armagan6e36a642020-11-10 21:18:41 +0000[diff] [blame]311 delegateData.m_OutputSlotForNode = std::vector<armnn::IOutputSlot*>(tfLiteContext->tensors_size, nullptr);
312
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]313 std::vector<armnn::BindingPointInfo> inputBindings;
314 std::vector<armnn::BindingPointInfo> outputBindings;
315
316 // Add input layer
317 auto status = AddInputLayer(delegateData, tfLiteContext, parameters->input_tensors, inputBindings);
318 if (status != kTfLiteOk)
319 {
320 throw armnn::Exception("TfLiteArmnnDelegate: Unable to add Inputs to the network!");
321 }
322
323 // Parse TfLite delegate nodes to ArmNN
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]324 for (int i = 0; i < parameters->nodes_to_replace->size; ++i)
325 {
326 const int nodeIndex = parameters->nodes_to_replace->data[i];
327
328 TfLiteNode* tfLiteNode = nullptr;
329 TfLiteRegistration* tfLiteRegistration = nullptr;
330 if (tfLiteContext->GetNodeAndRegistration(
331 tfLiteContext, nodeIndex, &tfLiteNode, &tfLiteRegistration) != kTfLiteOk)
332 {
Finn Williams6f9f9902020-11-13 13:23:15 +0000[diff] [blame]333 throw armnn::Exception(&"TfLiteArmnnDelegate: Unable to get node registration: " [ nodeIndex]);
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]334 }
335
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]336 if (VisitNode(delegateData, tfLiteContext, tfLiteRegistration, tfLiteNode, nodeIndex) != kTfLiteOk)
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]337 {
Finn Williams6f9f9902020-11-13 13:23:15 +0000[diff] [blame]338 throw armnn::Exception(&"TfLiteArmnnDelegate: Unable to parse node: " [ nodeIndex]);
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]339 }
340 }
341
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]342 // Add Output layer
343 status = AddOutputLayer(delegateData, tfLiteContext, parameters->output_tensors, outputBindings);
344 if (status != kTfLiteOk)
345 {
346 throw armnn::Exception("TfLiteArmnnDelegate: Unable to add Outputs to the network!");
347 }
348
349 // Optimize ArmNN network
350 armnn::IOptimizedNetworkPtr optNet(nullptr, nullptr);
351 try
352 {
Sadik Armagan6e36a642020-11-10 21:18:41 +0000[diff] [blame]353 optNet = armnn::Optimize(*(delegateData.m_Network.get()),
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]354 delegate->m_Options.GetBackends(),
Narumol Prangnawarat0b51d5a2021-01-20 15:58:29 +0000[diff] [blame]355 delegate->m_Runtime->GetDeviceSpec(),
356 delegate->m_Options.GetOptimizerOptions());
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]357 }
358 catch (std::exception &ex)
359 {
360 std::stringstream exMessage;
361 exMessage << "TfLiteArmnnDelegate: Exception (" << ex.what() << ") caught from optimize.";
362 throw armnn::Exception(exMessage.str());
363 }
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]364 if (!optNet)
365 {
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]366 // Optimize failed
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]367 throw armnn::Exception("TfLiteArmnnDelegate: Unable to optimize the network!");
368 }
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]369
Colm Donelan3e32a872021-10-04 22:55:37 +0100[diff] [blame^]370 // If set, we will serialize the optimized model into a dot file.
371 const std::string serializeToDotFile = delegate->m_Options.GetSerializeToDot();
372 if (!serializeToDotFile.empty())
373 {
374 fs::path filename = serializeToDotFile;
375 std::fstream file(filename.c_str(), std::ios_base::out);
376 optNet->SerializeToDot(file);
377 }
378
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]379 try
380 {
381 // Load graph into runtime
Narumol Prangnawarat0b51d5a2021-01-20 15:58:29 +0000[diff] [blame]382 std::string errorMessage;
Narumol Prangnawarat74a3cf52021-01-29 15:38:54 +0000[diff] [blame]383 armnn::Status loadingStatus;
Colm Donelan3e32a872021-10-04 22:55:37 +0100[diff] [blame^]384 armnn::MemorySource memorySource = armnn::MemorySource::Undefined;
Narumol Prangnawarat74a3cf52021-01-29 15:38:54 +0000[diff] [blame]385 if (delegate->m_Options.GetOptimizerOptions().m_ImportEnabled)
386 {
Colm Donelan3e32a872021-10-04 22:55:37 +0100[diff] [blame^]387 memorySource = armnn::MemorySource::Malloc;
Narumol Prangnawarat74a3cf52021-01-29 15:38:54 +0000[diff] [blame]388 }
Colm Donelan3e32a872021-10-04 22:55:37 +0100[diff] [blame^]389 armnn::INetworkProperties networkProperties(false,
390 memorySource,
391 memorySource,
392 delegate->m_Options.GetInternalProfilingState(),
393 delegate->m_Options.GetInternalProfilingDetail());
394 loadingStatus = delegate->m_Runtime->LoadNetwork(networkId,
395 std::move(optNet),
396 errorMessage,
397 networkProperties);
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]398 if (loadingStatus != armnn::Status::Success)
399 {
400 // Optimize failed
Narumol Prangnawarat0b51d5a2021-01-20 15:58:29 +0000[diff] [blame]401 throw armnn::Exception("TfLiteArmnnDelegate: Network could not be loaded:" + errorMessage);
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]402 }
403 }
404 catch (std::exception& ex)
405 {
406 std::stringstream exMessage;
407 exMessage << "TfLiteArmnnDelegate: Exception (" << ex.what() << ") caught from LoadNetwork.";
408 throw armnn::Exception(exMessage.str());
409 }
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]410
Narumol Prangnawarat0b51d5a2021-01-20 15:58:29 +0000[diff] [blame]411 // Register debug callback function
412 if (delegate->m_Options.GetDebugCallbackFunction().has_value())
413 {
414 delegate->m_Runtime->RegisterDebugCallback(networkId, delegate->m_Options.GetDebugCallbackFunction().value());
415 }
416
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]417 // Create a new SubGraph with networkId and runtime
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]418 return new ArmnnSubgraph(networkId, delegate->m_Runtime.get(), inputBindings, outputBindings);
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]419}
420
421TfLiteStatus ArmnnSubgraph::Prepare(TfLiteContext* tfLiteContext)
422{
Finn Williams6f9f9902020-11-13 13:23:15 +0000[diff] [blame]423 armnn::IgnoreUnused(tfLiteContext);
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]424 return kTfLiteOk;
425}
426
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]427TfLiteStatus ArmnnSubgraph::Invoke(TfLiteContext* tfLiteContext, TfLiteNode* tfLiteNode)
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]428{
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]429 // Prepare inputs
430 armnn::InputTensors inputTensors;
431 size_t inputIndex = 0;
432 for (auto inputIdx : tflite::TfLiteIntArrayView(tfLiteNode->inputs))
433 {
434 TfLiteTensor* tensor = &tfLiteContext->tensors[inputIdx];
435 if (tensor->allocation_type != kTfLiteMmapRo)
436 {
437 const armnn::BindingPointInfo& inputBinding = m_InputBindings[inputIndex];
438 const armnn::ConstTensor inputTensor(inputBinding.second, tensor->data.data);
439 inputTensors.emplace_back(inputIdx, inputTensor);
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]440
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]441 ++inputIndex;
442 }
443 }
444
445 // Prepare outputs
446 armnn::OutputTensors outputTensors;
447 size_t outputIndex = 0;
448 for (auto outputIdx : tflite::TfLiteIntArrayView(tfLiteNode->outputs))
449 {
450 const armnn::BindingPointInfo& outputBinding = m_OutputBindings[outputIndex];
451 TfLiteTensor* tensor = &tfLiteContext->tensors[outputIdx];
452 const armnn::Tensor outputTensor(outputBinding.second, tensor->data.data);
453 outputTensors.emplace_back(outputIdx, outputTensor);
454
455 ++outputIndex;
456 }
457
458 // Run graph
459 auto status = m_Runtime->EnqueueWorkload(m_NetworkId, inputTensors, outputTensors);
460 return (status == armnn::Status::Success) ? kTfLiteOk : kTfLiteError;
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]461}
462
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]463TfLiteStatus ArmnnSubgraph::VisitNode(DelegateData& delegateData,
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]464 TfLiteContext* tfLiteContext,
465 TfLiteRegistration* tfLiteRegistration,
466 TfLiteNode* tfLiteNode,
467 int nodeIndex)
468{
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]469 switch (tfLiteRegistration->builtin_code)
470 {
471 case kTfLiteBuiltinAbs:
472 return VisitElementwiseUnaryOperator(delegateData,
473 tfLiteContext,
474 tfLiteNode,
475 nodeIndex,
476 armnn::UnaryOperation::Abs);
477 case kTfLiteBuiltinAdd:
478 return VisitElementwiseBinaryOperator(delegateData,
479 tfLiteContext,
480 tfLiteNode,
481 nodeIndex,
482 kTfLiteBuiltinAdd);
483 case kTfLiteBuiltinArgMax:
484 return VisitArgMinMaxOperator(delegateData,
485 tfLiteContext,
486 tfLiteNode,
487 nodeIndex,
488 kTfLiteBuiltinArgMax);
489 case kTfLiteBuiltinArgMin:
490 return VisitArgMinMaxOperator(delegateData,
491 tfLiteContext,
492 tfLiteNode,
493 nodeIndex,
494 kTfLiteBuiltinArgMin);
495 case kTfLiteBuiltinAveragePool2d:
496 return VisitPoolingOperator(delegateData,
497 tfLiteContext,
498 tfLiteNode,
499 nodeIndex,
500 kTfLiteBuiltinAveragePool2d);
501 case kTfLiteBuiltinBatchToSpaceNd:
502 return VisitBatchToSpaceNdOperator(delegateData,
503 tfLiteContext,
504 tfLiteNode,
505 nodeIndex,
506 kTfLiteBuiltinBatchToSpaceNd);
Sadik Armagan937565b2021-04-21 14:03:28 +0100[diff] [blame]507 case kTfLiteBuiltinCast:
508 return VisitCastOperator(delegateData,
509 tfLiteContext,
510 tfLiteNode,
511 nodeIndex,
512 kTfLiteBuiltinCast);
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]513 case kTfLiteBuiltinConcatenation:
514 return VisitControlOperator(delegateData,
515 tfLiteContext,
516 tfLiteNode,
517 nodeIndex,
518 kTfLiteBuiltinConcatenation);
519 case kTfLiteBuiltinConv2d:
520 return VisitConvolutionOperator(delegateData,
521 tfLiteContext,
522 tfLiteNode,
523 nodeIndex,
524 kTfLiteBuiltinConv2d);
525 case kTfLiteBuiltinDepthToSpace:
526 return VisitDepthToSpaceOperator(delegateData,
527 tfLiteContext,
528 tfLiteNode,
529 nodeIndex,
530 kTfLiteBuiltinDepthToSpace);
531 case kTfLiteBuiltinDepthwiseConv2d:
532 return VisitConvolutionOperator(delegateData,
533 tfLiteContext,
534 tfLiteNode,
535 nodeIndex,
536 kTfLiteBuiltinDepthwiseConv2d);
537 case kTfLiteBuiltinDequantize:
538 return VisitDequantizeOperator(delegateData,
539 tfLiteContext,
540 tfLiteNode,
541 nodeIndex,
542 kTfLiteBuiltinDequantize);
543 case kTfLiteBuiltinDiv:
544 return VisitElementwiseBinaryOperator(delegateData,
545 tfLiteContext,
546 tfLiteNode,
547 nodeIndex,
548 kTfLiteBuiltinDiv);
549 case kTfLiteBuiltinElu:
550 return VisitActivationOperator(delegateData,
551 tfLiteContext,
552 tfLiteNode,
553 nodeIndex,
554 kTfLiteBuiltinElu);
555 case kTfLiteBuiltinEqual:
556 return VisitComparisonOperator(delegateData,
557 tfLiteContext,
558 tfLiteNode,
559 nodeIndex,
560 kTfLiteBuiltinEqual);
561 case kTfLiteBuiltinExp:
562 return VisitElementwiseUnaryOperator(delegateData,
563 tfLiteContext,
564 tfLiteNode,
565 nodeIndex,
566 armnn::UnaryOperation::Exp);
567 case kTfLiteBuiltinExpandDims:
568 return VisitExpandDimsOperator(delegateData,
569 tfLiteContext,
570 tfLiteNode,
571 nodeIndex,
572 kTfLiteBuiltinExpandDims);
573 case kTfLiteBuiltinFill:
574 return VisitFillOperator(delegateData,
575 tfLiteContext,
576 tfLiteNode,
577 nodeIndex,
578 kTfLiteBuiltinFill);
579 case kTfLiteBuiltinFloor:
580 return VisitFloorOperator(delegateData,
581 tfLiteContext,
582 tfLiteNode,
583 nodeIndex,
584 kTfLiteBuiltinFloor);
585 case kTfLiteBuiltinFullyConnected:
586 return VisitFullyConnectedOperator(delegateData,
587 tfLiteContext,
588 tfLiteNode,
589 nodeIndex,
590 kTfLiteBuiltinFullyConnected);
591 case kTfLiteBuiltinGather:
592 return VisitGatherOperator(delegateData,
593 tfLiteContext,
594 tfLiteNode,
595 nodeIndex,
596 kTfLiteBuiltinGather);
597 case kTfLiteBuiltinGatherNd:
598 return VisitGatherOperator(delegateData,
599 tfLiteContext,
600 tfLiteNode,
601 nodeIndex,
602 kTfLiteBuiltinGatherNd);
603 case kTfLiteBuiltinGreater:
604 return VisitComparisonOperator(delegateData,
605 tfLiteContext,
606 tfLiteNode,
607 nodeIndex,
608 kTfLiteBuiltinGreater);
609 case kTfLiteBuiltinGreaterEqual:
610 return VisitComparisonOperator(delegateData,
611 tfLiteContext,
612 tfLiteNode,
613 nodeIndex,
614 kTfLiteBuiltinGreaterEqual);
615 case kTfLiteBuiltinHardSwish:
616 return VisitActivationOperator(delegateData,
617 tfLiteContext,
618 tfLiteNode,
619 nodeIndex,
620 kTfLiteBuiltinHardSwish);
621 case kTfLiteBuiltinL2Normalization:
Sadik Armagan4b227bb2021-01-22 10:53:38 +0000[diff] [blame]622 return VisitL2NormalizationOperator(delegateData,
623 tfLiteContext,
624 tfLiteNode,
625 nodeIndex,
626 kTfLiteBuiltinL2Normalization);
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]627 case kTfLiteBuiltinL2Pool2d:
628 return VisitPoolingOperator(delegateData,
629 tfLiteContext,
630 tfLiteNode,
631 nodeIndex,
632 kTfLiteBuiltinL2Pool2d);
633 case kTfLiteBuiltinLess:
634 return VisitComparisonOperator(delegateData,
635 tfLiteContext,
636 tfLiteNode,
637 nodeIndex,
638 kTfLiteBuiltinLess);
639 case kTfLiteBuiltinLessEqual:
640 return VisitComparisonOperator(delegateData,
641 tfLiteContext,
642 tfLiteNode,
643 nodeIndex,
644 kTfLiteBuiltinLessEqual);
645 case kTfLiteBuiltinLocalResponseNormalization:
Sadik Armagan4b227bb2021-01-22 10:53:38 +0000[diff] [blame]646 return VisitLocalResponseNormalizationOperator(delegateData,
647 tfLiteContext,
648 tfLiteNode,
649 nodeIndex,
650 kTfLiteBuiltinLocalResponseNormalization);
Matthew Sloyanc8eb9552020-11-26 10:54:22 +0000[diff] [blame]651 case kTfLiteBuiltinLogicalAnd:
652 return VisitLogicalBinaryOperator(delegateData,
653 tfLiteContext,
654 tfLiteNode,
655 nodeIndex,
656 kTfLiteBuiltinLogicalAnd,
657 armnn::LogicalBinaryOperation::LogicalAnd);
658 case kTfLiteBuiltinLogicalNot:
659 return VisitElementwiseUnaryOperator(delegateData,
660 tfLiteContext,
661 tfLiteNode,
662 nodeIndex,
663 armnn::UnaryOperation::LogicalNot);
664 case kTfLiteBuiltinLogicalOr:
665 return VisitLogicalBinaryOperator(delegateData,
666 tfLiteContext,
667 tfLiteNode,
668 nodeIndex,
669 kTfLiteBuiltinLogicalOr,
670 armnn::LogicalBinaryOperation::LogicalOr);
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]671 case kTfLiteBuiltinLogistic:
672 return VisitActivationOperator(delegateData,
673 tfLiteContext,
674 tfLiteNode,
675 nodeIndex,
676 kTfLiteBuiltinLogistic);
677 case kTfLiteBuiltinLogSoftmax:
678 return VisitSoftmaxOperator(delegateData,
679 tfLiteContext,
680 tfLiteNode,
681 nodeIndex,
682 kTfLiteBuiltinLogSoftmax);
683 case kTfLiteBuiltinLstm:
684 return VisitLstmOperator(delegateData,
685 tfLiteContext,
686 tfLiteNode,
687 nodeIndex,
688 kTfLiteBuiltinLstm);
689 case kTfLiteBuiltinMaxPool2d:
690 return VisitPoolingOperator(delegateData,
691 tfLiteContext,
692 tfLiteNode,
693 nodeIndex,
694 kTfLiteBuiltinMaxPool2d);
695 case kTfLiteBuiltinMaximum:
696 return VisitElementwiseBinaryOperator(delegateData,
697 tfLiteContext,
698 tfLiteNode,
699 nodeIndex,
700 kTfLiteBuiltinMaximum);
701 case kTfLiteBuiltinMean:
702 return VisitControlOperator(delegateData,
703 tfLiteContext,
704 tfLiteNode,
705 nodeIndex,
706 kTfLiteBuiltinMean);
707 case kTfLiteBuiltinMinimum:
708 return VisitElementwiseBinaryOperator(delegateData,
709 tfLiteContext,
710 tfLiteNode,
711 nodeIndex,
712 kTfLiteBuiltinMinimum);
713 case kTfLiteBuiltinMul:
714 return VisitElementwiseBinaryOperator(delegateData,
715 tfLiteContext,
716 tfLiteNode,
717 nodeIndex,
718 kTfLiteBuiltinMul);
719 case kTfLiteBuiltinNeg:
720 return VisitElementwiseUnaryOperator(delegateData,
721 tfLiteContext,
722 tfLiteNode,
723 nodeIndex,
724 armnn::UnaryOperation::Neg);
725 case kTfLiteBuiltinNotEqual:
726 return VisitComparisonOperator(delegateData,
727 tfLiteContext,
728 tfLiteNode,
729 nodeIndex,
730 kTfLiteBuiltinNotEqual);
Matthew Sloyana7a12f52021-05-06 10:05:28 +0100[diff] [blame]731 case kTfLiteBuiltinPack:
732 return VisitPackOperator(delegateData,
733 tfLiteContext,
734 tfLiteNode,
735 nodeIndex,
736 kTfLiteBuiltinPack);
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]737 case kTfLiteBuiltinPad:
738 return VisitPadOperator(delegateData,
739 tfLiteContext,
740 tfLiteNode,
741 nodeIndex,
742 kTfLiteBuiltinPad);
743 case kTfLiteBuiltinPadv2:
744 return VisitPadOperator(delegateData,
745 tfLiteContext,
746 tfLiteNode,
747 nodeIndex,
748 kTfLiteBuiltinPadv2);
749 case kTfLiteBuiltinPrelu:
James Conroy39825482021-05-27 17:44:50 +0100[diff] [blame]750 return VisitPreluOperator(delegateData,
751 tfLiteContext,
752 tfLiteNode,
753 nodeIndex,
754 kTfLiteBuiltinPrelu);
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]755 case kTfLiteBuiltinQuantize:
756 return VisitQuantizeOperator(delegateData,
757 tfLiteContext,
758 tfLiteNode,
759 nodeIndex,
760 kTfLiteBuiltinQuantize);
761 case kTfLiteBuiltinRank:
762 return VisitControlOperator(delegateData,
763 tfLiteContext,
764 tfLiteNode,
765 nodeIndex,
766 kTfLiteBuiltinRank);
Sadik Armagana2747482021-02-09 10:28:54 +0000[diff] [blame]767 case kTfLiteBuiltinReduceMax:
768 return VisitReduceOperator(delegateData,
769 tfLiteContext,
770 tfLiteNode,
771 nodeIndex,
772 kTfLiteBuiltinReduceMax);
773 case kTfLiteBuiltinReduceMin:
774 return VisitReduceOperator(delegateData,
775 tfLiteContext,
776 tfLiteNode,
777 nodeIndex,
778 kTfLiteBuiltinReduceMin);
Teresa Charlin4e3e8312021-08-05 12:34:37 +0100[diff] [blame]779 case kTfLiteBuiltinReduceProd:
780 return VisitReduceOperator(delegateData,
781 tfLiteContext,
782 tfLiteNode,
783 nodeIndex,
784 kTfLiteBuiltinReduceProd);
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]785 case kTfLiteBuiltinRelu:
786 return VisitActivationOperator(delegateData,
787 tfLiteContext,
788 tfLiteNode,
789 nodeIndex,
790 kTfLiteBuiltinRelu);
791 case kTfLiteBuiltinReluN1To1:
792 return VisitActivationOperator(delegateData,
793 tfLiteContext,
794 tfLiteNode,
795 nodeIndex,
796 kTfLiteBuiltinReluN1To1);
797 case kTfLiteBuiltinRelu6:
798 return VisitActivationOperator(delegateData,
799 tfLiteContext,
800 tfLiteNode,
801 nodeIndex,
802 kTfLiteBuiltinRelu6);
803 case kTfLiteBuiltinReshape:
804 return VisitReshapeOperator(delegateData,
805 tfLiteContext,
806 tfLiteNode,
807 nodeIndex,
808 kTfLiteBuiltinReshape);
809 case kTfLiteBuiltinResizeBilinear:
810 return VisitResizeOperator(delegateData,
811 tfLiteContext,
812 tfLiteNode,
813 nodeIndex,
814 kTfLiteBuiltinResizeBilinear);
815 case kTfLiteBuiltinResizeNearestNeighbor:
816 return VisitResizeOperator(delegateData,
817 tfLiteContext,
818 tfLiteNode,
819 nodeIndex,
820 kTfLiteBuiltinResizeNearestNeighbor);
821 case kTfLiteBuiltinRsqrt:
822 return VisitElementwiseUnaryOperator(delegateData,
823 tfLiteContext,
824 tfLiteNode,
825 nodeIndex,
826 armnn::UnaryOperation::Rsqrt);
Keith Davis0176fd82021-06-01 17:36:32 +0100[diff] [blame]827 case kTfLiteBuiltinShape:
828 return VisitShapeOperator(delegateData,
829 tfLiteContext,
830 tfLiteNode,
831 nodeIndex,
832 kTfLiteBuiltinShape);
Sadik Armagan34fa1bd2020-11-27 12:40:52 +0000[diff] [blame]833 case kTfLiteBuiltinSplit:
834 return VisitSplitOperator(delegateData,
835 tfLiteContext,
836 tfLiteNode,
837 nodeIndex,
838 kTfLiteBuiltinSplit);
839 case kTfLiteBuiltinSplitV:
840 return VisitSplitVOperator(delegateData,
841 tfLiteContext,
842 tfLiteNode,
843 nodeIndex,
844 kTfLiteBuiltinSplitV);
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]845 case kTfLiteBuiltinSqrt:
846 return VisitElementwiseUnaryOperator(delegateData,
847 tfLiteContext,
848 tfLiteNode,
849 nodeIndex,
850 armnn::UnaryOperation::Sqrt);
851 case kTfLiteBuiltinSqueeze:
852 return VisitSqueezeOperator(delegateData,
853 tfLiteContext,
854 tfLiteNode,
855 nodeIndex,
856 kTfLiteBuiltinSqueeze);
857 case kTfLiteBuiltinStridedSlice:
858 return VisitSliceOperator(delegateData,
859 tfLiteContext,
860 tfLiteNode,
861 nodeIndex,
862 kTfLiteBuiltinStridedSlice);
Sadik Armagana2747482021-02-09 10:28:54 +0000[diff] [blame]863 case kTfLiteBuiltinSum:
864 return VisitReduceOperator(delegateData,
865 tfLiteContext,
866 tfLiteNode,
867 nodeIndex,
868 kTfLiteBuiltinSum);
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]869 case kTfLiteBuiltinTranspose:
870 return VisitTransposeOperator(delegateData,
871 tfLiteContext,
872 tfLiteNode,
873 nodeIndex,
874 kTfLiteBuiltinTranspose);
875 case kTfLiteBuiltinTransposeConv:
876 return VisitConvolutionOperator(delegateData,
877 tfLiteContext,
878 tfLiteNode,
879 nodeIndex,
880 kTfLiteBuiltinTransposeConv);
881 case kTfLiteBuiltinSoftmax:
882 return VisitSoftmaxOperator(delegateData,
883 tfLiteContext,
884 tfLiteNode,
885 nodeIndex,
886 kTfLiteBuiltinSoftmax);
887 case kTfLiteBuiltinSpaceToBatchNd:
888 return VisitSpaceToBatchNdOperator(delegateData,
889 tfLiteContext,
890 tfLiteNode,
891 nodeIndex,
892 kTfLiteBuiltinSpaceToBatchNd);
893 case kTfLiteBuiltinSpaceToDepth:
894 return VisitSpaceToDepthOperator(delegateData,
895 tfLiteContext,
896 tfLiteNode,
897 nodeIndex,
898 kTfLiteBuiltinSpaceToDepth);
899 case kTfLiteBuiltinSub:
900 return VisitElementwiseBinaryOperator(delegateData,
901 tfLiteContext,
902 tfLiteNode,
903 nodeIndex,
904 kTfLiteBuiltinSub);
905 case kTfLiteBuiltinTanh:
906 return VisitActivationOperator(delegateData,
907 tfLiteContext,
908 tfLiteNode,
909 nodeIndex,
910 kTfLiteBuiltinTanh);
Narumol Prangnawarat7684b182021-08-12 14:48:15 +0100[diff] [blame]911 case kTfLiteBuiltinUnidirectionalSequenceLstm:
912 return VisitUnidirectionalSequenceLstmOperator(delegateData,
913 tfLiteContext,
914 tfLiteNode,
915 nodeIndex,
916 kTfLiteBuiltinUnidirectionalSequenceLstm);
Kevin May8ab2d7a2021-05-07 09:32:51 +0100[diff] [blame]917 case kTfLiteBuiltinUnpack:
918 return VisitUnpackOperator(delegateData,
919 tfLiteContext,
920 tfLiteNode,
921 nodeIndex,
922 kTfLiteBuiltinUnpack);
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]923 default:
924 return kTfLiteError;
925 }
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]926}
927
928} // armnnDelegate namespace