Gitiles
Code Review Sign In
review.mlplatform.org / ml / armnn / 7684b18e8fec45355a49e7f7165c582efc553ab6 / . / delegate / src / armnn_delegate.cpp
blob: d3ccecb444f801416850535b62735dcf2c2e7e68 [file] [log] [blame]
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]1//
2// Copyright © 2020 Arm Ltd and Contributors. All rights reserved.
3// SPDX-License-Identifier: MIT
4//
5
6#include <armnn_delegate.hpp>
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]7
Matthew Sloyanac001ee2021-02-03 10:43:04 +0000[diff] [blame]8#include "Version.hpp"
9
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]10#include "Activation.hpp"
11#include "ArgMinMax.hpp"
12#include "BatchSpace.hpp"
13#include "Comparison.hpp"
14#include "Convolution.hpp"
15#include "Control.hpp"
16#include "ElementwiseBinary.hpp"
17#include "ElementwiseUnary.hpp"
18#include "Fill.hpp"
19#include "FullyConnected.hpp"
20#include "Gather.hpp"
Matthew Sloyanc8eb9552020-11-26 10:54:22 +0000[diff] [blame]21#include "LogicalBinary.hpp"
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]22#include "Lstm.hpp"
23#include "Normalization.hpp"
Matthew Sloyana7a12f52021-05-06 10:05:28 +0100[diff] [blame]24#include "Pack.hpp"
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]25#include "Pad.hpp"
26#include "Pooling.hpp"
James Conroy39825482021-05-27 17:44:50 +0100[diff] [blame]27#include "Prelu.hpp"
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]28#include "Quantization.hpp"
29#include "Redefine.hpp"
Sadik Armagana2747482021-02-09 10:28:54 +0000[diff] [blame]30#include "Reduce.hpp"
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]31#include "Resize.hpp"
32#include "Round.hpp"
Keith Davis0176fd82021-06-01 17:36:32 +0100[diff] [blame]33#include "Shape.hpp"
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]34#include "Slice.hpp"
35#include "Softmax.hpp"
36#include "SpaceDepth.hpp"
Sadik Armagan34fa1bd2020-11-27 12:40:52 +0000[diff] [blame]37#include "Split.hpp"
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]38#include "Transpose.hpp"
Narumol Prangnawarat7684b182021-08-12 14:48:15 +0100[diff] [blame^]39#include "UnidirectionalSequenceLstm.hpp"
Kevin May8ab2d7a2021-05-07 09:32:51 +0100[diff] [blame]40#include "Unpack.hpp"
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]41
42#include <flatbuffers/flatbuffers.h>
43#include <tensorflow/lite/context_util.h>
44
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]45#include <algorithm>
Matthew Sloyanac001ee2021-02-03 10:43:04 +0000[diff] [blame]46#include <iostream>
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]47#include <sstream>
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]48
49namespace armnnDelegate
50{
51
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]52DelegateOptions TfLiteArmnnDelegateOptionsDefault()
53{
54 DelegateOptions options(armnn::Compute::CpuRef);
55 return options;
56}
57
58TfLiteDelegate* TfLiteArmnnDelegateCreate(armnnDelegate::DelegateOptions options)
59{
60 auto* armnnDelegate = new ::armnnDelegate::Delegate(options);
61 return armnnDelegate->GetDelegate();
62}
63
64void TfLiteArmnnDelegateDelete(TfLiteDelegate* tfLiteDelegate)
65{
66 if (tfLiteDelegate != nullptr)
67 {
68 delete static_cast<::armnnDelegate::Delegate*>(tfLiteDelegate->data_);
69 }
70}
71
72TfLiteStatus DoPrepare(TfLiteContext* tfLiteContext, TfLiteDelegate* tfLiteDelegate)
73{
74 TfLiteIntArray* supportedOperators =
75 static_cast<::armnnDelegate::Delegate*>(tfLiteDelegate->data_)->IdentifyOperatorsToDelegate(tfLiteContext);
76
77 // ArmNN Delegate Registration
78 static const TfLiteRegistration kArmnnSubgraphRegistration = {
79 // ArmnnSubgraph Init
80 .init = [](TfLiteContext* tfLiteContext, const char* buffer, size_t length) -> void* {
Finn Williams6f9f9902020-11-13 13:23:15 +0000[diff] [blame]81 armnn::IgnoreUnused(length);
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]82 const TfLiteDelegateParams* parameters = reinterpret_cast<const TfLiteDelegateParams*>(buffer);
83
84 return static_cast<void*>(ArmnnSubgraph::Create(
85 tfLiteContext, parameters, static_cast<::armnnDelegate::Delegate*>(parameters->delegate->data_)));
86 },
87 // ArmnnSubgraph Free
88 .free = [](TfLiteContext* tfLiteContext, void* buffer) -> void {
Finn Williams6f9f9902020-11-13 13:23:15 +0000[diff] [blame]89 armnn::IgnoreUnused(tfLiteContext);
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]90 if (buffer != nullptr)
91 {
92 delete static_cast<ArmnnSubgraph*>(buffer);
93 }
94 },
95 // ArmnnSubgraph Prepare
96 .prepare = [](TfLiteContext* tfLiteContext, TfLiteNode* tfLiteNode) -> TfLiteStatus {
97 if (tfLiteNode->user_data == nullptr)
98 {
99 return kTfLiteError;
100 }
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]101 return static_cast<ArmnnSubgraph*>(tfLiteNode->user_data)->Prepare(tfLiteContext);
102 },
103 // ArmnnSubgraph Invoke
104 .invoke = [](TfLiteContext* tfLiteContext, TfLiteNode* tfLiteNode) -> TfLiteStatus {
105 if (tfLiteNode->user_data == nullptr)
106 {
107 return kTfLiteError;
108 }
109
110 return static_cast<ArmnnSubgraph*>(tfLiteNode->user_data)->Invoke(tfLiteContext, tfLiteNode);
111 },
112
113 .profiling_string = nullptr,
114 .builtin_code = kTfLiteBuiltinDelegate,
115 .custom_name = "TfLiteArmNnDelegate",
116 .version = 1,
117 };
118
119 const TfLiteStatus status =
120 tfLiteContext->ReplaceNodeSubsetsWithDelegateKernels(
121 tfLiteContext, kArmnnSubgraphRegistration, supportedOperators, tfLiteDelegate);
122
123 TfLiteIntArrayFree(supportedOperators);
124 return status;
125
126}
127
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]128Delegate::Delegate(armnnDelegate::DelegateOptions options)
129 : m_Runtime(nullptr, nullptr),
130 m_Options(std::move(options))
131{
Jan Eilers2cd18472020-12-15 10:42:38 +0000[diff] [blame]132 // Configures logging for ARMNN
133 if (options.IsLoggingEnabled())
134 {
135 armnn::ConfigureLogging(true, true, options.GetLoggingSeverity());
136 }
137
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]138 // Create ArmNN Runtime
139 armnn::IRuntime::CreationOptions runtimeOptions;
Sadik Armagan4189cc52020-11-11 18:01:48 +0000[diff] [blame]140
141 auto backendOptions = m_Options.GetBackendOptions();
142 if (!backendOptions.empty())
143 {
144 runtimeOptions.m_BackendOptions = backendOptions;
145 }
Narumol Prangnawarat0b51d5a2021-01-20 15:58:29 +0000[diff] [blame]146 else if (!m_Options.GetOptimizerOptions().m_ModelOptions.empty())
147 {
148 runtimeOptions.m_BackendOptions = m_Options.GetOptimizerOptions().m_ModelOptions;
149 }
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]150 m_Runtime = armnn::IRuntime::Create(runtimeOptions);
151
152 std::vector<armnn::BackendId> backends;
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]153 if (m_Runtime)
154 {
155 const armnn::BackendIdSet supportedDevices = m_Runtime->GetDeviceSpec().GetSupportedBackends();
156 for (auto& backend : m_Options.GetBackends())
157 {
158 if (std::find(supportedDevices.cbegin(), supportedDevices.cend(), backend) == supportedDevices.cend())
159 {
Sadik Armagan0534e032020-10-27 17:30:18 +0000[diff] [blame]160 TFLITE_LOG_PROD(tflite::TFLITE_LOG_INFO,
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]161 "TfLiteArmnnDelegate: Requested unknown backend %s", backend.Get().c_str());
162 }
163 else
164 {
165 backends.push_back(backend);
166 }
167 }
168 }
169
170 if (backends.empty())
171 {
172 // No known backend specified
173 throw armnn::InvalidArgumentException("TfLiteArmnnDelegate: No known backend specified.");
174 }
175 m_Options.SetBackends(backends);
176
177 TFLITE_LOG_PROD_ONCE(tflite::TFLITE_LOG_INFO, "TfLiteArmnnDelegate: Created TfLite ArmNN delegate.");
178}
179
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]180TfLiteIntArray* Delegate::IdentifyOperatorsToDelegate(TfLiteContext* tfLiteContext)
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]181{
182 TfLiteIntArray* executionPlan = nullptr;
183 if (tfLiteContext->GetExecutionPlan(tfLiteContext, &executionPlan) != kTfLiteOk)
184 {
185 TF_LITE_KERNEL_LOG(tfLiteContext, "TfLiteArmnnDelegate: Unable to get graph execution plan.");
186 return nullptr;
187 }
188
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]189 // Delegate data with null network
190 DelegateData delegateData(m_Options.GetBackends());
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]191
192 TfLiteIntArray* nodesToDelegate = TfLiteIntArrayCreate(executionPlan->size);
193 nodesToDelegate->size = 0;
194 for (int i = 0; i < executionPlan->size; ++i)
195 {
196 const int nodeIndex = executionPlan->data[i];
197
198 // If TfLite nodes can be delegated to ArmNN
199 TfLiteNode* tfLiteNode = nullptr;
200 TfLiteRegistration* tfLiteRegistration = nullptr;
201 if (tfLiteContext->GetNodeAndRegistration(
202 tfLiteContext, nodeIndex, &tfLiteNode, &tfLiteRegistration) != kTfLiteOk)
203 {
204 TF_LITE_KERNEL_LOG(tfLiteContext,
205 "TfLiteArmnnDelegate: Unable to get node and registration for node %d.",
206 nodeIndex);
207 continue;
208 }
209
210 if (ArmnnSubgraph::VisitNode(
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]211 delegateData, tfLiteContext, tfLiteRegistration, tfLiteNode, nodeIndex) != kTfLiteOk)
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]212 {
213 // node is not supported by ArmNN
214 continue;
215 }
216
217 nodesToDelegate->data[nodesToDelegate->size++] = nodeIndex;
218 }
219
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]220 std::sort(&nodesToDelegate->data[0], &nodesToDelegate->data[nodesToDelegate->size]);
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]221 return nodesToDelegate;
222}
223
224TfLiteDelegate* Delegate::GetDelegate()
225{
226 return &m_Delegate;
227}
228
Matthew Sloyanac001ee2021-02-03 10:43:04 +0000[diff] [blame]229const std::string Delegate::GetVersion()
230{
231 return DELEGATE_VERSION;
232}
233
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]234TfLiteStatus ArmnnSubgraph::AddInputLayer(DelegateData& delegateData,
235 TfLiteContext* tfLiteContext,
236 const TfLiteIntArray* inputs,
237 std::vector<armnn::BindingPointInfo>& inputBindings)
238{
Finn Williams6f9f9902020-11-13 13:23:15 +0000[diff] [blame]239 const size_t numInputs = static_cast<size_t>(inputs->size);
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]240 for (unsigned int i = 0; i < numInputs; ++i)
241 {
242 const int32_t tensorId = inputs->data[i];
243 const TfLiteTensor tensor = tfLiteContext->tensors[tensorId];
Sadik Armagan6e36a642020-11-10 21:18:41 +0000[diff] [blame]244 // Do not create bindings for constant inputs
245 if (tensor.allocation_type == kTfLiteMmapRo)
246 {
247 continue;
248 }
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]249
250 auto bindingId = static_cast<armnn::LayerBindingId>((tensorId));
251 armnn::IConnectableLayer* layer = delegateData.m_Network->AddInputLayer(bindingId);
252
253 auto tensorInfo = GetTensorInfoForTfLiteTensor(tensor);
254 armnn::IOutputSlot& outputSlot = layer->GetOutputSlot(0);
255 outputSlot.SetTensorInfo(tensorInfo);
256
257 // Store for creating connections
Finn Williams6f9f9902020-11-13 13:23:15 +0000[diff] [blame]258 delegateData.m_OutputSlotForNode[static_cast<unsigned long>(tensorId)] = &outputSlot;
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]259
Sadik Armagan6e36a642020-11-10 21:18:41 +0000[diff] [blame]260 inputBindings.push_back(std::make_pair(bindingId, tensorInfo));
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]261 }
Sadik Armagan6e36a642020-11-10 21:18:41 +0000[diff] [blame]262
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]263 return kTfLiteOk;
264}
265
266TfLiteStatus ArmnnSubgraph::AddOutputLayer(DelegateData& delegateData,
267 TfLiteContext* tfLiteContext,
268 const TfLiteIntArray* outputs,
269 std::vector<armnn::BindingPointInfo>& outputBindings)
270{
Finn Williams6f9f9902020-11-13 13:23:15 +0000[diff] [blame]271 const size_t numOutputs = static_cast<size_t>(outputs->size);
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]272 for (unsigned int i = 0; i < numOutputs; ++i)
273 {
274 const int32_t tensorId = outputs->data[i];
275 const TfLiteTensor tensor = tfLiteContext->tensors[tensorId];
276
277 auto bindingId = static_cast<armnn::LayerBindingId>((tensorId));
278 armnn::IConnectableLayer* layer = delegateData.m_Network->AddOutputLayer(bindingId);
279
280 auto tensorInfo = GetTensorInfoForTfLiteTensor(tensor);
Finn Williams6f9f9902020-11-13 13:23:15 +0000[diff] [blame]281 ARMNN_ASSERT(delegateData.m_OutputSlotForNode[static_cast<unsigned long>(tensorId)] != nullptr);
282 delegateData.m_OutputSlotForNode[static_cast<unsigned long>(tensorId)]->Connect(layer->GetInputSlot(0));
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]283 outputBindings.push_back(std::make_pair(bindingId, tensorInfo));
284 }
285
286 return kTfLiteOk;
287}
288
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]289ArmnnSubgraph* ArmnnSubgraph::Create(TfLiteContext* tfLiteContext,
290 const TfLiteDelegateParams* parameters,
291 const Delegate* delegate)
292{
293 TfLiteIntArray* executionPlan;
294 if (tfLiteContext->GetExecutionPlan(tfLiteContext, &executionPlan) != kTfLiteOk)
295 {
296 return nullptr;
297 }
298
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]299 // Initialize DelegateData holds network and output slots information
300 DelegateData delegateData(delegate->m_Options.GetBackends());
301
302 // Build ArmNN Network
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]303 armnn::NetworkOptions networkOptions = {};
304 armnn::NetworkId networkId;
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]305 delegateData.m_Network = armnn::INetwork::Create(networkOptions);
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]306
Sadik Armagan6e36a642020-11-10 21:18:41 +0000[diff] [blame]307 delegateData.m_OutputSlotForNode = std::vector<armnn::IOutputSlot*>(tfLiteContext->tensors_size, nullptr);
308
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]309 std::vector<armnn::BindingPointInfo> inputBindings;
310 std::vector<armnn::BindingPointInfo> outputBindings;
311
312 // Add input layer
313 auto status = AddInputLayer(delegateData, tfLiteContext, parameters->input_tensors, inputBindings);
314 if (status != kTfLiteOk)
315 {
316 throw armnn::Exception("TfLiteArmnnDelegate: Unable to add Inputs to the network!");
317 }
318
319 // Parse TfLite delegate nodes to ArmNN
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]320 for (int i = 0; i < parameters->nodes_to_replace->size; ++i)
321 {
322 const int nodeIndex = parameters->nodes_to_replace->data[i];
323
324 TfLiteNode* tfLiteNode = nullptr;
325 TfLiteRegistration* tfLiteRegistration = nullptr;
326 if (tfLiteContext->GetNodeAndRegistration(
327 tfLiteContext, nodeIndex, &tfLiteNode, &tfLiteRegistration) != kTfLiteOk)
328 {
Finn Williams6f9f9902020-11-13 13:23:15 +0000[diff] [blame]329 throw armnn::Exception(&"TfLiteArmnnDelegate: Unable to get node registration: " [ nodeIndex]);
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]330 }
331
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]332 if (VisitNode(delegateData, tfLiteContext, tfLiteRegistration, tfLiteNode, nodeIndex) != kTfLiteOk)
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]333 {
Finn Williams6f9f9902020-11-13 13:23:15 +0000[diff] [blame]334 throw armnn::Exception(&"TfLiteArmnnDelegate: Unable to parse node: " [ nodeIndex]);
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]335 }
336 }
337
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]338 // Add Output layer
339 status = AddOutputLayer(delegateData, tfLiteContext, parameters->output_tensors, outputBindings);
340 if (status != kTfLiteOk)
341 {
342 throw armnn::Exception("TfLiteArmnnDelegate: Unable to add Outputs to the network!");
343 }
344
345 // Optimize ArmNN network
346 armnn::IOptimizedNetworkPtr optNet(nullptr, nullptr);
347 try
348 {
Sadik Armagan6e36a642020-11-10 21:18:41 +0000[diff] [blame]349 optNet = armnn::Optimize(*(delegateData.m_Network.get()),
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]350 delegate->m_Options.GetBackends(),
Narumol Prangnawarat0b51d5a2021-01-20 15:58:29 +0000[diff] [blame]351 delegate->m_Runtime->GetDeviceSpec(),
352 delegate->m_Options.GetOptimizerOptions());
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]353 }
354 catch (std::exception &ex)
355 {
356 std::stringstream exMessage;
357 exMessage << "TfLiteArmnnDelegate: Exception (" << ex.what() << ") caught from optimize.";
358 throw armnn::Exception(exMessage.str());
359 }
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]360 if (!optNet)
361 {
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]362 // Optimize failed
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]363 throw armnn::Exception("TfLiteArmnnDelegate: Unable to optimize the network!");
364 }
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]365
366 try
367 {
368 // Load graph into runtime
Narumol Prangnawarat0b51d5a2021-01-20 15:58:29 +0000[diff] [blame]369 std::string errorMessage;
Narumol Prangnawarat74a3cf52021-01-29 15:38:54 +0000[diff] [blame]370 armnn::Status loadingStatus;
371 if (delegate->m_Options.GetOptimizerOptions().m_ImportEnabled)
372 {
Francis Murtagh73d3e2e2021-04-29 14:23:04 +0100[diff] [blame]373 armnn::INetworkProperties networkProperties(false,
374 armnn::MemorySource::Malloc,
375 armnn::MemorySource::Malloc);
Narumol Prangnawarat74a3cf52021-01-29 15:38:54 +0000[diff] [blame]376 loadingStatus = delegate->m_Runtime->LoadNetwork(networkId,
377 std::move(optNet),
378 errorMessage,
379 networkProperties);
380 }
381 else
382 {
383 loadingStatus = delegate->m_Runtime->LoadNetwork(networkId,
384 std::move(optNet),
385 errorMessage);
386 }
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]387 if (loadingStatus != armnn::Status::Success)
388 {
389 // Optimize failed
Narumol Prangnawarat0b51d5a2021-01-20 15:58:29 +0000[diff] [blame]390 throw armnn::Exception("TfLiteArmnnDelegate: Network could not be loaded:" + errorMessage);
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]391 }
392 }
393 catch (std::exception& ex)
394 {
395 std::stringstream exMessage;
396 exMessage << "TfLiteArmnnDelegate: Exception (" << ex.what() << ") caught from LoadNetwork.";
397 throw armnn::Exception(exMessage.str());
398 }
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]399
Narumol Prangnawarat0b51d5a2021-01-20 15:58:29 +0000[diff] [blame]400 // Register debug callback function
401 if (delegate->m_Options.GetDebugCallbackFunction().has_value())
402 {
403 delegate->m_Runtime->RegisterDebugCallback(networkId, delegate->m_Options.GetDebugCallbackFunction().value());
404 }
405
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]406 // Create a new SubGraph with networkId and runtime
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]407 return new ArmnnSubgraph(networkId, delegate->m_Runtime.get(), inputBindings, outputBindings);
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]408}
409
410TfLiteStatus ArmnnSubgraph::Prepare(TfLiteContext* tfLiteContext)
411{
Finn Williams6f9f9902020-11-13 13:23:15 +0000[diff] [blame]412 armnn::IgnoreUnused(tfLiteContext);
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]413 return kTfLiteOk;
414}
415
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]416TfLiteStatus ArmnnSubgraph::Invoke(TfLiteContext* tfLiteContext, TfLiteNode* tfLiteNode)
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]417{
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]418 // Prepare inputs
419 armnn::InputTensors inputTensors;
420 size_t inputIndex = 0;
421 for (auto inputIdx : tflite::TfLiteIntArrayView(tfLiteNode->inputs))
422 {
423 TfLiteTensor* tensor = &tfLiteContext->tensors[inputIdx];
424 if (tensor->allocation_type != kTfLiteMmapRo)
425 {
426 const armnn::BindingPointInfo& inputBinding = m_InputBindings[inputIndex];
427 const armnn::ConstTensor inputTensor(inputBinding.second, tensor->data.data);
428 inputTensors.emplace_back(inputIdx, inputTensor);
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]429
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]430 ++inputIndex;
431 }
432 }
433
434 // Prepare outputs
435 armnn::OutputTensors outputTensors;
436 size_t outputIndex = 0;
437 for (auto outputIdx : tflite::TfLiteIntArrayView(tfLiteNode->outputs))
438 {
439 const armnn::BindingPointInfo& outputBinding = m_OutputBindings[outputIndex];
440 TfLiteTensor* tensor = &tfLiteContext->tensors[outputIdx];
441 const armnn::Tensor outputTensor(outputBinding.second, tensor->data.data);
442 outputTensors.emplace_back(outputIdx, outputTensor);
443
444 ++outputIndex;
445 }
446
447 // Run graph
448 auto status = m_Runtime->EnqueueWorkload(m_NetworkId, inputTensors, outputTensors);
449 return (status == armnn::Status::Success) ? kTfLiteOk : kTfLiteError;
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]450}
451
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]452TfLiteStatus ArmnnSubgraph::VisitNode(DelegateData& delegateData,
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]453 TfLiteContext* tfLiteContext,
454 TfLiteRegistration* tfLiteRegistration,
455 TfLiteNode* tfLiteNode,
456 int nodeIndex)
457{
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]458 switch (tfLiteRegistration->builtin_code)
459 {
460 case kTfLiteBuiltinAbs:
461 return VisitElementwiseUnaryOperator(delegateData,
462 tfLiteContext,
463 tfLiteNode,
464 nodeIndex,
465 armnn::UnaryOperation::Abs);
466 case kTfLiteBuiltinAdd:
467 return VisitElementwiseBinaryOperator(delegateData,
468 tfLiteContext,
469 tfLiteNode,
470 nodeIndex,
471 kTfLiteBuiltinAdd);
472 case kTfLiteBuiltinArgMax:
473 return VisitArgMinMaxOperator(delegateData,
474 tfLiteContext,
475 tfLiteNode,
476 nodeIndex,
477 kTfLiteBuiltinArgMax);
478 case kTfLiteBuiltinArgMin:
479 return VisitArgMinMaxOperator(delegateData,
480 tfLiteContext,
481 tfLiteNode,
482 nodeIndex,
483 kTfLiteBuiltinArgMin);
484 case kTfLiteBuiltinAveragePool2d:
485 return VisitPoolingOperator(delegateData,
486 tfLiteContext,
487 tfLiteNode,
488 nodeIndex,
489 kTfLiteBuiltinAveragePool2d);
490 case kTfLiteBuiltinBatchToSpaceNd:
491 return VisitBatchToSpaceNdOperator(delegateData,
492 tfLiteContext,
493 tfLiteNode,
494 nodeIndex,
495 kTfLiteBuiltinBatchToSpaceNd);
Sadik Armagan937565b2021-04-21 14:03:28 +0100[diff] [blame]496 case kTfLiteBuiltinCast:
497 return VisitCastOperator(delegateData,
498 tfLiteContext,
499 tfLiteNode,
500 nodeIndex,
501 kTfLiteBuiltinCast);
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]502 case kTfLiteBuiltinConcatenation:
503 return VisitControlOperator(delegateData,
504 tfLiteContext,
505 tfLiteNode,
506 nodeIndex,
507 kTfLiteBuiltinConcatenation);
508 case kTfLiteBuiltinConv2d:
509 return VisitConvolutionOperator(delegateData,
510 tfLiteContext,
511 tfLiteNode,
512 nodeIndex,
513 kTfLiteBuiltinConv2d);
514 case kTfLiteBuiltinDepthToSpace:
515 return VisitDepthToSpaceOperator(delegateData,
516 tfLiteContext,
517 tfLiteNode,
518 nodeIndex,
519 kTfLiteBuiltinDepthToSpace);
520 case kTfLiteBuiltinDepthwiseConv2d:
521 return VisitConvolutionOperator(delegateData,
522 tfLiteContext,
523 tfLiteNode,
524 nodeIndex,
525 kTfLiteBuiltinDepthwiseConv2d);
526 case kTfLiteBuiltinDequantize:
527 return VisitDequantizeOperator(delegateData,
528 tfLiteContext,
529 tfLiteNode,
530 nodeIndex,
531 kTfLiteBuiltinDequantize);
532 case kTfLiteBuiltinDiv:
533 return VisitElementwiseBinaryOperator(delegateData,
534 tfLiteContext,
535 tfLiteNode,
536 nodeIndex,
537 kTfLiteBuiltinDiv);
538 case kTfLiteBuiltinElu:
539 return VisitActivationOperator(delegateData,
540 tfLiteContext,
541 tfLiteNode,
542 nodeIndex,
543 kTfLiteBuiltinElu);
544 case kTfLiteBuiltinEqual:
545 return VisitComparisonOperator(delegateData,
546 tfLiteContext,
547 tfLiteNode,
548 nodeIndex,
549 kTfLiteBuiltinEqual);
550 case kTfLiteBuiltinExp:
551 return VisitElementwiseUnaryOperator(delegateData,
552 tfLiteContext,
553 tfLiteNode,
554 nodeIndex,
555 armnn::UnaryOperation::Exp);
556 case kTfLiteBuiltinExpandDims:
557 return VisitExpandDimsOperator(delegateData,
558 tfLiteContext,
559 tfLiteNode,
560 nodeIndex,
561 kTfLiteBuiltinExpandDims);
562 case kTfLiteBuiltinFill:
563 return VisitFillOperator(delegateData,
564 tfLiteContext,
565 tfLiteNode,
566 nodeIndex,
567 kTfLiteBuiltinFill);
568 case kTfLiteBuiltinFloor:
569 return VisitFloorOperator(delegateData,
570 tfLiteContext,
571 tfLiteNode,
572 nodeIndex,
573 kTfLiteBuiltinFloor);
574 case kTfLiteBuiltinFullyConnected:
575 return VisitFullyConnectedOperator(delegateData,
576 tfLiteContext,
577 tfLiteNode,
578 nodeIndex,
579 kTfLiteBuiltinFullyConnected);
580 case kTfLiteBuiltinGather:
581 return VisitGatherOperator(delegateData,
582 tfLiteContext,
583 tfLiteNode,
584 nodeIndex,
585 kTfLiteBuiltinGather);
586 case kTfLiteBuiltinGatherNd:
587 return VisitGatherOperator(delegateData,
588 tfLiteContext,
589 tfLiteNode,
590 nodeIndex,
591 kTfLiteBuiltinGatherNd);
592 case kTfLiteBuiltinGreater:
593 return VisitComparisonOperator(delegateData,
594 tfLiteContext,
595 tfLiteNode,
596 nodeIndex,
597 kTfLiteBuiltinGreater);
598 case kTfLiteBuiltinGreaterEqual:
599 return VisitComparisonOperator(delegateData,
600 tfLiteContext,
601 tfLiteNode,
602 nodeIndex,
603 kTfLiteBuiltinGreaterEqual);
604 case kTfLiteBuiltinHardSwish:
605 return VisitActivationOperator(delegateData,
606 tfLiteContext,
607 tfLiteNode,
608 nodeIndex,
609 kTfLiteBuiltinHardSwish);
610 case kTfLiteBuiltinL2Normalization:
Sadik Armagan4b227bb2021-01-22 10:53:38 +0000[diff] [blame]611 return VisitL2NormalizationOperator(delegateData,
612 tfLiteContext,
613 tfLiteNode,
614 nodeIndex,
615 kTfLiteBuiltinL2Normalization);
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]616 case kTfLiteBuiltinL2Pool2d:
617 return VisitPoolingOperator(delegateData,
618 tfLiteContext,
619 tfLiteNode,
620 nodeIndex,
621 kTfLiteBuiltinL2Pool2d);
622 case kTfLiteBuiltinLess:
623 return VisitComparisonOperator(delegateData,
624 tfLiteContext,
625 tfLiteNode,
626 nodeIndex,
627 kTfLiteBuiltinLess);
628 case kTfLiteBuiltinLessEqual:
629 return VisitComparisonOperator(delegateData,
630 tfLiteContext,
631 tfLiteNode,
632 nodeIndex,
633 kTfLiteBuiltinLessEqual);
634 case kTfLiteBuiltinLocalResponseNormalization:
Sadik Armagan4b227bb2021-01-22 10:53:38 +0000[diff] [blame]635 return VisitLocalResponseNormalizationOperator(delegateData,
636 tfLiteContext,
637 tfLiteNode,
638 nodeIndex,
639 kTfLiteBuiltinLocalResponseNormalization);
Matthew Sloyanc8eb9552020-11-26 10:54:22 +0000[diff] [blame]640 case kTfLiteBuiltinLogicalAnd:
641 return VisitLogicalBinaryOperator(delegateData,
642 tfLiteContext,
643 tfLiteNode,
644 nodeIndex,
645 kTfLiteBuiltinLogicalAnd,
646 armnn::LogicalBinaryOperation::LogicalAnd);
647 case kTfLiteBuiltinLogicalNot:
648 return VisitElementwiseUnaryOperator(delegateData,
649 tfLiteContext,
650 tfLiteNode,
651 nodeIndex,
652 armnn::UnaryOperation::LogicalNot);
653 case kTfLiteBuiltinLogicalOr:
654 return VisitLogicalBinaryOperator(delegateData,
655 tfLiteContext,
656 tfLiteNode,
657 nodeIndex,
658 kTfLiteBuiltinLogicalOr,
659 armnn::LogicalBinaryOperation::LogicalOr);
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]660 case kTfLiteBuiltinLogistic:
661 return VisitActivationOperator(delegateData,
662 tfLiteContext,
663 tfLiteNode,
664 nodeIndex,
665 kTfLiteBuiltinLogistic);
666 case kTfLiteBuiltinLogSoftmax:
667 return VisitSoftmaxOperator(delegateData,
668 tfLiteContext,
669 tfLiteNode,
670 nodeIndex,
671 kTfLiteBuiltinLogSoftmax);
672 case kTfLiteBuiltinLstm:
673 return VisitLstmOperator(delegateData,
674 tfLiteContext,
675 tfLiteNode,
676 nodeIndex,
677 kTfLiteBuiltinLstm);
678 case kTfLiteBuiltinMaxPool2d:
679 return VisitPoolingOperator(delegateData,
680 tfLiteContext,
681 tfLiteNode,
682 nodeIndex,
683 kTfLiteBuiltinMaxPool2d);
684 case kTfLiteBuiltinMaximum:
685 return VisitElementwiseBinaryOperator(delegateData,
686 tfLiteContext,
687 tfLiteNode,
688 nodeIndex,
689 kTfLiteBuiltinMaximum);
690 case kTfLiteBuiltinMean:
691 return VisitControlOperator(delegateData,
692 tfLiteContext,
693 tfLiteNode,
694 nodeIndex,
695 kTfLiteBuiltinMean);
696 case kTfLiteBuiltinMinimum:
697 return VisitElementwiseBinaryOperator(delegateData,
698 tfLiteContext,
699 tfLiteNode,
700 nodeIndex,
701 kTfLiteBuiltinMinimum);
702 case kTfLiteBuiltinMul:
703 return VisitElementwiseBinaryOperator(delegateData,
704 tfLiteContext,
705 tfLiteNode,
706 nodeIndex,
707 kTfLiteBuiltinMul);
708 case kTfLiteBuiltinNeg:
709 return VisitElementwiseUnaryOperator(delegateData,
710 tfLiteContext,
711 tfLiteNode,
712 nodeIndex,
713 armnn::UnaryOperation::Neg);
714 case kTfLiteBuiltinNotEqual:
715 return VisitComparisonOperator(delegateData,
716 tfLiteContext,
717 tfLiteNode,
718 nodeIndex,
719 kTfLiteBuiltinNotEqual);
Matthew Sloyana7a12f52021-05-06 10:05:28 +0100[diff] [blame]720 case kTfLiteBuiltinPack:
721 return VisitPackOperator(delegateData,
722 tfLiteContext,
723 tfLiteNode,
724 nodeIndex,
725 kTfLiteBuiltinPack);
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]726 case kTfLiteBuiltinPad:
727 return VisitPadOperator(delegateData,
728 tfLiteContext,
729 tfLiteNode,
730 nodeIndex,
731 kTfLiteBuiltinPad);
732 case kTfLiteBuiltinPadv2:
733 return VisitPadOperator(delegateData,
734 tfLiteContext,
735 tfLiteNode,
736 nodeIndex,
737 kTfLiteBuiltinPadv2);
738 case kTfLiteBuiltinPrelu:
James Conroy39825482021-05-27 17:44:50 +0100[diff] [blame]739 return VisitPreluOperator(delegateData,
740 tfLiteContext,
741 tfLiteNode,
742 nodeIndex,
743 kTfLiteBuiltinPrelu);
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]744 case kTfLiteBuiltinQuantize:
745 return VisitQuantizeOperator(delegateData,
746 tfLiteContext,
747 tfLiteNode,
748 nodeIndex,
749 kTfLiteBuiltinQuantize);
750 case kTfLiteBuiltinRank:
751 return VisitControlOperator(delegateData,
752 tfLiteContext,
753 tfLiteNode,
754 nodeIndex,
755 kTfLiteBuiltinRank);
Sadik Armagana2747482021-02-09 10:28:54 +0000[diff] [blame]756 case kTfLiteBuiltinReduceMax:
757 return VisitReduceOperator(delegateData,
758 tfLiteContext,
759 tfLiteNode,
760 nodeIndex,
761 kTfLiteBuiltinReduceMax);
762 case kTfLiteBuiltinReduceMin:
763 return VisitReduceOperator(delegateData,
764 tfLiteContext,
765 tfLiteNode,
766 nodeIndex,
767 kTfLiteBuiltinReduceMin);
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]768 case kTfLiteBuiltinRelu:
769 return VisitActivationOperator(delegateData,
770 tfLiteContext,
771 tfLiteNode,
772 nodeIndex,
773 kTfLiteBuiltinRelu);
774 case kTfLiteBuiltinReluN1To1:
775 return VisitActivationOperator(delegateData,
776 tfLiteContext,
777 tfLiteNode,
778 nodeIndex,
779 kTfLiteBuiltinReluN1To1);
780 case kTfLiteBuiltinRelu6:
781 return VisitActivationOperator(delegateData,
782 tfLiteContext,
783 tfLiteNode,
784 nodeIndex,
785 kTfLiteBuiltinRelu6);
786 case kTfLiteBuiltinReshape:
787 return VisitReshapeOperator(delegateData,
788 tfLiteContext,
789 tfLiteNode,
790 nodeIndex,
791 kTfLiteBuiltinReshape);
792 case kTfLiteBuiltinResizeBilinear:
793 return VisitResizeOperator(delegateData,
794 tfLiteContext,
795 tfLiteNode,
796 nodeIndex,
797 kTfLiteBuiltinResizeBilinear);
798 case kTfLiteBuiltinResizeNearestNeighbor:
799 return VisitResizeOperator(delegateData,
800 tfLiteContext,
801 tfLiteNode,
802 nodeIndex,
803 kTfLiteBuiltinResizeNearestNeighbor);
804 case kTfLiteBuiltinRsqrt:
805 return VisitElementwiseUnaryOperator(delegateData,
806 tfLiteContext,
807 tfLiteNode,
808 nodeIndex,
809 armnn::UnaryOperation::Rsqrt);
Keith Davis0176fd82021-06-01 17:36:32 +0100[diff] [blame]810 case kTfLiteBuiltinShape:
811 return VisitShapeOperator(delegateData,
812 tfLiteContext,
813 tfLiteNode,
814 nodeIndex,
815 kTfLiteBuiltinShape);
Sadik Armagan34fa1bd2020-11-27 12:40:52 +0000[diff] [blame]816 case kTfLiteBuiltinSplit:
817 return VisitSplitOperator(delegateData,
818 tfLiteContext,
819 tfLiteNode,
820 nodeIndex,
821 kTfLiteBuiltinSplit);
822 case kTfLiteBuiltinSplitV:
823 return VisitSplitVOperator(delegateData,
824 tfLiteContext,
825 tfLiteNode,
826 nodeIndex,
827 kTfLiteBuiltinSplitV);
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]828 case kTfLiteBuiltinSqrt:
829 return VisitElementwiseUnaryOperator(delegateData,
830 tfLiteContext,
831 tfLiteNode,
832 nodeIndex,
833 armnn::UnaryOperation::Sqrt);
834 case kTfLiteBuiltinSqueeze:
835 return VisitSqueezeOperator(delegateData,
836 tfLiteContext,
837 tfLiteNode,
838 nodeIndex,
839 kTfLiteBuiltinSqueeze);
840 case kTfLiteBuiltinStridedSlice:
841 return VisitSliceOperator(delegateData,
842 tfLiteContext,
843 tfLiteNode,
844 nodeIndex,
845 kTfLiteBuiltinStridedSlice);
Sadik Armagana2747482021-02-09 10:28:54 +0000[diff] [blame]846 case kTfLiteBuiltinSum:
847 return VisitReduceOperator(delegateData,
848 tfLiteContext,
849 tfLiteNode,
850 nodeIndex,
851 kTfLiteBuiltinSum);
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]852 case kTfLiteBuiltinTranspose:
853 return VisitTransposeOperator(delegateData,
854 tfLiteContext,
855 tfLiteNode,
856 nodeIndex,
857 kTfLiteBuiltinTranspose);
858 case kTfLiteBuiltinTransposeConv:
859 return VisitConvolutionOperator(delegateData,
860 tfLiteContext,
861 tfLiteNode,
862 nodeIndex,
863 kTfLiteBuiltinTransposeConv);
864 case kTfLiteBuiltinSoftmax:
865 return VisitSoftmaxOperator(delegateData,
866 tfLiteContext,
867 tfLiteNode,
868 nodeIndex,
869 kTfLiteBuiltinSoftmax);
870 case kTfLiteBuiltinSpaceToBatchNd:
871 return VisitSpaceToBatchNdOperator(delegateData,
872 tfLiteContext,
873 tfLiteNode,
874 nodeIndex,
875 kTfLiteBuiltinSpaceToBatchNd);
876 case kTfLiteBuiltinSpaceToDepth:
877 return VisitSpaceToDepthOperator(delegateData,
878 tfLiteContext,
879 tfLiteNode,
880 nodeIndex,
881 kTfLiteBuiltinSpaceToDepth);
882 case kTfLiteBuiltinSub:
883 return VisitElementwiseBinaryOperator(delegateData,
884 tfLiteContext,
885 tfLiteNode,
886 nodeIndex,
887 kTfLiteBuiltinSub);
888 case kTfLiteBuiltinTanh:
889 return VisitActivationOperator(delegateData,
890 tfLiteContext,
891 tfLiteNode,
892 nodeIndex,
893 kTfLiteBuiltinTanh);
Narumol Prangnawarat7684b182021-08-12 14:48:15 +0100[diff] [blame^]894 case kTfLiteBuiltinUnidirectionalSequenceLstm:
895 return VisitUnidirectionalSequenceLstmOperator(delegateData,
896 tfLiteContext,
897 tfLiteNode,
898 nodeIndex,
899 kTfLiteBuiltinUnidirectionalSequenceLstm);
Kevin May8ab2d7a2021-05-07 09:32:51 +0100[diff] [blame]900 case kTfLiteBuiltinUnpack:
901 return VisitUnpackOperator(delegateData,
902 tfLiteContext,
903 tfLiteNode,
904 nodeIndex,
905 kTfLiteBuiltinUnpack);
Sadik Armagan62483be2020-10-23 17:14:43 +0100[diff] [blame]906 default:
907 return kTfLiteError;
908 }
Sadik Armagan3c24f432020-10-19 17:35:30 +0100[diff] [blame]909}
910
911} // armnnDelegate namespace