Gitiles
Code Review Sign In
review.mlplatform.org / ml / armnn / a638f101bcb51008932f922fe0a5cef28633bc66 / . / src / backends / neon / NeonBackend.cpp
blob: 3a398e03e636a7bb3fe898e4204c34817e8f42d7 [file] [log] [blame]
arovir014424b0a2018-10-04 10:46:04 +0100[diff] [blame]1//
Mike Kelly3ec30772023-03-08 13:47:17 +0000[diff] [blame]2// Copyright © 2017-2023 Arm Ltd and Contributors. All rights reserved.
arovir014424b0a2018-10-04 10:46:04 +0100[diff] [blame]3// SPDX-License-Identifier: MIT
4//
5
6#include "NeonBackend.hpp"
David Beck3e9e1152018-10-17 14:17:50 +0100[diff] [blame]7#include "NeonBackendId.hpp"
Sadik Armagan045f6be2020-09-10 13:37:32 +0100[diff] [blame]8#include "NeonBackendModelContext.hpp"
arovir01a0944792018-10-11 15:00:58 +0100[diff] [blame]9#include "NeonWorkloadFactory.hpp"
David Beck111b5d92018-11-12 14:59:37 +0000[diff] [blame]10#include "NeonLayerSupport.hpp"
Narumol Prangnawarat4e3e8182019-08-14 12:25:50 +0100[diff] [blame]11#include "NeonTensorHandleFactory.hpp"
arovir01a0944792018-10-11 15:00:58 +0100[diff] [blame]12
Matteo Martincighc601aa62019-10-29 15:03:22 +0000[diff] [blame]13#include <armnn/BackendRegistry.hpp>
Mike Kelly07810fc2020-11-12 10:58:48 +0000[diff] [blame]14#include <armnn/Descriptors.hpp>
Matteo Martincighc601aa62019-10-29 15:03:22 +0000[diff] [blame]15
Mike Kelly07810fc2020-11-12 10:58:48 +0000[diff] [blame]16#include <aclCommon/ArmComputeSubgraphUtils.hpp>
17#include <aclCommon/ArmComputeUtils.hpp>
Aron Virginas-Tar56055192018-11-12 18:10:43 +0000[diff] [blame]18#include <aclCommon/BaseMemoryManager.hpp>
19
Matteo Martincighe5b8eb92019-11-28 15:45:42 +0000[diff] [blame]20#include <armnn/backends/IBackendContext.hpp>
21#include <armnn/backends/IMemoryManager.hpp>
Aron Virginas-Tar56055192018-11-12 18:10:43 +0000[diff] [blame]22
Jan Eilers3c9e0452020-04-10 13:00:44 +0100[diff] [blame]23#include <armnn/utility/PolymorphicDowncast.hpp>
24
Francis Murtaghe8d7ccb2021-10-14 17:30:24 +0100[diff] [blame]25#include <neon/workloads/NeonAdditionWorkload.hpp>
26#include <neon/workloads/NeonBatchNormalizationWorkload.hpp>
27#include <neon/workloads/NeonConvolution2dWorkload.hpp>
28#include <neon/workloads/NeonDepthwiseConvolutionWorkload.hpp>
29#include <neon/workloads/NeonDivisionWorkload.hpp>
30#include <neon/workloads/NeonFullyConnectedWorkload.hpp>
31#include <neon/workloads/NeonMultiplicationWorkload.hpp>
32#include <neon/workloads/NeonReduceWorkload.hpp>
33#include <neon/workloads/NeonSubtractionWorkload.hpp>
34#include <backendsCommon/DefaultAllocator.hpp>
Mike Kelly07810fc2020-11-12 10:58:48 +0000[diff] [blame]35
David Beck263e3492018-11-09 14:46:40 +0000[diff] [blame]36#include <Optimizer.hpp>
arovir01a0944792018-10-11 15:00:58 +0100[diff] [blame]37
Mike Kelly07810fc2020-11-12 10:58:48 +0000[diff] [blame]38#include <arm_compute/core/Types.h>
Aron Virginas-Tar56055192018-11-12 18:10:43 +0000[diff] [blame]39#include <arm_compute/runtime/Allocator.h>
40
arovir014424b0a2018-10-04 10:46:04 +0100[diff] [blame]41namespace armnn
42{
43
David Beck3cc9a622018-10-12 10:38:31 +0100[diff] [blame]44const BackendId& NeonBackend::GetIdStatic()
arovir014424b0a2018-10-04 10:46:04 +0100[diff] [blame]45{
David Beck3e9e1152018-10-17 14:17:50 +0100[diff] [blame]46 static const BackendId s_Id{NeonBackendId()};
arovir014424b0a2018-10-04 10:46:04 +0100[diff] [blame]47 return s_Id;
48}
49
Aron Virginas-Tar56055192018-11-12 18:10:43 +0000[diff] [blame]50IBackendInternal::IMemoryManagerUniquePtr NeonBackend::CreateMemoryManager() const
arovir014424b0a2018-10-04 10:46:04 +0100[diff] [blame]51{
Aron Virginas-Tar56055192018-11-12 18:10:43 +0000[diff] [blame]52 return std::make_unique<NeonMemoryManager>(std::make_unique<arm_compute::Allocator>(),
Sadik Armagan13a9fa62019-04-26 16:04:34 +0100[diff] [blame]53 BaseMemoryManager::MemoryAffinity::Offset);
Aron Virginas-Tar56055192018-11-12 18:10:43 +0000[diff] [blame]54}
55
56IBackendInternal::IWorkloadFactoryPtr NeonBackend::CreateWorkloadFactory(
57 const IBackendInternal::IMemoryManagerSharedPtr& memoryManager) const
58{
59 return std::make_unique<NeonWorkloadFactory>(
Jan Eilers3c9e0452020-04-10 13:00:44 +0100[diff] [blame]60 PolymorphicPointerDowncast<NeonMemoryManager>(memoryManager));
arovir014424b0a2018-10-04 10:46:04 +0100[diff] [blame]61}
62
Narumol Prangnawarat4e3e8182019-08-14 12:25:50 +0100[diff] [blame]63IBackendInternal::IWorkloadFactoryPtr NeonBackend::CreateWorkloadFactory(
Sadik Armagan04a72972020-09-14 15:44:18 +0100[diff] [blame]64 const IBackendInternal::IMemoryManagerSharedPtr& memoryManager, const ModelOptions& modelOptions) const
65{
66 return std::make_unique<NeonWorkloadFactory>(
67 PolymorphicPointerDowncast<NeonMemoryManager>(memoryManager), CreateBackendSpecificModelContext(modelOptions));
68}
69
70IBackendInternal::IWorkloadFactoryPtr NeonBackend::CreateWorkloadFactory(
Narumol Prangnawarat4e3e8182019-08-14 12:25:50 +0100[diff] [blame]71 class TensorHandleFactoryRegistry& tensorHandleFactoryRegistry) const
72{
73 auto memoryManager = std::make_shared<NeonMemoryManager>(std::make_unique<arm_compute::Allocator>(),
74 BaseMemoryManager::MemoryAffinity::Offset);
75
76 tensorHandleFactoryRegistry.RegisterMemoryManager(memoryManager);
Narumol Prangnawarat77400452022-01-13 17:43:41 +0000[diff] [blame]77
78 auto factory = std::make_unique<NeonTensorHandleFactory>(memoryManager);
79 // Register copy and import factory pair
80 tensorHandleFactoryRegistry.RegisterCopyAndImportFactoryPair(factory->GetId(), factory->GetId());
81 // Register the factory
82 tensorHandleFactoryRegistry.RegisterFactory(std::move(factory));
83
Narumol Prangnawarat549cb7a2020-07-10 17:50:53 +0100[diff] [blame]84
Narumol Prangnawarat4e3e8182019-08-14 12:25:50 +0100[diff] [blame]85 return std::make_unique<NeonWorkloadFactory>(
Jan Eilers3c9e0452020-04-10 13:00:44 +0100[diff] [blame]86 PolymorphicPointerDowncast<NeonMemoryManager>(memoryManager));
Narumol Prangnawarat4e3e8182019-08-14 12:25:50 +0100[diff] [blame]87}
88
Sadik Armagan04a72972020-09-14 15:44:18 +0100[diff] [blame]89IBackendInternal::IWorkloadFactoryPtr NeonBackend::CreateWorkloadFactory(
90 TensorHandleFactoryRegistry& tensorHandleFactoryRegistry, const ModelOptions& modelOptions) const
91{
92 auto memoryManager = std::make_shared<NeonMemoryManager>(std::make_unique<arm_compute::Allocator>(),
93 BaseMemoryManager::MemoryAffinity::Offset);
94
95 tensorHandleFactoryRegistry.RegisterMemoryManager(memoryManager);
Narumol Prangnawarat77400452022-01-13 17:43:41 +0000[diff] [blame]96
97 auto factory = std::make_unique<NeonTensorHandleFactory>(memoryManager);
98 // Register copy and import factory pair
99 tensorHandleFactoryRegistry.RegisterCopyAndImportFactoryPair(factory->GetId(), factory->GetId());
100 // Register the factory
101 tensorHandleFactoryRegistry.RegisterFactory(std::move(factory));
Sadik Armagan04a72972020-09-14 15:44:18 +0100[diff] [blame]102
103 return std::make_unique<NeonWorkloadFactory>(
104 PolymorphicPointerDowncast<NeonMemoryManager>(memoryManager), CreateBackendSpecificModelContext(modelOptions));
105}
106
David Beck263e3492018-11-09 14:46:40 +0000[diff] [blame]107IBackendInternal::IBackendContextPtr NeonBackend::CreateBackendContext(const IRuntime::CreationOptions&) const
108{
109 return IBackendContextPtr{};
110}
111
Colm Donelane49755b2020-01-29 15:22:43 +0000[diff] [blame]112IBackendInternal::IBackendProfilingContextPtr NeonBackend::CreateBackendProfilingContext(
Colm Donelan1aff3932020-02-05 17:48:59 +0000[diff] [blame]113 const IRuntime::CreationOptions&, IBackendProfilingPtr&)
Colm Donelane49755b2020-01-29 15:22:43 +0000[diff] [blame]114{
115 return IBackendProfilingContextPtr{};
116}
117
Sadik Armagan045f6be2020-09-10 13:37:32 +0100[diff] [blame]118IBackendInternal::IBackendSpecificModelContextPtr NeonBackend::CreateBackendSpecificModelContext(
119 const ModelOptions& modelOptions) const
120{
121 return IBackendSpecificModelContextPtr{new NeonBackendModelContext{modelOptions}};
122}
123
David Beck111b5d92018-11-12 14:59:37 +0000[diff] [blame]124IBackendInternal::ILayerSupportSharedPtr NeonBackend::GetLayerSupport() const
125{
Sadik Armagan045f6be2020-09-10 13:37:32 +0100[diff] [blame]126 static ILayerSupportSharedPtr layerSupport
127 {
128 new NeonLayerSupport(IBackendInternal::IBackendSpecificModelContextPtr{})
129 };
130 return layerSupport;
131}
132
133IBackendInternal::ILayerSupportSharedPtr NeonBackend::GetLayerSupport(const ModelOptions& modelOptions) const
134{
135 static ILayerSupportSharedPtr layerSupport
136 {
137 new NeonLayerSupport(CreateBackendSpecificModelContext(modelOptions))
138 };
David Beck111b5d92018-11-12 14:59:37 +0000[diff] [blame]139 return layerSupport;
140}
141
Mike Kelly80512b02022-05-16 23:10:42 +0100[diff] [blame]142OptimizationViews NeonBackend::OptimizeSubgraphView(const SubgraphView& subgraph,
143 const ModelOptions& modelOptions) const
Matteo Martincighadddddb2019-01-24 14:06:23 +0000[diff] [blame]144{
Mike Kelly80512b02022-05-16 23:10:42 +0100[diff] [blame]145 OptimizationViews optimizationViews(modelOptions);
Matteo Martincighadddddb2019-01-24 14:06:23 +0000[diff] [blame]146
Francis Murtagh56ccf682021-12-13 18:48:12 +0000[diff] [blame]147 auto it = subgraph.endIConnectable();
Mike Kelly1ac690a2020-11-17 11:41:38 +0000[diff] [blame]148 std::map<LayerGuid, Layer*> untouched;
Mike Kelly07810fc2020-11-12 10:58:48 +0000[diff] [blame]149
Francis Murtagh56ccf682021-12-13 18:48:12 +0000[diff] [blame]150 while (it != subgraph.beginIConnectable())
Mike Kelly07810fc2020-11-12 10:58:48 +0000[diff] [blame]151 {
152 --it;
Francis Murtagh56ccf682021-12-13 18:48:12 +0000[diff] [blame]153 Layer& base = *(PolymorphicDowncast<Layer*>(*it));
Mike Kelly1ac690a2020-11-17 11:41:38 +0000[diff] [blame]154 untouched.insert({base.GetGuid(), &base});
155 }
156
Francis Murtagh56ccf682021-12-13 18:48:12 +0000[diff] [blame]157 it = subgraph.endIConnectable();
158 while (it != subgraph.beginIConnectable())
Mike Kelly1ac690a2020-11-17 11:41:38 +0000[diff] [blame]159 {
160 --it;
Francis Murtagh56ccf682021-12-13 18:48:12 +0000[diff] [blame]161 Layer& base = *(PolymorphicDowncast<Layer*>(*it));
Mike Kelly07810fc2020-11-12 10:58:48 +0000[diff] [blame]162
Matthew Sloyan5fc0fd62021-05-03 12:22:03 +0100[diff] [blame]163 // Fuse activation into previous layer if supported by backend
Mike Kelly07810fc2020-11-12 10:58:48 +0000[diff] [blame]164 if ((base.GetType() == LayerType::DepthwiseConvolution2d || base.GetType() == LayerType::Convolution2d
165 || base.GetType() == LayerType::BatchNormalization || base.GetType() == LayerType::FullyConnected
166 || base.GetType() == LayerType::Addition || base.GetType() == LayerType::Multiplication
Matthew Sloyanae123062021-05-07 14:18:01 +0000[diff] [blame]167 || base.GetType() == LayerType::Subtraction || base.GetType() == LayerType::Division)
Mike Kelly07810fc2020-11-12 10:58:48 +0000[diff] [blame]168 && (base.GetAdditionalInformation<ActivationDescriptor>() == nullptr))
169 {
170 for (auto output = base.BeginOutputSlots(); output != base.EndOutputSlots(); ++output)
171 {
172 if (output->GetNumConnections() == 1)
173 {
174 for (auto&& childInput : output->GetConnections())
175 {
Teresa Charlind672f5d2021-01-18 18:07:57 +0000[diff] [blame]176 if ((childInput->GetOwningLayer().GetType() == LayerType::Activation) &&
177 (checkDataTypeInputandOutput(childInput->GetOwningLayer())))
Mike Kelly07810fc2020-11-12 10:58:48 +0000[diff] [blame]178 {
179 Layer& child = childInput->GetOwningLayer();
180
181 auto* activationLayer = PolymorphicDowncast<ActivationLayer*>(&child);
182
183 const std::string name = std::string("fused-") + child.GetName() + std::string("-into-") +
184 base.GetName();
185
186 // Get params from activation layer
187 ActivationDescriptor activationDesc = activationLayer->GetParameters();
188
189 if (base.GetType() == LayerType::Convolution2d)
190 {
191 Convolution2dLayer* baseLayer = PolymorphicDowncast<Convolution2dLayer*>(&base);
192
193 Optional<TensorInfo> biases;
194
195 if (baseLayer->GetParameters().m_BiasEnabled)
196 {
Keith Davisb4dd5cc2022-04-07 11:32:00 +0100[diff] [blame]197 biases = baseLayer->GetInputSlot(2).GetConnectedOutputSlot()->GetTensorInfo();
Mike Kelly07810fc2020-11-12 10:58:48 +0000[diff] [blame]198 }
199
200 arm_compute::Status status = NeonConvolution2dWorkloadValidate(
201 baseLayer->GetInputSlot(0).GetConnectedOutputSlot()->GetTensorInfo(),
202 activationLayer->GetInputSlot(0).GetConnectedOutputSlot()->GetTensorInfo(),
203 baseLayer->GetParameters(),
Keith Davisb4dd5cc2022-04-07 11:32:00 +0100[diff] [blame]204 baseLayer->GetInputSlot(1).GetConnectedOutputSlot()->GetTensorInfo(),
Mike Kelly07810fc2020-11-12 10:58:48 +0000[diff] [blame]205 biases,
206 false,
207 &activationDesc);
208
209 if (status)
210 {
Cathal Corbettcbfd7182021-12-15 17:12:59 +0000[diff] [blame]211 FuseConvolution2dLayer<Convolution2dLayer>(optimizationViews,
212 baseLayer,
213 activationLayer,
214 activationDesc,
215 name);
Mike Kelly1ac690a2020-11-17 11:41:38 +0000[diff] [blame]216 untouched.erase(baseLayer->GetGuid());
217 untouched.erase(activationLayer->GetGuid());
Mike Kelly07810fc2020-11-12 10:58:48 +0000[diff] [blame]218 }
219 }
220 else if (base.GetType() == LayerType::DepthwiseConvolution2d)
221 {
222 DepthwiseConvolution2dLayer* baseLayer =
223 PolymorphicDowncast<DepthwiseConvolution2dLayer*>(&base);
224
225 Optional<TensorInfo> biases;
226
227 if (baseLayer->GetParameters().m_BiasEnabled)
228 {
Cathal Corbett06902652022-04-14 17:55:11 +0100[diff] [blame]229 biases = baseLayer->GetInputSlot(2).GetConnectedOutputSlot()->GetTensorInfo();
Mike Kelly07810fc2020-11-12 10:58:48 +0000[diff] [blame]230 }
231
232 arm_compute::Status status = NeonDepthwiseConvolutionWorkloadValidate(
233 baseLayer->GetInputSlot(0).GetConnectedOutputSlot()->GetTensorInfo(),
234 activationLayer->GetInputSlot(0).GetConnectedOutputSlot()->GetTensorInfo(),
235 baseLayer->GetParameters(),
Cathal Corbett06902652022-04-14 17:55:11 +0100[diff] [blame]236 baseLayer->GetInputSlot(1).GetConnectedOutputSlot()->GetTensorInfo(),
Mike Kelly07810fc2020-11-12 10:58:48 +0000[diff] [blame]237 biases,
238 &activationDesc);
239
240 if (status)
241 {
Cathal Corbettcbfd7182021-12-15 17:12:59 +0000[diff] [blame]242 FuseDepthwiseConvolution2dLayer<DepthwiseConvolution2dLayer>(optimizationViews,
243 baseLayer,
244 activationLayer,
245 activationDesc,
246 name);
Mike Kelly1ac690a2020-11-17 11:41:38 +0000[diff] [blame]247 untouched.erase(baseLayer->GetGuid());
248 untouched.erase(activationLayer->GetGuid());
Mike Kelly07810fc2020-11-12 10:58:48 +0000[diff] [blame]249 }
250 }
251 else if (base.GetType() == LayerType::FullyConnected)
252 {
253 FullyConnectedLayer* baseLayer = PolymorphicDowncast<FullyConnectedLayer*>(&base);
Cathal Corbett4452baf2022-05-13 09:55:59 +0100[diff] [blame]254 FullyConnectedDescriptor descriptor = baseLayer->GetParameters();
Matthew Bentham67d63902022-02-08 15:03:07 +0000[diff] [blame]255
Cathal Corbett4452baf2022-05-13 09:55:59 +0100[diff] [blame]256 // As bias is optional only try to get TensorInfo from input if bias is enabled.
257 Optional<TensorInfo> biases;
258 if (descriptor.m_BiasEnabled)
Matthew Bentham67d63902022-02-08 15:03:07 +0000[diff] [blame]259 {
Cathal Corbett4452baf2022-05-13 09:55:59 +0100[diff] [blame]260 biases = baseLayer->GetInputSlot(2).GetConnectedOutputSlot()->GetTensorInfo();
Matthew Bentham67d63902022-02-08 15:03:07 +0000[diff] [blame]261 }
Mike Kelly07810fc2020-11-12 10:58:48 +0000[diff] [blame]262
263 arm_compute::Status status = NeonFullyConnectedWorkloadValidate(
264 baseLayer->GetInputSlot(0).GetConnectedOutputSlot()->GetTensorInfo(),
265 activationLayer->GetInputSlot(0).GetConnectedOutputSlot()->GetTensorInfo(),
Cathal Corbett4452baf2022-05-13 09:55:59 +0100[diff] [blame]266 baseLayer->GetInputSlot(1).GetConnectedOutputSlot()->GetTensorInfo(),
Matthew Bentham67d63902022-02-08 15:03:07 +0000[diff] [blame]267 biases,
Mike Kelly07810fc2020-11-12 10:58:48 +0000[diff] [blame]268 baseLayer->GetParameters(),
269 &activationDesc);
270
271 if (status)
272 {
Cathal Corbettcbfd7182021-12-15 17:12:59 +0000[diff] [blame]273 FuseFullyConnectedLayer<FullyConnectedLayer>(optimizationViews,
274 baseLayer,
275 activationLayer,
276 activationDesc,
277 name);
Mike Kelly1ac690a2020-11-17 11:41:38 +0000[diff] [blame]278 untouched.erase(baseLayer->GetGuid());
279 untouched.erase(activationLayer->GetGuid());
Mike Kelly07810fc2020-11-12 10:58:48 +0000[diff] [blame]280 }
281 }
282 else if (base.GetType() == LayerType::BatchNormalization)
283 {
284 BatchNormalizationLayer* baseLayer =
285 PolymorphicDowncast<BatchNormalizationLayer*>(&base);
286
287 arm_compute::Status status = NeonBatchNormalizationValidate(
288 baseLayer->GetInputSlot(0).GetConnectedOutputSlot()->GetTensorInfo(),
289 activationLayer->GetInputSlot(0).GetConnectedOutputSlot()->GetTensorInfo(),
290 baseLayer->m_Mean->GetTensorInfo(),
291 baseLayer->m_Variance->GetTensorInfo(),
292 baseLayer->m_Beta->GetTensorInfo(),
293 baseLayer->m_Gamma->GetTensorInfo(),
294 baseLayer->GetParameters(),
295 &activationDesc);
296
297 if (status)
298 {
299 BatchNormalizationLayer* replacementLayer =
Cathal Corbettcbfd7182021-12-15 17:12:59 +0000[diff] [blame]300 FuseBatchNormalizationLayer<BatchNormalizationLayer>(optimizationViews,
301 baseLayer,
302 activationLayer,
303 activationDesc,
304 name);
Mike Kelly07810fc2020-11-12 10:58:48 +0000[diff] [blame]305
306 replacementLayer->m_Beta = std::move(baseLayer->m_Beta);
307 replacementLayer->m_Gamma = std::move(baseLayer->m_Gamma);
308 replacementLayer->m_Mean = std::move(baseLayer->m_Mean);
309 replacementLayer->m_Variance = std::move(baseLayer->m_Variance);
Mike Kelly1ac690a2020-11-17 11:41:38 +0000[diff] [blame]310 untouched.erase(baseLayer->GetGuid());
311 untouched.erase(activationLayer->GetGuid());
Mike Kelly07810fc2020-11-12 10:58:48 +0000[diff] [blame]312 }
313 }
314 else if (base.GetType() == LayerType::Addition)
315 {
316 AdditionLayer* baseLayer = PolymorphicDowncast<AdditionLayer*>(&base);
317
318 arm_compute::Status status = NeonAdditionWorkloadValidate(
319 baseLayer->GetInputSlot(0).GetConnectedOutputSlot()->GetTensorInfo(),
320 baseLayer->GetInputSlot(1).GetConnectedOutputSlot()->GetTensorInfo(),
321 activationLayer->GetInputSlot(0).GetConnectedOutputSlot()->GetTensorInfo(),
322 &activationDesc);
323
324 if (status)
325 {
Cathal Corbettcbfd7182021-12-15 17:12:59 +0000[diff] [blame]326 FuseAdditionLayer<AdditionLayer>(optimizationViews,
327 baseLayer,
328 activationLayer,
329 activationDesc,
330 name);
Mike Kelly1ac690a2020-11-17 11:41:38 +0000[diff] [blame]331 untouched.erase(baseLayer->GetGuid());
332 untouched.erase(activationLayer->GetGuid());
Mike Kelly07810fc2020-11-12 10:58:48 +0000[diff] [blame]333 }
334 }
335 else if (base.GetType() == LayerType::Division)
336 {
337 DivisionLayer* baseLayer = PolymorphicDowncast<DivisionLayer*>(&base);
338
339 arm_compute::Status status = NeonDivisionWorkloadValidate(
340 baseLayer->GetInputSlot(0).GetConnectedOutputSlot()->GetTensorInfo(),
341 baseLayer->GetInputSlot(1).GetConnectedOutputSlot()->GetTensorInfo(),
342 activationLayer->GetInputSlot(0).GetConnectedOutputSlot()->GetTensorInfo(),
343 &activationDesc);
344
345 if (status)
346 {
Cathal Corbettcbfd7182021-12-15 17:12:59 +0000[diff] [blame]347 FuseDivisionLayer<DivisionLayer>(optimizationViews,
348 baseLayer,
349 activationLayer,
350 activationDesc,
351 name);
Mike Kelly1ac690a2020-11-17 11:41:38 +0000[diff] [blame]352 untouched.erase(baseLayer->GetGuid());
353 untouched.erase(activationLayer->GetGuid());
Mike Kelly07810fc2020-11-12 10:58:48 +0000[diff] [blame]354 }
355 }
356 else if (base.GetType() == LayerType::Multiplication)
357 {
358 MultiplicationLayer* baseLayer = PolymorphicDowncast<MultiplicationLayer*>(&base);
359
360 arm_compute::Status status = NeonMultiplicationWorkloadValidate(
361 baseLayer->GetInputSlot(0).GetConnectedOutputSlot()->GetTensorInfo(),
362 baseLayer->GetInputSlot(1).GetConnectedOutputSlot()->GetTensorInfo(),
363 activationLayer->GetInputSlot(0).GetConnectedOutputSlot()->GetTensorInfo(),
364 &activationDesc);
365
366 if (status)
367 {
Cathal Corbettcbfd7182021-12-15 17:12:59 +0000[diff] [blame]368 FuseMultiplicationLayer<MultiplicationLayer>(optimizationViews,
369 baseLayer,
370 activationLayer,
371 activationDesc,
372 name);
Mike Kelly1ac690a2020-11-17 11:41:38 +0000[diff] [blame]373 untouched.erase(baseLayer->GetGuid());
374 untouched.erase(activationLayer->GetGuid());
Mike Kelly07810fc2020-11-12 10:58:48 +0000[diff] [blame]375 }
376 }
377 else if (base.GetType() == LayerType::Subtraction)
378 {
379 SubtractionLayer* baseLayer = PolymorphicDowncast<SubtractionLayer*>(&base);
380
381 arm_compute::Status status = NeonSubtractionWorkloadValidate(
382 baseLayer->GetInputSlot(0).GetConnectedOutputSlot()->GetTensorInfo(),
383 baseLayer->GetInputSlot(1).GetConnectedOutputSlot()->GetTensorInfo(),
384 activationLayer->GetInputSlot(0).GetConnectedOutputSlot()->GetTensorInfo(),
385 &activationDesc);
386
387 if (status)
388 {
Cathal Corbettcbfd7182021-12-15 17:12:59 +0000[diff] [blame]389 FuseSubtractionLayer<SubtractionLayer>(optimizationViews,
390 baseLayer,
391 activationLayer,
392 activationDesc,
393 name);
Mike Kelly1ac690a2020-11-17 11:41:38 +0000[diff] [blame]394 untouched.erase(baseLayer->GetGuid());
395 untouched.erase(activationLayer->GetGuid());
Mike Kelly07810fc2020-11-12 10:58:48 +0000[diff] [blame]396 }
397 }
Mike Kelly3ec30772023-03-08 13:47:17 +0000[diff] [blame]398 else if (base.GetType() == LayerType::ElementwiseBinary)
399 {
400 ElementwiseBinaryLayer* baseLayer = PolymorphicDowncast<ElementwiseBinaryLayer*>(&base);
401
402 if (baseLayer->GetParameters().m_Operation == BinaryOperation::Add)
403 {
404 arm_compute::Status status = NeonAdditionWorkloadValidate(
405 baseLayer->GetInputSlot(0).GetConnectedOutputSlot()->GetTensorInfo(),
406 baseLayer->GetInputSlot(1).GetConnectedOutputSlot()->GetTensorInfo(),
407 activationLayer->GetInputSlot(0).GetConnectedOutputSlot()->GetTensorInfo(),
408 &activationDesc);
409
410 if (status)
411 {
412 FuseElementwiseBinaryLayer<ElementwiseBinaryLayer>(optimizationViews,
413 baseLayer,
414 activationLayer,
415 activationDesc,
416 BinaryOperation::Add,
417 name);
418 untouched.erase(baseLayer->GetGuid());
419 untouched.erase(activationLayer->GetGuid());
420 }
421 }
422 else if (baseLayer->GetParameters().m_Operation == BinaryOperation::Div)
423 {
424 arm_compute::Status status = NeonDivisionWorkloadValidate(
425 baseLayer->GetInputSlot(0).GetConnectedOutputSlot()->GetTensorInfo(),
426 baseLayer->GetInputSlot(1).GetConnectedOutputSlot()->GetTensorInfo(),
427 activationLayer->GetInputSlot(0).GetConnectedOutputSlot()->GetTensorInfo(),
428 &activationDesc);
429
430 if (status)
431 {
432 FuseElementwiseBinaryLayer<ElementwiseBinaryLayer>(optimizationViews,
433 baseLayer,
434 activationLayer,
435 activationDesc,
436 BinaryOperation::Div,
437 name);
438 untouched.erase(baseLayer->GetGuid());
439 untouched.erase(activationLayer->GetGuid());
440 }
441 }
442 else if (baseLayer->GetParameters().m_Operation == BinaryOperation::Mul)
443 {
444 arm_compute::Status status = NeonMultiplicationWorkloadValidate(
445 baseLayer->GetInputSlot(0).GetConnectedOutputSlot()->GetTensorInfo(),
446 baseLayer->GetInputSlot(1).GetConnectedOutputSlot()->GetTensorInfo(),
447 activationLayer->GetInputSlot(0).GetConnectedOutputSlot()->GetTensorInfo(),
448 &activationDesc);
449
450 if (status)
451 {
452 FuseElementwiseBinaryLayer<ElementwiseBinaryLayer>(optimizationViews,
453 baseLayer,
454 activationLayer,
455 activationDesc,
456 BinaryOperation::Mul,
457 name);
458 untouched.erase(baseLayer->GetGuid());
459 untouched.erase(activationLayer->GetGuid());
460 }
461 }
462 else if (baseLayer->GetParameters().m_Operation == BinaryOperation::Sub)
463 {
464 arm_compute::Status status = NeonSubtractionWorkloadValidate(
465 baseLayer->GetInputSlot(0).GetConnectedOutputSlot()->GetTensorInfo(),
466 baseLayer->GetInputSlot(1).GetConnectedOutputSlot()->GetTensorInfo(),
467 activationLayer->GetInputSlot(0).GetConnectedOutputSlot()->GetTensorInfo(),
468 &activationDesc);
469
470 if (status)
471 {
472 FuseElementwiseBinaryLayer<ElementwiseBinaryLayer>(optimizationViews,
473 baseLayer,
474 activationLayer,
475 activationDesc,
476 BinaryOperation::Sub,
477 name);
478 untouched.erase(baseLayer->GetGuid());
479 untouched.erase(activationLayer->GetGuid());
480 }
481 }
482 // No fusion available for other BinaryOperations
483 }
Mike Kelly07810fc2020-11-12 10:58:48 +0000[diff] [blame]484 }
485 }
486 }
487 }
488 }
Matthew Sloyan5fc0fd62021-05-03 12:22:03 +0100[diff] [blame]489
490 // Separate reduce layer with multiple axes into multiple reduce layers with 1 axis.
491 if (base.GetType() == LayerType::Reduce)
492 {
493 ReduceLayer* baseLayer = PolymorphicDowncast<ReduceLayer*>(&base);
494 ReduceDescriptor reduceDescriptor = baseLayer->GetParameters();
495
496 if (!reduceDescriptor.m_vAxis.empty() && reduceDescriptor.m_vAxis.size() > 1)
497 {
498 // Add new layers to the graph and connect them.
Francis Murtagh56ccf682021-12-13 18:48:12 +0000[diff] [blame]499 std::vector<IConnectableLayer*> layers = ChainReduceLayers<ReduceLayer>(optimizationViews,
500 baseLayer,
501 reduceDescriptor);
Matthew Sloyan5fc0fd62021-05-03 12:22:03 +0100[diff] [blame]502
503 // Replace existing baselayer with new subgraph.
504 ReplaceLayers<ReduceLayer>(optimizationViews, baseLayer, layers);
505 untouched.erase(baseLayer->GetGuid());
506 }
507 }
Mike Kelly4cc341c2023-07-07 15:43:06 +0100[diff] [blame]508
509 // Remove Reshape where possible
510 if (base.GetType() == LayerType::Reshape)
511 {
512 ReshapeLayer* baseLayer = PolymorphicDowncast<ReshapeLayer*>(&base);
Mike Kelly4cc341c2023-07-07 15:43:06 +0100[diff] [blame]513
Mike Kellybe06f102023-07-17 17:49:55 +0100[diff] [blame]514 // Cannot remove a Reshape if it's connected to any layer that has an NCHW layout
515 if (ConnectedToLayerWithNCHW(baseLayer))
Mike Kelly4cc341c2023-07-07 15:43:06 +0100[diff] [blame]516 {
517 continue;
518 }
Mike Kellya638f102023-07-24 17:42:47 +0100[diff] [blame^]519 // Cannot remove a Reshape if it's connected to a SplitterLayer
520 if (ConnectedToLayerType(baseLayer, LayerType::Splitter))
Mike Kelly4cc341c2023-07-07 15:43:06 +0100[diff] [blame]521 {
522 continue;
523 }
524 RemoveReshapeLayer(baseLayer, untouched, optimizationViews);
525 }
Mike Kelly07810fc2020-11-12 10:58:48 +0000[diff] [blame]526 }
527
Mike Kelly4cc341c2023-07-07 15:43:06 +0100[diff] [blame]528 if (optimizationViews.GetSubstitutions().empty() && optimizationViews.GetDeletedSubgraphs().empty())
Mike Kelly07810fc2020-11-12 10:58:48 +0000[diff] [blame]529 {
530 optimizationViews.AddUntouchedSubgraph(SubgraphView(subgraph));
531 }
Mike Kelly1ac690a2020-11-17 11:41:38 +0000[diff] [blame]532 else
533 {
534 ReportUntouchedLayers(optimizationViews, untouched);
535 }
Matteo Martincighc3ba50e2019-05-22 14:28:16 +0100[diff] [blame]536
537 return optimizationViews;
Matteo Martincighadddddb2019-01-24 14:06:23 +0000[diff] [blame]538}
539
Narumol Prangnawarat4e3e8182019-08-14 12:25:50 +0100[diff] [blame]540std::vector<ITensorHandleFactory::FactoryId> NeonBackend::GetHandleFactoryPreferences() const
541{
Narumol Prangnawarat265e53e2020-10-30 16:06:55 +0000[diff] [blame]542 return std::vector<ITensorHandleFactory::FactoryId>() = { NeonTensorHandleFactory::GetIdStatic() };
Narumol Prangnawarat4e3e8182019-08-14 12:25:50 +0100[diff] [blame]543}
544
545void NeonBackend::RegisterTensorHandleFactories(class TensorHandleFactoryRegistry& registry)
546{
547 auto memoryManager = std::make_shared<NeonMemoryManager>(std::make_unique<arm_compute::Allocator>(),
548 BaseMemoryManager::MemoryAffinity::Offset);
549
550 registry.RegisterMemoryManager(memoryManager);
Narumol Prangnawarat77400452022-01-13 17:43:41 +0000[diff] [blame]551
552 auto factory = std::make_unique<NeonTensorHandleFactory>(memoryManager);
553 // Register copy and import factory pair
554 registry.RegisterCopyAndImportFactoryPair(factory->GetId(), factory->GetId());
555 // Register the factory
556 registry.RegisterFactory(std::move(factory));
Narumol Prangnawarat4e3e8182019-08-14 12:25:50 +0100[diff] [blame]557}
558
Francis Murtaghe8d7ccb2021-10-14 17:30:24 +0100[diff] [blame]559std::unique_ptr<ICustomAllocator> NeonBackend::GetDefaultAllocator() const
560{
561 return std::make_unique<DefaultAllocator>();
562}
563
564
Matthew Bentham42bad952018-12-17 09:23:36 +0000[diff] [blame]565} // namespace armnn