blob: f2ba94f5973d0179a9e0fa9686238e918c72e590 [file] [log] [blame]
Laurent Carlier749294b2020-06-01 09:03:17 +01001//
Teresa Charlin52664732020-06-29 16:27:03 +01002// Copyright © 2017 Arm Ltd and Contributors. All rights reserved.
David Beckecb56cd2018-09-05 12:52:57 +01003// SPDX-License-Identifier: MIT
telsoa014fcda012018-03-09 14:13:49 +00004//
Matteo Martincigh49124022019-01-11 13:25:59 +00005
telsoa014fcda012018-03-09 14:13:49 +00006#include "Network.hpp"
7#include "Graph.hpp"
8#include "Layer.hpp"
telsoa01c577f2c2018-08-31 09:22:23 +01009#include "DeviceSpec.hpp"
telsoa014fcda012018-03-09 14:13:49 +000010#include "Optimizer.hpp"
Derek Lambertiff05cc52019-04-26 13:05:17 +010011#include "SubgraphViewSelector.hpp"
Matteo Martincigh49124022019-01-11 13:25:59 +000012#include "BackendSettings.hpp"
David Beckac42efd2018-09-26 17:41:13 +010013#include "optimizations/All.hpp"
telsoa014fcda012018-03-09 14:13:49 +000014
Colm Donelan0c479742021-12-10 12:43:54 +000015#include <armnn/backends/TensorHandle.hpp>
16#include <armnn/backends/WorkloadFactory.hpp>
Matteo Martincighe5b8eb92019-11-28 15:45:42 +000017#include <armnn/backends/IBackendInternal.hpp>
Derek Lamberti84da38b2019-06-13 11:40:08 +010018#include <backendsCommon/TensorHandleFactoryRegistry.hpp>
David Beckac42efd2018-09-26 17:41:13 +010019
20#include <armnn/Exceptions.hpp>
telsoa014fcda012018-03-09 14:13:49 +000021#include <armnn/Utils.hpp>
telsoa01c577f2c2018-08-31 09:22:23 +010022#include <armnn/TypesUtils.hpp>
Matteo Martincighc601aa62019-10-29 15:03:22 +000023#include <armnn/BackendRegistry.hpp>
Matthew Benthamf48afc62020-01-15 17:55:08 +000024#include <armnn/Logging.hpp>
Narumol Prangnawaratac2770a2020-04-01 16:51:23 +010025#include <armnn/utility/Assert.hpp>
Jan Eilers8eb25602020-03-09 12:13:48 +000026#include <armnn/utility/IgnoreUnused.hpp>
Jan Eilersbb446e52020-04-02 13:56:54 +010027#include <armnn/utility/PolymorphicDowncast.hpp>
telsoa014fcda012018-03-09 14:13:49 +000028
Jim Flynn27761832022-03-20 21:52:17 +000029#include <client/include/IProfilingService.hpp>
Jan Eilers99d9d4a2019-11-06 10:02:16 +000030
Nikhil Raj77fe76b2021-06-09 14:55:32 +010031#include <common/include/ProfilingGuid.hpp>
32
Matthew Sloyan81beae32021-07-13 19:46:11 +010033#include <fmt/format.h>
34
telsoa014fcda012018-03-09 14:13:49 +000035#include <fcntl.h>
36#include <algorithm>
37#include <fstream>
38#include <memory>
telsoa01c577f2c2018-08-31 09:22:23 +010039#include <vector>
40#include <algorithm>
telsoa014fcda012018-03-09 14:13:49 +000041
telsoa014fcda012018-03-09 14:13:49 +000042namespace armnn
43{
44
Francis Murtagh3d2b4b22021-02-15 18:23:17 +000045INetwork::INetwork(NetworkOptions networkOptions) : pNetworkImpl(new NetworkImpl(networkOptions)) {}
46
47INetwork::~INetwork() = default;
48
49Status INetwork::PrintGraph()
50{
51 return pNetworkImpl->PrintGraph();
52}
53
54IConnectableLayer* INetwork::AddInputLayer(LayerBindingId id, const char* name)
55{
56 return pNetworkImpl->AddInputLayer(id, name);
57}
58
59
60IConnectableLayer* INetwork::AddArgMinMaxLayer(const ArgMinMaxDescriptor& desc,
61 const char* name)
62{
63 return pNetworkImpl->AddArgMinMaxLayer(desc, name);
64}
65
mathad01b392e982021-04-07 12:07:30 +010066IConnectableLayer* INetwork::AddCastLayer(const char* name)
67{
68 return pNetworkImpl->AddCastLayer(name);
69}
Francis Murtagh3d2b4b22021-02-15 18:23:17 +000070
71IConnectableLayer* INetwork::AddComparisonLayer(const ComparisonDescriptor& comparisonDescriptor,
72 const char* name)
73{
74 return pNetworkImpl->AddComparisonLayer(comparisonDescriptor, name);
75}
76
77
78IConnectableLayer* INetwork::AddConcatLayer(const ConcatDescriptor& concatDescriptor,
79 const char* name)
80{
81 return pNetworkImpl->AddConcatLayer(concatDescriptor, name);
82}
83
84
85IConnectableLayer* INetwork::AddConvolution2dLayer(const Convolution2dDescriptor& convolution2dDescriptor,
Francis Murtagh3d2b4b22021-02-15 18:23:17 +000086 const char* name)
87{
Keith Davisb4dd5cc2022-04-07 11:32:00 +010088 return pNetworkImpl->AddConvolution2dLayer(convolution2dDescriptor, name);
Francis Murtagh3d2b4b22021-02-15 18:23:17 +000089}
90
Keith Davisb4dd5cc2022-04-07 11:32:00 +010091ARMNN_NO_DEPRECATE_WARN_BEGIN
Francis Murtagh3d2b4b22021-02-15 18:23:17 +000092IConnectableLayer* INetwork::AddConvolution2dLayer(const Convolution2dDescriptor& convolution2dDescriptor,
Keith Davis721e6292022-05-17 10:06:53 +010093 const ConstTensor& weights,
94 const Optional<ConstTensor>& biases,
95 const char* name)
Francis Murtagh3d2b4b22021-02-15 18:23:17 +000096{
Francis Murtagh3d2b4b22021-02-15 18:23:17 +000097 return pNetworkImpl->AddConvolution2dLayer(convolution2dDescriptor,
98 weights,
99 armnn::Optional<ConstTensor>(biases),
100 name);
101}
Keith Davisb4dd5cc2022-04-07 11:32:00 +0100102ARMNN_NO_DEPRECATE_WARN_END
Francis Murtagh3d2b4b22021-02-15 18:23:17 +0000103
Matthew Sloyanb63a3112021-09-08 13:05:51 +0100104IConnectableLayer* INetwork::AddConvolution3dLayer(const Convolution3dDescriptor& convolution3dDescriptor,
Matthew Sloyanb63a3112021-09-08 13:05:51 +0100105 const char* name)
106{
Matthew Sloyan5d7b0a32021-10-18 13:07:49 +0100107 return pNetworkImpl->AddConvolution3dLayer(convolution3dDescriptor, name);
Matthew Sloyanb63a3112021-09-08 13:05:51 +0100108}
109
110
Francis Murtagh3d2b4b22021-02-15 18:23:17 +0000111IConnectableLayer* INetwork::AddDepthToSpaceLayer(const DepthToSpaceDescriptor& depthToSpaceDescriptor,
112 const char* name)
113{
114 return pNetworkImpl->AddDepthToSpaceLayer(depthToSpaceDescriptor, name);
115}
116
117
118IConnectableLayer* INetwork::AddDepthwiseConvolution2dLayer(
119 const DepthwiseConvolution2dDescriptor& convolution2dDescriptor,
Cathal Corbett06902652022-04-14 17:55:11 +0100120 const char* name)
121{
122 return pNetworkImpl->AddDepthwiseConvolution2dLayer(convolution2dDescriptor, name);
123}
124
125
126ARMNN_NO_DEPRECATE_WARN_BEGIN
127IConnectableLayer* INetwork::AddDepthwiseConvolution2dLayer(
128 const DepthwiseConvolution2dDescriptor& convolution2dDescriptor,
Francis Murtagh3d2b4b22021-02-15 18:23:17 +0000129 const ConstTensor& weights,
130 const Optional<ConstTensor>& biases,
131 const char* name)
132{
133 return pNetworkImpl->AddDepthwiseConvolution2dLayer(convolution2dDescriptor, weights, biases, name);
134}
Cathal Corbett06902652022-04-14 17:55:11 +0100135ARMNN_NO_DEPRECATE_WARN_END
Francis Murtagh3d2b4b22021-02-15 18:23:17 +0000136
137
Francis Murtagh3d2b4b22021-02-15 18:23:17 +0000138IConnectableLayer* INetwork::AddDequantizeLayer(const char* name)
139{
140 return pNetworkImpl->AddDequantizeLayer(name);
141}
142
143
144IConnectableLayer* INetwork::AddDetectionPostProcessLayer(
145 const DetectionPostProcessDescriptor& descriptor,
146 const ConstTensor& anchors,
147 const char* name)
148{
149 return pNetworkImpl->AddDetectionPostProcessLayer(descriptor, anchors, name);
150}
151
152
153IConnectableLayer* INetwork::AddElementwiseUnaryLayer(const ElementwiseUnaryDescriptor& elementwiseUnaryDescriptor,
154 const char* name)
155{
156 return pNetworkImpl->AddElementwiseUnaryLayer(elementwiseUnaryDescriptor, name);
157}
158
159
160IConnectableLayer* INetwork::AddFillLayer(const FillDescriptor& fillDescriptor,
161 const char* name)
162{
163 return pNetworkImpl->AddFillLayer(fillDescriptor, name);
164}
165
Francis Murtagh3d2b4b22021-02-15 18:23:17 +0000166IConnectableLayer* INetwork::AddFullyConnectedLayer(const FullyConnectedDescriptor& fullyConnectedDescriptor,
Matthew Sloyan81beae32021-07-13 19:46:11 +0100167 const char* name)
168{
169 return pNetworkImpl->AddFullyConnectedLayer(fullyConnectedDescriptor, name);
170}
171
Francis Murtagh3d2b4b22021-02-15 18:23:17 +0000172IConnectableLayer* INetwork::AddPermuteLayer(const PermuteDescriptor& permuteDescriptor,
173 const char* name)
174{
175 return pNetworkImpl->AddPermuteLayer(permuteDescriptor, name);
176}
177
178IConnectableLayer* INetwork::AddBatchToSpaceNdLayer(const BatchToSpaceNdDescriptor& batchToSpaceNdDescriptor,
179 const char* name)
180{
181 return pNetworkImpl->AddBatchToSpaceNdLayer(batchToSpaceNdDescriptor, name);
182}
183
184IConnectableLayer* INetwork::AddPooling2dLayer(const Pooling2dDescriptor& pooling2dDescriptor,
185 const char* name)
186{
187 return pNetworkImpl->AddPooling2dLayer(pooling2dDescriptor, name);
188}
189
Tamás Nyíri7b885b32021-10-26 14:47:57 +0100190IConnectableLayer* INetwork::AddPooling3dLayer(const Pooling3dDescriptor& pooling3dDescriptor,
191 const char* name)
192{
193 return pNetworkImpl->AddPooling3dLayer(pooling3dDescriptor, name);
194}
195
Cathal Corbett18655b82021-12-13 13:03:22 +0000196IConnectableLayer* INetwork::AddPrecompiledLayer(const PreCompiledDescriptor& preCompiledDescriptor,
Cathal Corbett3ea01072022-01-06 10:29:43 +0000197 CompiledBlobPtr compiledBlobPtr,
Cathal Corbettcbfd7182021-12-15 17:12:59 +0000198 const Optional<BackendId>& backend,
199 const char* name)
Cathal Corbett18655b82021-12-13 13:03:22 +0000200{
Cathal Corbett3ea01072022-01-06 10:29:43 +0000201 return pNetworkImpl->AddPrecompiledLayer(preCompiledDescriptor, std::move(compiledBlobPtr), backend, name);
Cathal Corbett18655b82021-12-13 13:03:22 +0000202}
203
Francis Murtagh3d2b4b22021-02-15 18:23:17 +0000204IConnectableLayer* INetwork::AddActivationLayer(const ActivationDescriptor& activationDescriptor,
205 const char* name)
206{
207 return pNetworkImpl->AddActivationLayer(activationDescriptor, name);
208}
209
210IConnectableLayer* INetwork::AddNormalizationLayer(const NormalizationDescriptor& normalizationDescriptor,
211 const char* name)
212{
213 return pNetworkImpl->AddNormalizationLayer(normalizationDescriptor, name);
214}
215
216IConnectableLayer* INetwork::AddSliceLayer(const SliceDescriptor& sliceDescriptor, const char* name)
217{
218 return pNetworkImpl->AddSliceLayer(sliceDescriptor, name);
219}
220IConnectableLayer* INetwork::AddSoftmaxLayer(const SoftmaxDescriptor& softmaxDescriptor,
221 const char* name)
222{
223 return pNetworkImpl->AddSoftmaxLayer(softmaxDescriptor, name);
224}
225
226IConnectableLayer* INetwork::AddSplitterLayer(const ViewsDescriptor& splitterDescriptor,
227 const char* name)
228{
229 return pNetworkImpl->AddSplitterLayer(splitterDescriptor, name);
230}
231
232IConnectableLayer* INetwork::AddMergeLayer(const char* name)
233{
234 return pNetworkImpl->AddMergeLayer(name);
235}
236
Francis Murtagh3d2b4b22021-02-15 18:23:17 +0000237IConnectableLayer* INetwork::AddAdditionLayer(const char* name)
238{
239 return pNetworkImpl->AddAdditionLayer(name);
240}
241
242IConnectableLayer* INetwork::AddMultiplicationLayer(const char* name)
243{
244 return pNetworkImpl->AddMultiplicationLayer(name);
245}
246
247IConnectableLayer* INetwork::AddBatchNormalizationLayer(const BatchNormalizationDescriptor& desc,
248 const ConstTensor& mean,
249 const ConstTensor& variance,
250 const ConstTensor& beta,
251 const ConstTensor& gamma,
252 const char* name)
253{
254 return pNetworkImpl->AddBatchNormalizationLayer(desc, mean, variance, beta, gamma, name);
255}
256
257IConnectableLayer* INetwork::AddRankLayer(const char* name)
258{
259 return pNetworkImpl->AddRankLayer(name);
260}
261
Francis Murtagh3d2b4b22021-02-15 18:23:17 +0000262IConnectableLayer* INetwork::AddResizeLayer(const ResizeDescriptor& resizeDescriptor,
263 const char* name)
264{
265 return pNetworkImpl->AddResizeLayer(resizeDescriptor, name);
266}
267
268IConnectableLayer* INetwork::AddReduceLayer(const ReduceDescriptor& reduceDescriptor,
269 const char* name)
270{
271 return pNetworkImpl->AddReduceLayer(reduceDescriptor, name);
272}
273
274IConnectableLayer* INetwork::AddInstanceNormalizationLayer(const InstanceNormalizationDescriptor& desc,
275 const char* name)
276{
277 return pNetworkImpl->AddInstanceNormalizationLayer(desc, name);
278}
279
280IConnectableLayer* INetwork::AddL2NormalizationLayer(const L2NormalizationDescriptor& desc,
281 const char* name)
282{
283 return pNetworkImpl->AddL2NormalizationLayer(desc, name);
284}
285
286IConnectableLayer* INetwork::AddLogSoftmaxLayer(const LogSoftmaxDescriptor& logSoftmaxDescriptor,
287 const char* name)
288{
289 return pNetworkImpl->AddLogSoftmaxLayer(logSoftmaxDescriptor, name);
290}
291
292IConnectableLayer* INetwork::AddConstantLayer(const ConstTensor& input,
293 const char* name)
294{
295 return pNetworkImpl->AddConstantLayer(input, name);
296}
297
298IConnectableLayer* INetwork::AddReshapeLayer(const ReshapeDescriptor& reshapeDescriptor,
299 const char* name)
300{
301 return pNetworkImpl->AddReshapeLayer(reshapeDescriptor, name);
302}
303
304IConnectableLayer* INetwork::AddSpaceToBatchNdLayer(const SpaceToBatchNdDescriptor& spaceToBatchNdDescriptor,
305 const char* name)
306{
307 return pNetworkImpl->AddSpaceToBatchNdLayer(spaceToBatchNdDescriptor, name);
308}
309
310IConnectableLayer* INetwork::AddSpaceToDepthLayer(const SpaceToDepthDescriptor& spaceToDepthDescriptor,
311 const char* name)
312{
313 return pNetworkImpl->AddSpaceToDepthLayer(spaceToDepthDescriptor, name);
314}
315
316IConnectableLayer* INetwork::AddFloorLayer(const char* name)
317{
318 return pNetworkImpl->AddFloorLayer(name);
319}
320IConnectableLayer* INetwork::AddOutputLayer(LayerBindingId id, const char* name)
321{
322 return pNetworkImpl->AddOutputLayer(id, name);
323}
324
325IConnectableLayer* INetwork::AddLstmLayer(const LstmDescriptor& descriptor,
326 const LstmInputParams& params,
327 const char* name)
328{
329 return pNetworkImpl->AddLstmLayer(descriptor, params, name);
330}
331
332IConnectableLayer* INetwork::AddDivisionLayer(const char* name)
333{
334 return pNetworkImpl->AddDivisionLayer(name);
335}
336
337IConnectableLayer* INetwork::AddSubtractionLayer(const char* name)
338{
339 return pNetworkImpl->AddSubtractionLayer(name);
340}
341
342IConnectableLayer* INetwork::AddMaximumLayer(const char* name)
343{
344 return pNetworkImpl->AddMaximumLayer(name);
345}
346
347IConnectableLayer* INetwork::AddMeanLayer(const MeanDescriptor& meanDescriptor, const char* name)
348{
349 return pNetworkImpl->AddMeanLayer(meanDescriptor, name);
350}
351
352IConnectableLayer* INetwork::AddPadLayer(const PadDescriptor& padDescriptor,
353 const char* name)
354{
355 return pNetworkImpl->AddPadLayer(padDescriptor, name);
356}
357
358IConnectableLayer* INetwork::AddQuantizeLayer(const char* name)
359{
360 return pNetworkImpl->AddQuantizeLayer(name);
361}
362
363IConnectableLayer* INetwork::AddStridedSliceLayer(const StridedSliceDescriptor& stridedSliceDescriptor,
364 const char* name)
365{
366 return pNetworkImpl->AddStridedSliceLayer(stridedSliceDescriptor, name);
367}
368
369IConnectableLayer* INetwork::AddMinimumLayer(const char* name)
370{
371 return pNetworkImpl->AddMinimumLayer(name);
372}
373
Francis Murtagh3d2b4b22021-02-15 18:23:17 +0000374IConnectableLayer* INetwork::AddGatherLayer(const GatherDescriptor& descriptor,
375 const char* name)
376{
377 return pNetworkImpl->AddGatherLayer(descriptor, name);
378}
379
Teresa Charlinb2d3ec52022-04-12 22:07:09 +0100380IConnectableLayer* INetwork::AddGatherNdLayer(const char* name)
381{
382 return pNetworkImpl->AddGatherNdLayer(name);
383}
384
Francis Murtagh3d2b4b22021-02-15 18:23:17 +0000385IConnectableLayer* INetwork::AddSwitchLayer(const char* name)
386{
387 return pNetworkImpl->AddSwitchLayer(name);
388}
389
390IConnectableLayer* INetwork::AddPreluLayer(const char* name)
391{
392 return pNetworkImpl->AddPreluLayer(name);
393}
394
395IConnectableLayer* INetwork::AddTransposeConvolution2dLayer(const TransposeConvolution2dDescriptor& descriptor,
396 const ConstTensor& weights,
397 const Optional<ConstTensor>& biases,
398 const char* name)
399{
400 return pNetworkImpl->AddTransposeConvolution2dLayer(descriptor, weights, biases, name);
401}
402
403IConnectableLayer* INetwork::AddTransposeLayer(const TransposeDescriptor& transposeDescriptor,
404 const char* name)
405{
406 return pNetworkImpl->AddTransposeLayer(transposeDescriptor, name);
407}
408
Keith Davis3ae3f972021-05-21 16:33:48 +0100409IConnectableLayer* INetwork::AddShapeLayer(const char* name)
410{
411 return pNetworkImpl->AddShapeLayer(name);
412}
413
Francis Murtagh3d2b4b22021-02-15 18:23:17 +0000414IConnectableLayer* INetwork::AddStackLayer(const StackDescriptor& descriptor,
415 const char* name)
416{
417 return pNetworkImpl->AddStackLayer(descriptor, name);
418}
419
420IConnectableLayer* INetwork::AddStandInLayer(const StandInDescriptor& descriptor,
421 const char* name)
422{
423 return pNetworkImpl->AddStandInLayer(descriptor, name);
424}
425
426IConnectableLayer* INetwork::AddQuantizedLstmLayer(const QuantizedLstmInputParams& params,
427 const char* name)
428{
429 return pNetworkImpl->AddQuantizedLstmLayer(params, name);
430}
431
432IConnectableLayer* INetwork::AddQLstmLayer(const QLstmDescriptor& descriptor,
433 const LstmInputParams& params,
434 const char* name)
435{
436 return pNetworkImpl->AddQLstmLayer(descriptor, params, name);
437}
438
439IConnectableLayer* INetwork::AddLogicalBinaryLayer(const LogicalBinaryDescriptor& descriptor,
440 const char* name)
441{
442 return pNetworkImpl->AddLogicalBinaryLayer(descriptor, name);
443}
444
Narumol Prangnawarat8ed39ae2021-07-15 16:16:25 +0100445IConnectableLayer* INetwork::AddUnidirectionalSequenceLstmLayer(
446 const UnidirectionalSequenceLstmDescriptor& descriptor,
447 const LstmInputParams& params,
448 const char* name)
449{
450 return pNetworkImpl->AddUnidirectionalSequenceLstmLayer(descriptor, params, name);
451}
452
Simon Obute51f67772021-09-03 15:50:13 +0100453IConnectableLayer* INetwork::AddChannelShuffleLayer(const ChannelShuffleDescriptor &descriptor,
454 const char* name)
455{
456 return pNetworkImpl->AddChannelShuffleLayer(descriptor, name);
457}
458
Jan Eilers1b2654f2021-09-24 15:45:46 +0100459ARMNN_NO_DEPRECATE_WARN_BEGIN
Francis Murtagh3d2b4b22021-02-15 18:23:17 +0000460void INetwork::Accept(ILayerVisitor& visitor) const
461{
462 return pNetworkImpl->Accept(visitor);
463}
Jan Eilers1b2654f2021-09-24 15:45:46 +0100464ARMNN_NO_DEPRECATE_WARN_END
Francis Murtagh3d2b4b22021-02-15 18:23:17 +0000465
466void INetwork::ExecuteStrategy(IStrategy& strategy) const
467{
468 return pNetworkImpl->ExecuteStrategy(strategy);
469}
470
Finn Williamsf24effa2020-07-03 10:12:03 +0100471armnn::INetwork* INetwork::CreateRaw(NetworkOptions networkOptions)
telsoa014fcda012018-03-09 14:13:49 +0000472{
Francis Murtagh3d2b4b22021-02-15 18:23:17 +0000473 return new INetwork(networkOptions);
telsoa014fcda012018-03-09 14:13:49 +0000474}
475
Finn Williamsf24effa2020-07-03 10:12:03 +0100476armnn::INetworkPtr INetwork::Create(NetworkOptions networkOptions)
telsoa014fcda012018-03-09 14:13:49 +0000477{
Finn Williamsf24effa2020-07-03 10:12:03 +0100478 return INetworkPtr(CreateRaw(networkOptions), &INetwork::Destroy);
telsoa014fcda012018-03-09 14:13:49 +0000479}
480
481void INetwork::Destroy(INetwork* network)
482{
Francis Murtagh3d2b4b22021-02-15 18:23:17 +0000483 delete network;
telsoa014fcda012018-03-09 14:13:49 +0000484}
485
Mike Kelly0d677db2021-06-27 22:39:21 +0100486IOptimizedNetwork::IOptimizedNetwork(const IOptimizedNetwork& other, const ModelOptions& modelOptions)
487 : pOptimizedNetworkImpl(new OptimizedNetworkImpl(*other.pOptimizedNetworkImpl.get(), modelOptions)) {}
Francis Murtagh3d2b4b22021-02-15 18:23:17 +0000488
489IOptimizedNetwork::IOptimizedNetwork(std::unique_ptr<Graph> graph)
490 : pOptimizedNetworkImpl(new OptimizedNetworkImpl(std::move(graph))) {}
491
492IOptimizedNetwork::IOptimizedNetwork(std::unique_ptr<OptimizedNetworkImpl> impl)
493 : pOptimizedNetworkImpl(std::move(impl)) {}
494
495IOptimizedNetwork::IOptimizedNetwork(std::unique_ptr<Graph> graph, const ModelOptions& modelOptions)
496 : pOptimizedNetworkImpl(new OptimizedNetworkImpl(std::move(graph), modelOptions)) {}
497
498IOptimizedNetwork::~IOptimizedNetwork() = default;
499
telsoa014fcda012018-03-09 14:13:49 +0000500void IOptimizedNetwork::Destroy(IOptimizedNetwork* network)
501{
Francis Murtagh3d2b4b22021-02-15 18:23:17 +0000502 delete network;
telsoa014fcda012018-03-09 14:13:49 +0000503}
504
Francis Murtagh3d2b4b22021-02-15 18:23:17 +0000505Status IOptimizedNetwork::PrintGraph()
506{
507 return pOptimizedNetworkImpl->PrintGraph();
508}
509
510Status IOptimizedNetwork::SerializeToDot(std::ostream& stream) const
511{
512 return pOptimizedNetworkImpl->SerializeToDot(stream);
513}
514
Derek Lambertie155bbf2021-10-13 14:32:12 +0100515const std::shared_ptr<IProfiler>& IOptimizedNetwork::GetProfiler() const
516{
517 return pOptimizedNetworkImpl->GetGraph().GetProfiler();
518}
519
Cathal Corbett5aa9fd72022-02-25 15:33:28 +0000520arm::pipe::ProfilingGuid IOptimizedNetwork::GetGuid() const
Francis Murtagh3d2b4b22021-02-15 18:23:17 +0000521{
522 return pOptimizedNetworkImpl->GetGuid();
523}
524
Sadik Armaganb7851f92021-10-06 16:37:02 +0100525size_t IOptimizedNetwork::GetNumInputs() const
526{
527 return pOptimizedNetworkImpl->GetNumInputs();
528}
529
530size_t IOptimizedNetwork::GetNumOutputs() const
531{
532 return pOptimizedNetworkImpl->GetNumOutputs();
533}
534
Francis Murtagh3d2b4b22021-02-15 18:23:17 +0000535Status OptimizedNetworkImpl::PrintGraph()
telsoa014fcda012018-03-09 14:13:49 +0000536{
537 m_Graph->Print();
538 return Status::Success;
539}
540
Francis Murtagh3d2b4b22021-02-15 18:23:17 +0000541Status OptimizedNetworkImpl::SerializeToDot(std::ostream& stream) const
surmeh01bceff2f2018-03-29 16:29:27 +0100542{
543 return m_Graph->SerializeToDot(stream);
544}
545
Sadik Armaganb7851f92021-10-06 16:37:02 +0100546size_t OptimizedNetworkImpl::GetNumInputs() const
547{
548 return m_Graph->GetNumInputs();
549}
550
551size_t OptimizedNetworkImpl::GetNumOutputs() const
552{
553 return m_Graph->GetNumOutputs();
554}
555
Matteo Martincigh49124022019-01-11 13:25:59 +0000556void ReportError(const std::string& errorMessage,
557 Optional<std::vector<std::string>&> errorMessages)
558{
559 std::stringstream fullErrorMessage;
560 fullErrorMessage << "ERROR: " << errorMessage;
Derek Lamberti08446972019-11-26 16:38:31 +0000561 ARMNN_LOG(warning) << fullErrorMessage.str();
Matteo Martincigh49124022019-01-11 13:25:59 +0000562 if (errorMessages)
563 {
564 errorMessages.value().push_back(fullErrorMessage.str());
565 }
566}
567
568void ReportWarning(const std::string& warningMessage,
569 Optional<std::vector<std::string>&> warningMessages)
570{
571 std::stringstream fullWarningMessage;
572 fullWarningMessage << "WARNING: " << warningMessage;
Derek Lamberti08446972019-11-26 16:38:31 +0000573 ARMNN_LOG(warning) << fullWarningMessage.str();
Matteo Martincigh49124022019-01-11 13:25:59 +0000574 if (warningMessages)
575 {
576 warningMessages.value().push_back(fullWarningMessage.str());
577 }
578}
579
Derek Lamberti4a9e24b2020-01-03 16:53:38 +0000580OptimizationResult ReturnWithError(OptimizationResult res,
581 const Layer* layer,
582 const BackendSettings& backendSettings,
583 Optional<std::vector<std::string>&> errMessages)
584{
585 std::stringstream failureMsg;
586 failureMsg << "Layer of type " << GetLayerTypeAsCString(layer->GetType())
587 << " is not supported on any preferred backend " << backendSettings.m_PreferredBackends;
588 ReportError(failureMsg.str(), errMessages);
589
590 res.m_Error = true;
591 return res;
592}
593
594
jimfly016b0b53d2018-10-08 14:43:01 +0100595bool CheckScaleSetOnQuantizedType(Layer* layer, Optional<std::vector<std::string>&> errMessages)
596{
597 bool noErrors = true;
598 unsigned int numOutputs = layer->GetNumOutputSlots();
599 for (unsigned int i = 0; i < numOutputs; i++) {
David Monahanb8554702019-04-25 16:03:38 +0100600 OutputSlot& outputSlot = layer->GetOutputSlot(i);
601 TensorInfo info = outputSlot.GetTensorInfo();
Derek Lambertif90c56d2020-01-10 17:14:08 +0000602 if (DataType::QAsymmU8 == info.GetDataType()) {
jimfly016b0b53d2018-10-08 14:43:01 +0100603 if (0.f == info.GetQuantizationScale()) {
604 noErrors = false;
605 std::stringstream ss;
Matteo Martincigh49124022019-01-11 13:25:59 +0000606 ss << "output " << i << " of layer " << GetLayerTypeAsCString(layer->GetType())
jimfly016b0b53d2018-10-08 14:43:01 +0100607 << " (" << layer->GetNameStr() << ") is of type"
608 << " Quantized 8 bit but its scale parameter has not been set";
Matteo Martincigh49124022019-01-11 13:25:59 +0000609 ReportError(ss.str(), errMessages);
jimfly016b0b53d2018-10-08 14:43:01 +0100610 }
David Monahanb8554702019-04-25 16:03:38 +0100611 // Softmax under QuantisedAsymm8 must always be scale (1.0f/256.0f) and offset 0
612 if ((info.GetQuantizationScale() != (1.0f / 256.0f) ||
613 info.GetQuantizationOffset() != 0) &&
614 layer->GetType() == armnn::LayerType::Softmax)
615 {
616 std::stringstream ss;
617 ss << "Quantization parameters for Softmax layer (Scale: " <<
618 info.GetQuantizationScale() << " and Offset: " << info.GetQuantizationOffset() <<
619 ") are incorrect and have been updated to Scale: 0.00390625 and Offset: 0";
Derek Lamberti08446972019-11-26 16:38:31 +0000620 ARMNN_LOG(warning) << ss.str();
David Monahanb8554702019-04-25 16:03:38 +0100621 info.SetQuantizationScale((1.0f /256.0f));
622 info.SetQuantizationOffset(0);
623 outputSlot.SetTensorInfo(info);
624 }
jimfly016b0b53d2018-10-08 14:43:01 +0100625 }
626 }
627 return noErrors;
628}
629
Narumol Prangnawarat250d3922020-03-30 16:11:04 +0100630template <typename LayerT>
631LayerT* ConvertBf16ToFp32Weight(Layer* l)
632{
Jan Eilersbb446e52020-04-02 13:56:54 +0100633 LayerT* layer = PolymorphicDowncast<LayerT*>(l);
Narumol Prangnawarat250d3922020-03-30 16:11:04 +0100634 if ((layer->GetType() == LayerType::Convolution2d || layer->GetType() == LayerType::FullyConnected)
635 && layer->m_Weight)
636 {
637 const TensorInfo& info = layer->m_Weight->GetTensorInfo();
638
639 if (info.GetDataType() == DataType::BFloat16)
640 {
641 std::vector<float> newValues(info.GetNumElements());
642
643 armnnUtils::FloatingPointConverter::ConvertBFloat16ToFloat32(
Finn Williams4422cec2021-03-22 17:51:06 +0000644 layer->m_Weight->template GetConstTensor<armnn::BFloat16>(), info.GetNumElements(), newValues.data());
Narumol Prangnawarat250d3922020-03-30 16:11:04 +0100645
646 TensorInfo newInfo(info.GetShape(), DataType::Float32);
647 ConstTensor newInput(newInfo, newValues);
James Conroy1f58f032021-04-27 17:13:27 +0100648 layer->m_Weight.reset(new ScopedTensorHandle(newInput));
Narumol Prangnawarat250d3922020-03-30 16:11:04 +0100649 }
650 }
651 return layer;
652}
653
Derek Lamberti4a9e24b2020-01-03 16:53:38 +0000654OptimizationResult AttemptBackendAssignment(BackendSettings& backendSettings,
655 Graph& graph,
656 Layer* layer,
657 BackendId backend,
658 DataType dataTypeIn,
659 DataType dataTypeOut,
660 const std::vector<BackendId>& availablePreferredBackends,
661 std::string& reasonIfUnsupported,
662 Optional<std::vector<std::string>&> errMessages)
663{
664 OptimizationResult result;
665
666 // Helper lambda to compose meaningful error message before returning with error
667 auto ReturnError = [&](const Layer* layer)
668 {
669 return ReturnWithError(result, layer, backendSettings, errMessages);
670 };
671
672 // need to set the compute device on the layer
673 // before we can check if it is supported
674 layer->SetBackendId(backend);
675 if (!IWorkloadFactory::IsLayerSupported(*layer, EmptyOptional(), reasonIfUnsupported))
676 {
677 if (dataTypeIn == DataType::Float16 || dataTypeOut == DataType::Float16)
678 {
679 if (IWorkloadFactory::IsLayerSupported(*layer, DataType::Float32, reasonIfUnsupported)
680 && layer->GetType() != LayerType::ConvertFp32ToFp16
681 && layer->GetType() != LayerType::ConvertFp16ToFp32)
682 {
Jan Eilers0c0019c2021-08-20 16:42:58 +0100683 auto ConstantLayerFromFp16ToFp32 = [](Layer& layer)
684 {
685 if (layer.GetType() == LayerType::Constant)
686 {
687 ConstantLayer* constantLayer = PolymorphicDowncast<ConstantLayer*>(&layer);
688
689 auto& info = constantLayer->m_LayerOutput->GetTensorInfo();
690
691 if (info.GetDataType() == DataType::Float16)
692 {
693 std::vector<float> newValues(info.GetNumElements());
694
695 armnnUtils::FloatingPointConverter::ConvertFloat16To32(
696 constantLayer->m_LayerOutput->GetConstTensor<Half>(),
697 info.GetNumElements(),
698 newValues.data());
699
700 TensorInfo newInfo(info);
701 newInfo.SetDataType(DataType::Float32);
702 ConstTensor newInput(newInfo, newValues);
703 constantLayer->m_LayerOutput.reset(new ScopedTensorHandle(newInput));
704
705 layer.GetOutputSlot(0).SetTensorInfo(newInfo);
706 }
707 }
708 };
709
710 bool checkType = false;
711
712 for (auto inputSlot : layer->GetInputSlots())
713 {
714 auto connectedOutputSlot = inputSlot.GetConnectedOutputSlot();
715 if (connectedOutputSlot->GetOwningLayer().GetType() == LayerType::Constant)
716 {
717 if (connectedOutputSlot->GetNumConnections() == 1)
718 {
719 checkType = true;
720 ConstantLayerFromFp16ToFp32(connectedOutputSlot->GetOwningLayer());
721 }
722 }
723 }
724
Derek Lamberti4a9e24b2020-01-03 16:53:38 +0000725 // Insert FP16 -> FP32 conversion layer before current layer
726 std::vector<ConvertFp16ToFp32Layer*> convertFp16ToFp32Layers;
727 if (dataTypeIn == DataType::Float16)
728 {
729 convertFp16ToFp32Layers =
Jan Eilers0c0019c2021-08-20 16:42:58 +0100730 InsertConvertFp16ToFp32LayersBefore(graph, *layer, checkType);
Derek Lamberti4a9e24b2020-01-03 16:53:38 +0000731 }
732
733 // Insert FP32 -> FP16 conversion layer after current layer
734 std::vector<ConvertFp32ToFp16Layer*> convertFp32ToFp16Layers;
735 if (dataTypeOut == DataType::Float16)
736 {
737 convertFp32ToFp16Layers =
738 InsertConvertFp32ToFp16LayersAfter(graph, *layer);
739 }
740
741 // Assign a supported backend to the newly introduced conversion layers
742 auto AssignFirstSupportedBackend = [&](Layer* layer, BackendId preferredBackend)
743 {
744 bool supportedBackendFound = false;
745 std::string reasonIfUnsupported;
746
747 // Try preferred backend first
748 layer->SetBackendId(preferredBackend);
749 if (IWorkloadFactory::IsLayerSupported(*layer,
750 EmptyOptional(),
751 reasonIfUnsupported))
752 {
753 supportedBackendFound = true;
754 }
755 else
756 {
757 for (const auto& backend : availablePreferredBackends)
758 {
759 // Skip preferred backend (we already determined that it is not supported)
760 if (backend == preferredBackend)
761 {
762 continue;
763 }
764
765 layer->SetBackendId(backend);
766 if (IWorkloadFactory::IsLayerSupported(*layer,
767 EmptyOptional(),
768 reasonIfUnsupported))
769 {
770 supportedBackendFound = true;
771 break;
772 }
773 }
774 }
775
776 return supportedBackendFound;
777 };
778
779 for (ConvertFp16ToFp32Layer* convertLayer : convertFp16ToFp32Layers)
780 {
781 if (!AssignFirstSupportedBackend(convertLayer, backend))
782 {
783 return ReturnError(convertLayer);
784 }
785 }
786
787 for (ConvertFp32ToFp16Layer* convertLayer : convertFp32ToFp16Layers)
788 {
789 if (!AssignFirstSupportedBackend(convertLayer, backend))
790 {
791 return ReturnError(convertLayer);
792 }
793 }
794
795 return result;
796 }
797 }
Narumol Prangnawaratbc7ffb52020-03-20 15:01:01 +0000798 else if (dataTypeIn == DataType::BFloat16 || dataTypeOut == DataType::BFloat16)
799 {
800 if (IWorkloadFactory::IsLayerSupported(*layer, DataType::Float32, reasonIfUnsupported)
801 && layer->GetType() != LayerType::ConvertFp32ToBf16
802 && layer->GetType() != LayerType::ConvertBf16ToFp32)
803 {
804 // Insert BF16 -> FP32 conversion layer before current layer
805 std::vector<ConvertBf16ToFp32Layer*> convertBf16ToFp32Layers;
806 if (dataTypeIn == DataType::BFloat16)
807 {
808 convertBf16ToFp32Layers =
809 InsertConvertBf16ToFp32LayersBefore(graph, *layer);
Narumol Prangnawarat250d3922020-03-30 16:11:04 +0100810 if (layer->GetType() == LayerType::Convolution2d)
811 {
812 ConvertBf16ToFp32Weight<Convolution2dLayer>(layer);
813 }
814 else if (layer->GetType() == LayerType::FullyConnected)
815 {
816 ConvertBf16ToFp32Weight<FullyConnectedLayer>(layer);
817 }
Narumol Prangnawaratbc7ffb52020-03-20 15:01:01 +0000818 }
819
820 // Insert FP32 -> BF16 conversion layer after current layer
821 std::vector<ConvertFp32ToBf16Layer*> convertFp32ToBf16Layers;
822 if (dataTypeOut == DataType::BFloat16)
823 {
824 convertFp32ToBf16Layers =
825 InsertConvertFp32ToBf16LayersAfter(graph, *layer);
826 }
827
828 // Assign a supported backend to the newly introduced conversion layers
829 auto AssignFirstSupportedBackend = [&](Layer* layer, BackendId preferredBackend)
830 {
831 bool supportedBackendFound = false;
832 std::string reasonIfUnsupported;
833
834 // Try preferred backend first
835 layer->SetBackendId(preferredBackend);
836 if (IWorkloadFactory::IsLayerSupported(*layer,
837 EmptyOptional(),
838 reasonIfUnsupported))
839 {
840 supportedBackendFound = true;
841 }
842 else
843 {
844 for (const auto& backend : availablePreferredBackends)
845 {
846 // Skip preferred backend (we already determined that it is not supported)
847 if (backend == preferredBackend)
848 {
849 continue;
850 }
851
852 layer->SetBackendId(backend);
853 if (IWorkloadFactory::IsLayerSupported(*layer,
854 EmptyOptional(),
855 reasonIfUnsupported))
856 {
857 supportedBackendFound = true;
858 break;
859 }
860 }
861 }
862
863 return supportedBackendFound;
864 };
865
866 for (ConvertBf16ToFp32Layer* convertLayer : convertBf16ToFp32Layers)
867 {
868 if (!AssignFirstSupportedBackend(convertLayer, backend))
869 {
870 return ReturnError(convertLayer);
871 }
872 }
873
874 for (ConvertFp32ToBf16Layer* convertLayer : convertFp32ToBf16Layers)
875 {
876 if (!AssignFirstSupportedBackend(convertLayer, backend))
877 {
878 return ReturnError(convertLayer);
879 }
880 }
881
882 return result;
883 }
884 }
885
Derek Lamberti4a9e24b2020-01-03 16:53:38 +0000886 std::stringstream warningMsg;
887 warningMsg << "Layer of type " << GetLayerTypeAsCString(layer->GetType())
888 << " is not supported on requested backend " << layer->GetBackendId().Get()
889 << " for input data type " << GetDataTypeName(dataTypeIn)
890 << " and output data type " << GetDataTypeName(dataTypeOut)
891 << " (reason: " << reasonIfUnsupported
892 << "), falling back to the next backend.";
893 ReportWarning(warningMsg.str(), errMessages);
894
895 return OptimizationResult(true, false);
896 }
897 else
898 {
899 return result;
900 }
901}
902
Francis Murtagh56ccf682021-12-13 18:48:12 +0000903// Refactor to allow passing the IConnectableLayer* rather than Layer Iterator
904// on Graph and SubgraphView which are different types.
905void AssignBackendsIConnectable(OptimizedNetworkImpl* optNetObjPtr,
906 IConnectableLayer* it,
907 Optional<std::vector<std::string>&> errMessages,
908 OptimizationResult& result,
909 BackendSettings& backendSettings,
910 std::vector<BackendId>& availablePreferredBackends)
911{
912 auto ReturnError = [&](const Layer* layer)
913 {
914 return ReturnWithError(result, layer, backendSettings, errMessages);
915 };
916
917 auto layer = PolymorphicDowncast<Layer*>(it);
918
919 if (layer->GetType() == LayerType::Input)
920 {
921 return;
922 }
923
924 DataType dataTypeIn = layer->GetNumInputSlots() == 0 ? DataType::Float32 :
925 layer->GetInputSlot(0).GetConnectedOutputSlot()->GetTensorInfo().GetDataType();
926 DataType dataTypeOut = layer->GetNumOutputSlots() == 0 ? DataType::Float32 :
927 layer->GetOutputSlot(0).GetTensorInfo().GetDataType();
928
929 std::string reasonIfUnsupported;
930 bool found = false;
931 if (!CheckScaleSetOnQuantizedType(layer, errMessages))
932 {
933 // don't bomb immediately, find all the quantized outputs
934 // which haven't had a scale set and report them all back.
935 result.m_Error = true;
936 }
937
938 // First try assign layer to hint backend
939 if (layer->GetBackendHint().has_value() &&
940 backendSettings.IsBackendSupported(layer->GetBackendHint().value()) &&
941 AttemptBackendAssignment(backendSettings,
942 optNetObjPtr->GetGraph(),
943 layer,
944 layer->GetBackendHint().value(),
945 dataTypeIn,
946 dataTypeOut,
947 availablePreferredBackends,
948 reasonIfUnsupported,
949 errMessages).IsOk())
950 {
951 found = true;
952 backendSettings.m_SelectedBackends.insert(layer->GetBackendHint().value());
953 }
954 else
955 {
956 // Try assign layer to prefered list of backends
957 for (const auto& backend : availablePreferredBackends)
958 {
959 if (layer->GetBackendHint().has_value() &&
960 layer->GetBackendHint().value() == backend)
961 {
962 continue; //Don't re-test the backend hint
963 }
964
965 OptimizationResult res = AttemptBackendAssignment(backendSettings,
966 optNetObjPtr->GetGraph(),
967 layer,
968 backend,
969 dataTypeIn,
970 dataTypeOut,
971 availablePreferredBackends,
972 reasonIfUnsupported,
973 errMessages);
974
975 if (res.IsOk())
976 {
977 found = true;
978 backendSettings.m_SelectedBackends.insert(backend);
979 break;
980 }
981 else if (res.IsError())
982 {
983 result = res; // Cannot continue.
984 // Note: we don't need to log the error as it would already
985 // be logged in AttemptBackendAssignment().
986 }
987 else
988 {
989 ARMNN_ASSERT_MSG(res.IsWarningOnly(), "OptimizationResult in unexpected state.");
990 }
991 }
992 }
993
994 // If the layer is unsupported by any devices, log and return a null network.
995 if (!found)
996 {
997 // NOTE: if the layer is not an operation queue type AND we have not got CpuRef as a
998 // fallback we should set the compute device on the layer to CpuRef (these are not
999 // available as accelerated operations, or are only available under certain
1000 // conditions, currently they comprise MemCopy, Constant, Permute)
1001 armnn::LayerType layerType = layer->GetType();
1002 if (!backendSettings.IsCpuRefUsed() && (layerType == armnn::LayerType::MemCopy ||
1003 layerType == armnn::LayerType::Constant ||
1004 layerType == armnn::LayerType::Permute))
1005 {
1006 BackendId cpuBackendId(armnn::Compute::CpuRef);
1007 layer->SetBackendId(cpuBackendId);
1008 backendSettings.m_SelectedBackends.insert(cpuBackendId);
1009 }
1010 else
1011 {
1012 result = ReturnError(layer);
1013 }
1014 }
1015
1016}
Derek Lamberti4a9e24b2020-01-03 16:53:38 +00001017
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001018OptimizationResult AssignBackends(OptimizedNetworkImpl* optNetObjPtr,
Matteo Martincigh49124022019-01-11 13:25:59 +00001019 BackendSettings& backendSettings,
1020 Graph::Iterator& firstLayer,
1021 Graph::Iterator& lastLayer,
1022 Optional<std::vector<std::string>&> errMessages)
telsoa014fcda012018-03-09 14:13:49 +00001023{
Derek Lambertif1e0ad32021-10-13 18:02:25 +01001024 ARMNN_SCOPED_PROFILING_EVENT(Compute::Undefined, "Optimizer_AssignBackends");
Matteo Martincigh49124022019-01-11 13:25:59 +00001025 OptimizationResult result;
telsoa014fcda012018-03-09 14:13:49 +00001026
Matteo Martincigh49124022019-01-11 13:25:59 +00001027 auto availablePreferredBackends = backendSettings.GetAvailablePreferredBackends();
1028 if (availablePreferredBackends.empty())
telsoa01c577f2c2018-08-31 09:22:23 +01001029 {
Matteo Martincigh49124022019-01-11 13:25:59 +00001030 std::stringstream failureMsg;
1031 failureMsg << "No preferred backends are available";
1032 ReportError(failureMsg.str(), errMessages);
1033
1034 result.m_Error = true;
1035 return result;
1036 }
1037
1038 for (auto it = firstLayer; it != lastLayer; ++it)
1039 {
Francis Murtagh56ccf682021-12-13 18:48:12 +00001040 AssignBackendsIConnectable(optNetObjPtr,
1041 *it,
1042 errMessages,
1043 result,
1044 backendSettings,
1045 availablePreferredBackends);
telsoa01c577f2c2018-08-31 09:22:23 +01001046 }
Matteo Martincigh49124022019-01-11 13:25:59 +00001047
Finn Williamsb1aad422021-10-28 19:07:32 +01001048 for (auto it = firstLayer; it != lastLayer; ++it)
1049 {
Francis Murtagh56ccf682021-12-13 18:48:12 +00001050 auto layer = PolymorphicDowncast<Layer*>(*it);
1051
1052 if(layer->GetType() == LayerType::Input)
1053 {
1054 BackendId connectedBackendId = layer->GetOutputSlot(0).GetConnection(0)->GetOwningLayer().GetBackendId();
1055 layer->SetBackendId(connectedBackendId);
1056 }
1057 }
1058
1059 return result;
1060}
1061
1062OptimizationResult AssignBackends(OptimizedNetworkImpl* optNetObjPtr,
1063 BackendSettings& backendSettings,
1064 SubgraphView::IConnectableLayerIterator& firstLayer,
1065 SubgraphView::IConnectableLayerIterator& lastLayer,
1066 Optional<std::vector<std::string>&> errMessages)
1067{
1068 ARMNN_SCOPED_PROFILING_EVENT(Compute::Undefined, "Optimizer_AssignBackends");
1069 OptimizationResult result;
1070
1071 auto availablePreferredBackends = backendSettings.GetAvailablePreferredBackends();
1072 if (availablePreferredBackends.empty())
1073 {
1074 std::stringstream failureMsg;
1075 failureMsg << "No preferred backends are available";
1076 ReportError(failureMsg.str(), errMessages);
1077
1078 result.m_Error = true;
1079 return result;
1080 }
1081
1082 for (auto it = firstLayer; it != lastLayer; ++it)
1083 {
1084 AssignBackendsIConnectable(optNetObjPtr,
1085 *it,
1086 errMessages,
1087 result,
1088 backendSettings,
1089 availablePreferredBackends);
1090 }
1091
1092 for (auto it = firstLayer; it != lastLayer; ++it)
1093 {
1094 auto layer = PolymorphicDowncast<Layer*>(*it);
Finn Williamsb1aad422021-10-28 19:07:32 +01001095
1096 if(layer->GetType() == LayerType::Input)
1097 {
1098 BackendId connectedBackendId = layer->GetOutputSlot(0).GetConnection(0)->GetOwningLayer().GetBackendId();
1099 layer->SetBackendId(connectedBackendId);
1100 }
1101 }
1102
Matteo Martincigh49124022019-01-11 13:25:59 +00001103 return result;
1104}
1105
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001106OptimizationResult AssignBackends(OptimizedNetworkImpl* optNetObjPtr,
Matteo Martincighadddddb2019-01-24 14:06:23 +00001107 BackendSettings& backendSettings,
Derek Lambertiff05cc52019-04-26 13:05:17 +01001108 SubgraphView& subgraph,
Matteo Martincighadddddb2019-01-24 14:06:23 +00001109 Optional<std::vector<std::string>&> errMessages)
Matteo Martincigh49124022019-01-11 13:25:59 +00001110{
Francis Murtagh56ccf682021-12-13 18:48:12 +00001111 SubgraphView::IConnectableLayerIterator firstLayer = subgraph.beginIConnectable();
1112 SubgraphView::IConnectableLayerIterator lastLayer = subgraph.endIConnectable();
Matteo Martincighadddddb2019-01-24 14:06:23 +00001113 return AssignBackends(optNetObjPtr,
1114 backendSettings,
1115 firstLayer,
1116 lastLayer,
1117 errMessages);
1118}
1119
Derek Lamberti84da38b2019-06-13 11:40:08 +01001120BackendsMap CreateSupportedBackends(TensorHandleFactoryRegistry& handleFactoryRegistry,
1121 BackendSettings& backendSettings)
1122{
1123 BackendsMap backends;
1124 auto const& backendRegistry = BackendRegistryInstance();
1125 for (auto&& selectedBackend : backendSettings.m_SupportedBackends)
1126 {
1127 auto backendFactory = backendRegistry.GetFactory(selectedBackend);
1128 auto backendObjPtr = backendFactory();
Narumol Prangnawaratac2770a2020-04-01 16:51:23 +01001129 ARMNN_ASSERT(backendObjPtr);
Derek Lamberti84da38b2019-06-13 11:40:08 +01001130
1131 backendObjPtr->RegisterTensorHandleFactories(handleFactoryRegistry);
1132
1133 backends[backendObjPtr->GetId()] = std::move(backendObjPtr);
1134 }
1135
1136 return backends;
1137}
1138
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001139OptimizationResult ApplyBackendOptimizations(OptimizedNetworkImpl* optNetObjPtr,
Matteo Martincighadddddb2019-01-24 14:06:23 +00001140 BackendSettings& backendSettings,
Derek Lamberti84da38b2019-06-13 11:40:08 +01001141 BackendsMap& backends,
Mike Kelly07810fc2020-11-12 10:58:48 +00001142 const ModelOptions& modelOptions,
Matteo Martincighadddddb2019-01-24 14:06:23 +00001143 Optional<std::vector<std::string>&> errMessages)
1144{
Narumol Prangnawaratac2770a2020-04-01 16:51:23 +01001145 ARMNN_ASSERT(optNetObjPtr);
Derek Lambertif1e0ad32021-10-13 18:02:25 +01001146 ARMNN_SCOPED_PROFILING_EVENT(Compute::Undefined, "Optimizer_ApplyBackendOptimizations")
Matteo Martincigh49124022019-01-11 13:25:59 +00001147 OptimizationResult result;
1148
Matteo Martincighadddddb2019-01-24 14:06:23 +00001149 // Get the optimized graph
1150 Graph& optGraph = optNetObjPtr->GetGraph();
Matteo Martincigh49124022019-01-11 13:25:59 +00001151
Matteo Martincighadddddb2019-01-24 14:06:23 +00001152 // Run backend specific optimizations
Matteo Martincighadddddb2019-01-24 14:06:23 +00001153 for (auto&& selectedBackend : backendSettings.m_SelectedBackends)
Matteo Martincigh49124022019-01-11 13:25:59 +00001154 {
Derek Lamberti84da38b2019-06-13 11:40:08 +01001155 auto backendObjPtr = backends.find(selectedBackend)->second.get();
Narumol Prangnawaratac2770a2020-04-01 16:51:23 +01001156 ARMNN_ASSERT(backendObjPtr);
Matteo Martincighadddddb2019-01-24 14:06:23 +00001157
Cathal Corbett4b19d222022-05-11 20:12:17 +01001158 if(selectedBackend == armnn::Compute::GpuAcc || selectedBackend == armnn::Compute::CpuAcc)
1159 {
1160 Optimizer::Pass(optGraph, MakeOptimizations(optimizations::PermuteDepthwiseConv2dWeights()));
Cathal Corbett541880f2022-05-16 15:20:56 +01001161 Optimizer::Pass(optGraph, MakeOptimizations(optimizations::FusePermuteIntoConstLayer()));
Cathal Corbett4b19d222022-05-11 20:12:17 +01001162 }
1163
Matteo Martincighadddddb2019-01-24 14:06:23 +00001164 // Select sub-graphs based on backend
Derek Lambertiff05cc52019-04-26 13:05:17 +01001165 SubgraphViewSelector::Subgraphs subgraphs =
Rob Hughes65c32262019-07-23 15:33:39 +01001166 SubgraphViewSelector::SelectSubgraphs(optGraph,
Matteo Martincigh602af092019-05-01 10:31:27 +01001167 // Select layers assigned to the requested backend
1168 [&backendObjPtr](const Layer& layer)
1169 {
Francis Murtagh56ccf682021-12-13 18:48:12 +00001170
Matteo Martincigh602af092019-05-01 10:31:27 +01001171 return layer.GetType() != LayerType::Input &&
1172 layer.GetType() != LayerType::Output &&
1173 layer.GetBackendId() == backendObjPtr->GetId();
1174 });
Derek Lambertiff05cc52019-04-26 13:05:17 +01001175 if (subgraphs.empty())
Matteo Martincigh49124022019-01-11 13:25:59 +00001176 {
Matteo Martincighadddddb2019-01-24 14:06:23 +00001177 // No sub-graphs found, try with next selected backend
1178 continue;
Matteo Martincigh49124022019-01-11 13:25:59 +00001179 }
Matteo Martincighadddddb2019-01-24 14:06:23 +00001180
1181 // Try to optimize each sub-graph
Derek Lambertiff05cc52019-04-26 13:05:17 +01001182 for (auto& subgraph : subgraphs)
Matteo Martincigh49124022019-01-11 13:25:59 +00001183 {
Matteo Martincighadddddb2019-01-24 14:06:23 +00001184 // Try to optimize the current sub-graph
Derek Lambertif1e0ad32021-10-13 18:02:25 +01001185 ARMNN_SCOPED_PROFILING_EVENT(backendObjPtr->GetId(), "Optimizer_OptimizeSubgraph");
Mike Kelly07810fc2020-11-12 10:58:48 +00001186 OptimizationViews optimizationViews = backendObjPtr->OptimizeSubgraphView(*subgraph, modelOptions);
Narumol Prangnawaratac2770a2020-04-01 16:51:23 +01001187 ARMNN_ASSERT(optimizationViews.Validate(*subgraph));
Matteo Martincighadddddb2019-01-24 14:06:23 +00001188
1189 // Optimization attempted, check the resulting optimized sub-graph
Matteo Martincigh84924332019-05-09 12:46:16 +01001190 for (auto& substitution : optimizationViews.GetSubstitutions())
Matteo Martincighadddddb2019-01-24 14:06:23 +00001191 {
1192 // Sub-graph optimized, substitute the sub-graph with the new optimized one in the main optimized graph
Matteo Martincigh84924332019-05-09 12:46:16 +01001193 SubgraphView& replacementSubgraph = substitution.m_ReplacementSubgraph;
1194 SubgraphView& substitutableSubgraph = substitution.m_SubstitutableSubgraph;
1195 optGraph.SubstituteSubgraph(substitutableSubgraph, replacementSubgraph);
Matteo Martincighadddddb2019-01-24 14:06:23 +00001196
1197 // Assign the current backend to the optimized sub-graph
Francis Murtagh56ccf682021-12-13 18:48:12 +00001198 const SubgraphView::IConnectableLayers& subgraphLayers = replacementSubgraph.GetIConnectableLayers();
1199 std::for_each(subgraphLayers.begin(), subgraphLayers.end(), [&selectedBackend](IConnectableLayer* l)
Derek Lambertic2fe5fb2019-05-08 10:23:08 +01001200 {
Narumol Prangnawaratac2770a2020-04-01 16:51:23 +01001201 ARMNN_ASSERT(l);
Francis Murtagh56ccf682021-12-13 18:48:12 +00001202 PolymorphicDowncast<Layer*>(l)->SetBackendId(selectedBackend);
Derek Lambertic2fe5fb2019-05-08 10:23:08 +01001203 });
Matteo Martincighadddddb2019-01-24 14:06:23 +00001204 }
Derek Lambertic2fe5fb2019-05-08 10:23:08 +01001205
Matteo Martincigh84924332019-05-09 12:46:16 +01001206 if (!optimizationViews.GetFailedSubgraphs().empty())
Matteo Martincighadddddb2019-01-24 14:06:23 +00001207 {
Matteo Martincighadddddb2019-01-24 14:06:23 +00001208 std::stringstream warningMsg;
Derek Lambertic2fe5fb2019-05-08 10:23:08 +01001209 warningMsg << "Some sub-graph(s) failed to optimized on " << backendObjPtr->GetId() << " backend.";
Matteo Martincighadddddb2019-01-24 14:06:23 +00001210 ReportWarning(warningMsg.str(), errMessages);
1211
1212 // Failed to optimize the given sub-graph, re-assign the sub-graph layers to other available backends
Derek Lambertic2fe5fb2019-05-08 10:23:08 +01001213 BackendSettings settingsCopy(backendSettings);
Matteo Martincighadddddb2019-01-24 14:06:23 +00001214 if (!backendObjPtr->GetId().IsCpuRef())
1215 {
1216 // Add the current backend to the list of backends to ignore
Derek Lambertic2fe5fb2019-05-08 10:23:08 +01001217 settingsCopy.m_IgnoredBackends.insert(backendObjPtr->GetId());
Matteo Martincighadddddb2019-01-24 14:06:23 +00001218 }
Derek Lambertic2fe5fb2019-05-08 10:23:08 +01001219
1220 int count=0;
Matteo Martincigh84924332019-05-09 12:46:16 +01001221 for (auto& failedSubgraph : optimizationViews.GetFailedSubgraphs())
Matteo Martincighadddddb2019-01-24 14:06:23 +00001222 {
Derek Lambertic2fe5fb2019-05-08 10:23:08 +01001223 // An error occurred: the optimization was attempted but not performed, try different backends
1224 std::stringstream subgraphMsg;
Francis Murtagh56ccf682021-12-13 18:48:12 +00001225 subgraphMsg << "Re-assigning backends to " << failedSubgraph.GetIConnectableLayers().size()
Derek Lambertic2fe5fb2019-05-08 10:23:08 +01001226 << " layers inside sub-graph " << count++;
Matteo Martincigh328d92b2019-07-04 17:52:55 +01001227 ReportWarning(subgraphMsg.str(), errMessages);
Derek Lambertic2fe5fb2019-05-08 10:23:08 +01001228
1229 OptimizationResult reassignmentResult = AssignBackends(optNetObjPtr,
1230 settingsCopy,
1231 *subgraph,
1232 errMessages);
1233 if (reassignmentResult.m_Error)
1234 {
1235 // Failed to re-assign one of the remaining backends to each layer of the sub-graph
1236 result.m_Error = true;
1237 return result;
1238 }
Matteo Martincighadddddb2019-01-24 14:06:23 +00001239 }
Matteo Martincigh49124022019-01-11 13:25:59 +00001240 }
1241 }
1242 }
1243
1244 return result;
1245}
1246
Derek Lamberti84da38b2019-06-13 11:40:08 +01001247bool RequiresCopy(ITensorHandleFactory::FactoryId src,
1248 ITensorHandleFactory::FactoryId dst,
1249 TensorHandleFactoryRegistry& registry)
1250{
1251 if (src != dst)
1252 {
1253 ITensorHandleFactory* srcFactory = registry.GetFactory(src);
1254 ITensorHandleFactory* dstFactory = registry.GetFactory(dst);
1255
Matteo Martincigha6539ed2019-08-27 13:43:32 +01001256 if (srcFactory && dstFactory &&
1257 (srcFactory->GetExportFlags() & dstFactory->GetImportFlags()) != 0)
Derek Lamberti84da38b2019-06-13 11:40:08 +01001258 {
1259 return false;
1260 }
1261 return true;
1262 }
1263 return false;
1264}
1265
1266// Find the handle factory for the input layer which results in fewest required copies.
1267ITensorHandleFactory::FactoryId CalculateSlotOptionForInput(BackendsMap& backends,
1268 OutputSlot& slot,
Narumol Prangnawarate5f0b242021-05-07 17:52:36 +01001269 TensorHandleFactoryRegistry& registry,
1270 bool importEnabled)
Derek Lamberti84da38b2019-06-13 11:40:08 +01001271{
1272 Layer& layer = slot.GetOwningLayer();
Narumol Prangnawaratac2770a2020-04-01 16:51:23 +01001273 ARMNN_ASSERT(layer.GetType() == LayerType::Input);
Derek Lamberti84da38b2019-06-13 11:40:08 +01001274
1275 // Explicitly select the tensorhandle factory for InputLayer because the rules for it are slightly different. It
1276 // doesn't matter which backend it is assigned to because they all use the same implementation, which
1277 // requires Map/Unmap support. This means that, so long as the handle type supports map/unmap semantics, we can
1278 // select a factory with maximum compatibility with the layers connected to the InputLayer.
1279
1280 // First ensure the from backends can support the TensorHandeAPI
1281 auto frmBackend = backends.find(layer.GetBackendId());
1282 if (frmBackend == backends.end() ||
1283 !frmBackend->second->SupportsTensorAllocatorAPI())
1284 {
1285 return ITensorHandleFactory::LegacyFactoryId;
1286 }
1287
1288 // Go through all connections to the output slot and determine the TensorHandleFactory which results in the
1289 // fewest copies.
1290 std::map<ITensorHandleFactory::FactoryId, int> factoryScores;
1291 int topScore = 0;
1292 ITensorHandleFactory::FactoryId topChoice = ITensorHandleFactory::LegacyFactoryId;
1293
1294 for (auto&& connection : slot.GetConnections())
1295 {
Narumol Prangnawarate5f0b242021-05-07 17:52:36 +01001296
Derek Lamberti84da38b2019-06-13 11:40:08 +01001297 const Layer& connectedLayer = connection->GetOwningLayer();
1298
1299 auto toBackend = backends.find(connectedLayer.GetBackendId());
Narumol Prangnawaratac2770a2020-04-01 16:51:23 +01001300 ARMNN_ASSERT_MSG(toBackend != backends.end(), "Backend id not found for the connected layer");
Derek Lamberti84da38b2019-06-13 11:40:08 +01001301
1302 if (!toBackend->second.get()->SupportsTensorAllocatorAPI())
1303 {
1304 // The destination backend does not support the tensor allocator API, move to the next one
1305 continue;
1306 }
1307
1308 auto dstPrefs = toBackend->second.get()->GetHandleFactoryPreferences();
1309 for (auto&& dst : dstPrefs)
1310 {
Derek Lambertif674aa02019-08-01 15:56:25 +01001311 // Input layers use the mem copy workload or import, so the selected factory must
1312 // support either the map/unmap API or Import API
Derek Lamberti84da38b2019-06-13 11:40:08 +01001313 ITensorHandleFactory* factory = registry.GetFactory(dst);
Narumol Prangnawarate5f0b242021-05-07 17:52:36 +01001314 if (importEnabled && factory->GetImportFlags() == 0)
Derek Lamberti84da38b2019-06-13 11:40:08 +01001315 {
Narumol Prangnawarate5f0b242021-05-07 17:52:36 +01001316 continue;
1317 }
1318 else if (!importEnabled && !factory->SupportsMapUnmap())
1319 {
Derek Lamberti84da38b2019-06-13 11:40:08 +01001320 continue;
1321 }
1322
1323 auto it = factoryScores.find(dst);
1324 if (it == factoryScores.end())
1325 {
1326 // Add new score to the table
1327 factoryScores[dst] = 0;
1328 if (topChoice == ITensorHandleFactory::LegacyFactoryId)
1329 {
1330 topChoice = dst;
1331 }
1332 }
1333 else
1334 {
1335 // Increase the score
1336 factoryScores[dst]++;
1337
1338 // Track the best option
1339 if (factoryScores[dst] > topScore)
1340 {
1341 topScore = factoryScores[dst];
1342 topChoice = dst;
1343 }
1344 }
1345 }
1346 }
1347
1348 return topChoice;
1349}
1350
1351// Find the handle factory for the output layer which results in fewest required copies.
1352ITensorHandleFactory::FactoryId CalculateSlotOptionForOutput(BackendsMap& backends,
1353 OutputSlot& slot,
1354 TensorHandleFactoryRegistry& registry)
1355{
Jan Eilers8eb25602020-03-09 12:13:48 +00001356 IgnoreUnused(backends, slot, registry);
Derek Lamberti94a88d22019-12-10 21:12:59 +00001357 return ITensorHandleFactory::DeferredFactoryId;
Derek Lamberti84da38b2019-06-13 11:40:08 +01001358}
1359
1360// For all handle factories supported on the source backend, we wish to find the one which requires the fewest copies
1361// when considering all connections.
1362ITensorHandleFactory::FactoryId CalculateSlotOption(BackendsMap& backends,
1363 OutputSlot& outputSlot,
Narumol Prangnawarate5f0b242021-05-07 17:52:36 +01001364 TensorHandleFactoryRegistry& registry,
James Conroya0f8b152022-06-21 11:31:47 +00001365 bool importEnabled)
Derek Lamberti84da38b2019-06-13 11:40:08 +01001366{
1367 // First ensure the from backends can support the TensorHandeAPI
1368 Layer& layer = outputSlot.GetOwningLayer();
1369 auto frmBackend = backends.find(layer.GetBackendId());
1370 if (frmBackend == backends.end() ||
1371 !frmBackend->second->SupportsTensorAllocatorAPI())
1372 {
1373 return ITensorHandleFactory::LegacyFactoryId;
1374 }
1375
Narumol Prangnawarate5f0b242021-05-07 17:52:36 +01001376 bool outputConnection = false;
Derek Lamberti84da38b2019-06-13 11:40:08 +01001377 for (auto&& connection : outputSlot.GetConnections())
1378 {
1379 const Layer& connectedLayer = connection->GetOwningLayer();
1380 if (connectedLayer.GetType() == LayerType::Output)
1381 {
Narumol Prangnawarate5f0b242021-05-07 17:52:36 +01001382 outputConnection = true;
Derek Lamberti84da38b2019-06-13 11:40:08 +01001383 }
1384 }
1385
1386 IBackendInternal* srcBackend = frmBackend->second.get();
1387 auto srcPrefs = srcBackend->GetHandleFactoryPreferences();
1388
1389 // Initialize the scores
1390 std::map<ITensorHandleFactory::FactoryId, int> factoryScores;
1391 for (auto&& pref : srcPrefs)
1392 {
James Conroya0f8b152022-06-21 11:31:47 +00001393 if (importEnabled)
Derek Lamberti84da38b2019-06-13 11:40:08 +01001394 {
1395 ITensorHandleFactory* factory = registry.GetFactory(pref);
Narumol Prangnawarate5f0b242021-05-07 17:52:36 +01001396 if (outputConnection)
1397 {
1398 // Check if this is fallback case
1399 bool fallbackConnection = false;
1400 for (auto&& inputSlot : layer.GetInputSlots())
1401 {
1402 if (inputSlot.GetConnectedOutputSlot()->GetOwningLayer().GetBackendId() != layer.GetBackendId())
1403 {
1404 fallbackConnection = true;
1405 }
1406 }
1407 if (fallbackConnection)
1408 {
1409 auto factoryCap = factory->GetCapabilities(&layer, &layer, CapabilityClass::FallbackImportDisabled);
1410 // Cannot use factory import if fallback import is not supported.
1411 if (!factoryCap.empty())
1412 {
1413 continue;
1414 }
1415 }
1416 else if (factory->GetExportFlags() == 0)
1417 {
1418 continue;
1419 }
1420 }
1421 if (!outputConnection)
1422 {
1423 auto factoryCap = factory->GetCapabilities(&layer, &layer, CapabilityClass::FallbackImportDisabled);
1424 // Cannot use factory import if fallback import is not supported.
1425 if (!factoryCap.empty())
1426 {
1427 continue;
1428 }
1429 }
1430
1431 }
1432 else
1433 {
1434 // Only consider factories that support map/unmap
1435 ITensorHandleFactory* factory = registry.GetFactory(pref);
Derek Lamberti84da38b2019-06-13 11:40:08 +01001436 if (!factory->SupportsMapUnmap())
1437 {
1438 // The current tensor handle factory does not support the map/unmap strategy, move to the next one
1439 continue;
1440 }
1441 }
1442
Narumol Prangnawarate5f0b242021-05-07 17:52:36 +01001443
Derek Lamberti84da38b2019-06-13 11:40:08 +01001444 auto it = factoryScores.find(pref);
1445 if (it == factoryScores.end())
1446 {
1447 // Add new score to the table
1448 factoryScores[pref] = 0;
1449 }
1450 }
1451
1452 // Score each handle factory based on how many times it requires copies on the slot connections
1453 for (auto&& connection : outputSlot.GetConnections())
1454 {
1455 const Layer& connectedLayer = connection->GetOwningLayer();
1456
1457 auto toBackend = backends.find(connectedLayer.GetBackendId());
Narumol Prangnawaratac2770a2020-04-01 16:51:23 +01001458 ARMNN_ASSERT_MSG(toBackend != backends.end(), "Backend id not found for the connected layer");
Derek Lamberti84da38b2019-06-13 11:40:08 +01001459
1460 auto dstPrefs = toBackend->second.get()->GetHandleFactoryPreferences();
1461 for (auto&& src : srcPrefs)
1462 {
1463 if (factoryScores.find(src) == factoryScores.end()) // Don't consider excluded factories
1464 {
1465 continue;
1466 }
1467
1468 for (auto&& dst : dstPrefs)
1469 {
1470 if (RequiresCopy(src, dst, registry))
1471 {
1472 // Copy avoided, increase the score
1473 factoryScores[src]++;
1474 break;
1475 }
1476 }
1477 }
1478 }
1479
1480 // Find the lowest score
1481 int minScore = std::numeric_limits<int>::max();
1482 for (auto it : factoryScores)
1483 {
1484 minScore = std::min(minScore, it.second);
1485 }
1486
1487 // Collect factories matching the best(lowest) score
1488 std::vector<ITensorHandleFactory::FactoryId> optimalFactories;
1489 for (auto it : factoryScores)
1490 {
1491 if (it.second == minScore)
1492 {
1493 optimalFactories.push_back(it.first);
1494 }
1495 }
1496
1497 // For all compatible Factories matching the best score, find the preferred one for the current layer.
1498 for (auto&& srcPref : srcPrefs)
1499 {
1500 for (auto&& comp : optimalFactories)
1501 {
1502 if (comp == srcPref)
1503 {
1504 return comp;
1505 }
1506 }
1507 }
1508
1509 return ITensorHandleFactory::LegacyFactoryId;
1510}
1511
Derek Lambertif674aa02019-08-01 15:56:25 +01001512EdgeStrategy CalculateEdgeStrategy(BackendsMap& backends,
1513 ITensorHandleFactory::FactoryId srcFactoryId,
1514 const Layer& layer,
1515 const Layer& connectedLayer,
Narumol Prangnawarata2493a02020-08-19 14:39:07 +01001516 TensorHandleFactoryRegistry& registry,
1517 bool importEnabled)
Derek Lamberti84da38b2019-06-13 11:40:08 +01001518{
1519 auto toBackend = backends.find(connectedLayer.GetBackendId());
Narumol Prangnawaratac2770a2020-04-01 16:51:23 +01001520 ARMNN_ASSERT_MSG(toBackend != backends.end(), "Backend id not found for the connected layer");
Derek Lamberti84da38b2019-06-13 11:40:08 +01001521
1522 auto dstPrefs = toBackend->second.get()->GetHandleFactoryPreferences();
1523
1524 // Legacy API check for backward compatibility
1525 if (srcFactoryId == ITensorHandleFactory::LegacyFactoryId || dstPrefs.empty())
1526 {
1527 if (layer.GetBackendId() != connectedLayer.GetBackendId())
1528 {
Derek Lambertif674aa02019-08-01 15:56:25 +01001529 return EdgeStrategy::CopyToTarget;
Derek Lamberti84da38b2019-06-13 11:40:08 +01001530 }
1531 else
1532 {
Derek Lambertif674aa02019-08-01 15:56:25 +01001533 return EdgeStrategy::DirectCompatibility;
Derek Lamberti84da38b2019-06-13 11:40:08 +01001534 }
1535 }
1536
1537 // TensorHandleFactory API present, so perform more sophisticated strategies.
Derek Lambertif674aa02019-08-01 15:56:25 +01001538 // Dst Output layers don't require copy because they use import or map/unmap
Derek Lamberti84da38b2019-06-13 11:40:08 +01001539 if (connectedLayer.GetType() == LayerType::Output)
1540 {
Derek Lambertif674aa02019-08-01 15:56:25 +01001541 return EdgeStrategy::DirectCompatibility;
Derek Lamberti84da38b2019-06-13 11:40:08 +01001542 }
1543
1544 // Search for direct match in prefs
1545 for (auto&& pref : dstPrefs)
1546 {
1547 if (pref == srcFactoryId)
1548 {
Derek Lambertif674aa02019-08-01 15:56:25 +01001549 return EdgeStrategy::DirectCompatibility;
Derek Lamberti84da38b2019-06-13 11:40:08 +01001550 }
1551 }
1552
1553 // Search for export/import options
1554 ITensorHandleFactory* srcFactory = registry.GetFactory(srcFactoryId);
Narumol Prangnawarata2493a02020-08-19 14:39:07 +01001555 if (srcFactory->GetExportFlags() != 0 && importEnabled)
Derek Lamberti84da38b2019-06-13 11:40:08 +01001556 {
1557 for (auto&& pref : dstPrefs)
1558 {
1559 ITensorHandleFactory* dstFactory = registry.GetFactory(pref);
James Conroyffab16f2019-11-07 14:37:09 +00001560
James Conroy47e863d2019-11-18 17:07:43 +00001561 // Handles cases when a destPref is not listed in TensorHandleFactoryRegistry
James Conroyffab16f2019-11-07 14:37:09 +00001562 if (!dstFactory) {
James Conroy47e863d2019-11-18 17:07:43 +00001563 continue;
James Conroyffab16f2019-11-07 14:37:09 +00001564 }
Derek Lambertif674aa02019-08-01 15:56:25 +01001565 if ((dstFactory->GetImportFlags() & srcFactory->GetExportFlags()) != 0)
Derek Lamberti84da38b2019-06-13 11:40:08 +01001566 {
Narumol Prangnawaratb8d771a2020-08-14 11:51:12 +01001567 auto srcCapability = srcFactory->GetCapabilities(&layer, &layer, CapabilityClass::PaddingRequired);
1568 auto dstCapability = dstFactory->GetCapabilities(&connectedLayer,
1569 &connectedLayer,
1570 CapabilityClass::PaddingRequired);
Narumol Prangnawarate5f0b242021-05-07 17:52:36 +01001571 auto srcFallback = srcFactory->GetCapabilities(&layer, &layer, CapabilityClass::FallbackImportDisabled);
1572 auto dstFallback = dstFactory->GetCapabilities(&connectedLayer,
1573 &connectedLayer,
1574 CapabilityClass::FallbackImportDisabled);
Narumol Prangnawaratb8d771a2020-08-14 11:51:12 +01001575 // Do not require memory copy if the source and destination do not require padding.
Narumol Prangnawarate5f0b242021-05-07 17:52:36 +01001576 if (srcCapability.empty() && dstCapability.empty() && srcFallback.empty() && dstFallback.empty())
Narumol Prangnawaratb8d771a2020-08-14 11:51:12 +01001577 {
1578 return EdgeStrategy::ExportToTarget;
1579 }
Derek Lamberti84da38b2019-06-13 11:40:08 +01001580 }
1581 }
1582 }
1583
1584 // Search for copy options via map/unmap
1585 if (srcFactory->SupportsMapUnmap())
1586 {
1587 for (auto&& pref : dstPrefs)
1588 {
1589 ITensorHandleFactory* dstFactory = registry.GetFactory(pref);
James Conroy47e863d2019-11-18 17:07:43 +00001590 if (dstFactory && dstFactory->SupportsMapUnmap())
Derek Lamberti84da38b2019-06-13 11:40:08 +01001591 {
Derek Lambertif674aa02019-08-01 15:56:25 +01001592 return EdgeStrategy::CopyToTarget;
Derek Lamberti84da38b2019-06-13 11:40:08 +01001593 }
1594 }
1595 }
1596
Derek Lambertif674aa02019-08-01 15:56:25 +01001597 return EdgeStrategy::Undefined;
Derek Lamberti84da38b2019-06-13 11:40:08 +01001598}
1599
1600// Select the TensorHandleFactories and the corresponding memory strategy
1601OptimizationResult SelectTensorHandleStrategy(Graph& optGraph,
1602 BackendsMap& backends,
1603 TensorHandleFactoryRegistry& registry,
Narumol Prangnawarata2493a02020-08-19 14:39:07 +01001604 bool importEnabled,
Derek Lamberti84da38b2019-06-13 11:40:08 +01001605 Optional<std::vector<std::string>&> errMessages)
1606{
Derek Lambertif1e0ad32021-10-13 18:02:25 +01001607 ARMNN_SCOPED_PROFILING_EVENT(Compute::Undefined, "Optimizer_SelectTensorHandleStrategy");
Derek Lamberti84da38b2019-06-13 11:40:08 +01001608 OptimizationResult result;
1609
James Conroya0f8b152022-06-21 11:31:47 +00001610 optGraph.ForEachLayer([&backends, &registry, &result, &errMessages, importEnabled](Layer* layer)
Derek Lamberti84da38b2019-06-13 11:40:08 +01001611 {
Narumol Prangnawaratac2770a2020-04-01 16:51:23 +01001612 ARMNN_ASSERT(layer);
Derek Lamberti84da38b2019-06-13 11:40:08 +01001613
1614 // Lets make sure the backend is in our list of supported backends. Something went wrong during backend
1615 // assignment if this check fails
Narumol Prangnawaratac2770a2020-04-01 16:51:23 +01001616 ARMNN_ASSERT(backends.find(layer->GetBackendId()) != backends.end());
Derek Lamberti84da38b2019-06-13 11:40:08 +01001617
1618 // Check each output separately
1619 for (unsigned int slotIdx = 0; slotIdx < layer->GetNumOutputSlots(); slotIdx++)
1620 {
1621 OutputSlot& outputSlot = layer->GetOutputSlot(slotIdx);
1622
1623 ITensorHandleFactory::FactoryId slotOption = ITensorHandleFactory::LegacyFactoryId;
1624
1625 // Calculate the factory to use which results in the fewest copies being made.
1626 switch(layer->GetType())
1627 {
1628 case LayerType::Input:
Narumol Prangnawarate5f0b242021-05-07 17:52:36 +01001629 slotOption = CalculateSlotOptionForInput(backends, outputSlot, registry, importEnabled);
Derek Lamberti84da38b2019-06-13 11:40:08 +01001630 break;
1631 case LayerType::Output:
1632 slotOption = CalculateSlotOptionForOutput(backends, outputSlot, registry);
1633 break;
1634 default:
James Conroya0f8b152022-06-21 11:31:47 +00001635 slotOption = CalculateSlotOption(backends, outputSlot, registry, importEnabled);
Derek Lamberti84da38b2019-06-13 11:40:08 +01001636 break;
1637 }
1638 outputSlot.SetTensorHandleFactory(slotOption);
1639
Derek Lambertif674aa02019-08-01 15:56:25 +01001640 // Now determine the "best" edge strategy for each connection given the slotOption.
Derek Lamberti84da38b2019-06-13 11:40:08 +01001641 unsigned int connectionIdx = 0;
1642 for (auto&& connection : outputSlot.GetConnections())
1643 {
1644 const Layer& connectedLayer = connection->GetOwningLayer();
1645
Narumol Prangnawarata2493a02020-08-19 14:39:07 +01001646 EdgeStrategy strategy = CalculateEdgeStrategy(backends, slotOption, *layer, connectedLayer,
1647 registry, importEnabled);
Derek Lamberti84da38b2019-06-13 11:40:08 +01001648
Derek Lambertif674aa02019-08-01 15:56:25 +01001649 if (strategy == EdgeStrategy::Undefined)
Derek Lamberti84da38b2019-06-13 11:40:08 +01001650 {
1651 result.m_Error = true;
1652 if (errMessages)
1653 {
1654 errMessages.value().emplace_back("Could not find valid strategy required for compatibility"
1655 " between backends.");
1656 }
1657 return;
1658 }
1659
Derek Lambertif674aa02019-08-01 15:56:25 +01001660 outputSlot.SetEdgeStrategy(connectionIdx, strategy);
Derek Lamberti84da38b2019-06-13 11:40:08 +01001661
1662 connectionIdx++;
1663 }
1664 }
1665 });
1666
1667 return result;
1668}
1669
Cathal Corbetta3f4fba2022-03-21 09:27:08 +00001670IOptimizedNetworkPtr Optimize(const Graph& inGraph,
Matteo Martincigh49124022019-01-11 13:25:59 +00001671 const std::vector<BackendId>& backendPreferences,
1672 const IDeviceSpec& deviceSpec,
1673 const OptimizerOptions& options,
Rob Hughes23214432019-11-05 11:27:36 +00001674 Optional<std::vector<std::string>&> messages)
Matteo Martincigh49124022019-01-11 13:25:59 +00001675{
Jan Eilers17d34da2021-12-08 16:15:12 +00001676 ARMNN_LOG(debug) << options.ToString();
Jan Eilers6a71bb52021-10-26 17:41:18 +01001677
Derek Lambertif1e0ad32021-10-13 18:02:25 +01001678 // Enable profiling
Cathal Corbetta3f4fba2022-03-21 09:27:08 +00001679 auto profiler = inGraph.GetProfiler();
Derek Lambertif1e0ad32021-10-13 18:02:25 +01001680 ProfilerManager::GetInstance().RegisterProfiler(profiler.get());
1681 profiler->EnableProfiling(options.m_ProfilingEnabled);
1682
1683 ARMNN_SCOPED_PROFILING_EVENT(Compute::Undefined, "Optimizer");
Matteo Martincigh49124022019-01-11 13:25:59 +00001684 if (backendPreferences.empty())
1685 {
Mike Kelly3a613cc2020-09-29 20:50:35 +01001686 throw InvalidArgumentException("Invoked Optimize with no backends specified");
Matteo Martincigh49124022019-01-11 13:25:59 +00001687 }
1688
Narumol Prangnawaratbc7ffb52020-03-20 15:01:01 +00001689 if (options.m_ReduceFp32ToFp16 && options.m_ReduceFp32ToBf16)
1690 {
1691 throw InvalidArgumentException("BFloat16 and Float16 optimization cannot be enabled at the same time.");
1692 }
1693
Cathal Corbett521032f2021-10-07 11:46:40 +01001694 // Ensure TensorInfo is set on all output slots of ConstantLayers in the graph
Cathal Corbetta3f4fba2022-03-21 09:27:08 +00001695 inGraph.VerifyConstantLayerSetTensorInfo();
Cathal Corbett521032f2021-10-07 11:46:40 +01001696
Cathal Corbetta3f4fba2022-03-21 09:27:08 +00001697 std::unique_ptr<Graph> graph = std::make_unique<Graph>(inGraph);
Matteo Martincigh49124022019-01-11 13:25:59 +00001698
James Conroya0f8b152022-06-21 11:31:47 +00001699 auto optNet = IOptimizedNetworkPtr(new IOptimizedNetwork(std::move(graph), options.m_ModelOptions),
Sadik Armagan045f6be2020-09-10 13:37:32 +01001700 &IOptimizedNetwork::Destroy);
Matteo Martincigh49124022019-01-11 13:25:59 +00001701
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001702 IOptimizedNetwork* optNetObjPtr = optNet.get();
Matteo Martincigh49124022019-01-11 13:25:59 +00001703
Matteo Martincighadddddb2019-01-24 14:06:23 +00001704 // Get the optimized graph
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001705 Graph& optGraph = optNetObjPtr->pOptimizedNetworkImpl->GetGraph();
Matteo Martincighadddddb2019-01-24 14:06:23 +00001706
Finn Williamsd218d982021-08-09 13:00:08 +01001707 if(options.m_shapeInferenceMethod == ShapeInferenceMethod::InferAndValidate)
1708 {
1709 // Infer the tensor infos for all output slots. Throws an exception on failure
1710 optGraph.InferTensorInfos();
1711 }
Finn Williams84e025a2021-08-05 17:29:32 +01001712
Narumol Prangnawarat16f82f92020-09-14 16:12:44 +01001713 // Perform AddBroadcastReshapeLayer optimisation
1714 using namespace optimizations;
1715 Optimizer::Pass(optGraph, MakeOptimizations(AddBroadcastReshapeLayer()));
1716
Finn Williamsd218d982021-08-09 13:00:08 +01001717 if(options.m_shapeInferenceMethod == ShapeInferenceMethod::ValidateOnly)
1718 {
1719 // Validate the tensor infos for all output slots. Throws an exception on failure
1720 optGraph.InferTensorInfos();
1721 }
1722
Cathal Corbett541880f2022-05-16 15:20:56 +01001723 // Need to FusePermuteIntoConstantLayer before FoldPadIntoDepthwiseConvolution2d or
1724 // FuseBatchNormIntoDepthwiseConvolution2D optimizations are called.
1725 Optimizer::Pass(optGraph, MakeOptimizations(FusePermuteIntoConstLayer()));
1726
Matteo Martincigh49124022019-01-11 13:25:59 +00001727 // Perform optimisation passes
Matteo Martincighadddddb2019-01-24 14:06:23 +00001728 Optimizer::Pass(optGraph, MakeOptimizations(SquashEqualPermuteSiblings(),
Mike Kelly490b7be2020-03-03 12:39:09 +00001729 SquashEqualTransposeSiblings(),
Matteo Martincighadddddb2019-01-24 14:06:23 +00001730 SquashEqualReshapeSiblings(),
1731 OptimizeInversePermutes(),
Mike Kelly490b7be2020-03-03 12:39:09 +00001732 OptimizeInverseTransposes(),
Matteo Martincighadddddb2019-01-24 14:06:23 +00001733 MovePermuteUp(),
Mike Kelly490b7be2020-03-03 12:39:09 +00001734 MoveTransposeUp(),
Matteo Martincighadddddb2019-01-24 14:06:23 +00001735 PermuteAsReshape(),
Mike Kelly490b7be2020-03-03 12:39:09 +00001736 TransposeAsReshape(),
Nina Drozd861985f2019-04-18 14:48:51 +01001737 OptimizeConsecutiveReshapes(),
Rob Hughes3a7d3a72019-09-24 16:59:56 +01001738 FoldPadIntoConvolution2d(),
Teresa Charlin5786eb72021-05-21 16:29:45 +01001739 FoldPadIntoDepthwiseConvolution2d(),
Diego Lopez Recasfe95d722021-03-19 12:40:16 +00001740 FoldPadIntoPooling2d(),
Mike Kelly490b7be2020-03-03 12:39:09 +00001741 PermuteAndBatchToSpaceAsDepthToSpace(),
Teresa Charlin06e03002020-10-15 13:16:07 +01001742 TransposeAndBatchToSpaceAsDepthToSpace(),
Mike Kelly90231b82020-11-05 15:44:56 +00001743 FuseBatchNormIntoConvolution2DFloat32(),
1744 FuseBatchNormIntoConvolution2DFloat16(),
1745 FuseBatchNormIntoDepthwiseConvolution2DFloat32(),
Cathal Corbett06902652022-04-14 17:55:11 +01001746 FuseBatchNormIntoDepthwiseConvolution2DFloat16(),
Cathal Corbett541880f2022-05-16 15:20:56 +01001747 ConvertConstDequantisationLayersToConstLayers()));
Matteo Martincigh49124022019-01-11 13:25:59 +00001748
Matteo Martincigh49124022019-01-11 13:25:59 +00001749 // If Fp32 to Fp16 optimization is set convert Fp32 network to Fp16
1750 if (options.m_ReduceFp32ToFp16)
1751 {
Derek Lambertif1e0ad32021-10-13 18:02:25 +01001752 ARMNN_SCOPED_PROFILING_EVENT(Compute::Undefined, "Optimizer_ReduceFp32ToFp16");
Matteo Martincighadddddb2019-01-24 14:06:23 +00001753 Optimizer::Pass(optGraph, MakeOptimizations(Fp32NetworkToFp16Converter()));
Derek Lambertidd6804b2019-11-27 09:29:57 +00001754 Optimizer::Pass(optGraph, MakeOptimizations(ConvertConstantsFloatToHalf()));
Matteo Martincigh49124022019-01-11 13:25:59 +00001755 }
1756
Narumol Prangnawaratbc7ffb52020-03-20 15:01:01 +00001757 // If Fp32 to Bf16 optimization is set convert Fp32 network to Bf16
Narumol Prangnawarat57ef0082020-03-26 09:20:43 +00001758 // Convert input of Convolution2d and FullyConnected from Fp32 to Bf16
1759 // Only Constant weight of Convolution2d and FullyConnected are converted from Fp32 to Bf16
Narumol Prangnawaratbc7ffb52020-03-20 15:01:01 +00001760 if (options.m_ReduceFp32ToBf16)
1761 {
Derek Lambertif1e0ad32021-10-13 18:02:25 +01001762 ARMNN_SCOPED_PROFILING_EVENT(Compute::Undefined, "Optimizer_ReduceFp32ToBf16");
Narumol Prangnawaratbc7ffb52020-03-20 15:01:01 +00001763 Optimizer::Pass(optGraph, MakeOptimizations(Fp32NetworkToBf16Converter()));
Narumol Prangnawaratbc7ffb52020-03-20 15:01:01 +00001764 }
1765
Matteo Martincigh49124022019-01-11 13:25:59 +00001766 // Initialize backend settings
1767 BackendSettings backendSettings(backendPreferences, deviceSpec);
1768 if (backendSettings.GetAvailablePreferredBackends().empty())
1769 {
1770 std::stringstream failureMsg;
1771 failureMsg << "None of the preferred backends " << backendPreferences
1772 << " are supported. Current platform provides " << backendSettings.m_SupportedBackends;
Rob Hughes23214432019-11-05 11:27:36 +00001773 ReportError(failureMsg.str(), messages);
Mike Kelly3a613cc2020-09-29 20:50:35 +01001774 throw InvalidArgumentException(failureMsg.str());
Matteo Martincigh49124022019-01-11 13:25:59 +00001775 }
1776
Derek Lamberti84da38b2019-06-13 11:40:08 +01001777 // Create a map to temporarily hold initialized backend objects
1778 TensorHandleFactoryRegistry tensorHandleFactoryRegistry;
1779 BackendsMap backends = CreateSupportedBackends(tensorHandleFactoryRegistry, backendSettings);
1780
Matteo Martincigh49124022019-01-11 13:25:59 +00001781 // Assign an available backend to each layer
Matteo Martincighadddddb2019-01-24 14:06:23 +00001782 Graph::Iterator firstLayer = optGraph.begin();
1783 Graph::Iterator lastLayer = optGraph.end();
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001784 OptimizationResult assignBackendsResult = AssignBackends(optNetObjPtr->pOptimizedNetworkImpl.get(),
Derek Lamberti84da38b2019-06-13 11:40:08 +01001785 backendSettings,
1786 firstLayer,
1787 lastLayer,
Rob Hughes23214432019-11-05 11:27:36 +00001788 messages);
Derek Lamberti84da38b2019-06-13 11:40:08 +01001789 if (assignBackendsResult.m_Error)
Matteo Martincigh49124022019-01-11 13:25:59 +00001790 {
1791 // Failed to assign a backend to each layer
Mike Kelly3a613cc2020-09-29 20:50:35 +01001792 throw InvalidArgumentException("Failed to assign a backend to each layer");
jimfly016b0b53d2018-10-08 14:43:01 +01001793 }
telsoa01c577f2c2018-08-31 09:22:23 +01001794
Matteo Martincighadddddb2019-01-24 14:06:23 +00001795 Optimizer::Pass(optGraph, MakeOptimizations(OptimizeInverseConversionsFp16(),
1796 OptimizeInverseConversionsFp32()));
telsoa01c577f2c2018-08-31 09:22:23 +01001797
Matteo Martincighadddddb2019-01-24 14:06:23 +00001798 // Apply the backend-specific optimizations
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001799 OptimizationResult backendOptimizationResult = ApplyBackendOptimizations(optNetObjPtr->pOptimizedNetworkImpl.get(),
Matteo Martincighadddddb2019-01-24 14:06:23 +00001800 backendSettings,
Derek Lamberti84da38b2019-06-13 11:40:08 +01001801 backends,
Mike Kelly07810fc2020-11-12 10:58:48 +00001802 options.m_ModelOptions,
Rob Hughes23214432019-11-05 11:27:36 +00001803 messages);
Matteo Martincighadddddb2019-01-24 14:06:23 +00001804 if (backendOptimizationResult.m_Error)
Matteo Martincigh49124022019-01-11 13:25:59 +00001805 {
Matteo Martincighadddddb2019-01-24 14:06:23 +00001806 // Failed to apply the backend-specific optimizations
Mike Kelly3a613cc2020-09-29 20:50:35 +01001807 throw InvalidArgumentException("Failed to apply the backend-specific optimizations");
Matteo Martincigh49124022019-01-11 13:25:59 +00001808 }
1809
Matteo Martincighadddddb2019-01-24 14:06:23 +00001810 // If the debug flag is set, then insert a DebugLayer after each layer
1811 // Doing this after applying the backend optimizations as they might have changed some layers
1812 if (options.m_Debug)
1813 {
1814 Optimizer::Pass(optGraph, MakeOptimizations(InsertDebugLayer()));
1815 }
1816
Derek Lamberti84da38b2019-06-13 11:40:08 +01001817 // Calculate the compatibility strategies for tensor handles
1818 OptimizationResult strategyResult = SelectTensorHandleStrategy(optGraph,
1819 backends,
1820 tensorHandleFactoryRegistry,
Narumol Prangnawarata2493a02020-08-19 14:39:07 +01001821 options.m_ImportEnabled,
Rob Hughes23214432019-11-05 11:27:36 +00001822 messages);
Derek Lamberti84da38b2019-06-13 11:40:08 +01001823 if (strategyResult.m_Error)
1824 {
1825 // Failed to apply the backend-specific optimizations
1826 return IOptimizedNetworkPtr(nullptr, &IOptimizedNetwork::Destroy);
1827 }
1828
1829 // Based on the tensor handle strategy determined above, insert copy layers where required.
Derek Lambertif1e0ad32021-10-13 18:02:25 +01001830 {
1831 ARMNN_SCOPED_PROFILING_EVENT(Compute::Undefined, "Optimizer_AddCompatibilityLayers");
1832 optGraph.AddCompatibilityLayers(backends, tensorHandleFactoryRegistry);
1833 }
telsoa01c577f2c2018-08-31 09:22:23 +01001834
1835 // Convert constants
Derek Lambertif1e0ad32021-10-13 18:02:25 +01001836 {
1837 ARMNN_SCOPED_PROFILING_EVENT(Compute::Undefined, "Optimizer_ConvertConstants");
1838 Optimizer::Pass(optGraph, MakeOptimizations(ConvertConstantsFloatToHalf()));
1839 Optimizer::Pass(optGraph, MakeOptimizations(ConvertConstantsHalfToFloat()));
Cathal Corbett8bd53602022-05-12 15:54:58 +01001840
1841 // Once the constants are converted we can now safely call RedirectMembersToConstantInputs
1842 Optimizer::Pass(optGraph, MakeOptimizations(RedirectMembersToConstantInputs()));
Derek Lambertif1e0ad32021-10-13 18:02:25 +01001843 }
telsoa01c577f2c2018-08-31 09:22:23 +01001844 return optNet;
telsoa014fcda012018-03-09 14:13:49 +00001845}
Cathal Corbetta3f4fba2022-03-21 09:27:08 +00001846
1847IOptimizedNetworkPtr Optimize(const INetwork& inNetwork,
1848 const std::vector<BackendId>& backendPreferences,
1849 const IDeviceSpec& deviceSpec,
1850 const OptimizerOptions& options,
1851 Optional<std::vector<std::string>&> messages)
1852{
1853 return Optimize(inNetwork.pNetworkImpl->GetGraph(),
1854 backendPreferences,
1855 deviceSpec,
1856 options,
1857 messages);
1858}
1859
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001860bool NetworkImpl::GetShapeInferenceMethod()
telsoa014fcda012018-03-09 14:13:49 +00001861{
Mike Kelly80512b02022-05-16 23:10:42 +01001862 bool shapeInferenceMethod = false;
Finn Williamsf24effa2020-07-03 10:12:03 +01001863
Mike Kelly80512b02022-05-16 23:10:42 +01001864 ParseOptions(m_NetworkOptions, "ShapeInferenceMethod", [&](std::string name, const BackendOptions::Var& value)
1865 {
1866 if (name == "InferAndValidate")
1867 {
1868 shapeInferenceMethod |= value.AsBool();
1869 }
1870 });
1871 return shapeInferenceMethod;
telsoa014fcda012018-03-09 14:13:49 +00001872}
Mike Kelly80512b02022-05-16 23:10:42 +01001873
1874bool NetworkImpl::GetAllowExpandedDims()
1875{
1876 bool allowExpandedDims = false;
1877
1878 ParseOptions(m_NetworkOptions, "AllowExpandedDims", [&](std::string name, const BackendOptions::Var& value)
1879 {
1880 if (name == "AllowExpandedDims")
1881 {
1882 allowExpandedDims |= value.AsBool();
1883 }
1884 });
1885 return allowExpandedDims;
1886}
1887
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001888NetworkImpl::NetworkImpl(NetworkOptions networkOptions)
Finn Williamsf24effa2020-07-03 10:12:03 +01001889: m_NetworkOptions(networkOptions),
Mike Kelly80512b02022-05-16 23:10:42 +01001890 m_Graph(std::make_unique<Graph>(GetShapeInferenceMethod(), GetAllowExpandedDims()))
Finn Williamsf24effa2020-07-03 10:12:03 +01001891{}
telsoa014fcda012018-03-09 14:13:49 +00001892
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001893NetworkImpl::~NetworkImpl()
telsoa014fcda012018-03-09 14:13:49 +00001894{
1895}
1896
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001897Status NetworkImpl::PrintGraph()
Jan Eilers99d9d4a2019-11-06 10:02:16 +00001898{
1899 m_Graph->Print();
1900 return Status::Success;
1901}
1902
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001903IConnectableLayer* NetworkImpl::AddInputLayer(LayerBindingId id, const char* name)
telsoa014fcda012018-03-09 14:13:49 +00001904{
1905 return m_Graph->AddLayer<InputLayer>(id, name);
1906}
1907
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001908IConnectableLayer* NetworkImpl::AddBatchToSpaceNdLayer(const BatchToSpaceNdDescriptor& batchToSpaceNdDescriptor,
Éanna Ó Catháin4e1e1362018-11-12 11:36:34 +00001909 const char* name)
1910{
1911 return m_Graph->AddLayer<BatchToSpaceNdLayer>(batchToSpaceNdDescriptor, name);
1912}
1913
mathad01b392e982021-04-07 12:07:30 +01001914IConnectableLayer* NetworkImpl::AddCastLayer(const char* name)
1915{
1916 return m_Graph->AddLayer<CastLayer>(name);
1917}
Simon Obute51f67772021-09-03 15:50:13 +01001918IConnectableLayer* NetworkImpl::AddChannelShuffleLayer(const ChannelShuffleDescriptor& channelShuffleDescriptor,
1919 const char* name)
1920{
1921 return m_Graph->AddLayer<ChannelShuffleLayer>(channelShuffleDescriptor, name);
1922}
mathad01b392e982021-04-07 12:07:30 +01001923
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001924IConnectableLayer* NetworkImpl::AddComparisonLayer(const ComparisonDescriptor& comparisonDescriptor,
Aron Virginas-Tar77bfb5e2019-10-16 17:45:38 +01001925 const char* name)
1926{
1927 return m_Graph->AddLayer<ComparisonLayer>(comparisonDescriptor, name);
1928}
1929
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001930IConnectableLayer* NetworkImpl::AddElementwiseUnaryLayer(const ElementwiseUnaryDescriptor& elementwiseUnaryDescriptor,
josh minor4a3c6102020-01-06 16:40:46 -06001931 const char* name)
1932{
1933 return m_Graph->AddLayer<ElementwiseUnaryLayer>(elementwiseUnaryDescriptor, name);
1934}
1935
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001936IConnectableLayer* NetworkImpl::AddFillLayer(const FillDescriptor& fillDescriptor,
Ryan OSheaec6c6802020-06-05 17:17:06 +01001937 const char* name)
1938{
1939 return m_Graph->AddLayer<FillLayer>(fillDescriptor, name);
1940}
1941
Matthew Sloyan81beae32021-07-13 19:46:11 +01001942IConnectableLayer* NetworkImpl::AddFullyConnectedLayer(const FullyConnectedDescriptor& fullyConnectedDescriptor,
1943 const char* name)
telsoa014fcda012018-03-09 14:13:49 +00001944{
Matthew Sloyan81beae32021-07-13 19:46:11 +01001945 return m_Graph->AddLayer<FullyConnectedLayer>(fullyConnectedDescriptor, name);
telsoa014fcda012018-03-09 14:13:49 +00001946}
1947
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001948IConnectableLayer* NetworkImpl::AddFullyConnectedLayer(const FullyConnectedDescriptor& fullyConnectedDescriptor,
Sadik Armaganf0a6dec2021-03-25 07:46:55 +00001949 const Optional<ConstTensor>& weights,
1950 const Optional<ConstTensor>& biases,
1951 const char* name)
1952{
Matthew Sloyan81beae32021-07-13 19:46:11 +01001953 ConstantLayer* weightsLayer = nullptr;
1954 ConstantLayer* biasLayer = nullptr;
1955 unsigned int numInputs = fullyConnectedDescriptor.GetNumInputs();
1956
1957 // Add a constant layer for weights
1958 if (weights.has_value())
1959 {
1960 weightsLayer = m_Graph->AddLayer<ConstantLayer>("Weights");
1961 weightsLayer->m_LayerOutput = std::make_shared<ScopedTensorHandle>(weights.value());
Matthew Sloyanb20d1d42021-08-09 15:33:41 +01001962
1963 TensorInfo weightsInfo = weightsLayer->m_LayerOutput->GetTensorInfo();
1964 weightsInfo.SetConstant();
1965
1966 weightsLayer->GetOutputSlot(0).SetTensorInfo(weightsInfo);
Matthew Sloyan81beae32021-07-13 19:46:11 +01001967 }
1968 else if (fullyConnectedDescriptor.m_ConstantWeights)
1969 {
1970 throw InvalidArgumentException("AddFullyConnectedLayer: Constant weights tensor is empty.");
1971 }
1972
1973 // Add a constant layer for biases
1974 if (biases.has_value() && fullyConnectedDescriptor.m_BiasEnabled)
1975 {
1976 biasLayer = m_Graph->AddLayer<ConstantLayer>("Biases");
1977 biasLayer->m_LayerOutput = std::make_shared<ScopedTensorHandle>(biases.value());
Matthew Sloyanb20d1d42021-08-09 15:33:41 +01001978
1979 TensorInfo biasInfo = biasLayer->m_LayerOutput->GetTensorInfo();
1980 biasInfo.SetConstant();
1981
1982 biasLayer->GetOutputSlot(0).SetTensorInfo(biasInfo);
Matthew Sloyan81beae32021-07-13 19:46:11 +01001983 }
1984
1985 if (numInputs < 2)
1986 {
1987 throw InvalidArgumentException("AddFullyConnectedLayer: Requires at least 2 input tensors: Input, Weights");
1988 }
1989
1990 auto layer = m_Graph->AddLayer<FullyConnectedLayer>(fullyConnectedDescriptor, name);
1991
1992 if (weightsLayer)
1993 {
1994 // Connect weights layer
1995 weightsLayer->GetOutputSlot(0).Connect(layer->GetInputSlot(1));
1996 }
1997
1998 if ( fullyConnectedDescriptor.m_BiasEnabled && numInputs == 3 )
1999 {
2000 if (biasLayer)
2001 {
2002 // Connect bias layer
2003 biasLayer->GetOutputSlot(0).Connect(layer->GetInputSlot(2));
2004 }
2005 }
2006 else if ( !fullyConnectedDescriptor.m_BiasEnabled && numInputs == 2 )
2007 {
2008 // Bias is disabled
2009 layer->m_Bias = nullptr;
2010 }
2011 else
2012 {
2013 throw InvalidArgumentException(fmt::format(
2014 "AddFullyConnectedLayer: Value mismatch. When bias is enabled in the "
2015 "descriptor the number of inputs is expected to be 3 otherwise 2. "
2016 "BiasEnabled={}, numInputs={}",
2017 fullyConnectedDescriptor.m_BiasEnabled,
2018 numInputs));
2019 }
2020
2021 return layer;
Sadik Armaganf0a6dec2021-03-25 07:46:55 +00002022}
2023
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002024IConnectableLayer* NetworkImpl::AddConcatLayer(const ConcatDescriptor& concatDescriptor,
Jim Flynn906f9462019-05-10 13:55:21 +01002025 const char* name)
2026{
Jim Flynne242f2d2019-05-22 14:24:13 +01002027 return m_Graph->AddLayer<ConcatLayer>(concatDescriptor, name);
Jim Flynn906f9462019-05-10 13:55:21 +01002028}
2029
Keith Davisb4dd5cc2022-04-07 11:32:00 +01002030IConnectableLayer* NetworkImpl::AddConvolution2dLayer(const Convolution2dDescriptor& convolution2dDescriptor,
2031 const char* name)
telsoa014fcda012018-03-09 14:13:49 +00002032{
Keith Davisb4dd5cc2022-04-07 11:32:00 +01002033 return m_Graph->AddLayer<Convolution2dLayer>(convolution2dDescriptor, name);
2034}
telsoa014fcda012018-03-09 14:13:49 +00002035
Keith Davisb4dd5cc2022-04-07 11:32:00 +01002036IConnectableLayer* NetworkImpl::AddConvolution2dLayer(const Convolution2dDescriptor& convolution2dDescriptor,
2037 const ConstTensor& weights,
2038 const Optional<ConstTensor>& biases,
2039 const char* name)
2040{
2041 auto layer = m_Graph->AddLayer<Convolution2dLayer>(convolution2dDescriptor, name);
2042 // Add a constant layer for weights
2043 ConstantLayer* weightsLayer = m_Graph->AddLayer<ConstantLayer>("Weights");
2044 weightsLayer->m_LayerOutput = std::make_shared<ScopedTensorHandle>(weights);
James Conroy1f58f032021-04-27 17:13:27 +01002045 layer->m_Weight = std::make_shared<ScopedTensorHandle>(weights);
Keith Davisb4dd5cc2022-04-07 11:32:00 +01002046 weightsLayer->GetOutputSlot(0).SetTensorInfo(weightsLayer->m_LayerOutput->GetTensorInfo());
2047 weightsLayer->GetOutputSlot(0).Connect(layer->GetInputSlot(1));
2048 // Add a constant layer for biases
2049 if (biases.has_value() && convolution2dDescriptor.m_BiasEnabled)
telsoa014fcda012018-03-09 14:13:49 +00002050 {
Keith Davisb4dd5cc2022-04-07 11:32:00 +01002051 ConstantLayer* biasLayer = m_Graph->AddLayer<ConstantLayer>("Bias");
2052 biasLayer->m_LayerOutput = std::make_shared<ScopedTensorHandle>(biases.value());
James Conroy1f58f032021-04-27 17:13:27 +01002053 layer->m_Bias = std::make_shared<ScopedTensorHandle>(biases.value());
Keith Davisb4dd5cc2022-04-07 11:32:00 +01002054 biasLayer->GetOutputSlot(0).SetTensorInfo(biasLayer->m_LayerOutput->GetTensorInfo());
2055 biasLayer->GetOutputSlot(0).Connect(layer->GetInputSlot(2));
telsoa014fcda012018-03-09 14:13:49 +00002056 }
telsoa014fcda012018-03-09 14:13:49 +00002057 return layer;
2058}
2059
Cathal Corbetta3f4fba2022-03-21 09:27:08 +00002060IConnectableLayer* NetworkImpl::AddConvertFp16ToFp32Layer(const char* name)
2061{
2062 return m_Graph->AddLayer<ConvertFp16ToFp32Layer>(name);
2063}
2064
2065IConnectableLayer* NetworkImpl::AddConvertFp32ToFp16Layer(const char* name)
2066{
2067 return m_Graph->AddLayer<ConvertFp32ToFp16Layer>(name);
2068}
2069
Matthew Sloyanb63a3112021-09-08 13:05:51 +01002070IConnectableLayer* NetworkImpl::AddConvolution3dLayer(const Convolution3dDescriptor& convolution3dDescriptor,
Matthew Sloyanb63a3112021-09-08 13:05:51 +01002071 const char* name)
2072{
Matthew Sloyan5d7b0a32021-10-18 13:07:49 +01002073 return m_Graph->AddLayer<Convolution3dLayer>(convolution3dDescriptor, name);
Matthew Sloyanb63a3112021-09-08 13:05:51 +01002074}
2075
2076IConnectableLayer* NetworkImpl::AddDepthToSpaceLayer(const DepthToSpaceDescriptor& depthToSpaceDescriptor,
2077 const char* name)
2078{
2079 return m_Graph->AddLayer<DepthToSpaceLayer>(depthToSpaceDescriptor, name);
2080}
2081
Cathal Corbett06902652022-04-14 17:55:11 +01002082IConnectableLayer* NetworkImpl::AddDepthwiseConvolution2dLayer(
2083 const DepthwiseConvolution2dDescriptor& convolution2dDescriptor,
2084 const char* name)
telsoa014fcda012018-03-09 14:13:49 +00002085{
Cathal Corbett06902652022-04-14 17:55:11 +01002086 return m_Graph->AddLayer<DepthwiseConvolution2dLayer>(convolution2dDescriptor, name);
telsoa014fcda012018-03-09 14:13:49 +00002087}
2088
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002089IConnectableLayer* NetworkImpl::AddDepthwiseConvolution2dLayer(
Cathal Corbett06902652022-04-14 17:55:11 +01002090 const DepthwiseConvolution2dDescriptor& convolution2dDescriptor,
2091 const ConstTensor& weights,
2092 const Optional<ConstTensor>& biases,
2093 const char* name)
Aron Virginas-Tarad402702019-02-22 17:03:44 +00002094{
Cathal Corbett06902652022-04-14 17:55:11 +01002095 auto layer = m_Graph->AddLayer<DepthwiseConvolution2dLayer>(convolution2dDescriptor, name);
2096
2097 // Add a constant layer for weights
2098 ConstantLayer* weightsLayer = m_Graph->AddLayer<ConstantLayer>("Weights");
2099 weightsLayer->m_LayerOutput = std::make_shared<ScopedTensorHandle>(weights);
2100 layer->m_Weight = std::make_shared<ScopedTensorHandle>(weights);
2101
2102 weightsLayer->GetOutputSlot(0).SetTensorInfo(weightsLayer->m_LayerOutput->GetTensorInfo());
2103 weightsLayer->GetOutputSlot(0).Connect(layer->GetInputSlot(1));
2104
2105 // Add a constant layer for biases
2106 if (biases.has_value() && convolution2dDescriptor.m_BiasEnabled)
2107 {
2108 ConstantLayer* biasLayer = m_Graph->AddLayer<ConstantLayer>("Bias");
2109 biasLayer->m_LayerOutput = std::make_shared<ScopedTensorHandle>(biases.value());
2110 layer->m_Bias = std::make_shared<ScopedTensorHandle>(biases.value());
2111
2112 biasLayer->GetOutputSlot(0).SetTensorInfo(biasLayer->m_LayerOutput->GetTensorInfo());
2113 biasLayer->GetOutputSlot(0).Connect(layer->GetInputSlot(2));
2114 }
2115
2116 return layer;
Aron Virginas-Tarad402702019-02-22 17:03:44 +00002117}
2118
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002119IConnectableLayer* NetworkImpl::AddDetectionPostProcessLayer(const armnn::DetectionPostProcessDescriptor& descriptor,
Narumol Prangnawarat6d302bf2019-02-04 11:46:26 +00002120 const ConstTensor& anchors, const char* name)
Narumol Prangnawarat94dd5d82019-01-23 18:06:26 +00002121{
Narumol Prangnawarat6d302bf2019-02-04 11:46:26 +00002122 const auto layer = m_Graph->AddLayer<DetectionPostProcessLayer>(descriptor, name);
2123
James Conroy1f58f032021-04-27 17:13:27 +01002124 layer->m_Anchors = std::make_shared<ScopedTensorHandle>(anchors);
Narumol Prangnawarat6d302bf2019-02-04 11:46:26 +00002125
2126 return layer;
Narumol Prangnawarat94dd5d82019-01-23 18:06:26 +00002127}
2128
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002129IConnectableLayer* NetworkImpl::AddPermuteLayer(const PermuteDescriptor& permuteDescriptor,
telsoa014fcda012018-03-09 14:13:49 +00002130 const char* name)
2131{
2132 return m_Graph->AddLayer<PermuteLayer>(permuteDescriptor, name);
2133}
2134
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002135IConnectableLayer* NetworkImpl::AddPooling2dLayer(const Pooling2dDescriptor& pooling2dDescriptor,
telsoa014fcda012018-03-09 14:13:49 +00002136 const char* name)
2137{
2138 return m_Graph->AddLayer<Pooling2dLayer>(pooling2dDescriptor, name);
2139}
2140
Tamás Nyíri7b885b32021-10-26 14:47:57 +01002141IConnectableLayer* NetworkImpl::AddPooling3dLayer(const Pooling3dDescriptor& pooling3dDescriptor,
2142 const char* name)
2143{
2144 return m_Graph->AddLayer<Pooling3dLayer>(pooling3dDescriptor, name);
2145}
2146
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002147IConnectableLayer* NetworkImpl::AddActivationLayer(const ActivationDescriptor& activationDescriptor,
telsoa014fcda012018-03-09 14:13:49 +00002148 const char* name)
2149{
2150 return m_Graph->AddLayer<ActivationLayer>(activationDescriptor, name);
2151}
2152
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002153IConnectableLayer* NetworkImpl::AddArgMinMaxLayer(const ArgMinMaxDescriptor& argMinMaxDescriptor,
Nikhil Rajee391d52019-09-05 17:50:44 +01002154 const char* name)
2155{
2156 return m_Graph->AddLayer<ArgMinMaxLayer>(argMinMaxDescriptor, name);
2157}
2158
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002159IConnectableLayer* NetworkImpl::AddNormalizationLayer(const NormalizationDescriptor&
telsoa01c577f2c2018-08-31 09:22:23 +01002160normalizationDescriptor,
telsoa014fcda012018-03-09 14:13:49 +00002161 const char* name)
2162{
2163 return m_Graph->AddLayer<NormalizationLayer>(normalizationDescriptor, name);
2164}
2165
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002166IConnectableLayer* NetworkImpl::AddSliceLayer(const SliceDescriptor& sliceDescriptor, const char* name)
Aron Virginas-Tar636ab402019-09-16 14:27:45 +01002167{
2168 return m_Graph->AddLayer<SliceLayer>(sliceDescriptor, name);
2169}
2170
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002171IConnectableLayer* NetworkImpl::AddSoftmaxLayer(const SoftmaxDescriptor& softmaxDescriptor,
telsoa014fcda012018-03-09 14:13:49 +00002172 const char* name)
2173{
2174 return m_Graph->AddLayer<SoftmaxLayer>(softmaxDescriptor, name);
2175}
2176
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002177IConnectableLayer* NetworkImpl::AddSplitterLayer(const ViewsDescriptor& splitterDescriptor,
telsoa014fcda012018-03-09 14:13:49 +00002178 const char* name)
2179{
2180 return m_Graph->AddLayer<SplitterLayer>(splitterDescriptor, name);
2181}
2182
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002183IConnectableLayer* NetworkImpl::AddMaximumLayer(const char* name)
Nattapat Chaimanowong5a4304a2018-11-28 10:44:37 +00002184{
2185 return m_Graph->AddLayer<MaximumLayer>(name);
2186}
2187
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002188IConnectableLayer* NetworkImpl::AddMinimumLayer(const char* name)
Éanna Ó Catháin20e58802018-12-04 10:29:06 +00002189{
2190 return m_Graph->AddLayer<MinimumLayer>(name);
2191}
2192
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002193IConnectableLayer* NetworkImpl::AddAdditionLayer(const char* name)
telsoa014fcda012018-03-09 14:13:49 +00002194{
2195 return m_Graph->AddLayer<AdditionLayer>(name);
2196}
2197
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002198IConnectableLayer* NetworkImpl::AddMultiplicationLayer(const char* name)
telsoa014fcda012018-03-09 14:13:49 +00002199{
2200 return m_Graph->AddLayer<MultiplicationLayer>(name);
2201}
2202
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002203IConnectableLayer* NetworkImpl::AddOutputLayer(LayerBindingId id, const char* name)
telsoa014fcda012018-03-09 14:13:49 +00002204{
2205 return m_Graph->AddLayer<OutputLayer>(id, name);
2206}
2207
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002208IConnectableLayer* NetworkImpl::AddBatchNormalizationLayer(const BatchNormalizationDescriptor& desc,
telsoa014fcda012018-03-09 14:13:49 +00002209 const ConstTensor& mean,
2210 const ConstTensor& variance,
2211 const ConstTensor& beta,
2212 const ConstTensor& gamma,
2213 const char* name)
2214{
2215 const auto layer = m_Graph->AddLayer<BatchNormalizationLayer>(desc, name);
2216
James Conroy1f58f032021-04-27 17:13:27 +01002217 layer->m_Mean = std::make_shared<ScopedTensorHandle>(mean);
2218 layer->m_Variance = std::make_shared<ScopedTensorHandle>(variance);
2219 layer->m_Beta = std::make_shared<ScopedTensorHandle>(beta);
2220 layer->m_Gamma = std::make_shared<ScopedTensorHandle>(gamma);
telsoa014fcda012018-03-09 14:13:49 +00002221
2222 return layer;
2223}
2224
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002225IConnectableLayer* NetworkImpl::AddRankLayer(const char* name)
Finn Williams2605b232020-06-10 15:53:46 +01002226{
2227 return m_Graph->AddLayer<RankLayer>(name);
2228}
2229
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002230IConnectableLayer* NetworkImpl::AddReduceLayer(const ReduceDescriptor& reduceDescriptor,
2231 const char* name)
Sadik Armagan0c3ea5b2021-02-03 09:29:30 +00002232{
2233 return m_Graph->AddLayer<ReduceLayer>(reduceDescriptor, name);
2234}
2235
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002236IConnectableLayer* NetworkImpl::AddResizeLayer(const ResizeDescriptor& resizeDescriptor, const char* name)
Teresa Charlina9075df2019-06-27 15:41:57 +01002237{
Aron Virginas-Tar169d2f12019-07-01 19:01:44 +01002238 return m_Graph->AddLayer<ResizeLayer>(resizeDescriptor, name);
Teresa Charlina9075df2019-06-27 15:41:57 +01002239}
2240
Keith Davis3ae3f972021-05-21 16:33:48 +01002241IConnectableLayer* NetworkImpl::AddShapeLayer(const char* name)
2242{
2243 return m_Graph->AddLayer<ShapeLayer>(name);
2244}
2245
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002246IConnectableLayer* NetworkImpl::AddInstanceNormalizationLayer(const InstanceNormalizationDescriptor& desc,
2247 const char* name)
Kevin Mayce5045a2019-10-02 14:07:47 +01002248{
2249 return m_Graph->AddLayer<InstanceNormalizationLayer>(desc, name);
2250}
2251
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002252IConnectableLayer* NetworkImpl::AddL2NormalizationLayer(const L2NormalizationDescriptor& desc,
2253 const char* name)
telsoa014fcda012018-03-09 14:13:49 +00002254{
Matteo Martincighbcd3c852018-09-28 14:14:12 +01002255 return m_Graph->AddLayer<L2NormalizationLayer>(desc, name);
telsoa014fcda012018-03-09 14:13:49 +00002256}
2257
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002258IConnectableLayer* NetworkImpl::AddLogSoftmaxLayer(const LogSoftmaxDescriptor& desc,
Aron Virginas-Tarf982dea2019-10-11 14:07:53 +01002259 const char* name)
2260{
2261 return m_Graph->AddLayer<LogSoftmaxLayer>(desc, name);
2262}
2263
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002264IConnectableLayer* NetworkImpl::AddConstantLayer(const ConstTensor& input, const char* name)
telsoa014fcda012018-03-09 14:13:49 +00002265{
telsoa01c577f2c2018-08-31 09:22:23 +01002266 auto layer = m_Graph->AddLayer<ConstantLayer>(name);
2267
James Conroy1f58f032021-04-27 17:13:27 +01002268 layer->m_LayerOutput = std::make_shared<ScopedTensorHandle>(input);
telsoa01c577f2c2018-08-31 09:22:23 +01002269
2270 return layer;
telsoa014fcda012018-03-09 14:13:49 +00002271}
2272
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002273IConnectableLayer* NetworkImpl::AddReshapeLayer(const ReshapeDescriptor& reshapeDescriptor,
telsoa01c577f2c2018-08-31 09:22:23 +01002274 const char* name)
telsoa014fcda012018-03-09 14:13:49 +00002275{
2276 return m_Graph->AddLayer<ReshapeLayer>(reshapeDescriptor, name);
2277}
2278
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002279IConnectableLayer* NetworkImpl::AddSpaceToBatchNdLayer(const SpaceToBatchNdDescriptor& spaceToBatchNdDescriptor,
Nattapat Chaimanowong207ef9a2018-11-02 10:57:25 +00002280 const char* name)
2281{
2282 return m_Graph->AddLayer<SpaceToBatchNdLayer>(spaceToBatchNdDescriptor, name);
2283}
2284
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002285IConnectableLayer* NetworkImpl::AddSpaceToDepthLayer(const SpaceToDepthDescriptor& spaceToDepthDescriptor,
Aron Virginas-Tar972af152019-06-11 14:14:03 +01002286 const char* name)
2287{
2288 return m_Graph->AddLayer<SpaceToDepthLayer>(spaceToDepthDescriptor, name);
2289}
2290
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002291IConnectableLayer* NetworkImpl::AddFloorLayer(const char* name)
telsoa014fcda012018-03-09 14:13:49 +00002292{
2293 return m_Graph->AddLayer<FloorLayer>(name);
2294}
2295
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002296IConnectableLayer* NetworkImpl::AddLstmLayer(const LstmDescriptor& descriptor,
telsoa01c577f2c2018-08-31 09:22:23 +01002297 const LstmInputParams& params,
2298 const char* name)
2299{
2300 const auto layer = m_Graph->AddLayer<LstmLayer>(descriptor, name);
2301
2302 //Lstm Basic Parameters
2303 layer->m_BasicParameters.m_InputToForgetWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002304 std::make_shared<ScopedTensorHandle>(*(params.m_InputToForgetWeights));
telsoa01c577f2c2018-08-31 09:22:23 +01002305 layer->m_BasicParameters.m_InputToCellWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002306 std::make_shared<ScopedTensorHandle>(*(params.m_InputToCellWeights));
telsoa01c577f2c2018-08-31 09:22:23 +01002307 layer->m_BasicParameters.m_InputToOutputWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002308 std::make_shared<ScopedTensorHandle>(*(params.m_InputToOutputWeights));
telsoa01c577f2c2018-08-31 09:22:23 +01002309 layer->m_BasicParameters.m_RecurrentToForgetWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002310 std::make_shared<ScopedTensorHandle>(*(params.m_RecurrentToForgetWeights));
telsoa01c577f2c2018-08-31 09:22:23 +01002311 layer->m_BasicParameters.m_RecurrentToCellWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002312 std::make_shared<ScopedTensorHandle>(*(params.m_RecurrentToCellWeights));
telsoa01c577f2c2018-08-31 09:22:23 +01002313 layer->m_BasicParameters.m_RecurrentToOutputWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002314 std::make_shared<ScopedTensorHandle>(*(params.m_RecurrentToOutputWeights));
telsoa01c577f2c2018-08-31 09:22:23 +01002315 layer->m_BasicParameters.m_ForgetGateBias =
James Conroy1f58f032021-04-27 17:13:27 +01002316 std::make_shared<ScopedTensorHandle>(*(params.m_ForgetGateBias));
telsoa01c577f2c2018-08-31 09:22:23 +01002317 layer->m_BasicParameters.m_CellBias =
James Conroy1f58f032021-04-27 17:13:27 +01002318 std::make_shared<ScopedTensorHandle>(*(params.m_CellBias));
telsoa01c577f2c2018-08-31 09:22:23 +01002319 layer->m_BasicParameters.m_OutputGateBias =
James Conroy1f58f032021-04-27 17:13:27 +01002320 std::make_shared<ScopedTensorHandle>(*(params.m_OutputGateBias));
telsoa01c577f2c2018-08-31 09:22:23 +01002321
2322 //Lstm Cifg parameters
2323 if(!descriptor.m_CifgEnabled)
2324 {
2325 if(params.m_InputToInputWeights == nullptr)
2326 {
Jan Eilerse2062cd2020-03-30 15:07:45 +01002327 throw InvalidArgumentException("AddLstmLayer: Input To Input Weights cannot be NULL "
2328 "when CIFG is disabled.");
telsoa01c577f2c2018-08-31 09:22:23 +01002329 }
2330 if(params.m_RecurrentToInputWeights == nullptr)
2331 {
2332 throw InvalidArgumentException(
Jan Eilerse2062cd2020-03-30 15:07:45 +01002333 "AddLstmLayer: Recurrent To Input Weights cannot be NULL "
2334 "when CIFG is disabled.");
telsoa01c577f2c2018-08-31 09:22:23 +01002335 }
2336 if(params.m_InputGateBias == nullptr)
2337 {
Jan Eilerse2062cd2020-03-30 15:07:45 +01002338 throw InvalidArgumentException("AddLstmLayer: Input Gate Bias cannot be NULL "
2339 "when CIFG is disabled.");
telsoa01c577f2c2018-08-31 09:22:23 +01002340 }
2341 layer->m_CifgParameters.m_InputToInputWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002342 std::make_shared<ScopedTensorHandle>(*(params.m_InputToInputWeights));
telsoa01c577f2c2018-08-31 09:22:23 +01002343 layer->m_CifgParameters.m_RecurrentToInputWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002344 std::make_shared<ScopedTensorHandle>(*(params.m_RecurrentToInputWeights));
telsoa01c577f2c2018-08-31 09:22:23 +01002345 layer->m_CifgParameters.m_InputGateBias =
James Conroy1f58f032021-04-27 17:13:27 +01002346 std::make_shared<ScopedTensorHandle>(*(params.m_InputGateBias));
telsoa01c577f2c2018-08-31 09:22:23 +01002347 }
2348
2349 //Lstm projection parameters
2350 if(descriptor.m_ProjectionEnabled)
2351 {
2352 if(params.m_ProjectionWeights == nullptr)
2353 {
Jan Eilerse2062cd2020-03-30 15:07:45 +01002354 throw InvalidArgumentException("AddLstmLayer: Projection Weights cannot be NULL "
2355 "when projection is enabled.");
telsoa01c577f2c2018-08-31 09:22:23 +01002356 }
2357 layer->m_ProjectionParameters.m_ProjectionWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002358 std::make_shared<ScopedTensorHandle>(*(params.m_ProjectionWeights));
telsoa01c577f2c2018-08-31 09:22:23 +01002359 if(params.m_ProjectionBias != nullptr)
2360 {
2361 layer->m_ProjectionParameters.m_ProjectionBias =
James Conroy1f58f032021-04-27 17:13:27 +01002362 std::make_shared<ScopedTensorHandle>(*(params.m_ProjectionBias));
telsoa01c577f2c2018-08-31 09:22:23 +01002363 }
2364 }
2365
2366 //Lstm Peephole params
2367 if(descriptor.m_PeepholeEnabled)
2368 {
Jan Eilerse2062cd2020-03-30 15:07:45 +01002369 if(!descriptor.m_CifgEnabled)
2370 {
2371 if(params.m_CellToInputWeights == nullptr)
2372 {
2373 throw InvalidArgumentException("AddLstmLayer: Cell To Input Weights cannot be NULL "
2374 "when Peephole is enabled and CIFG disabled.");
2375 }
2376
2377 layer->m_PeepholeParameters.m_CellToInputWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002378 std::make_shared<ScopedTensorHandle>(*(params.m_CellToInputWeights));
Jan Eilerse2062cd2020-03-30 15:07:45 +01002379 }
2380
telsoa01c577f2c2018-08-31 09:22:23 +01002381 if(params.m_CellToForgetWeights == nullptr)
2382 {
Jan Eilerse2062cd2020-03-30 15:07:45 +01002383 throw InvalidArgumentException("AddLstmLayer: Cell To Forget Weights cannot be NULL "
2384 "when Peephole is enabled.");
telsoa01c577f2c2018-08-31 09:22:23 +01002385 }
2386 if(params.m_CellToOutputWeights == nullptr)
2387 {
Jan Eilerse2062cd2020-03-30 15:07:45 +01002388 throw InvalidArgumentException("AddLstmLayer: Cell To Output Weights cannot be NULL "
2389 "when Peephole is enabled.");
telsoa01c577f2c2018-08-31 09:22:23 +01002390 }
Jan Eilerse2062cd2020-03-30 15:07:45 +01002391
telsoa01c577f2c2018-08-31 09:22:23 +01002392 layer->m_PeepholeParameters.m_CellToForgetWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002393 std::make_shared<ScopedTensorHandle>(*(params.m_CellToForgetWeights));
telsoa01c577f2c2018-08-31 09:22:23 +01002394 layer->m_PeepholeParameters.m_CellToOutputWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002395 std::make_shared<ScopedTensorHandle>(*(params.m_CellToOutputWeights));
telsoa01c577f2c2018-08-31 09:22:23 +01002396 }
Jan Eilersf8c62972019-07-17 11:07:49 +01002397
2398 //Lstm Layer Normalization params
2399 if(descriptor.m_LayerNormEnabled)
2400 {
2401 if(!descriptor.m_CifgEnabled)
2402 {
2403 if(params.m_InputLayerNormWeights == nullptr)
2404 {
Jan Eilerse2062cd2020-03-30 15:07:45 +01002405 throw InvalidArgumentException("AddLstmLayer: Input layer normalization weights cannot be NULL "
2406 "when layer normalization is enabled and CIFG disabled.");
Jan Eilersf8c62972019-07-17 11:07:49 +01002407 }
2408 layer->m_LayerNormParameters.m_InputLayerNormWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002409 std::make_shared<ScopedTensorHandle>(*(params.m_InputLayerNormWeights));
Jan Eilersf8c62972019-07-17 11:07:49 +01002410 }
2411
2412 if(params.m_ForgetLayerNormWeights == nullptr)
2413 {
Jan Eilerse2062cd2020-03-30 15:07:45 +01002414 throw InvalidArgumentException("AddLstmLayer: Forget layer normalization weights cannot be NULL "
2415 "when layer normalization is enabled.");
Jan Eilersf8c62972019-07-17 11:07:49 +01002416 }
2417 if(params.m_CellLayerNormWeights == nullptr)
2418 {
Jan Eilerse2062cd2020-03-30 15:07:45 +01002419 throw InvalidArgumentException("AddLstmLayer: Cell layer normalization weights cannot be NULL "
2420 "when layer normalization is enabled.");
Jan Eilersf8c62972019-07-17 11:07:49 +01002421 }
2422 if(params.m_OutputLayerNormWeights == nullptr)
2423 {
Jan Eilerse2062cd2020-03-30 15:07:45 +01002424 throw InvalidArgumentException("AddLstmLayer: Output layer normalization weights cannot be NULL "
2425 "when layer normalization is enabled.");
Jan Eilersf8c62972019-07-17 11:07:49 +01002426 }
2427 layer->m_LayerNormParameters.m_ForgetLayerNormWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002428 std::make_shared<ScopedTensorHandle>(*(params.m_ForgetLayerNormWeights));
Jan Eilersf8c62972019-07-17 11:07:49 +01002429 layer->m_LayerNormParameters.m_CellLayerNormWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002430 std::make_shared<ScopedTensorHandle>(*(params.m_CellLayerNormWeights));
Jan Eilersf8c62972019-07-17 11:07:49 +01002431 layer->m_LayerNormParameters.m_OutputLayerNormWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002432 std::make_shared<ScopedTensorHandle>(*(params.m_OutputLayerNormWeights));
Jan Eilersf8c62972019-07-17 11:07:49 +01002433 }
telsoa01c577f2c2018-08-31 09:22:23 +01002434 return layer;
2435}
2436
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002437IConnectableLayer* NetworkImpl::AddDivisionLayer(const char* name)
Francis Murtaghe7a86a42018-08-29 12:42:10 +01002438{
2439 return m_Graph->AddLayer<DivisionLayer>(name);
2440}
2441
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002442IConnectableLayer* NetworkImpl::AddSubtractionLayer(const char* name)
David Beck19526222018-09-12 16:00:08 +01002443{
2444 return m_Graph->AddLayer<SubtractionLayer>(name);
2445}
2446
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002447IConnectableLayer* NetworkImpl::AddMeanLayer(const MeanDescriptor& meanDescriptor, const char* name)
narpra0132b90462018-09-13 11:07:48 +01002448{
2449 return m_Graph->AddLayer<MeanLayer>(meanDescriptor,name);
2450}
2451
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002452IConnectableLayer* NetworkImpl::AddPadLayer(const PadDescriptor& padDescriptor, const char* name)
Mohamed Nour Abouelseoud5662c202018-09-24 13:30:09 +01002453{
2454 return m_Graph->AddLayer<PadLayer>(padDescriptor,name);
2455}
2456
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002457IConnectableLayer *NetworkImpl::AddQuantizeLayer(const char *name)
Derek Lambertia9cca6a2019-03-25 15:41:58 +00002458{
2459 return m_Graph->AddLayer<QuantizeLayer>(name);
2460}
2461
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002462IConnectableLayer* NetworkImpl::AddDequantizeLayer(const char* name)
Nattapat Chaimanowonge4294fd2019-03-28 09:56:53 +00002463{
2464 return m_Graph->AddLayer<DequantizeLayer>(name);
2465}
2466
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002467IConnectableLayer* NetworkImpl::AddStridedSliceLayer(const StridedSliceDescriptor& stridedSliceDescriptor,
Teresa Charlinb2d3ec52022-04-12 22:07:09 +01002468 const char* name)
Conor Kennedy430b5d82018-11-14 15:28:28 +00002469{
2470 return m_Graph->AddLayer<StridedSliceLayer>(stridedSliceDescriptor, name);
2471}
2472
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002473IConnectableLayer* NetworkImpl::AddGatherLayer(const GatherDescriptor& gatherDescriptor,
Teresa Charlinb2d3ec52022-04-12 22:07:09 +01002474 const char* name)
Teresa Charlin52664732020-06-29 16:27:03 +01002475{
2476 return m_Graph->AddLayer<GatherLayer>(gatherDescriptor, name);
narpra01b89b05f2019-01-16 09:53:09 +00002477}
2478
Teresa Charlinb2d3ec52022-04-12 22:07:09 +01002479IConnectableLayer* NetworkImpl::AddGatherNdLayer(const char* name)
2480{
2481 return m_Graph->AddLayer<GatherNdLayer>(name);
2482}
2483
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002484IConnectableLayer* NetworkImpl::AddMergeLayer(const char* name)
Nattapat Chaimanowong1f886302019-04-05 13:37:19 +01002485{
2486 return m_Graph->AddLayer<MergeLayer>(name);
2487}
2488
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002489IConnectableLayer* NetworkImpl::AddSwitchLayer(const char* name)
Sadik Armaganeff363d2019-04-05 15:25:46 +01002490{
2491 return m_Graph->AddLayer<SwitchLayer>(name);
2492}
2493
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002494IConnectableLayer* NetworkImpl::AddPreluLayer(const char* name)
Matteo Martincigh0e406ee2019-06-12 15:42:18 +01002495{
2496 return m_Graph->AddLayer<PreluLayer>(name);
2497}
2498
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002499IConnectableLayer* NetworkImpl::AddTransposeConvolution2dLayer(const TransposeConvolution2dDescriptor& descriptor,
Aron Virginas-Tar639fb042019-06-20 14:28:19 +01002500 const ConstTensor& weights,
2501 const Optional<ConstTensor>& biases,
2502 const char* name)
2503{
2504 if (descriptor.m_BiasEnabled && !biases.has_value())
2505 {
2506 throw InvalidArgumentException("AddTransposeConvolution2dLayer: Biases cannot be empty");
2507 }
2508
2509 const auto layer = m_Graph->AddLayer<TransposeConvolution2dLayer>(descriptor, name);
2510
James Conroy1f58f032021-04-27 17:13:27 +01002511 layer->m_Weight = std::make_shared<ScopedTensorHandle>(weights);
Aron Virginas-Tar639fb042019-06-20 14:28:19 +01002512
2513 if (descriptor.m_BiasEnabled)
2514 {
James Conroy1f58f032021-04-27 17:13:27 +01002515 layer->m_Bias = std::make_shared<ScopedTensorHandle>(biases.value());
Aron Virginas-Tar639fb042019-06-20 14:28:19 +01002516 }
2517
2518 return layer;
2519}
2520
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002521IConnectableLayer* NetworkImpl::AddTransposeLayer(const TransposeDescriptor& transposeDescriptor,
Mike Kellyc9ea45a2020-02-28 18:11:58 +00002522 const char* name)
2523{
2524 return m_Graph->AddLayer<TransposeLayer>(transposeDescriptor, name);
2525}
2526
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002527IConnectableLayer* NetworkImpl::AddStackLayer(const StackDescriptor& stackDescriptor,
Matthew Jackson2b8c1da2019-07-04 14:59:16 +01002528 const char* name)
2529{
2530 return m_Graph->AddLayer<StackLayer>(stackDescriptor, name);
2531}
2532
Derek Lamberti013c3902019-10-21 10:46:16 +01002533
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002534IConnectableLayer* NetworkImpl::AddStandInLayer(const StandInDescriptor& desc,
Derek Lamberti013c3902019-10-21 10:46:16 +01002535 const char* name)
2536{
2537 return m_Graph->AddLayer<StandInLayer>(desc, name);
2538}
2539
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002540IConnectableLayer* NetworkImpl::AddQuantizedLstmLayer(const QuantizedLstmInputParams& params,
James Conroyee18dc82019-07-17 11:27:46 +01002541 const char* name)
2542{
2543 const auto layer = m_Graph->AddLayer<QuantizedLstmLayer>(name);
2544
2545 // InputToX weights
2546 layer->m_QuantizedLstmParameters.m_InputToInputWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002547 std::make_shared<ScopedTensorHandle>(params.GetInputToInputWeights());
James Conroyee18dc82019-07-17 11:27:46 +01002548 layer->m_QuantizedLstmParameters.m_InputToForgetWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002549 std::make_shared<ScopedTensorHandle>(params.GetInputToForgetWeights());
James Conroyee18dc82019-07-17 11:27:46 +01002550 layer->m_QuantizedLstmParameters.m_InputToCellWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002551 std::make_shared<ScopedTensorHandle>(params.GetInputToCellWeights());
James Conroyee18dc82019-07-17 11:27:46 +01002552 layer->m_QuantizedLstmParameters.m_InputToOutputWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002553 std::make_shared<ScopedTensorHandle>(params.GetInputToOutputWeights());
James Conroyee18dc82019-07-17 11:27:46 +01002554
2555 // RecurrentToX weights
2556 layer->m_QuantizedLstmParameters.m_RecurrentToInputWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002557 std::make_shared<ScopedTensorHandle>(params.GetRecurrentToInputWeights());
James Conroyee18dc82019-07-17 11:27:46 +01002558 layer->m_QuantizedLstmParameters.m_RecurrentToForgetWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002559 std::make_shared<ScopedTensorHandle>(params.GetRecurrentToForgetWeights());
James Conroyee18dc82019-07-17 11:27:46 +01002560 layer->m_QuantizedLstmParameters.m_RecurrentToCellWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002561 std::make_shared<ScopedTensorHandle>(params.GetRecurrentToCellWeights());
James Conroyee18dc82019-07-17 11:27:46 +01002562 layer->m_QuantizedLstmParameters.m_RecurrentToOutputWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002563 std::make_shared<ScopedTensorHandle>(params.GetRecurrentToOutputWeights());
James Conroyee18dc82019-07-17 11:27:46 +01002564
2565 // Bias
2566 layer->m_QuantizedLstmParameters.m_InputGateBias =
James Conroy1f58f032021-04-27 17:13:27 +01002567 std::make_shared<ScopedTensorHandle>(params.GetInputGateBias());
James Conroyee18dc82019-07-17 11:27:46 +01002568 layer->m_QuantizedLstmParameters.m_ForgetGateBias =
James Conroy1f58f032021-04-27 17:13:27 +01002569 std::make_shared<ScopedTensorHandle>(params.GetForgetGateBias());
James Conroyee18dc82019-07-17 11:27:46 +01002570 layer->m_QuantizedLstmParameters.m_CellBias =
James Conroy1f58f032021-04-27 17:13:27 +01002571 std::make_shared<ScopedTensorHandle>(params.GetCellBias());
James Conroyee18dc82019-07-17 11:27:46 +01002572 layer->m_QuantizedLstmParameters.m_OutputGateBias =
James Conroy1f58f032021-04-27 17:13:27 +01002573 std::make_shared<ScopedTensorHandle>(params.GetOutputGateBias());
James Conroyee18dc82019-07-17 11:27:46 +01002574
2575 return layer;
2576}
2577
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002578IConnectableLayer* NetworkImpl::AddQLstmLayer(const QLstmDescriptor& descriptor,
James Conroy586a9aa2020-03-20 08:49:33 +00002579 const LstmInputParams& params,
2580 const char* name)
2581{
2582 const auto layer = m_Graph->AddLayer<QLstmLayer>(descriptor, name);
2583
2584 // QLstm Basic Parameters
2585 layer->m_BasicParameters.m_InputToForgetWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002586 std::make_shared<ScopedTensorHandle>(*(params.m_InputToForgetWeights));
James Conroy586a9aa2020-03-20 08:49:33 +00002587 layer->m_BasicParameters.m_InputToCellWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002588 std::make_shared<ScopedTensorHandle>(*(params.m_InputToCellWeights));
James Conroy586a9aa2020-03-20 08:49:33 +00002589 layer->m_BasicParameters.m_InputToOutputWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002590 std::make_shared<ScopedTensorHandle>(*(params.m_InputToOutputWeights));
James Conroy586a9aa2020-03-20 08:49:33 +00002591 layer->m_BasicParameters.m_RecurrentToForgetWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002592 std::make_shared<ScopedTensorHandle>(*(params.m_RecurrentToForgetWeights));
James Conroy586a9aa2020-03-20 08:49:33 +00002593 layer->m_BasicParameters.m_RecurrentToCellWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002594 std::make_shared<ScopedTensorHandle>(*(params.m_RecurrentToCellWeights));
James Conroy586a9aa2020-03-20 08:49:33 +00002595 layer->m_BasicParameters.m_RecurrentToOutputWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002596 std::make_shared<ScopedTensorHandle>(*(params.m_RecurrentToOutputWeights));
James Conroy586a9aa2020-03-20 08:49:33 +00002597 layer->m_BasicParameters.m_ForgetGateBias =
James Conroy1f58f032021-04-27 17:13:27 +01002598 std::make_shared<ScopedTensorHandle>(*(params.m_ForgetGateBias));
James Conroy586a9aa2020-03-20 08:49:33 +00002599 layer->m_BasicParameters.m_CellBias =
James Conroy1f58f032021-04-27 17:13:27 +01002600 std::make_shared<ScopedTensorHandle>(*(params.m_CellBias));
James Conroy586a9aa2020-03-20 08:49:33 +00002601 layer->m_BasicParameters.m_OutputGateBias =
James Conroy1f58f032021-04-27 17:13:27 +01002602 std::make_shared<ScopedTensorHandle>(*(params.m_OutputGateBias));
James Conroy586a9aa2020-03-20 08:49:33 +00002603
2604 // QLstm Cifg parameters
2605 if(!descriptor.m_CifgEnabled)
2606 {
2607 if(params.m_InputToInputWeights == nullptr)
2608 {
2609 throw InvalidArgumentException("AddQLstmLayer: Input To Input Weights cannot be NULL");
2610 }
2611
2612 if(params.m_RecurrentToInputWeights == nullptr)
2613 {
2614 throw InvalidArgumentException(
2615 "AddQLstmLayer: Recurrent To Input Weights cannot be NULL");
2616 }
2617
2618 if(params.m_InputGateBias == nullptr)
2619 {
2620 throw InvalidArgumentException("AddQLstmLayer: Input Gate Bias cannot be NULL");
2621 }
2622
2623 layer->m_CifgParameters.m_InputToInputWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002624 std::make_shared<ScopedTensorHandle>(*(params.m_InputToInputWeights));
James Conroy586a9aa2020-03-20 08:49:33 +00002625 layer->m_CifgParameters.m_RecurrentToInputWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002626 std::make_shared<ScopedTensorHandle>(*(params.m_RecurrentToInputWeights));
James Conroy586a9aa2020-03-20 08:49:33 +00002627 layer->m_CifgParameters.m_InputGateBias =
James Conroy1f58f032021-04-27 17:13:27 +01002628 std::make_shared<ScopedTensorHandle>(*(params.m_InputGateBias));
James Conroy586a9aa2020-03-20 08:49:33 +00002629 }
2630
2631 // QLstm Projection parameters
2632 if(descriptor.m_ProjectionEnabled)
2633 {
2634 if(params.m_ProjectionWeights == nullptr)
2635 {
2636 throw InvalidArgumentException("AddQLstmLayer: Projection Weights cannot be NULL");
2637 }
2638
James Conroy586a9aa2020-03-20 08:49:33 +00002639 layer->m_ProjectionParameters.m_ProjectionWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002640 std::make_shared<ScopedTensorHandle>(*(params.m_ProjectionWeights));
James Conroyed324052020-05-18 15:16:42 +01002641
2642 // Projection bias is optional even if projection is enabled
Cathal Corbett727c2b52022-05-06 12:11:37 +01002643 if(params.m_ProjectionBias != nullptr)
James Conroyed324052020-05-18 15:16:42 +01002644 {
2645 layer->m_ProjectionParameters.m_ProjectionBias =
James Conroy1f58f032021-04-27 17:13:27 +01002646 std::make_shared<ScopedTensorHandle>(*(params.m_ProjectionBias));
James Conroyed324052020-05-18 15:16:42 +01002647 }
2648
James Conroy586a9aa2020-03-20 08:49:33 +00002649 }
2650
2651 // QLstm Peephole params
2652 if(descriptor.m_PeepholeEnabled)
2653 {
2654 if(params.m_CellToForgetWeights == nullptr)
2655 {
2656 throw InvalidArgumentException("AddQLstmLayer: Cell To Forget Weights cannot be NULL");
2657 }
2658
2659 if(params.m_CellToOutputWeights == nullptr)
2660 {
2661 throw InvalidArgumentException("AddQLstmLayer: Cell To Output Weights cannot be NULL");
2662 }
2663
2664 if(!descriptor.m_CifgEnabled)
2665 {
2666 if(params.m_CellToInputWeights == nullptr)
2667 {
2668 throw InvalidArgumentException("AddQLstmLayer: Cell To Input Weights cannot be NULL");
2669 }
2670
2671 layer->m_PeepholeParameters.m_CellToInputWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002672 std::make_shared<ScopedTensorHandle>(*(params.m_CellToInputWeights));
James Conroy586a9aa2020-03-20 08:49:33 +00002673 }
2674
2675 layer->m_PeepholeParameters.m_CellToForgetWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002676 std::make_shared<ScopedTensorHandle>(*(params.m_CellToForgetWeights));
James Conroy586a9aa2020-03-20 08:49:33 +00002677 layer->m_PeepholeParameters.m_CellToOutputWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002678 std::make_shared<ScopedTensorHandle>(*(params.m_CellToOutputWeights));
James Conroy586a9aa2020-03-20 08:49:33 +00002679 }
2680
2681 // QLstm Layer Normalization params
2682 if(descriptor.m_LayerNormEnabled)
2683 {
2684 if(params.m_ForgetLayerNormWeights == nullptr)
2685 {
2686 throw InvalidArgumentException("AddQLstmLayer: Forget layer normalization weights cannot be NULL");
2687 }
2688
2689 if(params.m_CellLayerNormWeights == nullptr)
2690 {
2691 throw InvalidArgumentException("AddQLstmLayer: Cell layer normalization weights cannot be NULL");
2692 }
2693
2694 if(params.m_OutputLayerNormWeights == nullptr)
2695 {
2696 throw InvalidArgumentException("AddQLstmLayer: Output layer normalization weights cannot be NULL");
2697 }
2698
2699 if(!descriptor.m_CifgEnabled)
2700 {
2701 if(params.m_InputLayerNormWeights == nullptr)
2702 {
2703 throw InvalidArgumentException("AddQLstmLayer: Input layer normalization weights cannot be NULL");
2704 }
2705
2706 layer->m_LayerNormParameters.m_InputLayerNormWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002707 std::make_shared<ScopedTensorHandle>(*(params.m_InputLayerNormWeights));
James Conroy586a9aa2020-03-20 08:49:33 +00002708 }
2709
2710 layer->m_LayerNormParameters.m_ForgetLayerNormWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002711 std::make_shared<ScopedTensorHandle>(*(params.m_ForgetLayerNormWeights));
James Conroy586a9aa2020-03-20 08:49:33 +00002712 layer->m_LayerNormParameters.m_CellLayerNormWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002713 std::make_shared<ScopedTensorHandle>(*(params.m_CellLayerNormWeights));
James Conroy586a9aa2020-03-20 08:49:33 +00002714 layer->m_LayerNormParameters.m_OutputLayerNormWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002715 std::make_shared<ScopedTensorHandle>(*(params.m_OutputLayerNormWeights));
James Conroy586a9aa2020-03-20 08:49:33 +00002716 }
2717 return layer;
2718}
2719
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002720IConnectableLayer* NetworkImpl::AddLogicalBinaryLayer(const LogicalBinaryDescriptor& logicalBinaryDescriptor,
Narumol Prangnawarat8ed39ae2021-07-15 16:16:25 +01002721 const char* name)
James Conroyaba90cd2020-11-06 16:28:18 +00002722{
2723 return m_Graph->AddLayer<LogicalBinaryLayer>(logicalBinaryDescriptor, name);
2724}
2725
Narumol Prangnawarat8ed39ae2021-07-15 16:16:25 +01002726IConnectableLayer* NetworkImpl::AddUnidirectionalSequenceLstmLayer(
2727 const UnidirectionalSequenceLstmDescriptor& descriptor,
2728 const LstmInputParams& params,
2729 const char* name)
2730{
2731 const auto layer = m_Graph->AddLayer<UnidirectionalSequenceLstmLayer>(descriptor, name);
2732
2733 //Lstm Basic Parameters
2734 layer->m_BasicParameters.m_InputToForgetWeights =
2735 std::make_shared<ScopedTensorHandle>(*(params.m_InputToForgetWeights));
2736 layer->m_BasicParameters.m_InputToCellWeights =
2737 std::make_shared<ScopedTensorHandle>(*(params.m_InputToCellWeights));
2738 layer->m_BasicParameters.m_InputToOutputWeights =
2739 std::make_shared<ScopedTensorHandle>(*(params.m_InputToOutputWeights));
2740 layer->m_BasicParameters.m_RecurrentToForgetWeights =
2741 std::make_shared<ScopedTensorHandle>(*(params.m_RecurrentToForgetWeights));
2742 layer->m_BasicParameters.m_RecurrentToCellWeights =
2743 std::make_shared<ScopedTensorHandle>(*(params.m_RecurrentToCellWeights));
2744 layer->m_BasicParameters.m_RecurrentToOutputWeights =
2745 std::make_shared<ScopedTensorHandle>(*(params.m_RecurrentToOutputWeights));
2746 layer->m_BasicParameters.m_ForgetGateBias =
2747 std::make_shared<ScopedTensorHandle>(*(params.m_ForgetGateBias));
2748 layer->m_BasicParameters.m_CellBias =
2749 std::make_shared<ScopedTensorHandle>(*(params.m_CellBias));
2750 layer->m_BasicParameters.m_OutputGateBias =
2751 std::make_shared<ScopedTensorHandle>(*(params.m_OutputGateBias));
2752
2753 //Lstm Cifg parameters
2754 if(!descriptor.m_CifgEnabled)
2755 {
2756 if(params.m_InputToInputWeights == nullptr)
2757 {
2758 throw InvalidArgumentException("AddUnidirectionalSequenceLstmLayer: Input To Input Weights cannot be NULL "
2759 "when CIFG is disabled.");
2760 }
2761 if(params.m_RecurrentToInputWeights == nullptr)
2762 {
2763 throw InvalidArgumentException(
2764 "AddUnidirectionalSequenceLstmLayer: Recurrent To Input Weights cannot be NULL "
2765 "when CIFG is disabled.");
2766 }
2767 if(params.m_InputGateBias == nullptr)
2768 {
2769 throw InvalidArgumentException("AddUnidirectionalSequenceLstmLayer: Input Gate Bias cannot be NULL "
2770 "when CIFG is disabled.");
2771 }
2772 layer->m_CifgParameters.m_InputToInputWeights =
2773 std::make_shared<ScopedTensorHandle>(*(params.m_InputToInputWeights));
2774 layer->m_CifgParameters.m_RecurrentToInputWeights =
2775 std::make_shared<ScopedTensorHandle>(*(params.m_RecurrentToInputWeights));
2776 layer->m_CifgParameters.m_InputGateBias =
2777 std::make_shared<ScopedTensorHandle>(*(params.m_InputGateBias));
2778 }
2779
2780 //Lstm projection parameters
2781 if(descriptor.m_ProjectionEnabled)
2782 {
2783 if(params.m_ProjectionWeights == nullptr)
2784 {
2785 throw InvalidArgumentException("AddUnidirectionalSequenceLstmLayer: Projection Weights cannot be NULL "
2786 "when projection is enabled.");
2787 }
2788 layer->m_ProjectionParameters.m_ProjectionWeights =
2789 std::make_shared<ScopedTensorHandle>(*(params.m_ProjectionWeights));
2790 if(params.m_ProjectionBias != nullptr)
2791 {
2792 layer->m_ProjectionParameters.m_ProjectionBias =
2793 std::make_shared<ScopedTensorHandle>(*(params.m_ProjectionBias));
2794 }
2795 }
2796
2797 //Lstm Peephole params
2798 if(descriptor.m_PeepholeEnabled)
2799 {
2800 if(!descriptor.m_CifgEnabled)
2801 {
2802 if(params.m_CellToInputWeights == nullptr)
2803 {
2804 throw InvalidArgumentException("AddUnidirectionalSequenceLstmLayer: Cell To Input Weights "
2805 "cannot be NULL when Peephole is enabled and CIFG disabled.");
2806 }
2807
2808 layer->m_PeepholeParameters.m_CellToInputWeights =
2809 std::make_shared<ScopedTensorHandle>(*(params.m_CellToInputWeights));
2810 }
2811
2812 if(params.m_CellToForgetWeights == nullptr)
2813 {
2814 throw InvalidArgumentException("AddUnidirectionalSequenceLstmLayer: Cell To Forget Weights cannot be NULL "
2815 "when Peephole is enabled.");
2816 }
2817 if(params.m_CellToOutputWeights == nullptr)
2818 {
2819 throw InvalidArgumentException("AddUnidirectionalSequenceLstmLayer: Cell To Output Weights cannot be NULL "
2820 "when Peephole is enabled.");
2821 }
2822
2823 layer->m_PeepholeParameters.m_CellToForgetWeights =
2824 std::make_shared<ScopedTensorHandle>(*(params.m_CellToForgetWeights));
2825 layer->m_PeepholeParameters.m_CellToOutputWeights =
2826 std::make_shared<ScopedTensorHandle>(*(params.m_CellToOutputWeights));
2827 }
2828
2829 //Lstm Layer Normalization params
2830 if(descriptor.m_LayerNormEnabled)
2831 {
2832 if(!descriptor.m_CifgEnabled)
2833 {
2834 if(params.m_InputLayerNormWeights == nullptr)
2835 {
2836 throw InvalidArgumentException("AddUnidirectionalSequenceLstmLayer: Input layer normalization weights "
2837 "cannot be NULL when layer normalization is enabled and CIFG disabled.");
2838 }
2839 layer->m_LayerNormParameters.m_InputLayerNormWeights =
2840 std::make_shared<ScopedTensorHandle>(*(params.m_InputLayerNormWeights));
2841 }
2842
2843 if(params.m_ForgetLayerNormWeights == nullptr)
2844 {
2845 throw InvalidArgumentException("AddUnidirectionalSequenceLstmLayer: Forget layer normalization weights "
2846 "cannot be NULL when layer normalization is enabled.");
2847 }
2848 if(params.m_CellLayerNormWeights == nullptr)
2849 {
2850 throw InvalidArgumentException("AddUnidirectionalSequenceLstmLayer: Cell layer normalization weights "
2851 "cannot be NULL when layer normalization is enabled.");
2852 }
2853 if(params.m_OutputLayerNormWeights == nullptr)
2854 {
2855 throw InvalidArgumentException("AddUnidirectionalSequenceLstmLayer: Output layer normalization weights "
2856 "cannot be NULL when layer normalization is enabled.");
2857 }
2858 layer->m_LayerNormParameters.m_ForgetLayerNormWeights =
2859 std::make_shared<ScopedTensorHandle>(*(params.m_ForgetLayerNormWeights));
2860 layer->m_LayerNormParameters.m_CellLayerNormWeights =
2861 std::make_shared<ScopedTensorHandle>(*(params.m_CellLayerNormWeights));
2862 layer->m_LayerNormParameters.m_OutputLayerNormWeights =
2863 std::make_shared<ScopedTensorHandle>(*(params.m_OutputLayerNormWeights));
2864 }
2865 return layer;
2866}
2867
Cathal Corbett18655b82021-12-13 13:03:22 +00002868IConnectableLayer* NetworkImpl::AddPrecompiledLayer(const PreCompiledDescriptor& preCompiledDescriptor,
Cathal Corbett3ea01072022-01-06 10:29:43 +00002869 CompiledBlobPtr compiledBlobPtr,
Cathal Corbettcbfd7182021-12-15 17:12:59 +00002870 const Optional<BackendId>& backend,
2871 const char* name)
Cathal Corbett18655b82021-12-13 13:03:22 +00002872{
2873 // Method use is for backend users.
Cathal Corbettcbfd7182021-12-15 17:12:59 +00002874 PreCompiledLayer* layer;
2875 if (name)
2876 {
2877 layer = m_Graph->AddLayer<PreCompiledLayer>(preCompiledDescriptor, name);
2878 }
2879 else
2880 {
2881 layer = m_Graph->AddLayer<PreCompiledLayer>(preCompiledDescriptor, "pre-compiled");
2882 }
Cathal Corbett18655b82021-12-13 13:03:22 +00002883
2884 // Assign the pre-compiled object to layer
2885 // Pass only one compiled network, Arm NN does not handle multiple
2886 // pre-compiled objects in a single pre-compiled layer currently
2887 layer->SetPreCompiledObject(std::move(compiledBlobPtr));
2888
2889 if (backend.has_value())
2890 {
2891 layer->SetBackendId(backend.value());
2892 }
Francis Murtagh9d74ba62022-01-19 16:31:58 +00002893 else if (layer->GetBackendHint().has_value())
Cathal Corbett18655b82021-12-13 13:03:22 +00002894 {
2895 layer->SetBackendId(layer->GetBackendHint().value());
2896 }
2897
2898 return layer;
2899}
2900
Jan Eilers1b2654f2021-09-24 15:45:46 +01002901ARMNN_NO_DEPRECATE_WARN_BEGIN
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002902void NetworkImpl::Accept(ILayerVisitor& visitor) const
Mike Kelly8c1701a2019-02-11 17:01:27 +00002903{
2904 for (auto layer : GetGraph())
2905 {
2906 layer->Accept(visitor);
2907 };
2908}
Jan Eilers1b2654f2021-09-24 15:45:46 +01002909ARMNN_NO_DEPRECATE_WARN_END
Mike Kelly8c1701a2019-02-11 17:01:27 +00002910
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002911void NetworkImpl::ExecuteStrategy(IStrategy& strategy) const
Finn Williamsb454c5c2021-02-09 15:56:23 +00002912{
2913 for (auto layer : GetGraph())
2914 {
2915 layer->ExecuteStrategy(strategy);
2916 };
2917}
2918
Mike Kelly0d677db2021-06-27 22:39:21 +01002919OptimizedNetworkImpl::OptimizedNetworkImpl(const OptimizedNetworkImpl& other, const ModelOptions& modelOptions)
2920 : m_Graph(new Graph(*other.m_Graph.get()))
Jim Flynnaf947722022-03-02 11:04:47 +00002921 , m_Guid(arm::pipe::IProfilingService::GetNextGuid())
Mike Kelly0d677db2021-06-27 22:39:21 +01002922 , m_ModelOptions(modelOptions)
2923{
2924}
2925
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002926OptimizedNetworkImpl::OptimizedNetworkImpl(std::unique_ptr<Graph> graph)
Jim Flynnaf947722022-03-02 11:04:47 +00002927 : m_Graph(std::move(graph)), m_Guid(arm::pipe::IProfilingService::GetNextGuid())
telsoa014fcda012018-03-09 14:13:49 +00002928{
2929}
2930
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002931OptimizedNetworkImpl::OptimizedNetworkImpl(std::unique_ptr<Graph> graph, const ModelOptions& modelOptions)
Jim Flynnaf947722022-03-02 11:04:47 +00002932 : m_Graph(std::move(graph)), m_Guid(arm::pipe::IProfilingService::GetNextGuid()), m_ModelOptions(modelOptions)
Sadik Armagan045f6be2020-09-10 13:37:32 +01002933{
2934}
2935
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002936OptimizedNetworkImpl::~OptimizedNetworkImpl()
telsoa014fcda012018-03-09 14:13:49 +00002937{
2938}
2939
2940} // namespace armnn