blob: 365f1bdfa1af67d1edaa4b553720b68c309a73a5 [file] [log] [blame]
Laurent Carlier749294b2020-06-01 09:03:17 +01001//
Teresa Charlin52664732020-06-29 16:27:03 +01002// Copyright © 2017 Arm Ltd and Contributors. All rights reserved.
David Beckecb56cd2018-09-05 12:52:57 +01003// SPDX-License-Identifier: MIT
telsoa014fcda012018-03-09 14:13:49 +00004//
Matteo Martincigh49124022019-01-11 13:25:59 +00005
telsoa014fcda012018-03-09 14:13:49 +00006#include "Network.hpp"
7#include "Graph.hpp"
8#include "Layer.hpp"
telsoa01c577f2c2018-08-31 09:22:23 +01009#include "DeviceSpec.hpp"
telsoa014fcda012018-03-09 14:13:49 +000010#include "Optimizer.hpp"
Derek Lambertiff05cc52019-04-26 13:05:17 +010011#include "SubgraphViewSelector.hpp"
Matteo Martincigh49124022019-01-11 13:25:59 +000012#include "BackendSettings.hpp"
David Beckac42efd2018-09-26 17:41:13 +010013#include "optimizations/All.hpp"
telsoa014fcda012018-03-09 14:13:49 +000014
James Conroy1f58f032021-04-27 17:13:27 +010015#include <backendsCommon/TensorHandle.hpp>
Aron Virginas-Tarc9cc8042018-11-01 16:15:57 +000016#include <backendsCommon/WorkloadFactory.hpp>
Matteo Martincighe5b8eb92019-11-28 15:45:42 +000017#include <armnn/backends/IBackendInternal.hpp>
Derek Lamberti84da38b2019-06-13 11:40:08 +010018#include <backendsCommon/TensorHandleFactoryRegistry.hpp>
David Beckac42efd2018-09-26 17:41:13 +010019
20#include <armnn/Exceptions.hpp>
telsoa014fcda012018-03-09 14:13:49 +000021#include <armnn/Utils.hpp>
telsoa01c577f2c2018-08-31 09:22:23 +010022#include <armnn/TypesUtils.hpp>
Matteo Martincighc601aa62019-10-29 15:03:22 +000023#include <armnn/BackendRegistry.hpp>
Matthew Benthamf48afc62020-01-15 17:55:08 +000024#include <armnn/Logging.hpp>
Narumol Prangnawaratac2770a2020-04-01 16:51:23 +010025#include <armnn/utility/Assert.hpp>
Jan Eilers8eb25602020-03-09 12:13:48 +000026#include <armnn/utility/IgnoreUnused.hpp>
Jan Eilersbb446e52020-04-02 13:56:54 +010027#include <armnn/utility/PolymorphicDowncast.hpp>
telsoa014fcda012018-03-09 14:13:49 +000028
Jan Eilers99d9d4a2019-11-06 10:02:16 +000029#include <ProfilingService.hpp>
30
Nikhil Raj77fe76b2021-06-09 14:55:32 +010031#include <common/include/ProfilingGuid.hpp>
32
Matthew Sloyan81beae32021-07-13 19:46:11 +010033#include <fmt/format.h>
34
telsoa014fcda012018-03-09 14:13:49 +000035#include <fcntl.h>
36#include <algorithm>
37#include <fstream>
38#include <memory>
telsoa01c577f2c2018-08-31 09:22:23 +010039#include <vector>
40#include <algorithm>
telsoa014fcda012018-03-09 14:13:49 +000041
telsoa014fcda012018-03-09 14:13:49 +000042namespace armnn
43{
44
Francis Murtagh3d2b4b22021-02-15 18:23:17 +000045INetwork::INetwork(NetworkOptions networkOptions) : pNetworkImpl(new NetworkImpl(networkOptions)) {}
46
47INetwork::~INetwork() = default;
48
49Status INetwork::PrintGraph()
50{
51 return pNetworkImpl->PrintGraph();
52}
53
54IConnectableLayer* INetwork::AddInputLayer(LayerBindingId id, const char* name)
55{
56 return pNetworkImpl->AddInputLayer(id, name);
57}
58
59
60IConnectableLayer* INetwork::AddArgMinMaxLayer(const ArgMinMaxDescriptor& desc,
61 const char* name)
62{
63 return pNetworkImpl->AddArgMinMaxLayer(desc, name);
64}
65
mathad01b392e982021-04-07 12:07:30 +010066IConnectableLayer* INetwork::AddCastLayer(const char* name)
67{
68 return pNetworkImpl->AddCastLayer(name);
69}
Francis Murtagh3d2b4b22021-02-15 18:23:17 +000070
71IConnectableLayer* INetwork::AddComparisonLayer(const ComparisonDescriptor& comparisonDescriptor,
72 const char* name)
73{
74 return pNetworkImpl->AddComparisonLayer(comparisonDescriptor, name);
75}
76
77
78IConnectableLayer* INetwork::AddConcatLayer(const ConcatDescriptor& concatDescriptor,
79 const char* name)
80{
81 return pNetworkImpl->AddConcatLayer(concatDescriptor, name);
82}
83
84
85IConnectableLayer* INetwork::AddConvolution2dLayer(const Convolution2dDescriptor& convolution2dDescriptor,
86 const ConstTensor& weights,
87 const Optional<ConstTensor>& biases,
88 const char* name)
89{
90 return pNetworkImpl->AddConvolution2dLayer(convolution2dDescriptor, weights, biases, name);
91}
92
93
94IConnectableLayer* INetwork::AddConvolution2dLayer(const Convolution2dDescriptor& convolution2dDescriptor,
95 const ConstTensor& weights,
96 const char* name)
97{
98 Optional<ConstTensor> biases;
99 return pNetworkImpl->AddConvolution2dLayer(convolution2dDescriptor, weights, biases, name);
100}
101
102
103IConnectableLayer* INetwork::AddConvolution2dLayer(const Convolution2dDescriptor& convolution2dDescriptor,
104 const ConstTensor& weights,
105 const ConstTensor& biases,
106 const char* name )
107{
108
109 return pNetworkImpl->AddConvolution2dLayer(convolution2dDescriptor,
110 weights,
111 armnn::Optional<ConstTensor>(biases),
112 name);
113}
114
115
116IConnectableLayer* INetwork::AddDepthToSpaceLayer(const DepthToSpaceDescriptor& depthToSpaceDescriptor,
117 const char* name)
118{
119 return pNetworkImpl->AddDepthToSpaceLayer(depthToSpaceDescriptor, name);
120}
121
122
123IConnectableLayer* INetwork::AddDepthwiseConvolution2dLayer(
124 const DepthwiseConvolution2dDescriptor& convolution2dDescriptor,
125 const ConstTensor& weights,
126 const Optional<ConstTensor>& biases,
127 const char* name)
128{
129 return pNetworkImpl->AddDepthwiseConvolution2dLayer(convolution2dDescriptor, weights, biases, name);
130}
131
132
133IConnectableLayer* INetwork::AddDepthwiseConvolution2dLayer(
134 const DepthwiseConvolution2dDescriptor& convolution2dDescriptor,
135 const ConstTensor& weights,
136 const char* name)
137{
138 Optional<ConstTensor> biases;
139 return pNetworkImpl->AddDepthwiseConvolution2dLayer(convolution2dDescriptor, weights, biases, name);
140}
141
142
143IConnectableLayer* INetwork::AddDepthwiseConvolution2dLayer(
144 const DepthwiseConvolution2dDescriptor& convolution2dDescriptor,
145 const ConstTensor& weights,
146 const ConstTensor& biases,
147 const char* name)
148{
149 return pNetworkImpl->AddDepthwiseConvolution2dLayer(convolution2dDescriptor, weights,
150 armnn::Optional<ConstTensor>(biases), name);
151}
152
153
154IConnectableLayer* INetwork::AddDequantizeLayer(const char* name)
155{
156 return pNetworkImpl->AddDequantizeLayer(name);
157}
158
159
160IConnectableLayer* INetwork::AddDetectionPostProcessLayer(
161 const DetectionPostProcessDescriptor& descriptor,
162 const ConstTensor& anchors,
163 const char* name)
164{
165 return pNetworkImpl->AddDetectionPostProcessLayer(descriptor, anchors, name);
166}
167
168
169IConnectableLayer* INetwork::AddElementwiseUnaryLayer(const ElementwiseUnaryDescriptor& elementwiseUnaryDescriptor,
170 const char* name)
171{
172 return pNetworkImpl->AddElementwiseUnaryLayer(elementwiseUnaryDescriptor, name);
173}
174
175
176IConnectableLayer* INetwork::AddFillLayer(const FillDescriptor& fillDescriptor,
177 const char* name)
178{
179 return pNetworkImpl->AddFillLayer(fillDescriptor, name);
180}
181
Francis Murtagh3d2b4b22021-02-15 18:23:17 +0000182IConnectableLayer* INetwork::AddFullyConnectedLayer(const FullyConnectedDescriptor& fullyConnectedDescriptor,
Matthew Sloyan81beae32021-07-13 19:46:11 +0100183 const char* name)
184{
185 return pNetworkImpl->AddFullyConnectedLayer(fullyConnectedDescriptor, name);
186}
187
188IConnectableLayer* INetwork::AddFullyConnectedLayer(const FullyConnectedDescriptor& fullyConnectedDescriptor,
Francis Murtagh3d2b4b22021-02-15 18:23:17 +0000189 const ConstTensor& weights,
190 const Optional<ConstTensor>& biases,
191 const char* name)
192{
Sadik Armaganf0a6dec2021-03-25 07:46:55 +0000193 return pNetworkImpl->AddFullyConnectedLayer(fullyConnectedDescriptor,
194 armnn::Optional<ConstTensor>(weights),
195 biases,
196 name);
Francis Murtagh3d2b4b22021-02-15 18:23:17 +0000197}
198
199IConnectableLayer* INetwork::AddFullyConnectedLayer(const FullyConnectedDescriptor& fullyConnectedDescriptor,
Sadik Armaganf0a6dec2021-03-25 07:46:55 +0000200 const Optional<ConstTensor>& weights,
201 const Optional<ConstTensor>& biases,
202 const char* name)
203{
204 return pNetworkImpl->AddFullyConnectedLayer(fullyConnectedDescriptor, weights, biases, name);
Francis Murtagh3d2b4b22021-02-15 18:23:17 +0000205}
206
207IConnectableLayer* INetwork::AddPermuteLayer(const PermuteDescriptor& permuteDescriptor,
208 const char* name)
209{
210 return pNetworkImpl->AddPermuteLayer(permuteDescriptor, name);
211}
212
213IConnectableLayer* INetwork::AddBatchToSpaceNdLayer(const BatchToSpaceNdDescriptor& batchToSpaceNdDescriptor,
214 const char* name)
215{
216 return pNetworkImpl->AddBatchToSpaceNdLayer(batchToSpaceNdDescriptor, name);
217}
218
219IConnectableLayer* INetwork::AddPooling2dLayer(const Pooling2dDescriptor& pooling2dDescriptor,
220 const char* name)
221{
222 return pNetworkImpl->AddPooling2dLayer(pooling2dDescriptor, name);
223}
224
225IConnectableLayer* INetwork::AddActivationLayer(const ActivationDescriptor& activationDescriptor,
226 const char* name)
227{
228 return pNetworkImpl->AddActivationLayer(activationDescriptor, name);
229}
230
231IConnectableLayer* INetwork::AddNormalizationLayer(const NormalizationDescriptor& normalizationDescriptor,
232 const char* name)
233{
234 return pNetworkImpl->AddNormalizationLayer(normalizationDescriptor, name);
235}
236
237IConnectableLayer* INetwork::AddSliceLayer(const SliceDescriptor& sliceDescriptor, const char* name)
238{
239 return pNetworkImpl->AddSliceLayer(sliceDescriptor, name);
240}
241IConnectableLayer* INetwork::AddSoftmaxLayer(const SoftmaxDescriptor& softmaxDescriptor,
242 const char* name)
243{
244 return pNetworkImpl->AddSoftmaxLayer(softmaxDescriptor, name);
245}
246
247IConnectableLayer* INetwork::AddSplitterLayer(const ViewsDescriptor& splitterDescriptor,
248 const char* name)
249{
250 return pNetworkImpl->AddSplitterLayer(splitterDescriptor, name);
251}
252
253IConnectableLayer* INetwork::AddMergeLayer(const char* name)
254{
255 return pNetworkImpl->AddMergeLayer(name);
256}
257
258IConnectableLayer* INetwork::AddMergerLayer(const MergerDescriptor& mergerDescriptor,
259 const char* name)
260{
261 return pNetworkImpl->AddConcatLayer(mergerDescriptor, name);
262}
263
264IConnectableLayer* INetwork::AddAbsLayer(const char* name)
265{
266 return pNetworkImpl->AddElementwiseUnaryLayer(ElementwiseUnaryDescriptor(UnaryOperation::Abs), name);
267}
268
269IConnectableLayer* INetwork::AddAdditionLayer(const char* name)
270{
271 return pNetworkImpl->AddAdditionLayer(name);
272}
273
274IConnectableLayer* INetwork::AddMultiplicationLayer(const char* name)
275{
276 return pNetworkImpl->AddMultiplicationLayer(name);
277}
278
279IConnectableLayer* INetwork::AddBatchNormalizationLayer(const BatchNormalizationDescriptor& desc,
280 const ConstTensor& mean,
281 const ConstTensor& variance,
282 const ConstTensor& beta,
283 const ConstTensor& gamma,
284 const char* name)
285{
286 return pNetworkImpl->AddBatchNormalizationLayer(desc, mean, variance, beta, gamma, name);
287}
288
289IConnectableLayer* INetwork::AddRankLayer(const char* name)
290{
291 return pNetworkImpl->AddRankLayer(name);
292}
293
294IConnectableLayer* INetwork::AddResizeBilinearLayer(const ResizeBilinearDescriptor& descriptor,
295 const char* name)
296{
297 ResizeDescriptor resizeDescriptor;
298 resizeDescriptor.m_Method = ResizeMethod::Bilinear;
299 resizeDescriptor.m_DataLayout = descriptor.m_DataLayout;
300 resizeDescriptor.m_TargetWidth = descriptor.m_TargetWidth;
301 resizeDescriptor.m_TargetHeight = descriptor.m_TargetHeight;
302 resizeDescriptor.m_AlignCorners = descriptor.m_AlignCorners;
303 resizeDescriptor.m_HalfPixelCenters = descriptor.m_HalfPixelCenters;
304
305 return pNetworkImpl->AddResizeLayer(resizeDescriptor, name);
306}
307
308IConnectableLayer* INetwork::AddResizeLayer(const ResizeDescriptor& resizeDescriptor,
309 const char* name)
310{
311 return pNetworkImpl->AddResizeLayer(resizeDescriptor, name);
312}
313
314IConnectableLayer* INetwork::AddReduceLayer(const ReduceDescriptor& reduceDescriptor,
315 const char* name)
316{
317 return pNetworkImpl->AddReduceLayer(reduceDescriptor, name);
318}
319
320IConnectableLayer* INetwork::AddInstanceNormalizationLayer(const InstanceNormalizationDescriptor& desc,
321 const char* name)
322{
323 return pNetworkImpl->AddInstanceNormalizationLayer(desc, name);
324}
325
326IConnectableLayer* INetwork::AddL2NormalizationLayer(const L2NormalizationDescriptor& desc,
327 const char* name)
328{
329 return pNetworkImpl->AddL2NormalizationLayer(desc, name);
330}
331
332IConnectableLayer* INetwork::AddLogSoftmaxLayer(const LogSoftmaxDescriptor& logSoftmaxDescriptor,
333 const char* name)
334{
335 return pNetworkImpl->AddLogSoftmaxLayer(logSoftmaxDescriptor, name);
336}
337
338IConnectableLayer* INetwork::AddConstantLayer(const ConstTensor& input,
339 const char* name)
340{
341 return pNetworkImpl->AddConstantLayer(input, name);
342}
343
344IConnectableLayer* INetwork::AddReshapeLayer(const ReshapeDescriptor& reshapeDescriptor,
345 const char* name)
346{
347 return pNetworkImpl->AddReshapeLayer(reshapeDescriptor, name);
348}
349
350IConnectableLayer* INetwork::AddSpaceToBatchNdLayer(const SpaceToBatchNdDescriptor& spaceToBatchNdDescriptor,
351 const char* name)
352{
353 return pNetworkImpl->AddSpaceToBatchNdLayer(spaceToBatchNdDescriptor, name);
354}
355
356IConnectableLayer* INetwork::AddSpaceToDepthLayer(const SpaceToDepthDescriptor& spaceToDepthDescriptor,
357 const char* name)
358{
359 return pNetworkImpl->AddSpaceToDepthLayer(spaceToDepthDescriptor, name);
360}
361
362IConnectableLayer* INetwork::AddFloorLayer(const char* name)
363{
364 return pNetworkImpl->AddFloorLayer(name);
365}
366IConnectableLayer* INetwork::AddOutputLayer(LayerBindingId id, const char* name)
367{
368 return pNetworkImpl->AddOutputLayer(id, name);
369}
370
371IConnectableLayer* INetwork::AddLstmLayer(const LstmDescriptor& descriptor,
372 const LstmInputParams& params,
373 const char* name)
374{
375 return pNetworkImpl->AddLstmLayer(descriptor, params, name);
376}
377
378IConnectableLayer* INetwork::AddDivisionLayer(const char* name)
379{
380 return pNetworkImpl->AddDivisionLayer(name);
381}
382
383IConnectableLayer* INetwork::AddSubtractionLayer(const char* name)
384{
385 return pNetworkImpl->AddSubtractionLayer(name);
386}
387
388IConnectableLayer* INetwork::AddMaximumLayer(const char* name)
389{
390 return pNetworkImpl->AddMaximumLayer(name);
391}
392
393IConnectableLayer* INetwork::AddMeanLayer(const MeanDescriptor& meanDescriptor, const char* name)
394{
395 return pNetworkImpl->AddMeanLayer(meanDescriptor, name);
396}
397
398IConnectableLayer* INetwork::AddPadLayer(const PadDescriptor& padDescriptor,
399 const char* name)
400{
401 return pNetworkImpl->AddPadLayer(padDescriptor, name);
402}
403
404IConnectableLayer* INetwork::AddQuantizeLayer(const char* name)
405{
406 return pNetworkImpl->AddQuantizeLayer(name);
407}
408
409IConnectableLayer* INetwork::AddStridedSliceLayer(const StridedSliceDescriptor& stridedSliceDescriptor,
410 const char* name)
411{
412 return pNetworkImpl->AddStridedSliceLayer(stridedSliceDescriptor, name);
413}
414
415IConnectableLayer* INetwork::AddMinimumLayer(const char* name)
416{
417 return pNetworkImpl->AddMinimumLayer(name);
418}
419
420IConnectableLayer* INetwork::AddGreaterLayer(const char* name)
421{
422 return pNetworkImpl->AddComparisonLayer(ComparisonDescriptor(ComparisonOperation::Greater), name);
423}
424
425IConnectableLayer* INetwork::AddEqualLayer(const char* name)
426{
427 return pNetworkImpl->AddComparisonLayer(ComparisonDescriptor(ComparisonOperation::Equal), name);
428}
429
430IConnectableLayer* INetwork::AddRsqrtLayer(const char* name)
431{
432 return pNetworkImpl->AddElementwiseUnaryLayer(ElementwiseUnaryDescriptor(UnaryOperation::Rsqrt), name);
433}
434
435IConnectableLayer* INetwork::AddGatherLayer(const char* name)
436{
437 GatherDescriptor gatherDescriptor{};
438 return pNetworkImpl->AddGatherLayer(gatherDescriptor, name);
439}
440
441IConnectableLayer* INetwork::AddGatherLayer(const GatherDescriptor& descriptor,
442 const char* name)
443{
444 return pNetworkImpl->AddGatherLayer(descriptor, name);
445}
446
447IConnectableLayer* INetwork::AddSwitchLayer(const char* name)
448{
449 return pNetworkImpl->AddSwitchLayer(name);
450}
451
452IConnectableLayer* INetwork::AddPreluLayer(const char* name)
453{
454 return pNetworkImpl->AddPreluLayer(name);
455}
456
457IConnectableLayer* INetwork::AddTransposeConvolution2dLayer(const TransposeConvolution2dDescriptor& descriptor,
458 const ConstTensor& weights,
459 const Optional<ConstTensor>& biases,
460 const char* name)
461{
462 return pNetworkImpl->AddTransposeConvolution2dLayer(descriptor, weights, biases, name);
463}
464
465IConnectableLayer* INetwork::AddTransposeLayer(const TransposeDescriptor& transposeDescriptor,
466 const char* name)
467{
468 return pNetworkImpl->AddTransposeLayer(transposeDescriptor, name);
469}
470
Keith Davis3ae3f972021-05-21 16:33:48 +0100471IConnectableLayer* INetwork::AddShapeLayer(const char* name)
472{
473 return pNetworkImpl->AddShapeLayer(name);
474}
475
Francis Murtagh3d2b4b22021-02-15 18:23:17 +0000476IConnectableLayer* INetwork::AddStackLayer(const StackDescriptor& descriptor,
477 const char* name)
478{
479 return pNetworkImpl->AddStackLayer(descriptor, name);
480}
481
482IConnectableLayer* INetwork::AddStandInLayer(const StandInDescriptor& descriptor,
483 const char* name)
484{
485 return pNetworkImpl->AddStandInLayer(descriptor, name);
486}
487
488IConnectableLayer* INetwork::AddQuantizedLstmLayer(const QuantizedLstmInputParams& params,
489 const char* name)
490{
491 return pNetworkImpl->AddQuantizedLstmLayer(params, name);
492}
493
494IConnectableLayer* INetwork::AddQLstmLayer(const QLstmDescriptor& descriptor,
495 const LstmInputParams& params,
496 const char* name)
497{
498 return pNetworkImpl->AddQLstmLayer(descriptor, params, name);
499}
500
501IConnectableLayer* INetwork::AddLogicalBinaryLayer(const LogicalBinaryDescriptor& descriptor,
502 const char* name)
503{
504 return pNetworkImpl->AddLogicalBinaryLayer(descriptor, name);
505}
506
Narumol Prangnawarat8ed39ae2021-07-15 16:16:25 +0100507IConnectableLayer* INetwork::AddUnidirectionalSequenceLstmLayer(
508 const UnidirectionalSequenceLstmDescriptor& descriptor,
509 const LstmInputParams& params,
510 const char* name)
511{
512 return pNetworkImpl->AddUnidirectionalSequenceLstmLayer(descriptor, params, name);
513}
514
Francis Murtagh3d2b4b22021-02-15 18:23:17 +0000515void INetwork::Accept(ILayerVisitor& visitor) const
516{
517 return pNetworkImpl->Accept(visitor);
518}
519
520void INetwork::ExecuteStrategy(IStrategy& strategy) const
521{
522 return pNetworkImpl->ExecuteStrategy(strategy);
523}
524
Finn Williamsf24effa2020-07-03 10:12:03 +0100525armnn::INetwork* INetwork::CreateRaw(NetworkOptions networkOptions)
telsoa014fcda012018-03-09 14:13:49 +0000526{
Francis Murtagh3d2b4b22021-02-15 18:23:17 +0000527 return new INetwork(networkOptions);
telsoa014fcda012018-03-09 14:13:49 +0000528}
529
Finn Williamsf24effa2020-07-03 10:12:03 +0100530armnn::INetworkPtr INetwork::Create(NetworkOptions networkOptions)
telsoa014fcda012018-03-09 14:13:49 +0000531{
Finn Williamsf24effa2020-07-03 10:12:03 +0100532 return INetworkPtr(CreateRaw(networkOptions), &INetwork::Destroy);
telsoa014fcda012018-03-09 14:13:49 +0000533}
534
535void INetwork::Destroy(INetwork* network)
536{
Francis Murtagh3d2b4b22021-02-15 18:23:17 +0000537 delete network;
telsoa014fcda012018-03-09 14:13:49 +0000538}
539
Mike Kelly0d677db2021-06-27 22:39:21 +0100540IOptimizedNetwork::IOptimizedNetwork(const IOptimizedNetwork& other, const ModelOptions& modelOptions)
541 : pOptimizedNetworkImpl(new OptimizedNetworkImpl(*other.pOptimizedNetworkImpl.get(), modelOptions)) {}
Francis Murtagh3d2b4b22021-02-15 18:23:17 +0000542
543IOptimizedNetwork::IOptimizedNetwork(std::unique_ptr<Graph> graph)
544 : pOptimizedNetworkImpl(new OptimizedNetworkImpl(std::move(graph))) {}
545
546IOptimizedNetwork::IOptimizedNetwork(std::unique_ptr<OptimizedNetworkImpl> impl)
547 : pOptimizedNetworkImpl(std::move(impl)) {}
548
549IOptimizedNetwork::IOptimizedNetwork(std::unique_ptr<Graph> graph, const ModelOptions& modelOptions)
550 : pOptimizedNetworkImpl(new OptimizedNetworkImpl(std::move(graph), modelOptions)) {}
551
552IOptimizedNetwork::~IOptimizedNetwork() = default;
553
telsoa014fcda012018-03-09 14:13:49 +0000554void IOptimizedNetwork::Destroy(IOptimizedNetwork* network)
555{
Francis Murtagh3d2b4b22021-02-15 18:23:17 +0000556 delete network;
telsoa014fcda012018-03-09 14:13:49 +0000557}
558
Francis Murtagh3d2b4b22021-02-15 18:23:17 +0000559Status IOptimizedNetwork::PrintGraph()
560{
561 return pOptimizedNetworkImpl->PrintGraph();
562}
563
564Status IOptimizedNetwork::SerializeToDot(std::ostream& stream) const
565{
566 return pOptimizedNetworkImpl->SerializeToDot(stream);
567}
568
569profiling::ProfilingGuid IOptimizedNetwork::GetGuid() const
570{
571 return pOptimizedNetworkImpl->GetGuid();
572}
573
574Status OptimizedNetworkImpl::PrintGraph()
telsoa014fcda012018-03-09 14:13:49 +0000575{
576 m_Graph->Print();
577 return Status::Success;
578}
579
Francis Murtagh3d2b4b22021-02-15 18:23:17 +0000580Status OptimizedNetworkImpl::SerializeToDot(std::ostream& stream) const
surmeh01bceff2f2018-03-29 16:29:27 +0100581{
582 return m_Graph->SerializeToDot(stream);
583}
584
Matteo Martincigh49124022019-01-11 13:25:59 +0000585void ReportError(const std::string& errorMessage,
586 Optional<std::vector<std::string>&> errorMessages)
587{
588 std::stringstream fullErrorMessage;
589 fullErrorMessage << "ERROR: " << errorMessage;
Derek Lamberti08446972019-11-26 16:38:31 +0000590 ARMNN_LOG(warning) << fullErrorMessage.str();
Matteo Martincigh49124022019-01-11 13:25:59 +0000591 if (errorMessages)
592 {
593 errorMessages.value().push_back(fullErrorMessage.str());
594 }
595}
596
597void ReportWarning(const std::string& warningMessage,
598 Optional<std::vector<std::string>&> warningMessages)
599{
600 std::stringstream fullWarningMessage;
601 fullWarningMessage << "WARNING: " << warningMessage;
Derek Lamberti08446972019-11-26 16:38:31 +0000602 ARMNN_LOG(warning) << fullWarningMessage.str();
Matteo Martincigh49124022019-01-11 13:25:59 +0000603 if (warningMessages)
604 {
605 warningMessages.value().push_back(fullWarningMessage.str());
606 }
607}
608
Derek Lamberti4a9e24b2020-01-03 16:53:38 +0000609OptimizationResult ReturnWithError(OptimizationResult res,
610 const Layer* layer,
611 const BackendSettings& backendSettings,
612 Optional<std::vector<std::string>&> errMessages)
613{
614 std::stringstream failureMsg;
615 failureMsg << "Layer of type " << GetLayerTypeAsCString(layer->GetType())
616 << " is not supported on any preferred backend " << backendSettings.m_PreferredBackends;
617 ReportError(failureMsg.str(), errMessages);
618
619 res.m_Error = true;
620 return res;
621}
622
623
jimfly016b0b53d2018-10-08 14:43:01 +0100624bool CheckScaleSetOnQuantizedType(Layer* layer, Optional<std::vector<std::string>&> errMessages)
625{
626 bool noErrors = true;
627 unsigned int numOutputs = layer->GetNumOutputSlots();
628 for (unsigned int i = 0; i < numOutputs; i++) {
David Monahanb8554702019-04-25 16:03:38 +0100629 OutputSlot& outputSlot = layer->GetOutputSlot(i);
630 TensorInfo info = outputSlot.GetTensorInfo();
Derek Lambertif90c56d2020-01-10 17:14:08 +0000631 if (DataType::QAsymmU8 == info.GetDataType()) {
jimfly016b0b53d2018-10-08 14:43:01 +0100632 if (0.f == info.GetQuantizationScale()) {
633 noErrors = false;
634 std::stringstream ss;
Matteo Martincigh49124022019-01-11 13:25:59 +0000635 ss << "output " << i << " of layer " << GetLayerTypeAsCString(layer->GetType())
jimfly016b0b53d2018-10-08 14:43:01 +0100636 << " (" << layer->GetNameStr() << ") is of type"
637 << " Quantized 8 bit but its scale parameter has not been set";
Matteo Martincigh49124022019-01-11 13:25:59 +0000638 ReportError(ss.str(), errMessages);
jimfly016b0b53d2018-10-08 14:43:01 +0100639 }
David Monahanb8554702019-04-25 16:03:38 +0100640 // Softmax under QuantisedAsymm8 must always be scale (1.0f/256.0f) and offset 0
641 if ((info.GetQuantizationScale() != (1.0f / 256.0f) ||
642 info.GetQuantizationOffset() != 0) &&
643 layer->GetType() == armnn::LayerType::Softmax)
644 {
645 std::stringstream ss;
646 ss << "Quantization parameters for Softmax layer (Scale: " <<
647 info.GetQuantizationScale() << " and Offset: " << info.GetQuantizationOffset() <<
648 ") are incorrect and have been updated to Scale: 0.00390625 and Offset: 0";
Derek Lamberti08446972019-11-26 16:38:31 +0000649 ARMNN_LOG(warning) << ss.str();
David Monahanb8554702019-04-25 16:03:38 +0100650 info.SetQuantizationScale((1.0f /256.0f));
651 info.SetQuantizationOffset(0);
652 outputSlot.SetTensorInfo(info);
653 }
jimfly016b0b53d2018-10-08 14:43:01 +0100654 }
655 }
656 return noErrors;
657}
658
Narumol Prangnawarat250d3922020-03-30 16:11:04 +0100659template <typename LayerT>
660LayerT* ConvertBf16ToFp32Weight(Layer* l)
661{
Jan Eilersbb446e52020-04-02 13:56:54 +0100662 LayerT* layer = PolymorphicDowncast<LayerT*>(l);
Narumol Prangnawarat250d3922020-03-30 16:11:04 +0100663 if ((layer->GetType() == LayerType::Convolution2d || layer->GetType() == LayerType::FullyConnected)
664 && layer->m_Weight)
665 {
666 const TensorInfo& info = layer->m_Weight->GetTensorInfo();
667
668 if (info.GetDataType() == DataType::BFloat16)
669 {
670 std::vector<float> newValues(info.GetNumElements());
671
672 armnnUtils::FloatingPointConverter::ConvertBFloat16ToFloat32(
Finn Williams4422cec2021-03-22 17:51:06 +0000673 layer->m_Weight->template GetConstTensor<armnn::BFloat16>(), info.GetNumElements(), newValues.data());
Narumol Prangnawarat250d3922020-03-30 16:11:04 +0100674
675 TensorInfo newInfo(info.GetShape(), DataType::Float32);
676 ConstTensor newInput(newInfo, newValues);
James Conroy1f58f032021-04-27 17:13:27 +0100677 layer->m_Weight.reset(new ScopedTensorHandle(newInput));
Narumol Prangnawarat250d3922020-03-30 16:11:04 +0100678 }
679 }
680 return layer;
681}
682
Derek Lamberti4a9e24b2020-01-03 16:53:38 +0000683OptimizationResult AttemptBackendAssignment(BackendSettings& backendSettings,
684 Graph& graph,
685 Layer* layer,
686 BackendId backend,
687 DataType dataTypeIn,
688 DataType dataTypeOut,
689 const std::vector<BackendId>& availablePreferredBackends,
690 std::string& reasonIfUnsupported,
691 Optional<std::vector<std::string>&> errMessages)
692{
693 OptimizationResult result;
694
695 // Helper lambda to compose meaningful error message before returning with error
696 auto ReturnError = [&](const Layer* layer)
697 {
698 return ReturnWithError(result, layer, backendSettings, errMessages);
699 };
700
701 // need to set the compute device on the layer
702 // before we can check if it is supported
703 layer->SetBackendId(backend);
704 if (!IWorkloadFactory::IsLayerSupported(*layer, EmptyOptional(), reasonIfUnsupported))
705 {
706 if (dataTypeIn == DataType::Float16 || dataTypeOut == DataType::Float16)
707 {
708 if (IWorkloadFactory::IsLayerSupported(*layer, DataType::Float32, reasonIfUnsupported)
709 && layer->GetType() != LayerType::ConvertFp32ToFp16
710 && layer->GetType() != LayerType::ConvertFp16ToFp32)
711 {
712 // Insert FP16 -> FP32 conversion layer before current layer
713 std::vector<ConvertFp16ToFp32Layer*> convertFp16ToFp32Layers;
714 if (dataTypeIn == DataType::Float16)
715 {
716 convertFp16ToFp32Layers =
717 InsertConvertFp16ToFp32LayersBefore(graph, *layer);
718 }
719
720 // Insert FP32 -> FP16 conversion layer after current layer
721 std::vector<ConvertFp32ToFp16Layer*> convertFp32ToFp16Layers;
722 if (dataTypeOut == DataType::Float16)
723 {
724 convertFp32ToFp16Layers =
725 InsertConvertFp32ToFp16LayersAfter(graph, *layer);
726 }
727
728 // Assign a supported backend to the newly introduced conversion layers
729 auto AssignFirstSupportedBackend = [&](Layer* layer, BackendId preferredBackend)
730 {
731 bool supportedBackendFound = false;
732 std::string reasonIfUnsupported;
733
734 // Try preferred backend first
735 layer->SetBackendId(preferredBackend);
736 if (IWorkloadFactory::IsLayerSupported(*layer,
737 EmptyOptional(),
738 reasonIfUnsupported))
739 {
740 supportedBackendFound = true;
741 }
742 else
743 {
744 for (const auto& backend : availablePreferredBackends)
745 {
746 // Skip preferred backend (we already determined that it is not supported)
747 if (backend == preferredBackend)
748 {
749 continue;
750 }
751
752 layer->SetBackendId(backend);
753 if (IWorkloadFactory::IsLayerSupported(*layer,
754 EmptyOptional(),
755 reasonIfUnsupported))
756 {
757 supportedBackendFound = true;
758 break;
759 }
760 }
761 }
762
763 return supportedBackendFound;
764 };
765
766 for (ConvertFp16ToFp32Layer* convertLayer : convertFp16ToFp32Layers)
767 {
768 if (!AssignFirstSupportedBackend(convertLayer, backend))
769 {
770 return ReturnError(convertLayer);
771 }
772 }
773
774 for (ConvertFp32ToFp16Layer* convertLayer : convertFp32ToFp16Layers)
775 {
776 if (!AssignFirstSupportedBackend(convertLayer, backend))
777 {
778 return ReturnError(convertLayer);
779 }
780 }
781
782 return result;
783 }
784 }
Narumol Prangnawaratbc7ffb52020-03-20 15:01:01 +0000785 else if (dataTypeIn == DataType::BFloat16 || dataTypeOut == DataType::BFloat16)
786 {
787 if (IWorkloadFactory::IsLayerSupported(*layer, DataType::Float32, reasonIfUnsupported)
788 && layer->GetType() != LayerType::ConvertFp32ToBf16
789 && layer->GetType() != LayerType::ConvertBf16ToFp32)
790 {
791 // Insert BF16 -> FP32 conversion layer before current layer
792 std::vector<ConvertBf16ToFp32Layer*> convertBf16ToFp32Layers;
793 if (dataTypeIn == DataType::BFloat16)
794 {
795 convertBf16ToFp32Layers =
796 InsertConvertBf16ToFp32LayersBefore(graph, *layer);
Narumol Prangnawarat250d3922020-03-30 16:11:04 +0100797 if (layer->GetType() == LayerType::Convolution2d)
798 {
799 ConvertBf16ToFp32Weight<Convolution2dLayer>(layer);
800 }
801 else if (layer->GetType() == LayerType::FullyConnected)
802 {
803 ConvertBf16ToFp32Weight<FullyConnectedLayer>(layer);
804 }
Narumol Prangnawaratbc7ffb52020-03-20 15:01:01 +0000805 }
806
807 // Insert FP32 -> BF16 conversion layer after current layer
808 std::vector<ConvertFp32ToBf16Layer*> convertFp32ToBf16Layers;
809 if (dataTypeOut == DataType::BFloat16)
810 {
811 convertFp32ToBf16Layers =
812 InsertConvertFp32ToBf16LayersAfter(graph, *layer);
813 }
814
815 // Assign a supported backend to the newly introduced conversion layers
816 auto AssignFirstSupportedBackend = [&](Layer* layer, BackendId preferredBackend)
817 {
818 bool supportedBackendFound = false;
819 std::string reasonIfUnsupported;
820
821 // Try preferred backend first
822 layer->SetBackendId(preferredBackend);
823 if (IWorkloadFactory::IsLayerSupported(*layer,
824 EmptyOptional(),
825 reasonIfUnsupported))
826 {
827 supportedBackendFound = true;
828 }
829 else
830 {
831 for (const auto& backend : availablePreferredBackends)
832 {
833 // Skip preferred backend (we already determined that it is not supported)
834 if (backend == preferredBackend)
835 {
836 continue;
837 }
838
839 layer->SetBackendId(backend);
840 if (IWorkloadFactory::IsLayerSupported(*layer,
841 EmptyOptional(),
842 reasonIfUnsupported))
843 {
844 supportedBackendFound = true;
845 break;
846 }
847 }
848 }
849
850 return supportedBackendFound;
851 };
852
853 for (ConvertBf16ToFp32Layer* convertLayer : convertBf16ToFp32Layers)
854 {
855 if (!AssignFirstSupportedBackend(convertLayer, backend))
856 {
857 return ReturnError(convertLayer);
858 }
859 }
860
861 for (ConvertFp32ToBf16Layer* convertLayer : convertFp32ToBf16Layers)
862 {
863 if (!AssignFirstSupportedBackend(convertLayer, backend))
864 {
865 return ReturnError(convertLayer);
866 }
867 }
868
869 return result;
870 }
871 }
872
Derek Lamberti4a9e24b2020-01-03 16:53:38 +0000873 std::stringstream warningMsg;
874 warningMsg << "Layer of type " << GetLayerTypeAsCString(layer->GetType())
875 << " is not supported on requested backend " << layer->GetBackendId().Get()
876 << " for input data type " << GetDataTypeName(dataTypeIn)
877 << " and output data type " << GetDataTypeName(dataTypeOut)
878 << " (reason: " << reasonIfUnsupported
879 << "), falling back to the next backend.";
880 ReportWarning(warningMsg.str(), errMessages);
881
882 return OptimizationResult(true, false);
883 }
884 else
885 {
886 return result;
887 }
888}
889
890
Francis Murtagh3d2b4b22021-02-15 18:23:17 +0000891OptimizationResult AssignBackends(OptimizedNetworkImpl* optNetObjPtr,
Matteo Martincigh49124022019-01-11 13:25:59 +0000892 BackendSettings& backendSettings,
893 Graph::Iterator& firstLayer,
894 Graph::Iterator& lastLayer,
895 Optional<std::vector<std::string>&> errMessages)
telsoa014fcda012018-03-09 14:13:49 +0000896{
Matteo Martincigh49124022019-01-11 13:25:59 +0000897 OptimizationResult result;
telsoa014fcda012018-03-09 14:13:49 +0000898
Matteo Martincigh49124022019-01-11 13:25:59 +0000899 // Helper lambda to compose meaningful error message before returning with error
Derek Lamberti4a9e24b2020-01-03 16:53:38 +0000900 auto ReturnError = [&](const Layer* layer)
901 {
902 return ReturnWithError(result, layer, backendSettings, errMessages);
903 };
Matteo Martincigh49124022019-01-11 13:25:59 +0000904
telsoa01c577f2c2018-08-31 09:22:23 +0100905
Matteo Martincigh49124022019-01-11 13:25:59 +0000906 auto availablePreferredBackends = backendSettings.GetAvailablePreferredBackends();
907 if (availablePreferredBackends.empty())
telsoa01c577f2c2018-08-31 09:22:23 +0100908 {
Matteo Martincigh49124022019-01-11 13:25:59 +0000909 std::stringstream failureMsg;
910 failureMsg << "No preferred backends are available";
911 ReportError(failureMsg.str(), errMessages);
912
913 result.m_Error = true;
914 return result;
915 }
916
917 for (auto it = firstLayer; it != lastLayer; ++it)
918 {
919 auto layer = *it;
Aron Virginas-Tar87972be2019-11-13 15:16:28 +0000920
921 DataType dataTypeIn = layer->GetNumInputSlots() == 0 ? DataType::Float32 :
922 layer->GetInputSlot(0).GetConnectedOutputSlot()->GetTensorInfo().GetDataType();
923 DataType dataTypeOut = layer->GetNumOutputSlots() == 0 ? DataType::Float32 :
924 layer->GetOutputSlot(0).GetTensorInfo().GetDataType();
925
telsoa01c577f2c2018-08-31 09:22:23 +0100926 std::string reasonIfUnsupported;
927 bool found = false;
jimfly016b0b53d2018-10-08 14:43:01 +0100928 if (!CheckScaleSetOnQuantizedType(layer, errMessages))
929 {
930 // don't bomb immediately, find all the quantized outputs
931 // which haven't had a scale set and report them all back.
Matteo Martincigh49124022019-01-11 13:25:59 +0000932 result.m_Error = true;
jimfly016b0b53d2018-10-08 14:43:01 +0100933 }
Matteo Martincigh49124022019-01-11 13:25:59 +0000934
Derek Lamberti4a9e24b2020-01-03 16:53:38 +0000935 // First try assign layer to hint backend
936 if (layer->GetBackendHint().has_value() &&
937 backendSettings.IsBackendSupported(layer->GetBackendHint().value()) &&
938 AttemptBackendAssignment(backendSettings,
939 optNetObjPtr->GetGraph(),
940 layer,
941 layer->GetBackendHint().value(),
942 dataTypeIn,
943 dataTypeOut,
944 availablePreferredBackends,
945 reasonIfUnsupported,
946 errMessages).IsOk())
telsoa01c577f2c2018-08-31 09:22:23 +0100947 {
Derek Lamberti4a9e24b2020-01-03 16:53:38 +0000948 found = true;
949 backendSettings.m_SelectedBackends.insert(layer->GetBackendHint().value());
950 }
951 else
952 {
953 // Try assign layer to prefered list of backends
954 for (const auto& backend : availablePreferredBackends)
telsoa01c577f2c2018-08-31 09:22:23 +0100955 {
Derek Lamberti4a9e24b2020-01-03 16:53:38 +0000956 if (layer->GetBackendHint().has_value() &&
957 layer->GetBackendHint().value() == backend)
telsoa01c577f2c2018-08-31 09:22:23 +0100958 {
Derek Lamberti4a9e24b2020-01-03 16:53:38 +0000959 continue; //Don't re-test the backend hint
telsoa01c577f2c2018-08-31 09:22:23 +0100960 }
Derek Lamberti4a9e24b2020-01-03 16:53:38 +0000961
962 OptimizationResult res = AttemptBackendAssignment(backendSettings,
963 optNetObjPtr->GetGraph(),
964 layer,
965 backend,
966 dataTypeIn,
967 dataTypeOut,
968 availablePreferredBackends,
969 reasonIfUnsupported,
970 errMessages);
971
972 if (res.IsOk())
973 {
974 found = true;
975 backendSettings.m_SelectedBackends.insert(backend);
976 break;
977 }
978 else if (res.IsError())
979 {
980 return res; // Cannot continue.
981 // Note: we don't need to log the error as it would already
982 // be logged in AttemptBackendAssignment().
983 }
984 else
985 {
Narumol Prangnawaratac2770a2020-04-01 16:51:23 +0100986 ARMNN_ASSERT_MSG(res.IsWarningOnly(), "OptimizationResult in unexpected state.");
Derek Lamberti4a9e24b2020-01-03 16:53:38 +0000987 }
telsoa01c577f2c2018-08-31 09:22:23 +0100988 }
989 }
990
991 // If the layer is unsupported by any devices, log and return a null network.
Matteo Martincigh49124022019-01-11 13:25:59 +0000992 if (!found)
993 {
telsoa01c577f2c2018-08-31 09:22:23 +0100994 // NOTE: if the layer is not an operation queue type AND we have not got CpuRef as a
995 // fallback we should set the compute device on the layer to CpuRef (these are not
996 // available as accelerated operations, or are only available under certain
997 // conditions, currently they comprise MemCopy, Constant, Permute)
998 armnn::LayerType layerType = layer->GetType();
Matteo Martincigh49124022019-01-11 13:25:59 +0000999 if (!backendSettings.IsCpuRefUsed() && (layerType == armnn::LayerType::MemCopy ||
1000 layerType == armnn::LayerType::Constant ||
1001 layerType == armnn::LayerType::Permute))
telsoa01c577f2c2018-08-31 09:22:23 +01001002 {
Matteo Martincigh49124022019-01-11 13:25:59 +00001003 BackendId cpuBackendId(armnn::Compute::CpuRef);
1004 layer->SetBackendId(cpuBackendId);
1005 backendSettings.m_SelectedBackends.insert(cpuBackendId);
telsoa01c577f2c2018-08-31 09:22:23 +01001006 }
1007 else
1008 {
Derek Lamberti4a9e24b2020-01-03 16:53:38 +00001009 return ReturnError(layer);
telsoa01c577f2c2018-08-31 09:22:23 +01001010 }
1011 }
1012 }
Matteo Martincigh49124022019-01-11 13:25:59 +00001013
1014 return result;
1015}
1016
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001017OptimizationResult AssignBackends(OptimizedNetworkImpl* optNetObjPtr,
Matteo Martincighadddddb2019-01-24 14:06:23 +00001018 BackendSettings& backendSettings,
Derek Lambertiff05cc52019-04-26 13:05:17 +01001019 SubgraphView& subgraph,
Matteo Martincighadddddb2019-01-24 14:06:23 +00001020 Optional<std::vector<std::string>&> errMessages)
Matteo Martincigh49124022019-01-11 13:25:59 +00001021{
Derek Lambertiff05cc52019-04-26 13:05:17 +01001022 Graph::Iterator firstLayer = subgraph.begin();
1023 Graph::Iterator lastLayer = subgraph.end();
Matteo Martincighadddddb2019-01-24 14:06:23 +00001024 return AssignBackends(optNetObjPtr,
1025 backendSettings,
1026 firstLayer,
1027 lastLayer,
1028 errMessages);
1029}
1030
Derek Lamberti84da38b2019-06-13 11:40:08 +01001031BackendsMap CreateSupportedBackends(TensorHandleFactoryRegistry& handleFactoryRegistry,
1032 BackendSettings& backendSettings)
1033{
1034 BackendsMap backends;
1035 auto const& backendRegistry = BackendRegistryInstance();
1036 for (auto&& selectedBackend : backendSettings.m_SupportedBackends)
1037 {
1038 auto backendFactory = backendRegistry.GetFactory(selectedBackend);
1039 auto backendObjPtr = backendFactory();
Narumol Prangnawaratac2770a2020-04-01 16:51:23 +01001040 ARMNN_ASSERT(backendObjPtr);
Derek Lamberti84da38b2019-06-13 11:40:08 +01001041
1042 backendObjPtr->RegisterTensorHandleFactories(handleFactoryRegistry);
1043
1044 backends[backendObjPtr->GetId()] = std::move(backendObjPtr);
1045 }
1046
1047 return backends;
1048}
1049
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001050OptimizationResult ApplyBackendOptimizations(OptimizedNetworkImpl* optNetObjPtr,
Matteo Martincighadddddb2019-01-24 14:06:23 +00001051 BackendSettings& backendSettings,
Derek Lamberti84da38b2019-06-13 11:40:08 +01001052 BackendsMap& backends,
Mike Kelly07810fc2020-11-12 10:58:48 +00001053 const ModelOptions& modelOptions,
Matteo Martincighadddddb2019-01-24 14:06:23 +00001054 Optional<std::vector<std::string>&> errMessages)
1055{
Narumol Prangnawaratac2770a2020-04-01 16:51:23 +01001056 ARMNN_ASSERT(optNetObjPtr);
Matteo Martincigh49124022019-01-11 13:25:59 +00001057
1058 OptimizationResult result;
1059
Matteo Martincighadddddb2019-01-24 14:06:23 +00001060 // Get the optimized graph
1061 Graph& optGraph = optNetObjPtr->GetGraph();
Matteo Martincigh49124022019-01-11 13:25:59 +00001062
Matteo Martincighadddddb2019-01-24 14:06:23 +00001063 // Run backend specific optimizations
Matteo Martincighadddddb2019-01-24 14:06:23 +00001064 for (auto&& selectedBackend : backendSettings.m_SelectedBackends)
Matteo Martincigh49124022019-01-11 13:25:59 +00001065 {
Derek Lamberti84da38b2019-06-13 11:40:08 +01001066 auto backendObjPtr = backends.find(selectedBackend)->second.get();
Narumol Prangnawaratac2770a2020-04-01 16:51:23 +01001067 ARMNN_ASSERT(backendObjPtr);
Matteo Martincighadddddb2019-01-24 14:06:23 +00001068
1069 // Select sub-graphs based on backend
Derek Lambertiff05cc52019-04-26 13:05:17 +01001070 SubgraphViewSelector::Subgraphs subgraphs =
Rob Hughes65c32262019-07-23 15:33:39 +01001071 SubgraphViewSelector::SelectSubgraphs(optGraph,
Matteo Martincigh602af092019-05-01 10:31:27 +01001072 // Select layers assigned to the requested backend
1073 [&backendObjPtr](const Layer& layer)
1074 {
1075 return layer.GetType() != LayerType::Input &&
1076 layer.GetType() != LayerType::Output &&
1077 layer.GetBackendId() == backendObjPtr->GetId();
1078 });
Derek Lambertiff05cc52019-04-26 13:05:17 +01001079 if (subgraphs.empty())
Matteo Martincigh49124022019-01-11 13:25:59 +00001080 {
Matteo Martincighadddddb2019-01-24 14:06:23 +00001081 // No sub-graphs found, try with next selected backend
1082 continue;
Matteo Martincigh49124022019-01-11 13:25:59 +00001083 }
Matteo Martincighadddddb2019-01-24 14:06:23 +00001084
1085 // Try to optimize each sub-graph
Derek Lambertiff05cc52019-04-26 13:05:17 +01001086 for (auto& subgraph : subgraphs)
Matteo Martincigh49124022019-01-11 13:25:59 +00001087 {
Matteo Martincighadddddb2019-01-24 14:06:23 +00001088 // Try to optimize the current sub-graph
Mike Kelly07810fc2020-11-12 10:58:48 +00001089 OptimizationViews optimizationViews = backendObjPtr->OptimizeSubgraphView(*subgraph, modelOptions);
Narumol Prangnawaratac2770a2020-04-01 16:51:23 +01001090 ARMNN_ASSERT(optimizationViews.Validate(*subgraph));
Matteo Martincighadddddb2019-01-24 14:06:23 +00001091
1092 // Optimization attempted, check the resulting optimized sub-graph
Matteo Martincigh84924332019-05-09 12:46:16 +01001093 for (auto& substitution : optimizationViews.GetSubstitutions())
Matteo Martincighadddddb2019-01-24 14:06:23 +00001094 {
1095 // Sub-graph optimized, substitute the sub-graph with the new optimized one in the main optimized graph
Matteo Martincigh84924332019-05-09 12:46:16 +01001096 SubgraphView& replacementSubgraph = substitution.m_ReplacementSubgraph;
1097 SubgraphView& substitutableSubgraph = substitution.m_SubstitutableSubgraph;
1098 optGraph.SubstituteSubgraph(substitutableSubgraph, replacementSubgraph);
Matteo Martincighadddddb2019-01-24 14:06:23 +00001099
1100 // Assign the current backend to the optimized sub-graph
Matteo Martincigh84924332019-05-09 12:46:16 +01001101 std::for_each(replacementSubgraph.begin(), replacementSubgraph.end(), [&selectedBackend](Layer* l)
Derek Lambertic2fe5fb2019-05-08 10:23:08 +01001102 {
Narumol Prangnawaratac2770a2020-04-01 16:51:23 +01001103 ARMNN_ASSERT(l);
Derek Lambertic2fe5fb2019-05-08 10:23:08 +01001104 l->SetBackendId(selectedBackend);
1105 });
Matteo Martincighadddddb2019-01-24 14:06:23 +00001106 }
Derek Lambertic2fe5fb2019-05-08 10:23:08 +01001107
Matteo Martincigh84924332019-05-09 12:46:16 +01001108 if (!optimizationViews.GetFailedSubgraphs().empty())
Matteo Martincighadddddb2019-01-24 14:06:23 +00001109 {
Matteo Martincighadddddb2019-01-24 14:06:23 +00001110 std::stringstream warningMsg;
Derek Lambertic2fe5fb2019-05-08 10:23:08 +01001111 warningMsg << "Some sub-graph(s) failed to optimized on " << backendObjPtr->GetId() << " backend.";
Matteo Martincighadddddb2019-01-24 14:06:23 +00001112 ReportWarning(warningMsg.str(), errMessages);
1113
1114 // Failed to optimize the given sub-graph, re-assign the sub-graph layers to other available backends
Derek Lambertic2fe5fb2019-05-08 10:23:08 +01001115 BackendSettings settingsCopy(backendSettings);
Matteo Martincighadddddb2019-01-24 14:06:23 +00001116 if (!backendObjPtr->GetId().IsCpuRef())
1117 {
1118 // Add the current backend to the list of backends to ignore
Derek Lambertic2fe5fb2019-05-08 10:23:08 +01001119 settingsCopy.m_IgnoredBackends.insert(backendObjPtr->GetId());
Matteo Martincighadddddb2019-01-24 14:06:23 +00001120 }
Derek Lambertic2fe5fb2019-05-08 10:23:08 +01001121
1122 int count=0;
Matteo Martincigh84924332019-05-09 12:46:16 +01001123 for (auto& failedSubgraph : optimizationViews.GetFailedSubgraphs())
Matteo Martincighadddddb2019-01-24 14:06:23 +00001124 {
Derek Lambertic2fe5fb2019-05-08 10:23:08 +01001125 // An error occurred: the optimization was attempted but not performed, try different backends
1126 std::stringstream subgraphMsg;
1127 subgraphMsg << "Re-assigning backends to " << failedSubgraph.GetLayers().size()
1128 << " layers inside sub-graph " << count++;
Matteo Martincigh328d92b2019-07-04 17:52:55 +01001129 ReportWarning(subgraphMsg.str(), errMessages);
Derek Lambertic2fe5fb2019-05-08 10:23:08 +01001130
1131 OptimizationResult reassignmentResult = AssignBackends(optNetObjPtr,
1132 settingsCopy,
1133 *subgraph,
1134 errMessages);
1135 if (reassignmentResult.m_Error)
1136 {
1137 // Failed to re-assign one of the remaining backends to each layer of the sub-graph
1138 result.m_Error = true;
1139 return result;
1140 }
Matteo Martincighadddddb2019-01-24 14:06:23 +00001141 }
Matteo Martincigh49124022019-01-11 13:25:59 +00001142 }
1143 }
1144 }
1145
1146 return result;
1147}
1148
Derek Lamberti84da38b2019-06-13 11:40:08 +01001149bool RequiresCopy(ITensorHandleFactory::FactoryId src,
1150 ITensorHandleFactory::FactoryId dst,
1151 TensorHandleFactoryRegistry& registry)
1152{
1153 if (src != dst)
1154 {
1155 ITensorHandleFactory* srcFactory = registry.GetFactory(src);
1156 ITensorHandleFactory* dstFactory = registry.GetFactory(dst);
1157
Matteo Martincigha6539ed2019-08-27 13:43:32 +01001158 if (srcFactory && dstFactory &&
1159 (srcFactory->GetExportFlags() & dstFactory->GetImportFlags()) != 0)
Derek Lamberti84da38b2019-06-13 11:40:08 +01001160 {
1161 return false;
1162 }
1163 return true;
1164 }
1165 return false;
1166}
1167
1168// Find the handle factory for the input layer which results in fewest required copies.
1169ITensorHandleFactory::FactoryId CalculateSlotOptionForInput(BackendsMap& backends,
1170 OutputSlot& slot,
Narumol Prangnawarate5f0b242021-05-07 17:52:36 +01001171 TensorHandleFactoryRegistry& registry,
1172 bool importEnabled)
Derek Lamberti84da38b2019-06-13 11:40:08 +01001173{
1174 Layer& layer = slot.GetOwningLayer();
Narumol Prangnawaratac2770a2020-04-01 16:51:23 +01001175 ARMNN_ASSERT(layer.GetType() == LayerType::Input);
Derek Lamberti84da38b2019-06-13 11:40:08 +01001176
1177 // Explicitly select the tensorhandle factory for InputLayer because the rules for it are slightly different. It
1178 // doesn't matter which backend it is assigned to because they all use the same implementation, which
1179 // requires Map/Unmap support. This means that, so long as the handle type supports map/unmap semantics, we can
1180 // select a factory with maximum compatibility with the layers connected to the InputLayer.
1181
1182 // First ensure the from backends can support the TensorHandeAPI
1183 auto frmBackend = backends.find(layer.GetBackendId());
1184 if (frmBackend == backends.end() ||
1185 !frmBackend->second->SupportsTensorAllocatorAPI())
1186 {
1187 return ITensorHandleFactory::LegacyFactoryId;
1188 }
1189
1190 // Go through all connections to the output slot and determine the TensorHandleFactory which results in the
1191 // fewest copies.
1192 std::map<ITensorHandleFactory::FactoryId, int> factoryScores;
1193 int topScore = 0;
1194 ITensorHandleFactory::FactoryId topChoice = ITensorHandleFactory::LegacyFactoryId;
1195
1196 for (auto&& connection : slot.GetConnections())
1197 {
Narumol Prangnawarate5f0b242021-05-07 17:52:36 +01001198
Derek Lamberti84da38b2019-06-13 11:40:08 +01001199 const Layer& connectedLayer = connection->GetOwningLayer();
1200
1201 auto toBackend = backends.find(connectedLayer.GetBackendId());
Narumol Prangnawaratac2770a2020-04-01 16:51:23 +01001202 ARMNN_ASSERT_MSG(toBackend != backends.end(), "Backend id not found for the connected layer");
Derek Lamberti84da38b2019-06-13 11:40:08 +01001203
1204 if (!toBackend->second.get()->SupportsTensorAllocatorAPI())
1205 {
1206 // The destination backend does not support the tensor allocator API, move to the next one
1207 continue;
1208 }
1209
1210 auto dstPrefs = toBackend->second.get()->GetHandleFactoryPreferences();
1211 for (auto&& dst : dstPrefs)
1212 {
Derek Lambertif674aa02019-08-01 15:56:25 +01001213 // Input layers use the mem copy workload or import, so the selected factory must
1214 // support either the map/unmap API or Import API
Derek Lamberti84da38b2019-06-13 11:40:08 +01001215 ITensorHandleFactory* factory = registry.GetFactory(dst);
Narumol Prangnawarate5f0b242021-05-07 17:52:36 +01001216 if (importEnabled && factory->GetImportFlags() == 0)
Derek Lamberti84da38b2019-06-13 11:40:08 +01001217 {
Narumol Prangnawarate5f0b242021-05-07 17:52:36 +01001218 continue;
1219 }
1220 else if (!importEnabled && !factory->SupportsMapUnmap())
1221 {
Derek Lamberti84da38b2019-06-13 11:40:08 +01001222 continue;
1223 }
1224
1225 auto it = factoryScores.find(dst);
1226 if (it == factoryScores.end())
1227 {
1228 // Add new score to the table
1229 factoryScores[dst] = 0;
1230 if (topChoice == ITensorHandleFactory::LegacyFactoryId)
1231 {
1232 topChoice = dst;
1233 }
1234 }
1235 else
1236 {
1237 // Increase the score
1238 factoryScores[dst]++;
1239
1240 // Track the best option
1241 if (factoryScores[dst] > topScore)
1242 {
1243 topScore = factoryScores[dst];
1244 topChoice = dst;
1245 }
1246 }
1247 }
1248 }
1249
1250 return topChoice;
1251}
1252
1253// Find the handle factory for the output layer which results in fewest required copies.
1254ITensorHandleFactory::FactoryId CalculateSlotOptionForOutput(BackendsMap& backends,
1255 OutputSlot& slot,
1256 TensorHandleFactoryRegistry& registry)
1257{
Jan Eilers8eb25602020-03-09 12:13:48 +00001258 IgnoreUnused(backends, slot, registry);
Derek Lamberti94a88d22019-12-10 21:12:59 +00001259 return ITensorHandleFactory::DeferredFactoryId;
Derek Lamberti84da38b2019-06-13 11:40:08 +01001260}
1261
1262// For all handle factories supported on the source backend, we wish to find the one which requires the fewest copies
1263// when considering all connections.
1264ITensorHandleFactory::FactoryId CalculateSlotOption(BackendsMap& backends,
1265 OutputSlot& outputSlot,
Narumol Prangnawarate5f0b242021-05-07 17:52:36 +01001266 TensorHandleFactoryRegistry& registry,
1267 bool importEnabled)
Derek Lamberti84da38b2019-06-13 11:40:08 +01001268{
1269 // First ensure the from backends can support the TensorHandeAPI
1270 Layer& layer = outputSlot.GetOwningLayer();
1271 auto frmBackend = backends.find(layer.GetBackendId());
1272 if (frmBackend == backends.end() ||
1273 !frmBackend->second->SupportsTensorAllocatorAPI())
1274 {
1275 return ITensorHandleFactory::LegacyFactoryId;
1276 }
1277
Narumol Prangnawarate5f0b242021-05-07 17:52:36 +01001278 bool outputConnection = false;
Derek Lamberti84da38b2019-06-13 11:40:08 +01001279 for (auto&& connection : outputSlot.GetConnections())
1280 {
1281 const Layer& connectedLayer = connection->GetOwningLayer();
1282 if (connectedLayer.GetType() == LayerType::Output)
1283 {
Narumol Prangnawarate5f0b242021-05-07 17:52:36 +01001284 outputConnection = true;
Derek Lamberti84da38b2019-06-13 11:40:08 +01001285 }
1286 }
1287
1288 IBackendInternal* srcBackend = frmBackend->second.get();
1289 auto srcPrefs = srcBackend->GetHandleFactoryPreferences();
1290
1291 // Initialize the scores
1292 std::map<ITensorHandleFactory::FactoryId, int> factoryScores;
1293 for (auto&& pref : srcPrefs)
1294 {
Narumol Prangnawarate5f0b242021-05-07 17:52:36 +01001295 if (importEnabled)
Derek Lamberti84da38b2019-06-13 11:40:08 +01001296 {
1297 ITensorHandleFactory* factory = registry.GetFactory(pref);
Narumol Prangnawarate5f0b242021-05-07 17:52:36 +01001298 if (outputConnection)
1299 {
1300 // Check if this is fallback case
1301 bool fallbackConnection = false;
1302 for (auto&& inputSlot : layer.GetInputSlots())
1303 {
1304 if (inputSlot.GetConnectedOutputSlot()->GetOwningLayer().GetBackendId() != layer.GetBackendId())
1305 {
1306 fallbackConnection = true;
1307 }
1308 }
1309 if (fallbackConnection)
1310 {
1311 auto factoryCap = factory->GetCapabilities(&layer, &layer, CapabilityClass::FallbackImportDisabled);
1312 // Cannot use factory import if fallback import is not supported.
1313 if (!factoryCap.empty())
1314 {
1315 continue;
1316 }
1317 }
1318 else if (factory->GetExportFlags() == 0)
1319 {
1320 continue;
1321 }
1322 }
1323 if (!outputConnection)
1324 {
1325 auto factoryCap = factory->GetCapabilities(&layer, &layer, CapabilityClass::FallbackImportDisabled);
1326 // Cannot use factory import if fallback import is not supported.
1327 if (!factoryCap.empty())
1328 {
1329 continue;
1330 }
1331 }
1332
1333 }
1334 else
1335 {
1336 // Only consider factories that support map/unmap
1337 ITensorHandleFactory* factory = registry.GetFactory(pref);
Derek Lamberti84da38b2019-06-13 11:40:08 +01001338 if (!factory->SupportsMapUnmap())
1339 {
1340 // The current tensor handle factory does not support the map/unmap strategy, move to the next one
1341 continue;
1342 }
1343 }
1344
Narumol Prangnawarate5f0b242021-05-07 17:52:36 +01001345
Derek Lamberti84da38b2019-06-13 11:40:08 +01001346 auto it = factoryScores.find(pref);
1347 if (it == factoryScores.end())
1348 {
1349 // Add new score to the table
1350 factoryScores[pref] = 0;
1351 }
1352 }
1353
1354 // Score each handle factory based on how many times it requires copies on the slot connections
1355 for (auto&& connection : outputSlot.GetConnections())
1356 {
1357 const Layer& connectedLayer = connection->GetOwningLayer();
1358
1359 auto toBackend = backends.find(connectedLayer.GetBackendId());
Narumol Prangnawaratac2770a2020-04-01 16:51:23 +01001360 ARMNN_ASSERT_MSG(toBackend != backends.end(), "Backend id not found for the connected layer");
Derek Lamberti84da38b2019-06-13 11:40:08 +01001361
1362 auto dstPrefs = toBackend->second.get()->GetHandleFactoryPreferences();
1363 for (auto&& src : srcPrefs)
1364 {
1365 if (factoryScores.find(src) == factoryScores.end()) // Don't consider excluded factories
1366 {
1367 continue;
1368 }
1369
1370 for (auto&& dst : dstPrefs)
1371 {
1372 if (RequiresCopy(src, dst, registry))
1373 {
1374 // Copy avoided, increase the score
1375 factoryScores[src]++;
1376 break;
1377 }
1378 }
1379 }
1380 }
1381
1382 // Find the lowest score
1383 int minScore = std::numeric_limits<int>::max();
1384 for (auto it : factoryScores)
1385 {
1386 minScore = std::min(minScore, it.second);
1387 }
1388
1389 // Collect factories matching the best(lowest) score
1390 std::vector<ITensorHandleFactory::FactoryId> optimalFactories;
1391 for (auto it : factoryScores)
1392 {
1393 if (it.second == minScore)
1394 {
1395 optimalFactories.push_back(it.first);
1396 }
1397 }
1398
1399 // For all compatible Factories matching the best score, find the preferred one for the current layer.
1400 for (auto&& srcPref : srcPrefs)
1401 {
1402 for (auto&& comp : optimalFactories)
1403 {
1404 if (comp == srcPref)
1405 {
1406 return comp;
1407 }
1408 }
1409 }
1410
1411 return ITensorHandleFactory::LegacyFactoryId;
1412}
1413
Derek Lambertif674aa02019-08-01 15:56:25 +01001414EdgeStrategy CalculateEdgeStrategy(BackendsMap& backends,
1415 ITensorHandleFactory::FactoryId srcFactoryId,
1416 const Layer& layer,
1417 const Layer& connectedLayer,
Narumol Prangnawarata2493a02020-08-19 14:39:07 +01001418 TensorHandleFactoryRegistry& registry,
1419 bool importEnabled)
Derek Lamberti84da38b2019-06-13 11:40:08 +01001420{
1421 auto toBackend = backends.find(connectedLayer.GetBackendId());
Narumol Prangnawaratac2770a2020-04-01 16:51:23 +01001422 ARMNN_ASSERT_MSG(toBackend != backends.end(), "Backend id not found for the connected layer");
Derek Lamberti84da38b2019-06-13 11:40:08 +01001423
1424 auto dstPrefs = toBackend->second.get()->GetHandleFactoryPreferences();
1425
1426 // Legacy API check for backward compatibility
1427 if (srcFactoryId == ITensorHandleFactory::LegacyFactoryId || dstPrefs.empty())
1428 {
1429 if (layer.GetBackendId() != connectedLayer.GetBackendId())
1430 {
Derek Lambertif674aa02019-08-01 15:56:25 +01001431 return EdgeStrategy::CopyToTarget;
Derek Lamberti84da38b2019-06-13 11:40:08 +01001432 }
1433 else
1434 {
Derek Lambertif674aa02019-08-01 15:56:25 +01001435 return EdgeStrategy::DirectCompatibility;
Derek Lamberti84da38b2019-06-13 11:40:08 +01001436 }
1437 }
1438
1439 // TensorHandleFactory API present, so perform more sophisticated strategies.
Derek Lambertif674aa02019-08-01 15:56:25 +01001440 // Dst Output layers don't require copy because they use import or map/unmap
Derek Lamberti84da38b2019-06-13 11:40:08 +01001441 if (connectedLayer.GetType() == LayerType::Output)
1442 {
Derek Lambertif674aa02019-08-01 15:56:25 +01001443 return EdgeStrategy::DirectCompatibility;
Derek Lamberti84da38b2019-06-13 11:40:08 +01001444 }
1445
1446 // Search for direct match in prefs
1447 for (auto&& pref : dstPrefs)
1448 {
1449 if (pref == srcFactoryId)
1450 {
Derek Lambertif674aa02019-08-01 15:56:25 +01001451 return EdgeStrategy::DirectCompatibility;
Derek Lamberti84da38b2019-06-13 11:40:08 +01001452 }
1453 }
1454
1455 // Search for export/import options
1456 ITensorHandleFactory* srcFactory = registry.GetFactory(srcFactoryId);
Narumol Prangnawarata2493a02020-08-19 14:39:07 +01001457 if (srcFactory->GetExportFlags() != 0 && importEnabled)
Derek Lamberti84da38b2019-06-13 11:40:08 +01001458 {
1459 for (auto&& pref : dstPrefs)
1460 {
1461 ITensorHandleFactory* dstFactory = registry.GetFactory(pref);
James Conroyffab16f2019-11-07 14:37:09 +00001462
James Conroy47e863d2019-11-18 17:07:43 +00001463 // Handles cases when a destPref is not listed in TensorHandleFactoryRegistry
James Conroyffab16f2019-11-07 14:37:09 +00001464 if (!dstFactory) {
James Conroy47e863d2019-11-18 17:07:43 +00001465 continue;
James Conroyffab16f2019-11-07 14:37:09 +00001466 }
Derek Lambertif674aa02019-08-01 15:56:25 +01001467 if ((dstFactory->GetImportFlags() & srcFactory->GetExportFlags()) != 0)
Derek Lamberti84da38b2019-06-13 11:40:08 +01001468 {
Narumol Prangnawaratb8d771a2020-08-14 11:51:12 +01001469 auto srcCapability = srcFactory->GetCapabilities(&layer, &layer, CapabilityClass::PaddingRequired);
1470 auto dstCapability = dstFactory->GetCapabilities(&connectedLayer,
1471 &connectedLayer,
1472 CapabilityClass::PaddingRequired);
Narumol Prangnawarate5f0b242021-05-07 17:52:36 +01001473 auto srcFallback = srcFactory->GetCapabilities(&layer, &layer, CapabilityClass::FallbackImportDisabled);
1474 auto dstFallback = dstFactory->GetCapabilities(&connectedLayer,
1475 &connectedLayer,
1476 CapabilityClass::FallbackImportDisabled);
Narumol Prangnawaratb8d771a2020-08-14 11:51:12 +01001477 // Do not require memory copy if the source and destination do not require padding.
Narumol Prangnawarate5f0b242021-05-07 17:52:36 +01001478 if (srcCapability.empty() && dstCapability.empty() && srcFallback.empty() && dstFallback.empty())
Narumol Prangnawaratb8d771a2020-08-14 11:51:12 +01001479 {
1480 return EdgeStrategy::ExportToTarget;
1481 }
Derek Lamberti84da38b2019-06-13 11:40:08 +01001482 }
1483 }
1484 }
1485
1486 // Search for copy options via map/unmap
1487 if (srcFactory->SupportsMapUnmap())
1488 {
1489 for (auto&& pref : dstPrefs)
1490 {
1491 ITensorHandleFactory* dstFactory = registry.GetFactory(pref);
James Conroy47e863d2019-11-18 17:07:43 +00001492 if (dstFactory && dstFactory->SupportsMapUnmap())
Derek Lamberti84da38b2019-06-13 11:40:08 +01001493 {
Derek Lambertif674aa02019-08-01 15:56:25 +01001494 return EdgeStrategy::CopyToTarget;
Derek Lamberti84da38b2019-06-13 11:40:08 +01001495 }
1496 }
1497 }
1498
Derek Lambertif674aa02019-08-01 15:56:25 +01001499 return EdgeStrategy::Undefined;
Derek Lamberti84da38b2019-06-13 11:40:08 +01001500}
1501
1502// Select the TensorHandleFactories and the corresponding memory strategy
1503OptimizationResult SelectTensorHandleStrategy(Graph& optGraph,
1504 BackendsMap& backends,
1505 TensorHandleFactoryRegistry& registry,
Narumol Prangnawarata2493a02020-08-19 14:39:07 +01001506 bool importEnabled,
Derek Lamberti84da38b2019-06-13 11:40:08 +01001507 Optional<std::vector<std::string>&> errMessages)
1508{
1509 OptimizationResult result;
1510
Narumol Prangnawarata2493a02020-08-19 14:39:07 +01001511 optGraph.ForEachLayer([&backends, &registry, &result, &errMessages, importEnabled](Layer* layer)
Derek Lamberti84da38b2019-06-13 11:40:08 +01001512 {
Narumol Prangnawaratac2770a2020-04-01 16:51:23 +01001513 ARMNN_ASSERT(layer);
Derek Lamberti84da38b2019-06-13 11:40:08 +01001514
1515 // Lets make sure the backend is in our list of supported backends. Something went wrong during backend
1516 // assignment if this check fails
Narumol Prangnawaratac2770a2020-04-01 16:51:23 +01001517 ARMNN_ASSERT(backends.find(layer->GetBackendId()) != backends.end());
Derek Lamberti84da38b2019-06-13 11:40:08 +01001518
1519 // Check each output separately
1520 for (unsigned int slotIdx = 0; slotIdx < layer->GetNumOutputSlots(); slotIdx++)
1521 {
1522 OutputSlot& outputSlot = layer->GetOutputSlot(slotIdx);
1523
1524 ITensorHandleFactory::FactoryId slotOption = ITensorHandleFactory::LegacyFactoryId;
1525
1526 // Calculate the factory to use which results in the fewest copies being made.
1527 switch(layer->GetType())
1528 {
1529 case LayerType::Input:
Narumol Prangnawarate5f0b242021-05-07 17:52:36 +01001530 slotOption = CalculateSlotOptionForInput(backends, outputSlot, registry, importEnabled);
Derek Lamberti84da38b2019-06-13 11:40:08 +01001531 break;
1532 case LayerType::Output:
1533 slotOption = CalculateSlotOptionForOutput(backends, outputSlot, registry);
1534 break;
1535 default:
Narumol Prangnawarate5f0b242021-05-07 17:52:36 +01001536 slotOption = CalculateSlotOption(backends, outputSlot, registry, importEnabled);
Derek Lamberti84da38b2019-06-13 11:40:08 +01001537 break;
1538 }
1539 outputSlot.SetTensorHandleFactory(slotOption);
1540
Derek Lambertif674aa02019-08-01 15:56:25 +01001541 // Now determine the "best" edge strategy for each connection given the slotOption.
Derek Lamberti84da38b2019-06-13 11:40:08 +01001542 unsigned int connectionIdx = 0;
1543 for (auto&& connection : outputSlot.GetConnections())
1544 {
1545 const Layer& connectedLayer = connection->GetOwningLayer();
1546
Narumol Prangnawarata2493a02020-08-19 14:39:07 +01001547 EdgeStrategy strategy = CalculateEdgeStrategy(backends, slotOption, *layer, connectedLayer,
1548 registry, importEnabled);
Derek Lamberti84da38b2019-06-13 11:40:08 +01001549
Derek Lambertif674aa02019-08-01 15:56:25 +01001550 if (strategy == EdgeStrategy::Undefined)
Derek Lamberti84da38b2019-06-13 11:40:08 +01001551 {
1552 result.m_Error = true;
1553 if (errMessages)
1554 {
1555 errMessages.value().emplace_back("Could not find valid strategy required for compatibility"
1556 " between backends.");
1557 }
1558 return;
1559 }
1560
Derek Lambertif674aa02019-08-01 15:56:25 +01001561 outputSlot.SetEdgeStrategy(connectionIdx, strategy);
Derek Lamberti84da38b2019-06-13 11:40:08 +01001562
1563 connectionIdx++;
1564 }
1565 }
1566 });
1567
1568 return result;
1569}
1570
Matteo Martincigh49124022019-01-11 13:25:59 +00001571IOptimizedNetworkPtr Optimize(const INetwork& inNetwork,
1572 const std::vector<BackendId>& backendPreferences,
1573 const IDeviceSpec& deviceSpec,
1574 const OptimizerOptions& options,
Rob Hughes23214432019-11-05 11:27:36 +00001575 Optional<std::vector<std::string>&> messages)
Matteo Martincigh49124022019-01-11 13:25:59 +00001576{
1577 if (backendPreferences.empty())
1578 {
Mike Kelly3a613cc2020-09-29 20:50:35 +01001579 throw InvalidArgumentException("Invoked Optimize with no backends specified");
Matteo Martincigh49124022019-01-11 13:25:59 +00001580 }
1581
Narumol Prangnawaratbc7ffb52020-03-20 15:01:01 +00001582 if (options.m_ReduceFp32ToFp16 && options.m_ReduceFp32ToBf16)
1583 {
1584 throw InvalidArgumentException("BFloat16 and Float16 optimization cannot be enabled at the same time.");
1585 }
1586
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001587 std::unique_ptr<Graph> graph = std::make_unique<Graph>(inNetwork.pNetworkImpl->GetGraph());
Matteo Martincigh49124022019-01-11 13:25:59 +00001588
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001589 auto optNet = IOptimizedNetworkPtr(new IOptimizedNetwork(std::move(graph), options.m_ModelOptions),
Sadik Armagan045f6be2020-09-10 13:37:32 +01001590 &IOptimizedNetwork::Destroy);
Matteo Martincigh49124022019-01-11 13:25:59 +00001591
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001592 IOptimizedNetwork* optNetObjPtr = optNet.get();
Matteo Martincigh49124022019-01-11 13:25:59 +00001593
Matteo Martincighadddddb2019-01-24 14:06:23 +00001594 // Get the optimized graph
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001595 Graph& optGraph = optNetObjPtr->pOptimizedNetworkImpl->GetGraph();
Matteo Martincighadddddb2019-01-24 14:06:23 +00001596
Finn Williams84e025a2021-08-05 17:29:32 +01001597 // Infer the tensor infos for all output slots. Throws an exception on failure
1598 optGraph.InferTensorInfos();
1599
Narumol Prangnawarat16f82f92020-09-14 16:12:44 +01001600 // Perform AddBroadcastReshapeLayer optimisation
1601 using namespace optimizations;
1602 Optimizer::Pass(optGraph, MakeOptimizations(AddBroadcastReshapeLayer()));
1603
Matteo Martincigh49124022019-01-11 13:25:59 +00001604 // Perform optimisation passes
Matteo Martincighadddddb2019-01-24 14:06:23 +00001605 Optimizer::Pass(optGraph, MakeOptimizations(SquashEqualPermuteSiblings(),
Mike Kelly490b7be2020-03-03 12:39:09 +00001606 SquashEqualTransposeSiblings(),
Matteo Martincighadddddb2019-01-24 14:06:23 +00001607 SquashEqualReshapeSiblings(),
1608 OptimizeInversePermutes(),
Mike Kelly490b7be2020-03-03 12:39:09 +00001609 OptimizeInverseTransposes(),
Matteo Martincighadddddb2019-01-24 14:06:23 +00001610 MovePermuteUp(),
Mike Kelly490b7be2020-03-03 12:39:09 +00001611 MoveTransposeUp(),
Matteo Martincighadddddb2019-01-24 14:06:23 +00001612 PermuteAsReshape(),
Mike Kelly490b7be2020-03-03 12:39:09 +00001613 TransposeAsReshape(),
Nina Drozd861985f2019-04-18 14:48:51 +01001614 OptimizeConsecutiveReshapes(),
Matthew Sloyan33f89872021-06-30 10:20:17 +01001615 RedirectMembersToConstantInputs(),
Rob Hughes3a7d3a72019-09-24 16:59:56 +01001616 FoldPadIntoConvolution2d(),
Teresa Charlin5786eb72021-05-21 16:29:45 +01001617 FoldPadIntoDepthwiseConvolution2d(),
Diego Lopez Recasfe95d722021-03-19 12:40:16 +00001618 FoldPadIntoPooling2d(),
Mike Kelly490b7be2020-03-03 12:39:09 +00001619 PermuteAndBatchToSpaceAsDepthToSpace(),
Teresa Charlin06e03002020-10-15 13:16:07 +01001620 TransposeAndBatchToSpaceAsDepthToSpace(),
Mike Kelly90231b82020-11-05 15:44:56 +00001621 FuseBatchNormIntoConvolution2DFloat32(),
1622 FuseBatchNormIntoConvolution2DFloat16(),
1623 FuseBatchNormIntoDepthwiseConvolution2DFloat32(),
1624 FuseBatchNormIntoDepthwiseConvolution2DFloat16()));
Matteo Martincigh49124022019-01-11 13:25:59 +00001625
Matteo Martincigh49124022019-01-11 13:25:59 +00001626 // If Fp32 to Fp16 optimization is set convert Fp32 network to Fp16
1627 if (options.m_ReduceFp32ToFp16)
1628 {
Matteo Martincighadddddb2019-01-24 14:06:23 +00001629 Optimizer::Pass(optGraph, MakeOptimizations(Fp32NetworkToFp16Converter()));
Derek Lambertidd6804b2019-11-27 09:29:57 +00001630 Optimizer::Pass(optGraph, MakeOptimizations(ConvertConstantsFloatToHalf()));
Matteo Martincigh49124022019-01-11 13:25:59 +00001631 }
1632
Narumol Prangnawaratbc7ffb52020-03-20 15:01:01 +00001633 // If Fp32 to Bf16 optimization is set convert Fp32 network to Bf16
Narumol Prangnawarat57ef0082020-03-26 09:20:43 +00001634 // Convert input of Convolution2d and FullyConnected from Fp32 to Bf16
1635 // Only Constant weight of Convolution2d and FullyConnected are converted from Fp32 to Bf16
Narumol Prangnawaratbc7ffb52020-03-20 15:01:01 +00001636 if (options.m_ReduceFp32ToBf16)
1637 {
1638 Optimizer::Pass(optGraph, MakeOptimizations(Fp32NetworkToBf16Converter()));
Narumol Prangnawaratbc7ffb52020-03-20 15:01:01 +00001639 }
1640
Matteo Martincigh49124022019-01-11 13:25:59 +00001641 // Initialize backend settings
1642 BackendSettings backendSettings(backendPreferences, deviceSpec);
1643 if (backendSettings.GetAvailablePreferredBackends().empty())
1644 {
1645 std::stringstream failureMsg;
1646 failureMsg << "None of the preferred backends " << backendPreferences
1647 << " are supported. Current platform provides " << backendSettings.m_SupportedBackends;
Rob Hughes23214432019-11-05 11:27:36 +00001648 ReportError(failureMsg.str(), messages);
Mike Kelly3a613cc2020-09-29 20:50:35 +01001649 throw InvalidArgumentException(failureMsg.str());
Matteo Martincigh49124022019-01-11 13:25:59 +00001650 }
1651
Derek Lamberti84da38b2019-06-13 11:40:08 +01001652 // Create a map to temporarily hold initialized backend objects
1653 TensorHandleFactoryRegistry tensorHandleFactoryRegistry;
1654 BackendsMap backends = CreateSupportedBackends(tensorHandleFactoryRegistry, backendSettings);
1655
Matteo Martincigh49124022019-01-11 13:25:59 +00001656 // Assign an available backend to each layer
Matteo Martincighadddddb2019-01-24 14:06:23 +00001657 Graph::Iterator firstLayer = optGraph.begin();
1658 Graph::Iterator lastLayer = optGraph.end();
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001659 OptimizationResult assignBackendsResult = AssignBackends(optNetObjPtr->pOptimizedNetworkImpl.get(),
Derek Lamberti84da38b2019-06-13 11:40:08 +01001660 backendSettings,
1661 firstLayer,
1662 lastLayer,
Rob Hughes23214432019-11-05 11:27:36 +00001663 messages);
Derek Lamberti84da38b2019-06-13 11:40:08 +01001664 if (assignBackendsResult.m_Error)
Matteo Martincigh49124022019-01-11 13:25:59 +00001665 {
1666 // Failed to assign a backend to each layer
Mike Kelly3a613cc2020-09-29 20:50:35 +01001667 throw InvalidArgumentException("Failed to assign a backend to each layer");
jimfly016b0b53d2018-10-08 14:43:01 +01001668 }
telsoa01c577f2c2018-08-31 09:22:23 +01001669
Matteo Martincighadddddb2019-01-24 14:06:23 +00001670 Optimizer::Pass(optGraph, MakeOptimizations(OptimizeInverseConversionsFp16(),
1671 OptimizeInverseConversionsFp32()));
telsoa01c577f2c2018-08-31 09:22:23 +01001672
Matteo Martincighadddddb2019-01-24 14:06:23 +00001673 // Apply the backend-specific optimizations
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001674 OptimizationResult backendOptimizationResult = ApplyBackendOptimizations(optNetObjPtr->pOptimizedNetworkImpl.get(),
Matteo Martincighadddddb2019-01-24 14:06:23 +00001675 backendSettings,
Derek Lamberti84da38b2019-06-13 11:40:08 +01001676 backends,
Mike Kelly07810fc2020-11-12 10:58:48 +00001677 options.m_ModelOptions,
Rob Hughes23214432019-11-05 11:27:36 +00001678 messages);
Matteo Martincighadddddb2019-01-24 14:06:23 +00001679 if (backendOptimizationResult.m_Error)
Matteo Martincigh49124022019-01-11 13:25:59 +00001680 {
Matteo Martincighadddddb2019-01-24 14:06:23 +00001681 // Failed to apply the backend-specific optimizations
Mike Kelly3a613cc2020-09-29 20:50:35 +01001682 throw InvalidArgumentException("Failed to apply the backend-specific optimizations");
Matteo Martincigh49124022019-01-11 13:25:59 +00001683 }
1684
Matteo Martincighadddddb2019-01-24 14:06:23 +00001685 // If the debug flag is set, then insert a DebugLayer after each layer
1686 // Doing this after applying the backend optimizations as they might have changed some layers
1687 if (options.m_Debug)
1688 {
1689 Optimizer::Pass(optGraph, MakeOptimizations(InsertDebugLayer()));
1690 }
1691
Derek Lamberti84da38b2019-06-13 11:40:08 +01001692 // Calculate the compatibility strategies for tensor handles
1693 OptimizationResult strategyResult = SelectTensorHandleStrategy(optGraph,
1694 backends,
1695 tensorHandleFactoryRegistry,
Narumol Prangnawarata2493a02020-08-19 14:39:07 +01001696 options.m_ImportEnabled,
Rob Hughes23214432019-11-05 11:27:36 +00001697 messages);
Derek Lamberti84da38b2019-06-13 11:40:08 +01001698 if (strategyResult.m_Error)
1699 {
1700 // Failed to apply the backend-specific optimizations
1701 return IOptimizedNetworkPtr(nullptr, &IOptimizedNetwork::Destroy);
1702 }
1703
1704 // Based on the tensor handle strategy determined above, insert copy layers where required.
Derek Lambertif674aa02019-08-01 15:56:25 +01001705 optGraph.AddCompatibilityLayers(backends, tensorHandleFactoryRegistry);
telsoa01c577f2c2018-08-31 09:22:23 +01001706
1707 // Convert constants
Matteo Martincighadddddb2019-01-24 14:06:23 +00001708 Optimizer::Pass(optGraph, MakeOptimizations(ConvertConstantsFloatToHalf()));
1709 Optimizer::Pass(optGraph, MakeOptimizations(ConvertConstantsHalfToFloat()));
telsoa01c577f2c2018-08-31 09:22:23 +01001710
Derek Lamberti84da38b2019-06-13 11:40:08 +01001711 // Run backend specific optimizations (deprecated)
Matteo Martincigh49124022019-01-11 13:25:59 +00001712 for (auto&& chosenBackend : backendSettings.m_SelectedBackends)
David Beck263e3492018-11-09 14:46:40 +00001713 {
1714 auto factoryFun = BackendRegistryInstance().GetFactory(chosenBackend);
1715 auto backendPtr = factoryFun();
Narumol Prangnawaratac2770a2020-04-01 16:51:23 +01001716 ARMNN_ASSERT(backendPtr.get() != nullptr);
David Beck263e3492018-11-09 14:46:40 +00001717
Matteo Martincighed735042019-05-22 09:42:43 +01001718 ARMNN_NO_DEPRECATE_WARN_BEGIN
David Beck263e3492018-11-09 14:46:40 +00001719 auto backendSpecificOptimizations = backendPtr->GetOptimizations();
Matteo Martincighed735042019-05-22 09:42:43 +01001720 ARMNN_NO_DEPRECATE_WARN_END
1721
David Beck263e3492018-11-09 14:46:40 +00001722 if (!backendSpecificOptimizations.empty())
1723 {
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001724 Optimizer::Pass(optNetObjPtr->pOptimizedNetworkImpl->GetGraph(), backendSpecificOptimizations);
David Beck263e3492018-11-09 14:46:40 +00001725 }
1726 }
1727
telsoa01c577f2c2018-08-31 09:22:23 +01001728 return optNet;
telsoa014fcda012018-03-09 14:13:49 +00001729}
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001730bool NetworkImpl::GetShapeInferenceMethod()
telsoa014fcda012018-03-09 14:13:49 +00001731{
Finn Williamsf24effa2020-07-03 10:12:03 +01001732 if (m_NetworkOptions.size() > 0 && m_NetworkOptions[0].GetBackendId().Get() == "ShapeInferenceMethod")
1733 {
1734 return m_NetworkOptions[0].GetOption(0).GetValue().AsBool();
1735 }
1736
1737 return false;
telsoa014fcda012018-03-09 14:13:49 +00001738}
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001739NetworkImpl::NetworkImpl(NetworkOptions networkOptions)
Finn Williamsf24effa2020-07-03 10:12:03 +01001740: m_NetworkOptions(networkOptions),
1741 m_Graph(std::make_unique<Graph>(GetShapeInferenceMethod()))
1742{}
telsoa014fcda012018-03-09 14:13:49 +00001743
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001744NetworkImpl::~NetworkImpl()
telsoa014fcda012018-03-09 14:13:49 +00001745{
1746}
1747
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001748Status NetworkImpl::PrintGraph()
Jan Eilers99d9d4a2019-11-06 10:02:16 +00001749{
1750 m_Graph->Print();
1751 return Status::Success;
1752}
1753
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001754IConnectableLayer* NetworkImpl::AddInputLayer(LayerBindingId id, const char* name)
telsoa014fcda012018-03-09 14:13:49 +00001755{
1756 return m_Graph->AddLayer<InputLayer>(id, name);
1757}
1758
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001759IConnectableLayer* NetworkImpl::AddBatchToSpaceNdLayer(const BatchToSpaceNdDescriptor& batchToSpaceNdDescriptor,
Éanna Ó Catháin4e1e1362018-11-12 11:36:34 +00001760 const char* name)
1761{
1762 return m_Graph->AddLayer<BatchToSpaceNdLayer>(batchToSpaceNdDescriptor, name);
1763}
1764
mathad01b392e982021-04-07 12:07:30 +01001765IConnectableLayer* NetworkImpl::AddCastLayer(const char* name)
1766{
1767 return m_Graph->AddLayer<CastLayer>(name);
1768}
1769
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001770IConnectableLayer* NetworkImpl::AddComparisonLayer(const ComparisonDescriptor& comparisonDescriptor,
Aron Virginas-Tar77bfb5e2019-10-16 17:45:38 +01001771 const char* name)
1772{
1773 return m_Graph->AddLayer<ComparisonLayer>(comparisonDescriptor, name);
1774}
1775
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001776IConnectableLayer* NetworkImpl::AddElementwiseUnaryLayer(const ElementwiseUnaryDescriptor& elementwiseUnaryDescriptor,
josh minor4a3c6102020-01-06 16:40:46 -06001777 const char* name)
1778{
1779 return m_Graph->AddLayer<ElementwiseUnaryLayer>(elementwiseUnaryDescriptor, name);
1780}
1781
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001782IConnectableLayer* NetworkImpl::AddFillLayer(const FillDescriptor& fillDescriptor,
Ryan OSheaec6c6802020-06-05 17:17:06 +01001783 const char* name)
1784{
1785 return m_Graph->AddLayer<FillLayer>(fillDescriptor, name);
1786}
1787
Matthew Sloyan81beae32021-07-13 19:46:11 +01001788IConnectableLayer* NetworkImpl::AddFullyConnectedLayer(const FullyConnectedDescriptor& fullyConnectedDescriptor,
1789 const char* name)
telsoa014fcda012018-03-09 14:13:49 +00001790{
Matthew Sloyan81beae32021-07-13 19:46:11 +01001791 return m_Graph->AddLayer<FullyConnectedLayer>(fullyConnectedDescriptor, name);
telsoa014fcda012018-03-09 14:13:49 +00001792}
1793
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001794IConnectableLayer* NetworkImpl::AddFullyConnectedLayer(const FullyConnectedDescriptor& fullyConnectedDescriptor,
Sadik Armaganf0a6dec2021-03-25 07:46:55 +00001795 const Optional<ConstTensor>& weights,
1796 const Optional<ConstTensor>& biases,
1797 const char* name)
1798{
Matthew Sloyan81beae32021-07-13 19:46:11 +01001799 ConstantLayer* weightsLayer = nullptr;
1800 ConstantLayer* biasLayer = nullptr;
1801 unsigned int numInputs = fullyConnectedDescriptor.GetNumInputs();
1802
1803 // Add a constant layer for weights
1804 if (weights.has_value())
1805 {
1806 weightsLayer = m_Graph->AddLayer<ConstantLayer>("Weights");
1807 weightsLayer->m_LayerOutput = std::make_shared<ScopedTensorHandle>(weights.value());
Matthew Sloyanb20d1d42021-08-09 15:33:41 +01001808
1809 TensorInfo weightsInfo = weightsLayer->m_LayerOutput->GetTensorInfo();
1810 weightsInfo.SetConstant();
1811
1812 weightsLayer->GetOutputSlot(0).SetTensorInfo(weightsInfo);
Matthew Sloyan81beae32021-07-13 19:46:11 +01001813 }
1814 else if (fullyConnectedDescriptor.m_ConstantWeights)
1815 {
1816 throw InvalidArgumentException("AddFullyConnectedLayer: Constant weights tensor is empty.");
1817 }
1818
1819 // Add a constant layer for biases
1820 if (biases.has_value() && fullyConnectedDescriptor.m_BiasEnabled)
1821 {
1822 biasLayer = m_Graph->AddLayer<ConstantLayer>("Biases");
1823 biasLayer->m_LayerOutput = std::make_shared<ScopedTensorHandle>(biases.value());
Matthew Sloyanb20d1d42021-08-09 15:33:41 +01001824
1825 TensorInfo biasInfo = biasLayer->m_LayerOutput->GetTensorInfo();
1826 biasInfo.SetConstant();
1827
1828 biasLayer->GetOutputSlot(0).SetTensorInfo(biasInfo);
Matthew Sloyan81beae32021-07-13 19:46:11 +01001829 }
1830
1831 if (numInputs < 2)
1832 {
1833 throw InvalidArgumentException("AddFullyConnectedLayer: Requires at least 2 input tensors: Input, Weights");
1834 }
1835
1836 auto layer = m_Graph->AddLayer<FullyConnectedLayer>(fullyConnectedDescriptor, name);
1837
1838 if (weightsLayer)
1839 {
1840 // Connect weights layer
1841 weightsLayer->GetOutputSlot(0).Connect(layer->GetInputSlot(1));
1842 }
1843
1844 if ( fullyConnectedDescriptor.m_BiasEnabled && numInputs == 3 )
1845 {
1846 if (biasLayer)
1847 {
1848 // Connect bias layer
1849 biasLayer->GetOutputSlot(0).Connect(layer->GetInputSlot(2));
1850 }
1851 }
1852 else if ( !fullyConnectedDescriptor.m_BiasEnabled && numInputs == 2 )
1853 {
1854 // Bias is disabled
1855 layer->m_Bias = nullptr;
1856 }
1857 else
1858 {
1859 throw InvalidArgumentException(fmt::format(
1860 "AddFullyConnectedLayer: Value mismatch. When bias is enabled in the "
1861 "descriptor the number of inputs is expected to be 3 otherwise 2. "
1862 "BiasEnabled={}, numInputs={}",
1863 fullyConnectedDescriptor.m_BiasEnabled,
1864 numInputs));
1865 }
1866
1867 return layer;
Sadik Armaganf0a6dec2021-03-25 07:46:55 +00001868}
1869
1870IConnectableLayer* NetworkImpl::AddFullyConnectedLayer(const FullyConnectedDescriptor& fullyConnectedDescriptor,
Matthew Sloyan81beae32021-07-13 19:46:11 +01001871 const ConstTensor& weights,
1872 const Optional<ConstTensor>& biases,
1873 const char* name)
telsoa014fcda012018-03-09 14:13:49 +00001874{
Sadik Armaganf0a6dec2021-03-25 07:46:55 +00001875 Optional<ConstTensor> optionalWeights(weights);
Matthew Sloyan81beae32021-07-13 19:46:11 +01001876 return AddFullyConnectedLayer(fullyConnectedDescriptor, optionalWeights, biases, name);
telsoa014fcda012018-03-09 14:13:49 +00001877}
1878
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001879IConnectableLayer* NetworkImpl::AddConcatLayer(const ConcatDescriptor& concatDescriptor,
Jim Flynn906f9462019-05-10 13:55:21 +01001880 const char* name)
1881{
Jim Flynne242f2d2019-05-22 14:24:13 +01001882 return m_Graph->AddLayer<ConcatLayer>(concatDescriptor, name);
Jim Flynn906f9462019-05-10 13:55:21 +01001883}
1884
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001885IConnectableLayer* NetworkImpl::AddConvolution2dLayerImpl(const Convolution2dDescriptor& convolution2dDescriptor,
1886 const ConstTensor& weights,
1887 const Optional<ConstTensor>& biases,
1888 const char* name)
telsoa014fcda012018-03-09 14:13:49 +00001889{
Aron Virginas-Tarad402702019-02-22 17:03:44 +00001890 if (convolution2dDescriptor.m_BiasEnabled && !biases.has_value())
telsoa014fcda012018-03-09 14:13:49 +00001891 {
Aron Virginas-Tarad402702019-02-22 17:03:44 +00001892 throw InvalidArgumentException("AddConvolution2dLayer: biases cannot be empty");
telsoa014fcda012018-03-09 14:13:49 +00001893 }
1894
1895 const auto layer = m_Graph->AddLayer<Convolution2dLayer>(convolution2dDescriptor, name);
1896
James Conroy1f58f032021-04-27 17:13:27 +01001897 layer->m_Weight = std::make_shared<ScopedTensorHandle>(weights);
telsoa014fcda012018-03-09 14:13:49 +00001898
1899 if (convolution2dDescriptor.m_BiasEnabled)
1900 {
James Conroy1f58f032021-04-27 17:13:27 +01001901 layer->m_Bias = std::make_shared<ScopedTensorHandle>(biases.value());
telsoa014fcda012018-03-09 14:13:49 +00001902 }
1903
1904 return layer;
1905}
1906
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001907IConnectableLayer* NetworkImpl::AddConvolution2dLayer(const Convolution2dDescriptor& convolution2dDescriptor,
telsoa01c577f2c2018-08-31 09:22:23 +01001908 const ConstTensor& weights,
Aron Virginas-Tarad402702019-02-22 17:03:44 +00001909 const Optional<ConstTensor>& biases,
telsoa01c577f2c2018-08-31 09:22:23 +01001910 const char* name)
telsoa014fcda012018-03-09 14:13:49 +00001911{
Aron Virginas-Tarad402702019-02-22 17:03:44 +00001912 return AddConvolution2dLayerImpl(convolution2dDescriptor, weights, biases, name);
telsoa014fcda012018-03-09 14:13:49 +00001913}
Aron Virginas-Tarad402702019-02-22 17:03:44 +00001914
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001915IConnectableLayer* NetworkImpl::AddConvolution2dLayer(const Convolution2dDescriptor& convolution2dDescriptor,
Aron Virginas-Tarad402702019-02-22 17:03:44 +00001916 const ConstTensor& weights,
1917 const char* name)
1918{
Matteo Martincighfc598e12019-05-14 10:36:13 +01001919 Optional<ConstTensor> biases;
Aron Virginas-Tarad402702019-02-22 17:03:44 +00001920 return AddConvolution2dLayerImpl(convolution2dDescriptor, weights, biases, name);
1921}
1922
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001923IConnectableLayer* NetworkImpl::AddConvolution2dLayer(const Convolution2dDescriptor& convolution2dDescriptor,
telsoa01c577f2c2018-08-31 09:22:23 +01001924 const ConstTensor& weights,
1925 const ConstTensor& biases,
1926 const char* name)
telsoa014fcda012018-03-09 14:13:49 +00001927{
Aron Virginas-Tarad402702019-02-22 17:03:44 +00001928 Optional<ConstTensor> optionalBiases(biases);
1929 return AddConvolution2dLayerImpl(convolution2dDescriptor, weights, optionalBiases, name);
telsoa014fcda012018-03-09 14:13:49 +00001930}
1931
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001932IConnectableLayer* NetworkImpl::AddDepthwiseConvolution2dLayerImpl(
telsoa014fcda012018-03-09 14:13:49 +00001933 const DepthwiseConvolution2dDescriptor& convolution2dDescriptor,
1934 const ConstTensor& weights,
Aron Virginas-Tarad402702019-02-22 17:03:44 +00001935 const Optional<ConstTensor>& biases,
telsoa014fcda012018-03-09 14:13:49 +00001936 const char* name)
1937{
Aron Virginas-Tarad402702019-02-22 17:03:44 +00001938 if (convolution2dDescriptor.m_BiasEnabled && !biases.has_value())
telsoa014fcda012018-03-09 14:13:49 +00001939 {
Aron Virginas-Tarad402702019-02-22 17:03:44 +00001940 throw InvalidArgumentException("AddDepthwiseConvolution2dLayer: biases cannot be empty");
telsoa014fcda012018-03-09 14:13:49 +00001941 }
1942
Matteo Martincigh3d6898c2019-01-15 16:11:44 +00001943 const auto layer = m_Graph->AddLayer<DepthwiseConvolution2dLayer>(convolution2dDescriptor, name);
telsoa014fcda012018-03-09 14:13:49 +00001944
James Conroy1f58f032021-04-27 17:13:27 +01001945 layer->m_Weight = std::make_shared<ScopedTensorHandle>(weights);
telsoa014fcda012018-03-09 14:13:49 +00001946
1947 if (convolution2dDescriptor.m_BiasEnabled)
1948 {
James Conroy1f58f032021-04-27 17:13:27 +01001949 layer->m_Bias = std::make_shared<ScopedTensorHandle>(biases.value());
telsoa014fcda012018-03-09 14:13:49 +00001950 }
1951
1952 return layer;
1953}
1954
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001955IConnectableLayer* NetworkImpl::AddDepthToSpaceLayer(const DepthToSpaceDescriptor& depthToSpaceDescriptor,
Aron Virginas-Tardd6247f2019-09-19 14:31:17 +01001956 const char* name)
1957{
1958 return m_Graph->AddLayer<DepthToSpaceLayer>(depthToSpaceDescriptor, name);
1959}
1960
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001961IConnectableLayer* NetworkImpl::AddDepthwiseConvolution2dLayer(
Aron Virginas-Tarad402702019-02-22 17:03:44 +00001962 const DepthwiseConvolution2dDescriptor& convolution2dDescriptor,
1963 const ConstTensor& weights,
1964 const Optional<ConstTensor>& biases,
1965 const char* name)
1966{
1967 return AddDepthwiseConvolution2dLayerImpl(convolution2dDescriptor, weights, biases, name);
1968}
1969
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001970IConnectableLayer* NetworkImpl::AddDepthwiseConvolution2dLayer(
telsoa014fcda012018-03-09 14:13:49 +00001971 const DepthwiseConvolution2dDescriptor& convolution2dDescriptor,
1972 const ConstTensor& weights,
1973 const char* name)
1974{
Matteo Martincighfc598e12019-05-14 10:36:13 +01001975 Optional<ConstTensor> biases;
Aron Virginas-Tarad402702019-02-22 17:03:44 +00001976 return AddDepthwiseConvolution2dLayerImpl(convolution2dDescriptor, weights, biases, name);
telsoa014fcda012018-03-09 14:13:49 +00001977}
Aron Virginas-Tarad402702019-02-22 17:03:44 +00001978
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001979IConnectableLayer* NetworkImpl::AddDepthwiseConvolution2dLayer(
telsoa014fcda012018-03-09 14:13:49 +00001980 const DepthwiseConvolution2dDescriptor& convolution2dDescriptor,
1981 const ConstTensor& weights,
1982 const ConstTensor& biases,
1983 const char* name)
1984{
Aron Virginas-Tarad402702019-02-22 17:03:44 +00001985 Optional<ConstTensor> optionalBiases(biases);
1986 return AddDepthwiseConvolution2dLayerImpl(convolution2dDescriptor, weights, optionalBiases, name);
telsoa014fcda012018-03-09 14:13:49 +00001987}
1988
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001989IConnectableLayer* NetworkImpl::AddDetectionPostProcessLayer(const armnn::DetectionPostProcessDescriptor& descriptor,
Narumol Prangnawarat6d302bf2019-02-04 11:46:26 +00001990 const ConstTensor& anchors, const char* name)
Narumol Prangnawarat94dd5d82019-01-23 18:06:26 +00001991{
Narumol Prangnawarat6d302bf2019-02-04 11:46:26 +00001992 const auto layer = m_Graph->AddLayer<DetectionPostProcessLayer>(descriptor, name);
1993
James Conroy1f58f032021-04-27 17:13:27 +01001994 layer->m_Anchors = std::make_shared<ScopedTensorHandle>(anchors);
Narumol Prangnawarat6d302bf2019-02-04 11:46:26 +00001995
1996 return layer;
Narumol Prangnawarat94dd5d82019-01-23 18:06:26 +00001997}
1998
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00001999IConnectableLayer* NetworkImpl::AddPermuteLayer(const PermuteDescriptor& permuteDescriptor,
telsoa014fcda012018-03-09 14:13:49 +00002000 const char* name)
2001{
2002 return m_Graph->AddLayer<PermuteLayer>(permuteDescriptor, name);
2003}
2004
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002005IConnectableLayer* NetworkImpl::AddPooling2dLayer(const Pooling2dDescriptor& pooling2dDescriptor,
telsoa014fcda012018-03-09 14:13:49 +00002006 const char* name)
2007{
2008 return m_Graph->AddLayer<Pooling2dLayer>(pooling2dDescriptor, name);
2009}
2010
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002011IConnectableLayer* NetworkImpl::AddActivationLayer(const ActivationDescriptor& activationDescriptor,
telsoa014fcda012018-03-09 14:13:49 +00002012 const char* name)
2013{
2014 return m_Graph->AddLayer<ActivationLayer>(activationDescriptor, name);
2015}
2016
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002017IConnectableLayer* NetworkImpl::AddArgMinMaxLayer(const ArgMinMaxDescriptor& argMinMaxDescriptor,
Nikhil Rajee391d52019-09-05 17:50:44 +01002018 const char* name)
2019{
2020 return m_Graph->AddLayer<ArgMinMaxLayer>(argMinMaxDescriptor, name);
2021}
2022
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002023IConnectableLayer* NetworkImpl::AddNormalizationLayer(const NormalizationDescriptor&
telsoa01c577f2c2018-08-31 09:22:23 +01002024normalizationDescriptor,
telsoa014fcda012018-03-09 14:13:49 +00002025 const char* name)
2026{
2027 return m_Graph->AddLayer<NormalizationLayer>(normalizationDescriptor, name);
2028}
2029
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002030IConnectableLayer* NetworkImpl::AddSliceLayer(const SliceDescriptor& sliceDescriptor, const char* name)
Aron Virginas-Tar636ab402019-09-16 14:27:45 +01002031{
2032 return m_Graph->AddLayer<SliceLayer>(sliceDescriptor, name);
2033}
2034
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002035IConnectableLayer* NetworkImpl::AddSoftmaxLayer(const SoftmaxDescriptor& softmaxDescriptor,
telsoa014fcda012018-03-09 14:13:49 +00002036 const char* name)
2037{
2038 return m_Graph->AddLayer<SoftmaxLayer>(softmaxDescriptor, name);
2039}
2040
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002041IConnectableLayer* NetworkImpl::AddSplitterLayer(const ViewsDescriptor& splitterDescriptor,
telsoa014fcda012018-03-09 14:13:49 +00002042 const char* name)
2043{
2044 return m_Graph->AddLayer<SplitterLayer>(splitterDescriptor, name);
2045}
2046
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002047IConnectableLayer* NetworkImpl::AddMaximumLayer(const char* name)
Nattapat Chaimanowong5a4304a2018-11-28 10:44:37 +00002048{
2049 return m_Graph->AddLayer<MaximumLayer>(name);
2050}
2051
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002052IConnectableLayer* NetworkImpl::AddMinimumLayer(const char* name)
Éanna Ó Catháin20e58802018-12-04 10:29:06 +00002053{
2054 return m_Graph->AddLayer<MinimumLayer>(name);
2055}
2056
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002057IConnectableLayer* NetworkImpl::AddMergerLayer(const MergerDescriptor& mergerDescriptor,
Jim Flynn906f9462019-05-10 13:55:21 +01002058 const char* name)
telsoa014fcda012018-03-09 14:13:49 +00002059{
Jim Flynne242f2d2019-05-22 14:24:13 +01002060 return AddConcatLayer(mergerDescriptor, name);
telsoa014fcda012018-03-09 14:13:49 +00002061}
2062
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002063IConnectableLayer* NetworkImpl::AddAbsLayer(const char * name)
Kevin May868eb142019-09-04 17:29:31 +01002064{
josh minor4a3c6102020-01-06 16:40:46 -06002065 return AddElementwiseUnaryLayer(ElementwiseUnaryDescriptor(UnaryOperation::Abs), name);
Kevin May868eb142019-09-04 17:29:31 +01002066}
2067
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002068IConnectableLayer* NetworkImpl::AddAdditionLayer(const char* name)
telsoa014fcda012018-03-09 14:13:49 +00002069{
2070 return m_Graph->AddLayer<AdditionLayer>(name);
2071}
2072
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002073IConnectableLayer* NetworkImpl::AddMultiplicationLayer(const char* name)
telsoa014fcda012018-03-09 14:13:49 +00002074{
2075 return m_Graph->AddLayer<MultiplicationLayer>(name);
2076}
2077
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002078IConnectableLayer* NetworkImpl::AddOutputLayer(LayerBindingId id, const char* name)
telsoa014fcda012018-03-09 14:13:49 +00002079{
2080 return m_Graph->AddLayer<OutputLayer>(id, name);
2081}
2082
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002083IConnectableLayer* NetworkImpl::AddBatchNormalizationLayer(const BatchNormalizationDescriptor& desc,
telsoa014fcda012018-03-09 14:13:49 +00002084 const ConstTensor& mean,
2085 const ConstTensor& variance,
2086 const ConstTensor& beta,
2087 const ConstTensor& gamma,
2088 const char* name)
2089{
2090 const auto layer = m_Graph->AddLayer<BatchNormalizationLayer>(desc, name);
2091
James Conroy1f58f032021-04-27 17:13:27 +01002092 layer->m_Mean = std::make_shared<ScopedTensorHandle>(mean);
2093 layer->m_Variance = std::make_shared<ScopedTensorHandle>(variance);
2094 layer->m_Beta = std::make_shared<ScopedTensorHandle>(beta);
2095 layer->m_Gamma = std::make_shared<ScopedTensorHandle>(gamma);
telsoa014fcda012018-03-09 14:13:49 +00002096
2097 return layer;
2098}
2099
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002100IConnectableLayer* NetworkImpl::AddRankLayer(const char* name)
Finn Williams2605b232020-06-10 15:53:46 +01002101{
2102 return m_Graph->AddLayer<RankLayer>(name);
2103}
2104
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002105IConnectableLayer* NetworkImpl::AddReduceLayer(const ReduceDescriptor& reduceDescriptor,
2106 const char* name)
Sadik Armagan0c3ea5b2021-02-03 09:29:30 +00002107{
2108 return m_Graph->AddLayer<ReduceLayer>(reduceDescriptor, name);
2109}
2110
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002111IConnectableLayer* NetworkImpl::AddResizeBilinearLayer(const ResizeBilinearDescriptor& descriptor,
2112 const char* name)
telsoa014fcda012018-03-09 14:13:49 +00002113{
Aron Virginas-Tar169d2f12019-07-01 19:01:44 +01002114 ResizeDescriptor resizeDescriptor;
David Monahan4a0c9b92020-05-30 09:48:39 +01002115 resizeDescriptor.m_Method = ResizeMethod::Bilinear;
2116 resizeDescriptor.m_DataLayout = descriptor.m_DataLayout;
2117 resizeDescriptor.m_TargetWidth = descriptor.m_TargetWidth;
2118 resizeDescriptor.m_TargetHeight = descriptor.m_TargetHeight;
2119 resizeDescriptor.m_AlignCorners = descriptor.m_AlignCorners;
2120 resizeDescriptor.m_HalfPixelCenters = descriptor.m_HalfPixelCenters;
Aron Virginas-Tar169d2f12019-07-01 19:01:44 +01002121
2122 return m_Graph->AddLayer<ResizeLayer>(resizeDescriptor, name);
telsoa014fcda012018-03-09 14:13:49 +00002123}
2124
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002125IConnectableLayer* NetworkImpl::AddResizeLayer(const ResizeDescriptor& resizeDescriptor, const char* name)
Teresa Charlina9075df2019-06-27 15:41:57 +01002126{
Aron Virginas-Tar169d2f12019-07-01 19:01:44 +01002127 return m_Graph->AddLayer<ResizeLayer>(resizeDescriptor, name);
Teresa Charlina9075df2019-06-27 15:41:57 +01002128}
2129
Keith Davis3ae3f972021-05-21 16:33:48 +01002130IConnectableLayer* NetworkImpl::AddShapeLayer(const char* name)
2131{
2132 return m_Graph->AddLayer<ShapeLayer>(name);
2133}
2134
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002135IConnectableLayer* NetworkImpl::AddInstanceNormalizationLayer(const InstanceNormalizationDescriptor& desc,
2136 const char* name)
Kevin Mayce5045a2019-10-02 14:07:47 +01002137{
2138 return m_Graph->AddLayer<InstanceNormalizationLayer>(desc, name);
2139}
2140
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002141IConnectableLayer* NetworkImpl::AddL2NormalizationLayer(const L2NormalizationDescriptor& desc,
2142 const char* name)
telsoa014fcda012018-03-09 14:13:49 +00002143{
Matteo Martincighbcd3c852018-09-28 14:14:12 +01002144 return m_Graph->AddLayer<L2NormalizationLayer>(desc, name);
telsoa014fcda012018-03-09 14:13:49 +00002145}
2146
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002147IConnectableLayer* NetworkImpl::AddLogSoftmaxLayer(const LogSoftmaxDescriptor& desc,
Aron Virginas-Tarf982dea2019-10-11 14:07:53 +01002148 const char* name)
2149{
2150 return m_Graph->AddLayer<LogSoftmaxLayer>(desc, name);
2151}
2152
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002153IConnectableLayer* NetworkImpl::AddConstantLayer(const ConstTensor& input, const char* name)
telsoa014fcda012018-03-09 14:13:49 +00002154{
telsoa01c577f2c2018-08-31 09:22:23 +01002155 auto layer = m_Graph->AddLayer<ConstantLayer>(name);
2156
James Conroy1f58f032021-04-27 17:13:27 +01002157 layer->m_LayerOutput = std::make_shared<ScopedTensorHandle>(input);
telsoa01c577f2c2018-08-31 09:22:23 +01002158
2159 return layer;
telsoa014fcda012018-03-09 14:13:49 +00002160}
2161
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002162IConnectableLayer* NetworkImpl::AddReshapeLayer(const ReshapeDescriptor& reshapeDescriptor,
telsoa01c577f2c2018-08-31 09:22:23 +01002163 const char* name)
telsoa014fcda012018-03-09 14:13:49 +00002164{
2165 return m_Graph->AddLayer<ReshapeLayer>(reshapeDescriptor, name);
2166}
2167
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002168IConnectableLayer* NetworkImpl::AddSpaceToBatchNdLayer(const SpaceToBatchNdDescriptor& spaceToBatchNdDescriptor,
Nattapat Chaimanowong207ef9a2018-11-02 10:57:25 +00002169 const char* name)
2170{
2171 return m_Graph->AddLayer<SpaceToBatchNdLayer>(spaceToBatchNdDescriptor, name);
2172}
2173
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002174IConnectableLayer* NetworkImpl::AddSpaceToDepthLayer(const SpaceToDepthDescriptor& spaceToDepthDescriptor,
Aron Virginas-Tar972af152019-06-11 14:14:03 +01002175 const char* name)
2176{
2177 return m_Graph->AddLayer<SpaceToDepthLayer>(spaceToDepthDescriptor, name);
2178}
2179
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002180IConnectableLayer* NetworkImpl::AddFloorLayer(const char* name)
telsoa014fcda012018-03-09 14:13:49 +00002181{
2182 return m_Graph->AddLayer<FloorLayer>(name);
2183}
2184
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002185IConnectableLayer* NetworkImpl::AddLstmLayer(const LstmDescriptor& descriptor,
telsoa01c577f2c2018-08-31 09:22:23 +01002186 const LstmInputParams& params,
2187 const char* name)
2188{
2189 const auto layer = m_Graph->AddLayer<LstmLayer>(descriptor, name);
2190
2191 //Lstm Basic Parameters
2192 layer->m_BasicParameters.m_InputToForgetWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002193 std::make_shared<ScopedTensorHandle>(*(params.m_InputToForgetWeights));
telsoa01c577f2c2018-08-31 09:22:23 +01002194 layer->m_BasicParameters.m_InputToCellWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002195 std::make_shared<ScopedTensorHandle>(*(params.m_InputToCellWeights));
telsoa01c577f2c2018-08-31 09:22:23 +01002196 layer->m_BasicParameters.m_InputToOutputWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002197 std::make_shared<ScopedTensorHandle>(*(params.m_InputToOutputWeights));
telsoa01c577f2c2018-08-31 09:22:23 +01002198 layer->m_BasicParameters.m_RecurrentToForgetWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002199 std::make_shared<ScopedTensorHandle>(*(params.m_RecurrentToForgetWeights));
telsoa01c577f2c2018-08-31 09:22:23 +01002200 layer->m_BasicParameters.m_RecurrentToCellWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002201 std::make_shared<ScopedTensorHandle>(*(params.m_RecurrentToCellWeights));
telsoa01c577f2c2018-08-31 09:22:23 +01002202 layer->m_BasicParameters.m_RecurrentToOutputWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002203 std::make_shared<ScopedTensorHandle>(*(params.m_RecurrentToOutputWeights));
telsoa01c577f2c2018-08-31 09:22:23 +01002204 layer->m_BasicParameters.m_ForgetGateBias =
James Conroy1f58f032021-04-27 17:13:27 +01002205 std::make_shared<ScopedTensorHandle>(*(params.m_ForgetGateBias));
telsoa01c577f2c2018-08-31 09:22:23 +01002206 layer->m_BasicParameters.m_CellBias =
James Conroy1f58f032021-04-27 17:13:27 +01002207 std::make_shared<ScopedTensorHandle>(*(params.m_CellBias));
telsoa01c577f2c2018-08-31 09:22:23 +01002208 layer->m_BasicParameters.m_OutputGateBias =
James Conroy1f58f032021-04-27 17:13:27 +01002209 std::make_shared<ScopedTensorHandle>(*(params.m_OutputGateBias));
telsoa01c577f2c2018-08-31 09:22:23 +01002210
2211 //Lstm Cifg parameters
2212 if(!descriptor.m_CifgEnabled)
2213 {
2214 if(params.m_InputToInputWeights == nullptr)
2215 {
Jan Eilerse2062cd2020-03-30 15:07:45 +01002216 throw InvalidArgumentException("AddLstmLayer: Input To Input Weights cannot be NULL "
2217 "when CIFG is disabled.");
telsoa01c577f2c2018-08-31 09:22:23 +01002218 }
2219 if(params.m_RecurrentToInputWeights == nullptr)
2220 {
2221 throw InvalidArgumentException(
Jan Eilerse2062cd2020-03-30 15:07:45 +01002222 "AddLstmLayer: Recurrent To Input Weights cannot be NULL "
2223 "when CIFG is disabled.");
telsoa01c577f2c2018-08-31 09:22:23 +01002224 }
2225 if(params.m_InputGateBias == nullptr)
2226 {
Jan Eilerse2062cd2020-03-30 15:07:45 +01002227 throw InvalidArgumentException("AddLstmLayer: Input Gate Bias cannot be NULL "
2228 "when CIFG is disabled.");
telsoa01c577f2c2018-08-31 09:22:23 +01002229 }
2230 layer->m_CifgParameters.m_InputToInputWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002231 std::make_shared<ScopedTensorHandle>(*(params.m_InputToInputWeights));
telsoa01c577f2c2018-08-31 09:22:23 +01002232 layer->m_CifgParameters.m_RecurrentToInputWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002233 std::make_shared<ScopedTensorHandle>(*(params.m_RecurrentToInputWeights));
telsoa01c577f2c2018-08-31 09:22:23 +01002234 layer->m_CifgParameters.m_InputGateBias =
James Conroy1f58f032021-04-27 17:13:27 +01002235 std::make_shared<ScopedTensorHandle>(*(params.m_InputGateBias));
telsoa01c577f2c2018-08-31 09:22:23 +01002236 }
2237
2238 //Lstm projection parameters
2239 if(descriptor.m_ProjectionEnabled)
2240 {
2241 if(params.m_ProjectionWeights == nullptr)
2242 {
Jan Eilerse2062cd2020-03-30 15:07:45 +01002243 throw InvalidArgumentException("AddLstmLayer: Projection Weights cannot be NULL "
2244 "when projection is enabled.");
telsoa01c577f2c2018-08-31 09:22:23 +01002245 }
2246 layer->m_ProjectionParameters.m_ProjectionWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002247 std::make_shared<ScopedTensorHandle>(*(params.m_ProjectionWeights));
telsoa01c577f2c2018-08-31 09:22:23 +01002248 if(params.m_ProjectionBias != nullptr)
2249 {
2250 layer->m_ProjectionParameters.m_ProjectionBias =
James Conroy1f58f032021-04-27 17:13:27 +01002251 std::make_shared<ScopedTensorHandle>(*(params.m_ProjectionBias));
telsoa01c577f2c2018-08-31 09:22:23 +01002252 }
2253 }
2254
2255 //Lstm Peephole params
2256 if(descriptor.m_PeepholeEnabled)
2257 {
Jan Eilerse2062cd2020-03-30 15:07:45 +01002258 if(!descriptor.m_CifgEnabled)
2259 {
2260 if(params.m_CellToInputWeights == nullptr)
2261 {
2262 throw InvalidArgumentException("AddLstmLayer: Cell To Input Weights cannot be NULL "
2263 "when Peephole is enabled and CIFG disabled.");
2264 }
2265
2266 layer->m_PeepholeParameters.m_CellToInputWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002267 std::make_shared<ScopedTensorHandle>(*(params.m_CellToInputWeights));
Jan Eilerse2062cd2020-03-30 15:07:45 +01002268 }
2269
telsoa01c577f2c2018-08-31 09:22:23 +01002270 if(params.m_CellToForgetWeights == nullptr)
2271 {
Jan Eilerse2062cd2020-03-30 15:07:45 +01002272 throw InvalidArgumentException("AddLstmLayer: Cell To Forget Weights cannot be NULL "
2273 "when Peephole is enabled.");
telsoa01c577f2c2018-08-31 09:22:23 +01002274 }
2275 if(params.m_CellToOutputWeights == nullptr)
2276 {
Jan Eilerse2062cd2020-03-30 15:07:45 +01002277 throw InvalidArgumentException("AddLstmLayer: Cell To Output Weights cannot be NULL "
2278 "when Peephole is enabled.");
telsoa01c577f2c2018-08-31 09:22:23 +01002279 }
Jan Eilerse2062cd2020-03-30 15:07:45 +01002280
telsoa01c577f2c2018-08-31 09:22:23 +01002281 layer->m_PeepholeParameters.m_CellToForgetWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002282 std::make_shared<ScopedTensorHandle>(*(params.m_CellToForgetWeights));
telsoa01c577f2c2018-08-31 09:22:23 +01002283 layer->m_PeepholeParameters.m_CellToOutputWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002284 std::make_shared<ScopedTensorHandle>(*(params.m_CellToOutputWeights));
telsoa01c577f2c2018-08-31 09:22:23 +01002285 }
Jan Eilersf8c62972019-07-17 11:07:49 +01002286
2287 //Lstm Layer Normalization params
2288 if(descriptor.m_LayerNormEnabled)
2289 {
2290 if(!descriptor.m_CifgEnabled)
2291 {
2292 if(params.m_InputLayerNormWeights == nullptr)
2293 {
Jan Eilerse2062cd2020-03-30 15:07:45 +01002294 throw InvalidArgumentException("AddLstmLayer: Input layer normalization weights cannot be NULL "
2295 "when layer normalization is enabled and CIFG disabled.");
Jan Eilersf8c62972019-07-17 11:07:49 +01002296 }
2297 layer->m_LayerNormParameters.m_InputLayerNormWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002298 std::make_shared<ScopedTensorHandle>(*(params.m_InputLayerNormWeights));
Jan Eilersf8c62972019-07-17 11:07:49 +01002299 }
2300
2301 if(params.m_ForgetLayerNormWeights == nullptr)
2302 {
Jan Eilerse2062cd2020-03-30 15:07:45 +01002303 throw InvalidArgumentException("AddLstmLayer: Forget layer normalization weights cannot be NULL "
2304 "when layer normalization is enabled.");
Jan Eilersf8c62972019-07-17 11:07:49 +01002305 }
2306 if(params.m_CellLayerNormWeights == nullptr)
2307 {
Jan Eilerse2062cd2020-03-30 15:07:45 +01002308 throw InvalidArgumentException("AddLstmLayer: Cell layer normalization weights cannot be NULL "
2309 "when layer normalization is enabled.");
Jan Eilersf8c62972019-07-17 11:07:49 +01002310 }
2311 if(params.m_OutputLayerNormWeights == nullptr)
2312 {
Jan Eilerse2062cd2020-03-30 15:07:45 +01002313 throw InvalidArgumentException("AddLstmLayer: Output layer normalization weights cannot be NULL "
2314 "when layer normalization is enabled.");
Jan Eilersf8c62972019-07-17 11:07:49 +01002315 }
2316 layer->m_LayerNormParameters.m_ForgetLayerNormWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002317 std::make_shared<ScopedTensorHandle>(*(params.m_ForgetLayerNormWeights));
Jan Eilersf8c62972019-07-17 11:07:49 +01002318 layer->m_LayerNormParameters.m_CellLayerNormWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002319 std::make_shared<ScopedTensorHandle>(*(params.m_CellLayerNormWeights));
Jan Eilersf8c62972019-07-17 11:07:49 +01002320 layer->m_LayerNormParameters.m_OutputLayerNormWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002321 std::make_shared<ScopedTensorHandle>(*(params.m_OutputLayerNormWeights));
Jan Eilersf8c62972019-07-17 11:07:49 +01002322 }
telsoa01c577f2c2018-08-31 09:22:23 +01002323 return layer;
2324}
2325
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002326IConnectableLayer* NetworkImpl::AddDivisionLayer(const char* name)
Francis Murtaghe7a86a42018-08-29 12:42:10 +01002327{
2328 return m_Graph->AddLayer<DivisionLayer>(name);
2329}
2330
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002331IConnectableLayer* NetworkImpl::AddSubtractionLayer(const char* name)
David Beck19526222018-09-12 16:00:08 +01002332{
2333 return m_Graph->AddLayer<SubtractionLayer>(name);
2334}
2335
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002336IConnectableLayer* NetworkImpl::AddMeanLayer(const MeanDescriptor& meanDescriptor, const char* name)
narpra0132b90462018-09-13 11:07:48 +01002337{
2338 return m_Graph->AddLayer<MeanLayer>(meanDescriptor,name);
2339}
2340
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002341IConnectableLayer* NetworkImpl::AddPadLayer(const PadDescriptor& padDescriptor, const char* name)
Mohamed Nour Abouelseoud5662c202018-09-24 13:30:09 +01002342{
2343 return m_Graph->AddLayer<PadLayer>(padDescriptor,name);
2344}
2345
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002346IConnectableLayer *NetworkImpl::AddQuantizeLayer(const char *name)
Derek Lambertia9cca6a2019-03-25 15:41:58 +00002347{
2348 return m_Graph->AddLayer<QuantizeLayer>(name);
2349}
2350
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002351IConnectableLayer* NetworkImpl::AddDequantizeLayer(const char* name)
Nattapat Chaimanowonge4294fd2019-03-28 09:56:53 +00002352{
2353 return m_Graph->AddLayer<DequantizeLayer>(name);
2354}
2355
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002356IConnectableLayer* NetworkImpl::AddStridedSliceLayer(const StridedSliceDescriptor& stridedSliceDescriptor,
Conor Kennedy430b5d82018-11-14 15:28:28 +00002357 const char* name)
2358{
2359 return m_Graph->AddLayer<StridedSliceLayer>(stridedSliceDescriptor, name);
2360}
2361
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002362IConnectableLayer* NetworkImpl::AddGreaterLayer(const char* name)
Matteo Martincigh59a950c2018-12-13 12:48:25 +00002363{
Aron Virginas-Tar77bfb5e2019-10-16 17:45:38 +01002364 return AddComparisonLayer(ComparisonDescriptor(ComparisonOperation::Greater), name);
Matteo Martincigh59a950c2018-12-13 12:48:25 +00002365}
2366
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002367IConnectableLayer* NetworkImpl::AddEqualLayer(const char* name)
FrancisMurtagh20995952018-12-17 12:11:36 +00002368{
Aron Virginas-Tar77bfb5e2019-10-16 17:45:38 +01002369 return AddComparisonLayer(ComparisonDescriptor(ComparisonOperation::Equal), name);
FrancisMurtagh20995952018-12-17 12:11:36 +00002370}
2371
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002372IConnectableLayer* NetworkImpl::AddRsqrtLayer(const char * name)
Mohamed Nour Abouelseouda1d3c6a2018-12-27 12:39:16 +00002373{
josh minor4a3c6102020-01-06 16:40:46 -06002374 return AddElementwiseUnaryLayer(ElementwiseUnaryDescriptor(UnaryOperation::Rsqrt), name);
Mohamed Nour Abouelseouda1d3c6a2018-12-27 12:39:16 +00002375}
2376
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002377IConnectableLayer* NetworkImpl::AddGatherLayer(const char* name)
narpra01b89b05f2019-01-16 09:53:09 +00002378{
Teresa Charlin52664732020-06-29 16:27:03 +01002379 GatherDescriptor gatherDescriptor{};
2380 return AddGatherLayer(gatherDescriptor, name);
2381}
2382
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002383IConnectableLayer* NetworkImpl::AddGatherLayer(const GatherDescriptor& gatherDescriptor,
Teresa Charlin52664732020-06-29 16:27:03 +01002384 const char* name)
2385{
2386 return m_Graph->AddLayer<GatherLayer>(gatherDescriptor, name);
narpra01b89b05f2019-01-16 09:53:09 +00002387}
2388
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002389IConnectableLayer* NetworkImpl::AddMergeLayer(const char* name)
Nattapat Chaimanowong1f886302019-04-05 13:37:19 +01002390{
2391 return m_Graph->AddLayer<MergeLayer>(name);
2392}
2393
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002394IConnectableLayer* NetworkImpl::AddSwitchLayer(const char* name)
Sadik Armaganeff363d2019-04-05 15:25:46 +01002395{
2396 return m_Graph->AddLayer<SwitchLayer>(name);
2397}
2398
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002399IConnectableLayer* NetworkImpl::AddPreluLayer(const char* name)
Matteo Martincigh0e406ee2019-06-12 15:42:18 +01002400{
2401 return m_Graph->AddLayer<PreluLayer>(name);
2402}
2403
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002404IConnectableLayer* NetworkImpl::AddTransposeConvolution2dLayer(const TransposeConvolution2dDescriptor& descriptor,
Aron Virginas-Tar639fb042019-06-20 14:28:19 +01002405 const ConstTensor& weights,
2406 const Optional<ConstTensor>& biases,
2407 const char* name)
2408{
2409 if (descriptor.m_BiasEnabled && !biases.has_value())
2410 {
2411 throw InvalidArgumentException("AddTransposeConvolution2dLayer: Biases cannot be empty");
2412 }
2413
2414 const auto layer = m_Graph->AddLayer<TransposeConvolution2dLayer>(descriptor, name);
2415
James Conroy1f58f032021-04-27 17:13:27 +01002416 layer->m_Weight = std::make_shared<ScopedTensorHandle>(weights);
Aron Virginas-Tar639fb042019-06-20 14:28:19 +01002417
2418 if (descriptor.m_BiasEnabled)
2419 {
James Conroy1f58f032021-04-27 17:13:27 +01002420 layer->m_Bias = std::make_shared<ScopedTensorHandle>(biases.value());
Aron Virginas-Tar639fb042019-06-20 14:28:19 +01002421 }
2422
2423 return layer;
2424}
2425
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002426IConnectableLayer* NetworkImpl::AddTransposeLayer(const TransposeDescriptor& transposeDescriptor,
Mike Kellyc9ea45a2020-02-28 18:11:58 +00002427 const char* name)
2428{
2429 return m_Graph->AddLayer<TransposeLayer>(transposeDescriptor, name);
2430}
2431
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002432IConnectableLayer* NetworkImpl::AddStackLayer(const StackDescriptor& stackDescriptor,
Matthew Jackson2b8c1da2019-07-04 14:59:16 +01002433 const char* name)
2434{
2435 return m_Graph->AddLayer<StackLayer>(stackDescriptor, name);
2436}
2437
Derek Lamberti013c3902019-10-21 10:46:16 +01002438
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002439IConnectableLayer* NetworkImpl::AddStandInLayer(const StandInDescriptor& desc,
Derek Lamberti013c3902019-10-21 10:46:16 +01002440 const char* name)
2441{
2442 return m_Graph->AddLayer<StandInLayer>(desc, name);
2443}
2444
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002445IConnectableLayer* NetworkImpl::AddQuantizedLstmLayer(const QuantizedLstmInputParams& params,
James Conroyee18dc82019-07-17 11:27:46 +01002446 const char* name)
2447{
2448 const auto layer = m_Graph->AddLayer<QuantizedLstmLayer>(name);
2449
2450 // InputToX weights
2451 layer->m_QuantizedLstmParameters.m_InputToInputWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002452 std::make_shared<ScopedTensorHandle>(params.GetInputToInputWeights());
James Conroyee18dc82019-07-17 11:27:46 +01002453 layer->m_QuantizedLstmParameters.m_InputToForgetWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002454 std::make_shared<ScopedTensorHandle>(params.GetInputToForgetWeights());
James Conroyee18dc82019-07-17 11:27:46 +01002455 layer->m_QuantizedLstmParameters.m_InputToCellWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002456 std::make_shared<ScopedTensorHandle>(params.GetInputToCellWeights());
James Conroyee18dc82019-07-17 11:27:46 +01002457 layer->m_QuantizedLstmParameters.m_InputToOutputWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002458 std::make_shared<ScopedTensorHandle>(params.GetInputToOutputWeights());
James Conroyee18dc82019-07-17 11:27:46 +01002459
2460 // RecurrentToX weights
2461 layer->m_QuantizedLstmParameters.m_RecurrentToInputWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002462 std::make_shared<ScopedTensorHandle>(params.GetRecurrentToInputWeights());
James Conroyee18dc82019-07-17 11:27:46 +01002463 layer->m_QuantizedLstmParameters.m_RecurrentToForgetWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002464 std::make_shared<ScopedTensorHandle>(params.GetRecurrentToForgetWeights());
James Conroyee18dc82019-07-17 11:27:46 +01002465 layer->m_QuantizedLstmParameters.m_RecurrentToCellWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002466 std::make_shared<ScopedTensorHandle>(params.GetRecurrentToCellWeights());
James Conroyee18dc82019-07-17 11:27:46 +01002467 layer->m_QuantizedLstmParameters.m_RecurrentToOutputWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002468 std::make_shared<ScopedTensorHandle>(params.GetRecurrentToOutputWeights());
James Conroyee18dc82019-07-17 11:27:46 +01002469
2470 // Bias
2471 layer->m_QuantizedLstmParameters.m_InputGateBias =
James Conroy1f58f032021-04-27 17:13:27 +01002472 std::make_shared<ScopedTensorHandle>(params.GetInputGateBias());
James Conroyee18dc82019-07-17 11:27:46 +01002473 layer->m_QuantizedLstmParameters.m_ForgetGateBias =
James Conroy1f58f032021-04-27 17:13:27 +01002474 std::make_shared<ScopedTensorHandle>(params.GetForgetGateBias());
James Conroyee18dc82019-07-17 11:27:46 +01002475 layer->m_QuantizedLstmParameters.m_CellBias =
James Conroy1f58f032021-04-27 17:13:27 +01002476 std::make_shared<ScopedTensorHandle>(params.GetCellBias());
James Conroyee18dc82019-07-17 11:27:46 +01002477 layer->m_QuantizedLstmParameters.m_OutputGateBias =
James Conroy1f58f032021-04-27 17:13:27 +01002478 std::make_shared<ScopedTensorHandle>(params.GetOutputGateBias());
James Conroyee18dc82019-07-17 11:27:46 +01002479
2480 return layer;
2481}
2482
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002483IConnectableLayer* NetworkImpl::AddQLstmLayer(const QLstmDescriptor& descriptor,
James Conroy586a9aa2020-03-20 08:49:33 +00002484 const LstmInputParams& params,
2485 const char* name)
2486{
2487 const auto layer = m_Graph->AddLayer<QLstmLayer>(descriptor, name);
2488
2489 // QLstm Basic Parameters
2490 layer->m_BasicParameters.m_InputToForgetWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002491 std::make_shared<ScopedTensorHandle>(*(params.m_InputToForgetWeights));
James Conroy586a9aa2020-03-20 08:49:33 +00002492 layer->m_BasicParameters.m_InputToCellWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002493 std::make_shared<ScopedTensorHandle>(*(params.m_InputToCellWeights));
James Conroy586a9aa2020-03-20 08:49:33 +00002494 layer->m_BasicParameters.m_InputToOutputWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002495 std::make_shared<ScopedTensorHandle>(*(params.m_InputToOutputWeights));
James Conroy586a9aa2020-03-20 08:49:33 +00002496 layer->m_BasicParameters.m_RecurrentToForgetWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002497 std::make_shared<ScopedTensorHandle>(*(params.m_RecurrentToForgetWeights));
James Conroy586a9aa2020-03-20 08:49:33 +00002498 layer->m_BasicParameters.m_RecurrentToCellWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002499 std::make_shared<ScopedTensorHandle>(*(params.m_RecurrentToCellWeights));
James Conroy586a9aa2020-03-20 08:49:33 +00002500 layer->m_BasicParameters.m_RecurrentToOutputWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002501 std::make_shared<ScopedTensorHandle>(*(params.m_RecurrentToOutputWeights));
James Conroy586a9aa2020-03-20 08:49:33 +00002502 layer->m_BasicParameters.m_ForgetGateBias =
James Conroy1f58f032021-04-27 17:13:27 +01002503 std::make_shared<ScopedTensorHandle>(*(params.m_ForgetGateBias));
James Conroy586a9aa2020-03-20 08:49:33 +00002504 layer->m_BasicParameters.m_CellBias =
James Conroy1f58f032021-04-27 17:13:27 +01002505 std::make_shared<ScopedTensorHandle>(*(params.m_CellBias));
James Conroy586a9aa2020-03-20 08:49:33 +00002506 layer->m_BasicParameters.m_OutputGateBias =
James Conroy1f58f032021-04-27 17:13:27 +01002507 std::make_shared<ScopedTensorHandle>(*(params.m_OutputGateBias));
James Conroy586a9aa2020-03-20 08:49:33 +00002508
2509 // QLstm Cifg parameters
2510 if(!descriptor.m_CifgEnabled)
2511 {
2512 if(params.m_InputToInputWeights == nullptr)
2513 {
2514 throw InvalidArgumentException("AddQLstmLayer: Input To Input Weights cannot be NULL");
2515 }
2516
2517 if(params.m_RecurrentToInputWeights == nullptr)
2518 {
2519 throw InvalidArgumentException(
2520 "AddQLstmLayer: Recurrent To Input Weights cannot be NULL");
2521 }
2522
2523 if(params.m_InputGateBias == nullptr)
2524 {
2525 throw InvalidArgumentException("AddQLstmLayer: Input Gate Bias cannot be NULL");
2526 }
2527
2528 layer->m_CifgParameters.m_InputToInputWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002529 std::make_shared<ScopedTensorHandle>(*(params.m_InputToInputWeights));
James Conroy586a9aa2020-03-20 08:49:33 +00002530 layer->m_CifgParameters.m_RecurrentToInputWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002531 std::make_shared<ScopedTensorHandle>(*(params.m_RecurrentToInputWeights));
James Conroy586a9aa2020-03-20 08:49:33 +00002532 layer->m_CifgParameters.m_InputGateBias =
James Conroy1f58f032021-04-27 17:13:27 +01002533 std::make_shared<ScopedTensorHandle>(*(params.m_InputGateBias));
James Conroy586a9aa2020-03-20 08:49:33 +00002534 }
2535
2536 // QLstm Projection parameters
2537 if(descriptor.m_ProjectionEnabled)
2538 {
2539 if(params.m_ProjectionWeights == nullptr)
2540 {
2541 throw InvalidArgumentException("AddQLstmLayer: Projection Weights cannot be NULL");
2542 }
2543
James Conroy586a9aa2020-03-20 08:49:33 +00002544 layer->m_ProjectionParameters.m_ProjectionWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002545 std::make_shared<ScopedTensorHandle>(*(params.m_ProjectionWeights));
James Conroyed324052020-05-18 15:16:42 +01002546
2547 // Projection bias is optional even if projection is enabled
2548 if(params.m_ProjectionWeights != nullptr)
2549 {
2550 layer->m_ProjectionParameters.m_ProjectionBias =
James Conroy1f58f032021-04-27 17:13:27 +01002551 std::make_shared<ScopedTensorHandle>(*(params.m_ProjectionBias));
James Conroyed324052020-05-18 15:16:42 +01002552 }
2553
James Conroy586a9aa2020-03-20 08:49:33 +00002554 }
2555
2556 // QLstm Peephole params
2557 if(descriptor.m_PeepholeEnabled)
2558 {
2559 if(params.m_CellToForgetWeights == nullptr)
2560 {
2561 throw InvalidArgumentException("AddQLstmLayer: Cell To Forget Weights cannot be NULL");
2562 }
2563
2564 if(params.m_CellToOutputWeights == nullptr)
2565 {
2566 throw InvalidArgumentException("AddQLstmLayer: Cell To Output Weights cannot be NULL");
2567 }
2568
2569 if(!descriptor.m_CifgEnabled)
2570 {
2571 if(params.m_CellToInputWeights == nullptr)
2572 {
2573 throw InvalidArgumentException("AddQLstmLayer: Cell To Input Weights cannot be NULL");
2574 }
2575
2576 layer->m_PeepholeParameters.m_CellToInputWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002577 std::make_shared<ScopedTensorHandle>(*(params.m_CellToInputWeights));
James Conroy586a9aa2020-03-20 08:49:33 +00002578 }
2579
2580 layer->m_PeepholeParameters.m_CellToForgetWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002581 std::make_shared<ScopedTensorHandle>(*(params.m_CellToForgetWeights));
James Conroy586a9aa2020-03-20 08:49:33 +00002582 layer->m_PeepholeParameters.m_CellToOutputWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002583 std::make_shared<ScopedTensorHandle>(*(params.m_CellToOutputWeights));
James Conroy586a9aa2020-03-20 08:49:33 +00002584 }
2585
2586 // QLstm Layer Normalization params
2587 if(descriptor.m_LayerNormEnabled)
2588 {
2589 if(params.m_ForgetLayerNormWeights == nullptr)
2590 {
2591 throw InvalidArgumentException("AddQLstmLayer: Forget layer normalization weights cannot be NULL");
2592 }
2593
2594 if(params.m_CellLayerNormWeights == nullptr)
2595 {
2596 throw InvalidArgumentException("AddQLstmLayer: Cell layer normalization weights cannot be NULL");
2597 }
2598
2599 if(params.m_OutputLayerNormWeights == nullptr)
2600 {
2601 throw InvalidArgumentException("AddQLstmLayer: Output layer normalization weights cannot be NULL");
2602 }
2603
2604 if(!descriptor.m_CifgEnabled)
2605 {
2606 if(params.m_InputLayerNormWeights == nullptr)
2607 {
2608 throw InvalidArgumentException("AddQLstmLayer: Input layer normalization weights cannot be NULL");
2609 }
2610
2611 layer->m_LayerNormParameters.m_InputLayerNormWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002612 std::make_shared<ScopedTensorHandle>(*(params.m_InputLayerNormWeights));
James Conroy586a9aa2020-03-20 08:49:33 +00002613 }
2614
2615 layer->m_LayerNormParameters.m_ForgetLayerNormWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002616 std::make_shared<ScopedTensorHandle>(*(params.m_ForgetLayerNormWeights));
James Conroy586a9aa2020-03-20 08:49:33 +00002617 layer->m_LayerNormParameters.m_CellLayerNormWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002618 std::make_shared<ScopedTensorHandle>(*(params.m_CellLayerNormWeights));
James Conroy586a9aa2020-03-20 08:49:33 +00002619 layer->m_LayerNormParameters.m_OutputLayerNormWeights =
James Conroy1f58f032021-04-27 17:13:27 +01002620 std::make_shared<ScopedTensorHandle>(*(params.m_OutputLayerNormWeights));
James Conroy586a9aa2020-03-20 08:49:33 +00002621 }
2622 return layer;
2623}
2624
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002625IConnectableLayer* NetworkImpl::AddLogicalBinaryLayer(const LogicalBinaryDescriptor& logicalBinaryDescriptor,
Narumol Prangnawarat8ed39ae2021-07-15 16:16:25 +01002626 const char* name)
James Conroyaba90cd2020-11-06 16:28:18 +00002627{
2628 return m_Graph->AddLayer<LogicalBinaryLayer>(logicalBinaryDescriptor, name);
2629}
2630
Narumol Prangnawarat8ed39ae2021-07-15 16:16:25 +01002631IConnectableLayer* NetworkImpl::AddUnidirectionalSequenceLstmLayer(
2632 const UnidirectionalSequenceLstmDescriptor& descriptor,
2633 const LstmInputParams& params,
2634 const char* name)
2635{
2636 const auto layer = m_Graph->AddLayer<UnidirectionalSequenceLstmLayer>(descriptor, name);
2637
2638 //Lstm Basic Parameters
2639 layer->m_BasicParameters.m_InputToForgetWeights =
2640 std::make_shared<ScopedTensorHandle>(*(params.m_InputToForgetWeights));
2641 layer->m_BasicParameters.m_InputToCellWeights =
2642 std::make_shared<ScopedTensorHandle>(*(params.m_InputToCellWeights));
2643 layer->m_BasicParameters.m_InputToOutputWeights =
2644 std::make_shared<ScopedTensorHandle>(*(params.m_InputToOutputWeights));
2645 layer->m_BasicParameters.m_RecurrentToForgetWeights =
2646 std::make_shared<ScopedTensorHandle>(*(params.m_RecurrentToForgetWeights));
2647 layer->m_BasicParameters.m_RecurrentToCellWeights =
2648 std::make_shared<ScopedTensorHandle>(*(params.m_RecurrentToCellWeights));
2649 layer->m_BasicParameters.m_RecurrentToOutputWeights =
2650 std::make_shared<ScopedTensorHandle>(*(params.m_RecurrentToOutputWeights));
2651 layer->m_BasicParameters.m_ForgetGateBias =
2652 std::make_shared<ScopedTensorHandle>(*(params.m_ForgetGateBias));
2653 layer->m_BasicParameters.m_CellBias =
2654 std::make_shared<ScopedTensorHandle>(*(params.m_CellBias));
2655 layer->m_BasicParameters.m_OutputGateBias =
2656 std::make_shared<ScopedTensorHandle>(*(params.m_OutputGateBias));
2657
2658 //Lstm Cifg parameters
2659 if(!descriptor.m_CifgEnabled)
2660 {
2661 if(params.m_InputToInputWeights == nullptr)
2662 {
2663 throw InvalidArgumentException("AddUnidirectionalSequenceLstmLayer: Input To Input Weights cannot be NULL "
2664 "when CIFG is disabled.");
2665 }
2666 if(params.m_RecurrentToInputWeights == nullptr)
2667 {
2668 throw InvalidArgumentException(
2669 "AddUnidirectionalSequenceLstmLayer: Recurrent To Input Weights cannot be NULL "
2670 "when CIFG is disabled.");
2671 }
2672 if(params.m_InputGateBias == nullptr)
2673 {
2674 throw InvalidArgumentException("AddUnidirectionalSequenceLstmLayer: Input Gate Bias cannot be NULL "
2675 "when CIFG is disabled.");
2676 }
2677 layer->m_CifgParameters.m_InputToInputWeights =
2678 std::make_shared<ScopedTensorHandle>(*(params.m_InputToInputWeights));
2679 layer->m_CifgParameters.m_RecurrentToInputWeights =
2680 std::make_shared<ScopedTensorHandle>(*(params.m_RecurrentToInputWeights));
2681 layer->m_CifgParameters.m_InputGateBias =
2682 std::make_shared<ScopedTensorHandle>(*(params.m_InputGateBias));
2683 }
2684
2685 //Lstm projection parameters
2686 if(descriptor.m_ProjectionEnabled)
2687 {
2688 if(params.m_ProjectionWeights == nullptr)
2689 {
2690 throw InvalidArgumentException("AddUnidirectionalSequenceLstmLayer: Projection Weights cannot be NULL "
2691 "when projection is enabled.");
2692 }
2693 layer->m_ProjectionParameters.m_ProjectionWeights =
2694 std::make_shared<ScopedTensorHandle>(*(params.m_ProjectionWeights));
2695 if(params.m_ProjectionBias != nullptr)
2696 {
2697 layer->m_ProjectionParameters.m_ProjectionBias =
2698 std::make_shared<ScopedTensorHandle>(*(params.m_ProjectionBias));
2699 }
2700 }
2701
2702 //Lstm Peephole params
2703 if(descriptor.m_PeepholeEnabled)
2704 {
2705 if(!descriptor.m_CifgEnabled)
2706 {
2707 if(params.m_CellToInputWeights == nullptr)
2708 {
2709 throw InvalidArgumentException("AddUnidirectionalSequenceLstmLayer: Cell To Input Weights "
2710 "cannot be NULL when Peephole is enabled and CIFG disabled.");
2711 }
2712
2713 layer->m_PeepholeParameters.m_CellToInputWeights =
2714 std::make_shared<ScopedTensorHandle>(*(params.m_CellToInputWeights));
2715 }
2716
2717 if(params.m_CellToForgetWeights == nullptr)
2718 {
2719 throw InvalidArgumentException("AddUnidirectionalSequenceLstmLayer: Cell To Forget Weights cannot be NULL "
2720 "when Peephole is enabled.");
2721 }
2722 if(params.m_CellToOutputWeights == nullptr)
2723 {
2724 throw InvalidArgumentException("AddUnidirectionalSequenceLstmLayer: Cell To Output Weights cannot be NULL "
2725 "when Peephole is enabled.");
2726 }
2727
2728 layer->m_PeepholeParameters.m_CellToForgetWeights =
2729 std::make_shared<ScopedTensorHandle>(*(params.m_CellToForgetWeights));
2730 layer->m_PeepholeParameters.m_CellToOutputWeights =
2731 std::make_shared<ScopedTensorHandle>(*(params.m_CellToOutputWeights));
2732 }
2733
2734 //Lstm Layer Normalization params
2735 if(descriptor.m_LayerNormEnabled)
2736 {
2737 if(!descriptor.m_CifgEnabled)
2738 {
2739 if(params.m_InputLayerNormWeights == nullptr)
2740 {
2741 throw InvalidArgumentException("AddUnidirectionalSequenceLstmLayer: Input layer normalization weights "
2742 "cannot be NULL when layer normalization is enabled and CIFG disabled.");
2743 }
2744 layer->m_LayerNormParameters.m_InputLayerNormWeights =
2745 std::make_shared<ScopedTensorHandle>(*(params.m_InputLayerNormWeights));
2746 }
2747
2748 if(params.m_ForgetLayerNormWeights == nullptr)
2749 {
2750 throw InvalidArgumentException("AddUnidirectionalSequenceLstmLayer: Forget layer normalization weights "
2751 "cannot be NULL when layer normalization is enabled.");
2752 }
2753 if(params.m_CellLayerNormWeights == nullptr)
2754 {
2755 throw InvalidArgumentException("AddUnidirectionalSequenceLstmLayer: Cell layer normalization weights "
2756 "cannot be NULL when layer normalization is enabled.");
2757 }
2758 if(params.m_OutputLayerNormWeights == nullptr)
2759 {
2760 throw InvalidArgumentException("AddUnidirectionalSequenceLstmLayer: Output layer normalization weights "
2761 "cannot be NULL when layer normalization is enabled.");
2762 }
2763 layer->m_LayerNormParameters.m_ForgetLayerNormWeights =
2764 std::make_shared<ScopedTensorHandle>(*(params.m_ForgetLayerNormWeights));
2765 layer->m_LayerNormParameters.m_CellLayerNormWeights =
2766 std::make_shared<ScopedTensorHandle>(*(params.m_CellLayerNormWeights));
2767 layer->m_LayerNormParameters.m_OutputLayerNormWeights =
2768 std::make_shared<ScopedTensorHandle>(*(params.m_OutputLayerNormWeights));
2769 }
2770 return layer;
2771}
2772
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002773void NetworkImpl::Accept(ILayerVisitor& visitor) const
Mike Kelly8c1701a2019-02-11 17:01:27 +00002774{
2775 for (auto layer : GetGraph())
2776 {
2777 layer->Accept(visitor);
2778 };
2779}
2780
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002781void NetworkImpl::ExecuteStrategy(IStrategy& strategy) const
Finn Williamsb454c5c2021-02-09 15:56:23 +00002782{
2783 for (auto layer : GetGraph())
2784 {
2785 layer->ExecuteStrategy(strategy);
2786 };
2787}
2788
Mike Kelly0d677db2021-06-27 22:39:21 +01002789OptimizedNetworkImpl::OptimizedNetworkImpl(const OptimizedNetworkImpl& other, const ModelOptions& modelOptions)
2790 : m_Graph(new Graph(*other.m_Graph.get()))
2791 , m_Guid(profiling::ProfilingService::GetNextGuid())
2792 , m_ModelOptions(modelOptions)
2793{
2794}
2795
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002796OptimizedNetworkImpl::OptimizedNetworkImpl(std::unique_ptr<Graph> graph)
Sadik Armagan3184c902020-03-18 10:57:30 +00002797 : m_Graph(std::move(graph)), m_Guid(profiling::ProfilingService::GetNextGuid())
telsoa014fcda012018-03-09 14:13:49 +00002798{
2799}
2800
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002801OptimizedNetworkImpl::OptimizedNetworkImpl(std::unique_ptr<Graph> graph, const ModelOptions& modelOptions)
Sadik Armagan045f6be2020-09-10 13:37:32 +01002802 : m_Graph(std::move(graph)), m_Guid(profiling::ProfilingService::GetNextGuid()), m_ModelOptions(modelOptions)
2803{
2804}
2805
Francis Murtagh3d2b4b22021-02-15 18:23:17 +00002806OptimizedNetworkImpl::~OptimizedNetworkImpl()
telsoa014fcda012018-03-09 14:13:49 +00002807{
2808}
2809
2810} // namespace armnn