blob: 640e5947e2b33c17712507be16458b5480f57fed [file] [log] [blame]
Laurent Carlier749294b2020-06-01 09:03:17 +01001//
Jim Flynn6398a982020-05-27 17:05:21 +01002// Copyright © 2017 Arm Ltd and Contributors. All rights reserved.
David Beckecb56cd2018-09-05 12:52:57 +01003// SPDX-License-Identifier: MIT
telsoa014fcda012018-03-09 14:13:49 +00004//
5#include "Runtime.hpp"
6
David Beck056be3c2018-10-22 13:16:00 +01007#include <armnn/Version.hpp>
Matteo Martincighc601aa62019-10-29 15:03:22 +00008#include <armnn/BackendRegistry.hpp>
Jan Eilers15fcc7e2021-07-14 13:50:15 +01009#include <armnn/BackendHelper.hpp>
Matthew Benthamf48afc62020-01-15 17:55:08 +000010#include <armnn/Logging.hpp>
alered01a7227ac2020-05-07 14:58:29 +010011#include <armnn/utility/Timer.hpp>
Matteo Martincighe54aa062019-08-05 14:12:11 +010012
Matteo Martincighe5b8eb92019-11-28 15:45:42 +000013#include <armnn/backends/IBackendContext.hpp>
Matteo Martincighe54aa062019-08-05 14:12:11 +010014#include <backendsCommon/DynamicBackendUtils.hpp>
Jim Flynne1fdd282021-10-26 21:26:10 +010015#include <backendsCommon/memoryOptimizerStrategyLibrary/MemoryOptimizerStrategyLibrary.hpp>
Jan Eilersbb446e52020-04-02 13:56:54 +010016#include <armnn/utility/PolymorphicDowncast.hpp>
Jim Flynn4c9ed1d2022-01-23 23:57:20 +000017#include <ProfilingOptionsConverter.hpp>
telsoa014fcda012018-03-09 14:13:49 +000018
Nikhil Raj77fe76b2021-06-09 14:55:32 +010019#include <common/include/LabelsAndEventClasses.hpp>
20
surmeh013537c2c2018-05-18 16:31:43 +010021#include <iostream>
22
Colm Donelan1aff3932020-02-05 17:48:59 +000023#include <backends/BackendProfiling.hpp>
telsoa014fcda012018-03-09 14:13:49 +000024
25using namespace armnn;
26using namespace std;
27
28namespace armnn
29{
Kevin Mayd92a6e42021-02-04 10:27:41 +000030IRuntime::IRuntime() : pRuntimeImpl( new RuntimeImpl(armnn::IRuntime::CreationOptions())) {}
31
32IRuntime::IRuntime(const IRuntime::CreationOptions& options) : pRuntimeImpl(new RuntimeImpl(options)) {}
33
34IRuntime::~IRuntime() = default;
telsoa014fcda012018-03-09 14:13:49 +000035
36IRuntime* IRuntime::CreateRaw(const CreationOptions& options)
37{
Kevin Mayd92a6e42021-02-04 10:27:41 +000038 return new IRuntime(options);
telsoa014fcda012018-03-09 14:13:49 +000039}
40
41IRuntimePtr IRuntime::Create(const CreationOptions& options)
42{
43 return IRuntimePtr(CreateRaw(options), &IRuntime::Destroy);
44}
45
46void IRuntime::Destroy(IRuntime* runtime)
47{
Kevin Mayd92a6e42021-02-04 10:27:41 +000048 delete runtime;
telsoa014fcda012018-03-09 14:13:49 +000049}
50
Kevin Mayd92a6e42021-02-04 10:27:41 +000051Status IRuntime::LoadNetwork(NetworkId& networkIdOut, IOptimizedNetworkPtr network)
52{
53 return pRuntimeImpl->LoadNetwork(networkIdOut, std::move(network));
54}
55
56Status IRuntime::LoadNetwork(NetworkId& networkIdOut,
57 IOptimizedNetworkPtr network,
58 std::string& errorMessage)
59{
60 return pRuntimeImpl->LoadNetwork(networkIdOut, std::move(network), errorMessage);
61}
62
63Status IRuntime::LoadNetwork(NetworkId& networkIdOut,
64 IOptimizedNetworkPtr network,
65 std::string& errorMessage,
66 const INetworkProperties& networkProperties)
67{
68 return pRuntimeImpl->LoadNetwork(networkIdOut, std::move(network), errorMessage, networkProperties);
69}
70
Cathal Corbett5b8093c2021-10-22 11:12:07 +010071armnn::TensorInfo IRuntime::GetInputTensorInfo(NetworkId networkId, LayerBindingId layerId) const
Kevin Mayd92a6e42021-02-04 10:27:41 +000072{
73 return pRuntimeImpl->GetInputTensorInfo(networkId, layerId);
74}
75
Cathal Corbett5b8093c2021-10-22 11:12:07 +010076armnn::TensorInfo IRuntime::GetOutputTensorInfo(NetworkId networkId, LayerBindingId layerId) const
Kevin Mayd92a6e42021-02-04 10:27:41 +000077{
78 return pRuntimeImpl->GetOutputTensorInfo(networkId, layerId);
79}
80
Narumol Prangnawarate2af6f42022-01-28 17:59:18 +000081std::vector<ImportedInputId> IRuntime::ImportInputs(NetworkId networkId, const InputTensors& inputTensors,
82 MemorySource forceImportMemorySource)
Finn Williamsf37b9702021-09-01 18:06:04 +010083{
Narumol Prangnawarate2af6f42022-01-28 17:59:18 +000084 return pRuntimeImpl->ImportInputs(networkId, inputTensors, forceImportMemorySource);
Finn Williamsf37b9702021-09-01 18:06:04 +010085}
86
Narumol Prangnawarate2af6f42022-01-28 17:59:18 +000087std::vector<ImportedOutputId> IRuntime::ImportOutputs(NetworkId networkId, const OutputTensors& outputTensors,
88 MemorySource forceImportMemorySource)
Finn Williams8636bc72021-10-02 15:06:39 +010089{
Narumol Prangnawarate2af6f42022-01-28 17:59:18 +000090 return pRuntimeImpl->ImportOutputs(networkId, outputTensors, forceImportMemorySource);
Finn Williams8636bc72021-10-02 15:06:39 +010091}
92
93void IRuntime::ClearImportedInputs(NetworkId networkId, const std::vector<ImportedInputId> inputIds)
94{
95 return pRuntimeImpl->ClearImportedInputs(networkId, inputIds);
96}
97void IRuntime::ClearImportedOutputs(NetworkId networkId, const std::vector<ImportedOutputId> outputIds)
98{
99 return pRuntimeImpl->ClearImportedOutputs(networkId, outputIds);
100}
Finn Williamsf37b9702021-09-01 18:06:04 +0100101
Kevin Mayd92a6e42021-02-04 10:27:41 +0000102Status IRuntime::EnqueueWorkload(NetworkId networkId,
103 const InputTensors& inputTensors,
Narumol Prangnawarate2af6f42022-01-28 17:59:18 +0000104 const OutputTensors& outputTensors,
105 std::vector<ImportedInputId> preImportedInputIds,
106 std::vector<ImportedOutputId> preImportedOutputIds)
Kevin Mayd92a6e42021-02-04 10:27:41 +0000107{
Narumol Prangnawarate2af6f42022-01-28 17:59:18 +0000108 return pRuntimeImpl->EnqueueWorkload(networkId, inputTensors, outputTensors,
109 preImportedInputIds, preImportedOutputIds);
Kevin Mayd92a6e42021-02-04 10:27:41 +0000110}
111
Mike Kelly55a8ffd2021-04-07 20:10:49 +0100112Status IRuntime::Execute(IWorkingMemHandle& workingMemHandle,
113 const InputTensors& inputTensors,
Finn Williamsf37b9702021-09-01 18:06:04 +0100114 const OutputTensors& outputTensors,
Finn Williams8636bc72021-10-02 15:06:39 +0100115 std::vector<ImportedInputId> preImportedInputs,
116 std::vector<ImportedOutputId> preImportedOutputs)
Mike Kelly55a8ffd2021-04-07 20:10:49 +0100117{
Finn Williams8636bc72021-10-02 15:06:39 +0100118 return pRuntimeImpl->Execute(workingMemHandle, inputTensors, outputTensors, preImportedInputs, preImportedOutputs);
Mike Kelly55a8ffd2021-04-07 20:10:49 +0100119}
120
Kevin Mayd92a6e42021-02-04 10:27:41 +0000121Status IRuntime::UnloadNetwork(NetworkId networkId)
122{
123 return pRuntimeImpl->UnloadNetwork(networkId);
124}
125
126const IDeviceSpec& IRuntime::GetDeviceSpec() const
127{
128 return pRuntimeImpl->GetDeviceSpec();
129}
130
Mike Kelly55a8ffd2021-04-07 20:10:49 +0100131std::unique_ptr<IWorkingMemHandle> IRuntime::CreateWorkingMemHandle(NetworkId networkId)
132{
133 return pRuntimeImpl->CreateWorkingMemHandle(networkId);
134}
135
Kevin Mayd92a6e42021-02-04 10:27:41 +0000136const std::shared_ptr<IProfiler> IRuntime::GetProfiler(NetworkId networkId) const
137{
138 return pRuntimeImpl->GetProfiler(networkId);
139}
140
141void IRuntime::RegisterDebugCallback(NetworkId networkId, const DebugCallbackFunction& func)
142{
143 return pRuntimeImpl->RegisterDebugCallback(networkId, func);
144}
145
146int RuntimeImpl::GenerateNetworkId()
telsoa014fcda012018-03-09 14:13:49 +0000147{
148 return m_NetworkIdCounter++;
149}
150
Kevin Mayd92a6e42021-02-04 10:27:41 +0000151Status RuntimeImpl::LoadNetwork(NetworkId& networkIdOut, IOptimizedNetworkPtr inNetwork)
telsoa014fcda012018-03-09 14:13:49 +0000152{
telsoa01c577f2c2018-08-31 09:22:23 +0100153 std::string ignoredErrorMessage;
154 return LoadNetwork(networkIdOut, std::move(inNetwork), ignoredErrorMessage);
155}
156
Kevin Mayd92a6e42021-02-04 10:27:41 +0000157Status RuntimeImpl::LoadNetwork(NetworkId& networkIdOut,
158 IOptimizedNetworkPtr inNetwork,
159 std::string& errorMessage)
David Monahan4f1e8e42019-09-04 09:22:10 +0100160{
Jan Eilersc1c872f2021-07-22 13:17:04 +0100161 INetworkProperties networkProperties(
162 false, MemorySource::Undefined, MemorySource::Undefined);
David Monahan4f1e8e42019-09-04 09:22:10 +0100163 return LoadNetwork(networkIdOut, std::move(inNetwork), errorMessage, networkProperties);
164}
165
Kevin Mayd92a6e42021-02-04 10:27:41 +0000166Status RuntimeImpl::LoadNetwork(NetworkId& networkIdOut,
167 IOptimizedNetworkPtr inNetwork,
168 std::string& errorMessage,
169 const INetworkProperties& networkProperties)
telsoa01c577f2c2018-08-31 09:22:23 +0100170{
Derek Lambertie155bbf2021-10-13 14:32:12 +0100171 // Register the profiler
172 auto profiler = inNetwork->GetProfiler();
173 ProfilerManager::GetInstance().RegisterProfiler(profiler.get());
174
telsoa014fcda012018-03-09 14:13:49 +0000175 IOptimizedNetwork* rawNetwork = inNetwork.release();
David Beck1b61be52018-11-08 09:19:14 +0000176
177 networkIdOut = GenerateNetworkId();
178
179 for (auto&& context : m_BackendContexts)
180 {
181 context.second->BeforeLoadNetwork(networkIdOut);
182 }
183
telsoa014fcda012018-03-09 14:13:49 +0000184 unique_ptr<LoadedNetwork> loadedNetwork = LoadedNetwork::MakeLoadedNetwork(
Francis Murtagh3d2b4b22021-02-15 18:23:17 +0000185 std::unique_ptr<IOptimizedNetwork>(rawNetwork),
David Monahan4f1e8e42019-09-04 09:22:10 +0100186 errorMessage,
Sadik Armagan3184c902020-03-18 10:57:30 +0000187 networkProperties,
Finn Williamsf364d532021-06-09 17:07:33 +0100188 m_ProfilingService);
telsoa014fcda012018-03-09 14:13:49 +0000189
190 if (!loadedNetwork)
191 {
192 return Status::Failure;
193 }
194
telsoa01c577f2c2018-08-31 09:22:23 +0100195 {
196 std::lock_guard<std::mutex> lockGuard(m_Mutex);
197
198 // Stores the network
199 m_LoadedNetworks[networkIdOut] = std::move(loadedNetwork);
200 }
telsoa014fcda012018-03-09 14:13:49 +0000201
David Beck1b61be52018-11-08 09:19:14 +0000202 for (auto&& context : m_BackendContexts)
203 {
204 context.second->AfterLoadNetwork(networkIdOut);
205 }
206
Sadik Armagan3184c902020-03-18 10:57:30 +0000207 if (m_ProfilingService.IsProfilingEnabled())
Keith Davise394bd92019-12-02 15:12:19 +0000208 {
Cathal Corbett5aa9fd72022-02-25 15:33:28 +0000209 m_ProfilingService.IncrementCounterValue(arm::pipe::NETWORK_LOADS);
Keith Davise394bd92019-12-02 15:12:19 +0000210 }
211
telsoa014fcda012018-03-09 14:13:49 +0000212 return Status::Success;
telsoa014fcda012018-03-09 14:13:49 +0000213}
214
Kevin Mayd92a6e42021-02-04 10:27:41 +0000215Status RuntimeImpl::UnloadNetwork(NetworkId networkId)
telsoa014fcda012018-03-09 14:13:49 +0000216{
David Beck1b61be52018-11-08 09:19:14 +0000217 bool unloadOk = true;
218 for (auto&& context : m_BackendContexts)
David Beck9efb57d2018-11-05 13:40:33 +0000219 {
David Beck1b61be52018-11-08 09:19:14 +0000220 unloadOk &= context.second->BeforeUnloadNetwork(networkId);
David Beck9efb57d2018-11-05 13:40:33 +0000221 }
David Beck1b61be52018-11-08 09:19:14 +0000222
223 if (!unloadOk)
224 {
Kevin Mayd92a6e42021-02-04 10:27:41 +0000225 ARMNN_LOG(warning) << "RuntimeImpl::UnloadNetwork(): failed to unload "
Derek Lamberti08446972019-11-26 16:38:31 +0000226 "network with ID:" << networkId << " because BeforeUnloadNetwork failed";
David Beck1b61be52018-11-08 09:19:14 +0000227 return Status::Failure;
228 }
David Beck9efb57d2018-11-05 13:40:33 +0000229
Cathal Corbett5aa9fd72022-02-25 15:33:28 +0000230 std::unique_ptr<arm::pipe::TimelineUtilityMethods> timelineUtils =
231 arm::pipe::TimelineUtilityMethods::GetTimelineUtils(m_ProfilingService);
telsoa014fcda012018-03-09 14:13:49 +0000232 {
telsoa01c577f2c2018-08-31 09:22:23 +0100233 std::lock_guard<std::mutex> lockGuard(m_Mutex);
234
Jim Flynnf7713212020-07-14 09:50:59 +0100235 // If timeline recording is on mark the Network end of life
236 if (timelineUtils)
237 {
238 auto search = m_LoadedNetworks.find(networkId);
239 if (search != m_LoadedNetworks.end())
240 {
Cathal Corbett5aa9fd72022-02-25 15:33:28 +0000241 arm::pipe::ProfilingGuid networkGuid = search->second->GetNetworkGuid();
Jim Flynnf7713212020-07-14 09:50:59 +0100242 timelineUtils->RecordEvent(networkGuid,
Cathal Corbett5aa9fd72022-02-25 15:33:28 +0000243 arm::pipe::LabelsAndEventClasses::ARMNN_PROFILING_EOL_EVENT_CLASS);
Jim Flynnf7713212020-07-14 09:50:59 +0100244 }
245 }
Narumol Prangnawaratec5463d2022-02-04 17:50:20 +0000246
telsoa01c577f2c2018-08-31 09:22:23 +0100247 if (m_LoadedNetworks.erase(networkId) == 0)
248 {
Kevin Mayd92a6e42021-02-04 10:27:41 +0000249 ARMNN_LOG(warning) << "WARNING: RuntimeImpl::UnloadNetwork(): " << networkId << " not found!";
telsoa01c577f2c2018-08-31 09:22:23 +0100250 return Status::Failure;
251 }
Sadik Armagan3184c902020-03-18 10:57:30 +0000252
253 if (m_ProfilingService.IsProfilingEnabled())
Keith Davise394bd92019-12-02 15:12:19 +0000254 {
Cathal Corbett5aa9fd72022-02-25 15:33:28 +0000255 m_ProfilingService.IncrementCounterValue(arm::pipe::NETWORK_UNLOADS);
Keith Davise394bd92019-12-02 15:12:19 +0000256 }
David Beck1b61be52018-11-08 09:19:14 +0000257 }
David Beck9efb57d2018-11-05 13:40:33 +0000258
David Beck1b61be52018-11-08 09:19:14 +0000259 for (auto&& context : m_BackendContexts)
260 {
261 context.second->AfterUnloadNetwork(networkId);
telsoa01c577f2c2018-08-31 09:22:23 +0100262 }
263
Derek Lambertie155bbf2021-10-13 14:32:12 +0100264 // Unregister the profiler
265 ProfilerManager::GetInstance().RegisterProfiler(nullptr);
266
Kevin Mayd92a6e42021-02-04 10:27:41 +0000267 ARMNN_LOG(debug) << "RuntimeImpl::UnloadNetwork(): Unloaded network with ID: " << networkId;
telsoa014fcda012018-03-09 14:13:49 +0000268 return Status::Success;
269}
270
Kevin Mayd92a6e42021-02-04 10:27:41 +0000271const std::shared_ptr<IProfiler> RuntimeImpl::GetProfiler(NetworkId networkId) const
telsoa01c577f2c2018-08-31 09:22:23 +0100272{
273 auto it = m_LoadedNetworks.find(networkId);
274 if (it != m_LoadedNetworks.end())
275 {
276 auto& loadedNetwork = it->second;
277 return loadedNetwork->GetProfiler();
278 }
279
280 return nullptr;
281}
282
Cathal Corbett5aa9fd72022-02-25 15:33:28 +0000283void RuntimeImpl::ReportStructure() // arm::pipe::IProfilingService& profilingService as param
Keith Davis33ed2212020-03-30 10:43:41 +0100284{
285 // No-op for the time being, but this may be useful in future to have the profilingService available
286 // if (profilingService.IsProfilingEnabled()){}
287
288 LoadedNetworks::iterator it = m_LoadedNetworks.begin();
289 while (it != m_LoadedNetworks.end())
290 {
291 auto& loadedNetwork = it->second;
292 loadedNetwork->SendNetworkStructure();
293 // Increment the Iterator to point to next entry
294 it++;
295 }
296}
297
Kevin Mayd92a6e42021-02-04 10:27:41 +0000298RuntimeImpl::RuntimeImpl(const IRuntime::CreationOptions& options)
Keith Davis33ed2212020-03-30 10:43:41 +0100299 : m_NetworkIdCounter(0),
300 m_ProfilingService(*this)
telsoa014fcda012018-03-09 14:13:49 +0000301{
alered01a7227ac2020-05-07 14:58:29 +0100302 const auto start_time = armnn::GetTimeNow();
Sadik Armagan4a0844d2022-01-26 09:57:05 +0000303 ARMNN_LOG(info) << "ArmNN v" << ARMNN_VERSION;
Keith Davis33ed2212020-03-30 10:43:41 +0100304 if ( options.m_ProfilingOptions.m_TimelineEnabled && !options.m_ProfilingOptions.m_EnableProfiling )
305 {
Jan Eilersc1c872f2021-07-22 13:17:04 +0100306 throw RuntimeException(
307 "It is not possible to enable timeline reporting without profiling being enabled");
Keith Davis33ed2212020-03-30 10:43:41 +0100308 }
309
Matteo Martincighe54aa062019-08-05 14:12:11 +0100310 // Load any available/compatible dynamic backend before the runtime
311 // goes through the backend registry
312 LoadDynamicBackends(options.m_DynamicBackendsPath);
313
Cathal Corbett5aa9fd72022-02-25 15:33:28 +0000314 armnn::BackendIdSet supportedBackends;
David Beck1b61be52018-11-08 09:19:14 +0000315 for (const auto& id : BackendRegistryInstance().GetBackendIds())
316 {
317 // Store backend contexts for the supported ones
Matthew Bentham9a61fa62020-02-04 10:03:55 +0000318 try {
David Beck1b61be52018-11-08 09:19:14 +0000319 auto factoryFun = BackendRegistryInstance().GetFactory(id);
Colm Donelan380c1a02021-08-17 00:52:23 +0100320 ARMNN_ASSERT(factoryFun != nullptr);
David Beck1b61be52018-11-08 09:19:14 +0000321 auto backend = factoryFun();
Colm Donelan380c1a02021-08-17 00:52:23 +0100322 ARMNN_ASSERT(backend != nullptr);
Narumol Prangnawaratac2770a2020-04-01 16:51:23 +0100323 ARMNN_ASSERT(backend.get() != nullptr);
David Beck1b61be52018-11-08 09:19:14 +0000324
Jan Eilersc1c872f2021-07-22 13:17:04 +0100325 auto customAllocatorMapIterator = options.m_CustomAllocatorMap.find(id);
Francis Murtagh62573b62021-08-12 11:55:21 +0100326 if (customAllocatorMapIterator != options.m_CustomAllocatorMap.end() &&
327 customAllocatorMapIterator->second == nullptr)
328 {
Colm Donelan380c1a02021-08-17 00:52:23 +0100329 // We need to manually clean up the dynamic backends before throwing an exception.
330 DynamicBackendUtils::DeregisterDynamicBackends(m_DeviceSpec.GetDynamicBackends());
331 m_DeviceSpec.ClearDynamicBackends();
Francis Murtagh62573b62021-08-12 11:55:21 +0100332 throw armnn::Exception("Allocator associated with id " + id.Get() + " is null");
333 }
Jan Eilersc1c872f2021-07-22 13:17:04 +0100334
Jan Eilers15fcc7e2021-07-14 13:50:15 +0100335 // If the runtime is created in protected mode only add backends that support this mode
336 if (options.m_ProtectedMode)
337 {
338 // check if backend supports ProtectedMode
339 using BackendCapability = BackendOptions::BackendOption;
340 BackendCapability protectedContentCapability {"ProtectedContentAllocation", true};
341 if (!HasCapability(protectedContentCapability, id))
342 {
343 // Protected Content Allocation is not supported by the backend
344 // backend should not be registered
345 ARMNN_LOG(warning) << "Backend "
346 << id
Sadik Armagan4a0844d2022-01-26 09:57:05 +0000347 << " is not registered as does not support protected content allocation.";
Jan Eilers15fcc7e2021-07-14 13:50:15 +0100348 continue;
349 }
Jan Eilersc1c872f2021-07-22 13:17:04 +0100350 // The user is responsible to provide a custom memory allocator which allows to allocate
351 // protected memory
352 if (customAllocatorMapIterator != options.m_CustomAllocatorMap.end())
Jan Eilers15fcc7e2021-07-14 13:50:15 +0100353 {
Jan Eilersc1c872f2021-07-22 13:17:04 +0100354 std::string err;
355 if (customAllocatorMapIterator->second->GetMemorySourceType()
356 == armnn::MemorySource::DmaBufProtected)
357 {
358 if (!backend->UseCustomMemoryAllocator(customAllocatorMapIterator->second, err))
359 {
360 ARMNN_LOG(error) << "The backend "
361 << id
362 << " reported an error when entering protected mode. Backend won't be"
363 << " used. ErrorMsg: " << err;
364 continue;
365 }
366 // No errors so register the Custom Allocator with the BackendRegistry
367 BackendRegistryInstance().RegisterAllocator(id, customAllocatorMapIterator->second);
368 }
369 else
370 {
371 ARMNN_LOG(error) << "The CustomAllocator provided with the runtime options doesn't support "
372 "protected memory. Protected mode can't be activated. The backend "
Jan Eilers15fcc7e2021-07-14 13:50:15 +0100373 << id
Jan Eilersc1c872f2021-07-22 13:17:04 +0100374 << " is not going to be used. MemorySource must be MemorySource::DmaBufProtected";
375 continue;
376 }
377 }
378 else
379 {
380 ARMNN_LOG(error) << "Protected mode can't be activated for backend: "
381 << id
382 << " no custom allocator was provided to the runtime options.";
Jan Eilers15fcc7e2021-07-14 13:50:15 +0100383 continue;
384 }
385 }
Jan Eilersc1c872f2021-07-22 13:17:04 +0100386 else
387 {
388 // If a custom memory allocator is provided make the backend use that instead of the default
389 if (customAllocatorMapIterator != options.m_CustomAllocatorMap.end())
390 {
391 std::string err;
392 if (!backend->UseCustomMemoryAllocator(customAllocatorMapIterator->second, err))
393 {
394 ARMNN_LOG(error) << "The backend "
395 << id
396 << " reported an error when trying to use the provided custom allocator."
397 " Backend won't be used."
398 << " ErrorMsg: " << err;
399 continue;
400 }
401 // No errors so register the Custom Allocator with the BackendRegistry
402 BackendRegistryInstance().RegisterAllocator(id, customAllocatorMapIterator->second);
403 }
404 }
Sadik Armaganb8a26d82021-10-04 15:13:11 +0100405
406 // check if custom memory optimizer strategy map is set
407 if (!options.m_MemoryOptimizerStrategyMap.empty())
408 {
409 auto customMemoryOptimizerStrategyMapIterator = options.m_MemoryOptimizerStrategyMap.find(id);
410 // if a memory optimizer strategy is provided make the backend use that instead of the default
411 if (customMemoryOptimizerStrategyMapIterator != options.m_MemoryOptimizerStrategyMap.end())
412 {
413 // no errors.. register the memory optimizer strategy with the BackendRegistry
414 BackendRegistryInstance().RegisterMemoryOptimizerStrategy(
415 id, customMemoryOptimizerStrategyMapIterator->second);
416
417 ARMNN_LOG(info) << "MemoryOptimizerStrategy "
418 << customMemoryOptimizerStrategyMapIterator->second->GetName()
419 << " set for the backend " << id << ".";
420 }
421 }
422 else
423 {
424 // check if to use one of the existing memory optimizer strategies is set
425 std::string memoryOptimizerStrategyName = "";
426 ParseOptions(options.m_BackendOptions, id, [&](std::string name, const BackendOptions::Var& value)
427 {
428 if (name == "MemoryOptimizerStrategy")
429 {
430 memoryOptimizerStrategyName = ParseStringBackendOption(value, "");
431 }
432 });
433 if (memoryOptimizerStrategyName != "")
434 {
Jim Flynne1fdd282021-10-26 21:26:10 +0100435 std::shared_ptr<IMemoryOptimizerStrategy> strategy =
436 GetMemoryOptimizerStrategy(memoryOptimizerStrategyName);
437
438 if (!strategy)
Sadik Armaganb8a26d82021-10-04 15:13:11 +0100439 {
Jim Flynne1fdd282021-10-26 21:26:10 +0100440 ARMNN_LOG(warning) << "MemoryOptimizerStrategy: " << memoryOptimizerStrategyName
Sadik Armagan4a0844d2022-01-26 09:57:05 +0000441 << " was not found.";
Jim Flynne1fdd282021-10-26 21:26:10 +0100442 }
443 else
444 {
445 using BackendCapability = BackendOptions::BackendOption;
446 auto strategyType = GetMemBlockStrategyTypeName(strategy->GetMemBlockStrategyType());
447 BackendCapability memOptimizeStrategyCapability {strategyType, true};
448 if (HasCapability(memOptimizeStrategyCapability, id))
449 {
450 BackendRegistryInstance().RegisterMemoryOptimizerStrategy(id, strategy);
451
452 ARMNN_LOG(info) << "MemoryOptimizerStrategy: "
453 << memoryOptimizerStrategyName << " set for the backend " << id << ".";
454 }
455 else
456 {
457 ARMNN_LOG(warning) << "Backend "
458 << id
459 << " does not have multi-axis packing capability and cannot support"
Sadik Armagan4a0844d2022-01-26 09:57:05 +0000460 << "MemoryOptimizerStrategy: " << memoryOptimizerStrategyName << ".";
Jim Flynne1fdd282021-10-26 21:26:10 +0100461 }
Sadik Armaganb8a26d82021-10-04 15:13:11 +0100462 }
463 }
464 }
465
David Beck1b61be52018-11-08 09:19:14 +0000466 auto context = backend->CreateBackendContext(options);
467
468 // backends are allowed to return nullptrs if they
469 // don't wish to create a backend specific context
470 if (context)
471 {
472 m_BackendContexts.emplace(std::make_pair(id, std::move(context)));
473 }
Matthew Bentham9a61fa62020-02-04 10:03:55 +0000474 supportedBackends.emplace(id);
Colm Donelan1aff3932020-02-05 17:48:59 +0000475
Cathal Corbett5aa9fd72022-02-25 15:33:28 +0000476 unique_ptr<arm::pipe::IBackendProfiling> profilingIface =
477 std::make_unique<arm::pipe::BackendProfiling>(arm::pipe::BackendProfiling(
478 arm::pipe::ConvertExternalProfilingOptions(options.m_ProfilingOptions), m_ProfilingService, id));
Colm Donelan1aff3932020-02-05 17:48:59 +0000479
480 // Backends may also provide a profiling context. Ask for it now.
481 auto profilingContext = backend->CreateBackendProfilingContext(options, profilingIface);
482 // Backends that don't support profiling will return a null profiling context.
483 if (profilingContext)
484 {
Finn Williamsfe5a24b2020-04-09 16:05:28 +0100485 // Pass the context onto the profiling service.
486 m_ProfilingService.AddBackendProfilingContext(id, profilingContext);
Colm Donelan1aff3932020-02-05 17:48:59 +0000487 }
David Beck1b61be52018-11-08 09:19:14 +0000488 }
Matthew Bentham9a61fa62020-02-04 10:03:55 +0000489 catch (const BackendUnavailableException&)
490 {
491 // Ignore backends which are unavailable
492 }
David Beck1b61be52018-11-08 09:19:14 +0000493 }
Finn Williamsfe5a24b2020-04-09 16:05:28 +0100494
Finn Williams45a73622020-05-15 18:41:05 +0100495 BackendRegistryInstance().SetProfilingService(m_ProfilingService);
Finn Williamsfe5a24b2020-04-09 16:05:28 +0100496 // pass configuration info to the profiling service
Cathal Corbett5aa9fd72022-02-25 15:33:28 +0000497 m_ProfilingService.ConfigureProfilingService(
498 arm::pipe::ConvertExternalProfilingOptions(options.m_ProfilingOptions));
Jim Flynn6398a982020-05-27 17:05:21 +0100499 if (options.m_ProfilingOptions.m_EnableProfiling)
500 {
501 // try to wait for the profiling service to initialise
502 m_ProfilingService.WaitForProfilingServiceActivation(3000);
503 }
Finn Williamsfe5a24b2020-04-09 16:05:28 +0100504
Matthew Bentham9a61fa62020-02-04 10:03:55 +0000505 m_DeviceSpec.AddSupportedBackends(supportedBackends);
alered01a7227ac2020-05-07 14:58:29 +0100506
507 ARMNN_LOG(info) << "Initialization time: " << std::setprecision(2)
Sadik Armagan4a0844d2022-01-26 09:57:05 +0000508 << std::fixed << armnn::GetTimeDuration(start_time).count() << " ms.";
surmeh01bceff2f2018-03-29 16:29:27 +0100509}
510
Kevin Mayd92a6e42021-02-04 10:27:41 +0000511RuntimeImpl::~RuntimeImpl()
surmeh01bceff2f2018-03-29 16:29:27 +0100512{
Jan Eilers17d34da2021-12-08 16:15:12 +0000513 const auto startTime = armnn::GetTimeNow();
surmeh01bceff2f2018-03-29 16:29:27 +0100514 std::vector<int> networkIDs;
surmeh013537c2c2018-05-18 16:31:43 +0100515 try
516 {
517 // Coverity fix: The following code may throw an exception of type std::length_error.
518 std::transform(m_LoadedNetworks.begin(), m_LoadedNetworks.end(),
519 std::back_inserter(networkIDs),
520 [](const auto &pair) { return pair.first; });
521 }
522 catch (const std::exception& e)
523 {
524 // Coverity fix: BOOST_LOG_TRIVIAL (typically used to report errors) may throw an
525 // exception of type std::length_error.
526 // Using stderr instead in this context as there is no point in nesting try-catch blocks here.
527 std::cerr << "WARNING: An error has occurred when getting the IDs of the networks to unload: " << e.what()
528 << "\nSome of the loaded networks may not be unloaded" << std::endl;
529 }
530 // We then proceed to unload all the networks which IDs have been appended to the list
531 // up to the point the exception was thrown (if any).
surmeh01bceff2f2018-03-29 16:29:27 +0100532
533 for (auto networkID : networkIDs)
534 {
surmeh013537c2c2018-05-18 16:31:43 +0100535 try
536 {
537 // Coverity fix: UnloadNetwork() may throw an exception of type std::length_error,
538 // boost::log::v2s_mt_posix::odr_violation or boost::log::v2s_mt_posix::system_error
539 UnloadNetwork(networkID);
540 }
541 catch (const std::exception& e)
542 {
543 // Coverity fix: BOOST_LOG_TRIVIAL (typically used to report errors) may throw an
544 // exception of type std::length_error.
545 // Using stderr instead in this context as there is no point in nesting try-catch blocks here.
546 std::cerr << "WARNING: An error has occurred when unloading network " << networkID << ": " << e.what()
547 << std::endl;
548 }
telsoa014fcda012018-03-09 14:13:49 +0000549 }
Narumol Prangnawarat60a20fb2019-12-09 17:24:41 +0000550
551 // Clear all dynamic backends.
552 DynamicBackendUtils::DeregisterDynamicBackends(m_DeviceSpec.GetDynamicBackends());
553 m_DeviceSpec.ClearDynamicBackends();
Colm Donelan1aff3932020-02-05 17:48:59 +0000554 m_BackendContexts.clear();
Finn Williams45a73622020-05-15 18:41:05 +0100555
556 BackendRegistryInstance().SetProfilingService(armnn::EmptyOptional());
alered01a7227ac2020-05-07 14:58:29 +0100557 ARMNN_LOG(info) << "Shutdown time: " << std::setprecision(2)
Sadik Armagan4a0844d2022-01-26 09:57:05 +0000558 << std::fixed << armnn::GetTimeDuration(startTime).count() << " ms.";
telsoa014fcda012018-03-09 14:13:49 +0000559}
560
Kevin Mayd92a6e42021-02-04 10:27:41 +0000561LoadedNetwork* RuntimeImpl::GetLoadedNetworkPtr(NetworkId networkId) const
surmeh013537c2c2018-05-18 16:31:43 +0100562{
563 std::lock_guard<std::mutex> lockGuard(m_Mutex);
564 return m_LoadedNetworks.at(networkId).get();
565}
566
Kevin Mayd92a6e42021-02-04 10:27:41 +0000567TensorInfo RuntimeImpl::GetInputTensorInfo(NetworkId networkId, LayerBindingId layerId) const
telsoa014fcda012018-03-09 14:13:49 +0000568{
surmeh013537c2c2018-05-18 16:31:43 +0100569 return GetLoadedNetworkPtr(networkId)->GetInputTensorInfo(layerId);
telsoa014fcda012018-03-09 14:13:49 +0000570}
571
Kevin Mayd92a6e42021-02-04 10:27:41 +0000572TensorInfo RuntimeImpl::GetOutputTensorInfo(NetworkId networkId, LayerBindingId layerId) const
telsoa014fcda012018-03-09 14:13:49 +0000573{
surmeh013537c2c2018-05-18 16:31:43 +0100574 return GetLoadedNetworkPtr(networkId)->GetOutputTensorInfo(layerId);
telsoa014fcda012018-03-09 14:13:49 +0000575}
576
Narumol Prangnawarate2af6f42022-01-28 17:59:18 +0000577std::vector<ImportedInputId> RuntimeImpl::ImportInputs(NetworkId networkId, const InputTensors& inputTensors,
578 MemorySource forceImportMemorySource)
Finn Williamsf37b9702021-09-01 18:06:04 +0100579{
Narumol Prangnawarate2af6f42022-01-28 17:59:18 +0000580 return GetLoadedNetworkPtr(networkId)->ImportInputs(inputTensors, forceImportMemorySource);
Finn Williamsf37b9702021-09-01 18:06:04 +0100581}
582
Narumol Prangnawarate2af6f42022-01-28 17:59:18 +0000583std::vector<ImportedOutputId> RuntimeImpl::ImportOutputs(NetworkId networkId, const OutputTensors& outputTensors,
584 MemorySource forceImportMemorySource)
Finn Williams8636bc72021-10-02 15:06:39 +0100585{
Narumol Prangnawarate2af6f42022-01-28 17:59:18 +0000586 return GetLoadedNetworkPtr(networkId)->ImportOutputs(outputTensors, forceImportMemorySource);
Finn Williams8636bc72021-10-02 15:06:39 +0100587}
Finn Williamsf37b9702021-09-01 18:06:04 +0100588
Finn Williams8636bc72021-10-02 15:06:39 +0100589void RuntimeImpl::ClearImportedInputs(NetworkId networkId, const std::vector<ImportedInputId> inputIds)
590{
591 return GetLoadedNetworkPtr(networkId)->ClearImportedInputs(inputIds);
592}
593void RuntimeImpl::ClearImportedOutputs(NetworkId networkId, const std::vector<ImportedOutputId> outputIds)
594{
595 return GetLoadedNetworkPtr(networkId)->ClearImportedOutputs(outputIds);
596}
Derek Lamberti03614f62018-10-02 15:52:46 +0100597
Kevin Mayd92a6e42021-02-04 10:27:41 +0000598Status RuntimeImpl::EnqueueWorkload(NetworkId networkId,
telsoa01c577f2c2018-08-31 09:22:23 +0100599 const InputTensors& inputTensors,
Narumol Prangnawarate2af6f42022-01-28 17:59:18 +0000600 const OutputTensors& outputTensors,
601 std::vector<ImportedInputId> preImportedInputIds,
602 std::vector<ImportedOutputId> preImportedOutputIds)
telsoa014fcda012018-03-09 14:13:49 +0000603{
Jan Eilers17d34da2021-12-08 16:15:12 +0000604 const auto startTime = armnn::GetTimeNow();
605
surmeh013537c2c2018-05-18 16:31:43 +0100606 LoadedNetwork* loadedNetwork = GetLoadedNetworkPtr(networkId);
Mike Kelly55a8ffd2021-04-07 20:10:49 +0100607
608 if (!loadedNetwork)
609 {
Sadik Armagan4a0844d2022-01-26 09:57:05 +0000610 ARMNN_LOG(error) << "A Network with an id of " << networkId << " does not exist.";
Mike Kelly55a8ffd2021-04-07 20:10:49 +0100611 return Status::Failure;
612 }
613 if (loadedNetwork->IsAsyncEnabled())
614 {
Sadik Armagan4a0844d2022-01-26 09:57:05 +0000615 ARMNN_LOG(error) << "Network " << networkId << " is async enabled.";
Mike Kelly55a8ffd2021-04-07 20:10:49 +0100616 return Status::Failure;
617 }
Narumol Prangnawarat5b4d0d52020-06-23 11:45:56 +0100618 ProfilerManager::GetInstance().RegisterProfiler(loadedNetwork->GetProfiler().get());
619
620 ARMNN_SCOPED_PROFILING_EVENT(Compute::Undefined, "EnqueueWorkload");
Derek Lamberti03614f62018-10-02 15:52:46 +0100621
622 static thread_local NetworkId lastId = networkId;
623 if (lastId != networkId)
624 {
625 LoadedNetworkFuncSafe(lastId, [](LoadedNetwork* network)
626 {
627 network->FreeWorkingMemory();
628 });
629 }
630 lastId=networkId;
631
Narumol Prangnawarate2af6f42022-01-28 17:59:18 +0000632 auto status = loadedNetwork->EnqueueWorkload(inputTensors, outputTensors,
633 preImportedInputIds, preImportedOutputIds);
Jan Eilers17d34da2021-12-08 16:15:12 +0000634
David Monahan1fc448a2022-02-24 15:55:56 +0000635
636 // Check if we imported, if not there's no need to call the After EnqueueWorkload events
637 if (!preImportedInputIds.empty() || !preImportedOutputIds.empty())
638 {
639 // Call After EnqueueWorkload events
640 for (auto&& context : m_BackendContexts)
641 {
642 context.second->AfterEnqueueWorkload(networkId);
643 }
644 }
Jan Eilers17d34da2021-12-08 16:15:12 +0000645 ARMNN_LOG(info) << "Execution time: " << std::setprecision(2)
Sadik Armagan4a0844d2022-01-26 09:57:05 +0000646 << std::fixed << armnn::GetTimeDuration(startTime).count() << " ms.";
Jan Eilers17d34da2021-12-08 16:15:12 +0000647 return status;
telsoa014fcda012018-03-09 14:13:49 +0000648}
649
Mike Kelly55a8ffd2021-04-07 20:10:49 +0100650Status RuntimeImpl::Execute(IWorkingMemHandle& iWorkingMemHandle,
651 const InputTensors& inputTensors,
Finn Williamsf37b9702021-09-01 18:06:04 +0100652 const OutputTensors& outputTensors,
Finn Williams8636bc72021-10-02 15:06:39 +0100653 std::vector<ImportedInputId> preImportedInputs,
654 std::vector<ImportedOutputId> preImportedOutputs)
Mike Kelly55a8ffd2021-04-07 20:10:49 +0100655{
Jan Eilers17d34da2021-12-08 16:15:12 +0000656 const auto startTime = armnn::GetTimeNow();
657
Mike Kelly55a8ffd2021-04-07 20:10:49 +0100658 NetworkId networkId = iWorkingMemHandle.GetNetworkId();
659 LoadedNetwork* loadedNetwork = GetLoadedNetworkPtr(networkId);
660
661 if (!loadedNetwork)
662 {
Sadik Armagan4a0844d2022-01-26 09:57:05 +0000663 ARMNN_LOG(error) << "A Network with an id of " << networkId << " does not exist.";
Mike Kelly55a8ffd2021-04-07 20:10:49 +0100664 return Status::Failure;
665 }
666 if (!loadedNetwork->IsAsyncEnabled())
667 {
Sadik Armagan4a0844d2022-01-26 09:57:05 +0000668 ARMNN_LOG(error) << "Attempting execute " << networkId << " when it is not async enabled.";
Mike Kelly55a8ffd2021-04-07 20:10:49 +0100669 return Status::Failure;
670 }
671 ProfilerManager::GetInstance().RegisterProfiler(loadedNetwork->GetProfiler().get());
672
673 ARMNN_SCOPED_PROFILING_EVENT(Compute::Undefined, "Execute");
674
Jan Eilers17d34da2021-12-08 16:15:12 +0000675 auto status = loadedNetwork->Execute(inputTensors,
676 outputTensors,
677 iWorkingMemHandle,
678 preImportedInputs,
679 preImportedOutputs);
680
681 ARMNN_LOG(info) << "Execution time: " << std::setprecision(2)
Sadik Armagan4a0844d2022-01-26 09:57:05 +0000682 << std::fixed << armnn::GetTimeDuration(startTime).count() << " ms.";
Jan Eilers17d34da2021-12-08 16:15:12 +0000683
684 return status;
Mike Kelly55a8ffd2021-04-07 20:10:49 +0100685}
686
687/// Create a new unique WorkingMemHandle object. Create multiple handles if you wish to have
688/// overlapped Execution by calling this function from different threads.
689std::unique_ptr<IWorkingMemHandle> RuntimeImpl::CreateWorkingMemHandle(NetworkId networkId)
690{
691 LoadedNetwork* loadedNetwork = GetLoadedNetworkPtr(networkId);
692
693 if (!loadedNetwork)
694 {
Sadik Armagan4a0844d2022-01-26 09:57:05 +0000695 ARMNN_LOG(error) << "A Network with an id of " << networkId << " does not exist.";
Mike Kelly55a8ffd2021-04-07 20:10:49 +0100696 return nullptr;
697 }
698 if (!loadedNetwork->IsAsyncEnabled())
699 {
Sadik Armagan4a0844d2022-01-26 09:57:05 +0000700 ARMNN_LOG(error) << "Network " << networkId << " is not async enabled.";
Mike Kelly55a8ffd2021-04-07 20:10:49 +0100701 return nullptr;
702 }
703 ProfilerManager::GetInstance().RegisterProfiler(loadedNetwork->GetProfiler().get());
704
705 ARMNN_SCOPED_PROFILING_EVENT(Compute::Undefined, "CreateWorkingMemHandle");
706
707 static thread_local NetworkId lastId = networkId;
708 if (lastId != networkId)
709 {
710 LoadedNetworkFuncSafe(lastId, [](LoadedNetwork* network)
711 {
712 network->FreeWorkingMemory();
713 });
714 }
715 lastId=networkId;
716
717 return loadedNetwork->CreateWorkingMemHandle(networkId);
718}
719
Kevin Mayd92a6e42021-02-04 10:27:41 +0000720void RuntimeImpl::RegisterDebugCallback(NetworkId networkId, const DebugCallbackFunction& func)
Nattapat Chaimanowong6e948202019-03-22 14:01:46 +0000721{
722 LoadedNetwork* loadedNetwork = GetLoadedNetworkPtr(networkId);
723 loadedNetwork->RegisterDebugCallback(func);
724}
725
Kevin Mayd92a6e42021-02-04 10:27:41 +0000726void RuntimeImpl::LoadDynamicBackends(const std::string& overrideBackendPath)
Matteo Martincighe54aa062019-08-05 14:12:11 +0100727{
728 // Get the paths where to load the dynamic backends from
729 std::vector<std::string> backendPaths = DynamicBackendUtils::GetBackendPaths(overrideBackendPath);
730
731 // Get the shared objects to try to load as dynamic backends
732 std::vector<std::string> sharedObjects = DynamicBackendUtils::GetSharedObjects(backendPaths);
733
734 // Create a list of dynamic backends
Matteo Martincigh0c2b2892019-08-05 14:12:11 +0100735 m_DynamicBackends = DynamicBackendUtils::CreateDynamicBackends(sharedObjects);
736
737 // Register the dynamic backends in the backend registry
Cathal Corbett5aa9fd72022-02-25 15:33:28 +0000738 armnn::BackendIdSet registeredBackendIds = DynamicBackendUtils::RegisterDynamicBackends(m_DynamicBackends);
Matteo Martincigh89533902019-08-15 12:08:06 +0100739
740 // Add the registered dynamic backend ids to the list of supported backends
Narumol Prangnawarat60a20fb2019-12-09 17:24:41 +0000741 m_DeviceSpec.AddSupportedBackends(registeredBackendIds, true);
telsoa014fcda012018-03-09 14:13:49 +0000742}
Matteo Martincighe54aa062019-08-05 14:12:11 +0100743
744} // namespace armnn