Laurent Carlier | 749294b | 2020-06-01 09:03:17 +0100 | [diff] [blame] | 1 | // |
Jim Flynn | 6398a98 | 2020-05-27 17:05:21 +0100 | [diff] [blame] | 2 | // Copyright © 2017 Arm Ltd and Contributors. All rights reserved. |
David Beck | ecb56cd | 2018-09-05 12:52:57 +0100 | [diff] [blame] | 3 | // SPDX-License-Identifier: MIT |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 4 | // |
| 5 | |
| 6 | #include "LoadedNetwork.hpp" |
| 7 | #include "Layer.hpp" |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 8 | #include "Graph.hpp" |
| 9 | #include "Network.hpp" |
Jim Flynn | f771321 | 2020-07-14 09:50:59 +0100 | [diff] [blame] | 10 | #include <Processes.hpp> |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 11 | #include "Profiling.hpp" |
surmeh01 | 3537c2c | 2018-05-18 16:31:43 +0100 | [diff] [blame] | 12 | #include "HeapProfiling.hpp" |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 13 | |
Matteo Martincigh | c601aa6 | 2019-10-29 15:03:22 +0000 | [diff] [blame] | 14 | #include <armnn/BackendRegistry.hpp> |
Matthew Bentham | f48afc6 | 2020-01-15 17:55:08 +0000 | [diff] [blame] | 15 | #include <armnn/Logging.hpp> |
Narumol Prangnawarat | ac2770a | 2020-04-01 16:51:23 +0100 | [diff] [blame] | 16 | #include <armnn/utility/Assert.hpp> |
Matteo Martincigh | c601aa6 | 2019-10-29 15:03:22 +0000 | [diff] [blame] | 17 | |
Aron Virginas-Tar | c9cc804 | 2018-11-01 16:15:57 +0000 | [diff] [blame] | 18 | #include <backendsCommon/CpuTensorHandle.hpp> |
Matteo Martincigh | e5b8eb9 | 2019-11-28 15:45:42 +0000 | [diff] [blame] | 19 | #include <armnn/backends/IMemoryManager.hpp> |
Derek Lamberti | f674aa0 | 2019-08-01 15:56:25 +0100 | [diff] [blame] | 20 | #include <backendsCommon/MemCopyWorkload.hpp> |
| 21 | #include <backendsCommon/MemSyncWorkload.hpp> |
Matteo Martincigh | e5b8eb9 | 2019-11-28 15:45:42 +0000 | [diff] [blame] | 22 | |
Narumol Prangnawarat | df31cfe | 2019-11-22 11:26:06 +0000 | [diff] [blame] | 23 | #include <LabelsAndEventClasses.hpp> |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 24 | |
Colm Donelan | 5b5c222 | 2020-09-09 12:48:16 +0100 | [diff] [blame] | 25 | #include <fmt/format.h> |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 26 | |
| 27 | namespace armnn |
| 28 | { |
| 29 | |
| 30 | using namespace std; |
Narumol Prangnawarat | df31cfe | 2019-11-22 11:26:06 +0000 | [diff] [blame] | 31 | using namespace armnn::profiling; |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 32 | |
telsoa01 | c577f2c | 2018-08-31 09:22:23 +0100 | [diff] [blame] | 33 | namespace |
| 34 | { |
| 35 | |
| 36 | template <typename ExceptionType> |
| 37 | std::string ToErrorMessage(const char * prefix, const ExceptionType & error) |
| 38 | { |
| 39 | std::stringstream ss; |
| 40 | ss << prefix << " " << error.what(); |
| 41 | return ss.str(); |
| 42 | } |
| 43 | |
Narumol Prangnawarat | df31cfe | 2019-11-22 11:26:06 +0000 | [diff] [blame] | 44 | void AddLayerStructure(std::unique_ptr<TimelineUtilityMethods>& timelineUtils, |
| 45 | const Layer& layer, |
| 46 | ProfilingGuid networkGuid) |
| 47 | { |
| 48 | // Add layer to the post-optimisation network structure |
| 49 | std::string layerName = layer.GetNameStr().empty() ? "<Unnamed>" : layer.GetNameStr(); |
| 50 | timelineUtils->CreateNamedTypedChildEntity(layer.GetGuid(), |
| 51 | networkGuid, |
| 52 | layerName, |
| 53 | LabelsAndEventClasses::LAYER_GUID); |
| 54 | for (auto&& input : layer.GetInputSlots()) |
| 55 | { |
| 56 | const IOutputSlot* source = input.GetConnectedOutputSlot(); |
Narumol Prangnawarat | ac2770a | 2020-04-01 16:51:23 +0100 | [diff] [blame] | 57 | ARMNN_ASSERT(source != NULL); |
Narumol Prangnawarat | df31cfe | 2019-11-22 11:26:06 +0000 | [diff] [blame] | 58 | timelineUtils->CreateConnectionRelationship(ProfilingRelationshipType::RetentionLink, |
| 59 | source->GetOwningLayerGuid(), |
| 60 | layer.GetGuid()); |
| 61 | } |
| 62 | } |
| 63 | |
| 64 | void AddWorkloadStructure(std::unique_ptr<TimelineUtilityMethods>& timelineUtils, |
| 65 | std::unique_ptr<IWorkload>& workload, |
| 66 | const Layer& layer) |
| 67 | { |
| 68 | // Add workload to the post-optimisation network structure |
| 69 | timelineUtils->CreateTypedEntity(workload->GetGuid(), LabelsAndEventClasses::WORKLOAD_GUID); |
| 70 | timelineUtils->MarkEntityWithLabel(workload->GetGuid(), |
| 71 | layer.GetBackendId().Get(), |
| 72 | LabelsAndEventClasses::BACKENDID_GUID); |
| 73 | |
| 74 | // Link the workload to the layer |
| 75 | timelineUtils->CreateRelationship(ProfilingRelationshipType::RetentionLink, |
| 76 | layer.GetGuid(), |
Jim Flynn | 6398a98 | 2020-05-27 17:05:21 +0100 | [diff] [blame] | 77 | workload->GetGuid(), |
| 78 | LabelsAndEventClasses::CHILD_GUID); |
Narumol Prangnawarat | df31cfe | 2019-11-22 11:26:06 +0000 | [diff] [blame] | 79 | } |
| 80 | |
telsoa01 | c577f2c | 2018-08-31 09:22:23 +0100 | [diff] [blame] | 81 | } // anonymous |
| 82 | |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 83 | std::unique_ptr<LoadedNetwork> LoadedNetwork::MakeLoadedNetwork(std::unique_ptr<OptimizedNetwork> net, |
David Monahan | 4f1e8e4 | 2019-09-04 09:22:10 +0100 | [diff] [blame] | 84 | std::string& errorMessage, |
Sadik Armagan | 3184c90 | 2020-03-18 10:57:30 +0000 | [diff] [blame] | 85 | const INetworkProperties& networkProperties, |
| 86 | profiling::ProfilingService& profilingService) |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 87 | { |
| 88 | std::unique_ptr<LoadedNetwork> loadedNetwork; |
| 89 | |
Aron Virginas-Tar | a8e06ed | 2018-10-19 16:46:15 +0100 | [diff] [blame] | 90 | auto Fail = [&](const std::exception& error) -> std::unique_ptr<LoadedNetwork> |
| 91 | { |
| 92 | errorMessage = ToErrorMessage("An error occurred when preparing the network workloads: ", error); |
Derek Lamberti | 0844697 | 2019-11-26 16:38:31 +0000 | [diff] [blame] | 93 | ARMNN_LOG(error) << errorMessage; |
Aron Virginas-Tar | a8e06ed | 2018-10-19 16:46:15 +0100 | [diff] [blame] | 94 | |
| 95 | return std::unique_ptr<LoadedNetwork>(); |
| 96 | }; |
| 97 | |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 98 | try |
| 99 | { |
Sadik Armagan | 3184c90 | 2020-03-18 10:57:30 +0000 | [diff] [blame] | 100 | loadedNetwork.reset(new LoadedNetwork(std::move(net), networkProperties, profilingService)); |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 101 | } |
Aron Virginas-Tar | a8e06ed | 2018-10-19 16:46:15 +0100 | [diff] [blame] | 102 | catch (const armnn::RuntimeException& error) |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 103 | { |
Aron Virginas-Tar | a8e06ed | 2018-10-19 16:46:15 +0100 | [diff] [blame] | 104 | return Fail(error); |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 105 | } |
| 106 | catch (const armnn::Exception& error) |
| 107 | { |
Aron Virginas-Tar | a8e06ed | 2018-10-19 16:46:15 +0100 | [diff] [blame] | 108 | return Fail(error); |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 109 | } |
Aron Virginas-Tar | a8e06ed | 2018-10-19 16:46:15 +0100 | [diff] [blame] | 110 | catch (const std::runtime_error& error) |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 111 | { |
Aron Virginas-Tar | a8e06ed | 2018-10-19 16:46:15 +0100 | [diff] [blame] | 112 | return Fail(error); |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 113 | } |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 114 | |
| 115 | return loadedNetwork; |
| 116 | } |
| 117 | |
David Monahan | 4f1e8e4 | 2019-09-04 09:22:10 +0100 | [diff] [blame] | 118 | LoadedNetwork::LoadedNetwork(std::unique_ptr<OptimizedNetwork> net, |
Sadik Armagan | 3184c90 | 2020-03-18 10:57:30 +0000 | [diff] [blame] | 119 | const INetworkProperties& networkProperties, |
| 120 | profiling::ProfilingService& profilingService) : |
David Monahan | 4f1e8e4 | 2019-09-04 09:22:10 +0100 | [diff] [blame] | 121 | m_OptimizedNetwork(std::move(net)), |
| 122 | m_IsImportEnabled(networkProperties.m_ImportEnabled), |
Sadik Armagan | 3184c90 | 2020-03-18 10:57:30 +0000 | [diff] [blame] | 123 | m_IsExportEnabled(networkProperties.m_ExportEnabled), |
Narumol Prangnawarat | 549cb7a | 2020-07-10 17:50:53 +0100 | [diff] [blame] | 124 | m_TensorHandleFactoryRegistry(), |
Sadik Armagan | 3184c90 | 2020-03-18 10:57:30 +0000 | [diff] [blame] | 125 | m_ProfilingService(profilingService) |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 126 | { |
telsoa01 | c577f2c | 2018-08-31 09:22:23 +0100 | [diff] [blame] | 127 | // Create a profiler and register it for the current thread. |
| 128 | m_Profiler = std::make_shared<Profiler>(); |
| 129 | ProfilerManager::GetInstance().RegisterProfiler(m_Profiler.get()); |
| 130 | |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 131 | Graph& order = m_OptimizedNetwork->GetGraph().TopologicalSort(); |
David Beck | 29c75de | 2018-10-23 13:35:58 +0100 | [diff] [blame] | 132 | //First create tensor handlers, backends and workload factories. |
telsoa01 | c577f2c | 2018-08-31 09:22:23 +0100 | [diff] [blame] | 133 | //Handlers are created before workloads are. |
| 134 | //Because workload creation can modify some of the handlers, |
Jim Flynn | e242f2d | 2019-05-22 14:24:13 +0100 | [diff] [blame] | 135 | //(for example the splitter and concat layers). |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 136 | for (auto&& layer : order) |
| 137 | { |
Derek Lamberti | 84da38b | 2019-06-13 11:40:08 +0100 | [diff] [blame] | 138 | auto const& backendId = layer->GetBackendId(); |
| 139 | if (m_Backends.count(backendId) == 0) |
David Beck | 29c75de | 2018-10-23 13:35:58 +0100 | [diff] [blame] | 140 | { |
Derek Lamberti | 84da38b | 2019-06-13 11:40:08 +0100 | [diff] [blame] | 141 | auto createBackend = BackendRegistryInstance().GetFactory(backendId); |
| 142 | auto it = m_Backends.emplace(std::make_pair(backendId, createBackend())); |
Aron Virginas-Tar | 5605519 | 2018-11-12 18:10:43 +0000 | [diff] [blame] | 143 | |
Derek Lamberti | 84da38b | 2019-06-13 11:40:08 +0100 | [diff] [blame] | 144 | IBackendInternal* backend = it.first->second.get(); |
Aron Virginas-Tar | 5605519 | 2018-11-12 18:10:43 +0000 | [diff] [blame] | 145 | |
Derek Lamberti | 84da38b | 2019-06-13 11:40:08 +0100 | [diff] [blame] | 146 | if (backend->SupportsTensorAllocatorAPI()) |
| 147 | { |
Sadik Armagan | 04a7297 | 2020-09-14 15:44:18 +0100 | [diff] [blame] | 148 | auto workloadFactory = backend->CreateWorkloadFactory( |
| 149 | m_TensorHandleFactoryRegistry, m_OptimizedNetwork->GetModelOptions()); |
Derek Lamberti | 84da38b | 2019-06-13 11:40:08 +0100 | [diff] [blame] | 150 | m_WorkloadFactories.emplace( |
| 151 | std::make_pair(backendId, std::make_pair(std::move(workloadFactory), nullptr))); |
| 152 | } |
| 153 | else |
| 154 | { |
| 155 | IBackendInternal::IMemoryManagerSharedPtr memoryManager = backend->CreateMemoryManager(); |
Sadik Armagan | 04a7297 | 2020-09-14 15:44:18 +0100 | [diff] [blame] | 156 | auto workloadFactory = backend->CreateWorkloadFactory( |
| 157 | memoryManager, m_OptimizedNetwork->GetModelOptions()); |
Derek Lamberti | 84da38b | 2019-06-13 11:40:08 +0100 | [diff] [blame] | 158 | |
| 159 | m_WorkloadFactories.emplace( |
| 160 | std::make_pair(backendId, std::make_pair(std::move(workloadFactory), memoryManager))); |
| 161 | } |
David Beck | 29c75de | 2018-10-23 13:35:58 +0100 | [diff] [blame] | 162 | } |
Derek Lamberti | 84da38b | 2019-06-13 11:40:08 +0100 | [diff] [blame] | 163 | } |
| 164 | |
| 165 | for (auto&& layer : order) |
| 166 | { |
David Monahan | 3fb7e10 | 2019-08-20 11:25:29 +0100 | [diff] [blame] | 167 | auto& workloadFactory = GetWorkloadFactory(*layer); |
| 168 | |
| 169 | switch (layer->GetType()) |
| 170 | { |
| 171 | case LayerType::Input: |
Narumol Prangnawarat | b8d771a | 2020-08-14 11:51:12 +0100 | [diff] [blame] | 172 | case LayerType::MemImport: |
David Monahan | 3fb7e10 | 2019-08-20 11:25:29 +0100 | [diff] [blame] | 173 | { |
| 174 | // If IsImportEnabled is true then we need to set IsMemoryManaged to false when creating TensorHandles |
| 175 | layer->CreateTensorHandles(m_TensorHandleFactoryRegistry, workloadFactory, !m_IsImportEnabled); |
| 176 | break; |
| 177 | } |
| 178 | default: |
| 179 | { |
Ferran Balaguer | 83239f9 | 2019-09-19 11:49:25 +0100 | [diff] [blame] | 180 | // Look for the layer with 1 OutputSlot which has 1 connection and that connection is an Output Layer |
| 181 | // If Export is enabled disable memory management so we can export, otherwise we do a copy |
| 182 | if((layer->GetNumOutputSlots() == 1) && |
| 183 | (layer->GetOutputSlots()[0].GetNumConnections() == 1) && |
| 184 | (layer->GetOutputSlots()[0].GetConnection(0)->GetOwningLayer().GetType() == LayerType::Output)) |
| 185 | { |
| 186 | layer->CreateTensorHandles(m_TensorHandleFactoryRegistry, workloadFactory, !m_IsExportEnabled); |
| 187 | } |
| 188 | else |
| 189 | { |
| 190 | layer->CreateTensorHandles(m_TensorHandleFactoryRegistry, workloadFactory); |
| 191 | } |
David Monahan | 3fb7e10 | 2019-08-20 11:25:29 +0100 | [diff] [blame] | 192 | } |
| 193 | } |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 194 | } |
| 195 | |
Narumol Prangnawarat | df31cfe | 2019-11-22 11:26:06 +0000 | [diff] [blame] | 196 | ProfilingGuid networkGuid = m_OptimizedNetwork->GetGuid(); |
Sadik Armagan | 3184c90 | 2020-03-18 10:57:30 +0000 | [diff] [blame] | 197 | std::unique_ptr<TimelineUtilityMethods> timelineUtils = |
| 198 | TimelineUtilityMethods::GetTimelineUtils(m_ProfilingService); |
Narumol Prangnawarat | df31cfe | 2019-11-22 11:26:06 +0000 | [diff] [blame] | 199 | if (timelineUtils) |
| 200 | { |
| 201 | timelineUtils->CreateTypedEntity(networkGuid, LabelsAndEventClasses::NETWORK_GUID); |
Jim Flynn | f771321 | 2020-07-14 09:50:59 +0100 | [diff] [blame] | 202 | // Mark the network with a start of life event |
| 203 | timelineUtils->RecordEvent(networkGuid, LabelsAndEventClasses::ARMNN_PROFILING_SOL_EVENT_CLASS); |
| 204 | // and with the process ID |
| 205 | int processID = armnnUtils::Processes::GetCurrentId(); |
| 206 | std::stringstream ss; |
| 207 | ss << processID; |
| 208 | timelineUtils->MarkEntityWithLabel(networkGuid, ss.str(), LabelsAndEventClasses::PROCESS_ID_GUID); |
Narumol Prangnawarat | df31cfe | 2019-11-22 11:26:06 +0000 | [diff] [blame] | 209 | } |
| 210 | |
telsoa01 | c577f2c | 2018-08-31 09:22:23 +0100 | [diff] [blame] | 211 | //Then create workloads. |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 212 | for (auto&& layer : order) |
| 213 | { |
Narumol Prangnawarat | df31cfe | 2019-11-22 11:26:06 +0000 | [diff] [blame] | 214 | if (timelineUtils) |
| 215 | { |
| 216 | // Add layer to the post-optimisation network structure |
| 217 | AddLayerStructure(timelineUtils, *layer, networkGuid); |
| 218 | } |
| 219 | |
surmeh01 | 3537c2c | 2018-05-18 16:31:43 +0100 | [diff] [blame] | 220 | const IWorkloadFactory& workloadFactory = GetWorkloadFactory(*layer); |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 221 | |
| 222 | switch (layer->GetType()) |
| 223 | { |
| 224 | case LayerType::Input: |
| 225 | case LayerType::Output: |
| 226 | { |
telsoa01 | c577f2c | 2018-08-31 09:22:23 +0100 | [diff] [blame] | 227 | // Inputs and outputs are treated in a special way - see EnqueueInput() and EnqueueOutput(). |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 228 | break; |
| 229 | } |
| 230 | default: |
| 231 | { |
Derek Lamberti | 94a88d2 | 2019-12-10 21:12:59 +0000 | [diff] [blame] | 232 | auto workload = layer->CreateWorkload(workloadFactory); |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 233 | |
| 234 | if (!workload) |
| 235 | { |
Narumol Prangnawarat | df31cfe | 2019-11-22 11:26:06 +0000 | [diff] [blame] | 236 | const char* const layerName = |
| 237 | layer->GetNameStr().length() != 0 ? layer->GetName() : "<Unnamed>"; |
Colm Donelan | 5b5c222 | 2020-09-09 12:48:16 +0100 | [diff] [blame] | 238 | throw InvalidArgumentException( |
| 239 | fmt::format("No workload created for layer (name: '{0}' type: '{1}') (compute '{2}')", |
| 240 | layerName, static_cast<int>(layer->GetType()), layer->GetBackendId().Get() |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 241 | )); |
| 242 | } |
| 243 | |
Narumol Prangnawarat | df31cfe | 2019-11-22 11:26:06 +0000 | [diff] [blame] | 244 | if (timelineUtils) |
| 245 | { |
| 246 | // Add workload to the post-optimisation network structure |
| 247 | AddWorkloadStructure(timelineUtils, workload, *layer); |
| 248 | } |
| 249 | |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 250 | m_WorkloadQueue.push_back(move(workload)); |
telsoa01 | c577f2c | 2018-08-31 09:22:23 +0100 | [diff] [blame] | 251 | // release the constant data in the layer.. |
| 252 | layer->ReleaseConstantData(); |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 253 | break; |
| 254 | } |
| 255 | } |
| 256 | } |
| 257 | |
Narumol Prangnawarat | df31cfe | 2019-11-22 11:26:06 +0000 | [diff] [blame] | 258 | if (timelineUtils) |
| 259 | { |
| 260 | // Commit to send the post-optimisation network structure |
| 261 | timelineUtils->Commit(); |
| 262 | } |
| 263 | |
telsoa01 | c577f2c | 2018-08-31 09:22:23 +0100 | [diff] [blame] | 264 | // Set up memory. |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 265 | m_OptimizedNetwork->GetGraph().AllocateDynamicBuffers(); |
Derek Lamberti | f30f7d3 | 2019-04-09 10:25:02 +0100 | [diff] [blame] | 266 | |
| 267 | // Now that the intermediate tensor memory has been set-up, do any post allocation configuration for each workload. |
| 268 | for (auto& workload : m_WorkloadQueue) |
| 269 | { |
| 270 | workload->PostAllocationConfigure(); |
| 271 | } |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 272 | } |
| 273 | |
Keith Davis | 33ed221 | 2020-03-30 10:43:41 +0100 | [diff] [blame] | 274 | void LoadedNetwork::SendNetworkStructure() |
| 275 | { |
| 276 | Graph& order = m_OptimizedNetwork->GetGraph().TopologicalSort(); |
| 277 | ProfilingGuid networkGuid = m_OptimizedNetwork->GetGuid(); |
| 278 | |
| 279 | std::unique_ptr<TimelineUtilityMethods> timelineUtils = |
| 280 | TimelineUtilityMethods::GetTimelineUtils(m_ProfilingService); |
| 281 | |
| 282 | timelineUtils->CreateTypedEntity(networkGuid, LabelsAndEventClasses::NETWORK_GUID); |
| 283 | |
| 284 | for (auto&& layer : order) |
| 285 | { |
| 286 | // Add layer to the post-optimisation network structure |
| 287 | AddLayerStructure(timelineUtils, *layer, networkGuid); |
| 288 | switch (layer->GetType()) |
| 289 | { |
| 290 | case LayerType::Input: |
| 291 | case LayerType::Output: |
| 292 | { |
| 293 | // Inputs and outputs are treated in a special way - see EnqueueInput() and EnqueueOutput(). |
| 294 | break; |
| 295 | } |
| 296 | default: |
| 297 | { |
| 298 | for (auto& workload : m_WorkloadQueue) |
| 299 | { |
| 300 | // Add workload to the post-optimisation network structure |
| 301 | AddWorkloadStructure(timelineUtils, workload, *layer); |
| 302 | } |
| 303 | break; |
| 304 | } |
| 305 | } |
| 306 | } |
| 307 | // Commit to send the post-optimisation network structure |
| 308 | timelineUtils->Commit(); |
| 309 | } |
| 310 | |
Jim Flynn | f771321 | 2020-07-14 09:50:59 +0100 | [diff] [blame] | 311 | profiling::ProfilingGuid LoadedNetwork::GetNetworkGuid() |
| 312 | { |
| 313 | return m_OptimizedNetwork->GetGuid(); |
| 314 | } |
| 315 | |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 316 | TensorInfo LoadedNetwork::GetInputTensorInfo(LayerBindingId layerId) const |
| 317 | { |
| 318 | for (auto&& inputLayer : m_OptimizedNetwork->GetGraph().GetInputLayers()) |
| 319 | { |
Narumol Prangnawarat | ac2770a | 2020-04-01 16:51:23 +0100 | [diff] [blame] | 320 | ARMNN_ASSERT_MSG(inputLayer->GetNumOutputSlots() == 1, "Input layer should have exactly 1 output slot"); |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 321 | if (inputLayer->GetBindingId() == layerId) |
| 322 | { |
| 323 | return inputLayer->GetOutputSlot(0).GetTensorInfo(); |
| 324 | } |
| 325 | } |
| 326 | |
Colm Donelan | 5b5c222 | 2020-09-09 12:48:16 +0100 | [diff] [blame] | 327 | throw InvalidArgumentException(fmt::format("No input layer is associated with id {}", layerId)); |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 328 | } |
| 329 | |
| 330 | TensorInfo LoadedNetwork::GetOutputTensorInfo(LayerBindingId layerId) const |
| 331 | { |
| 332 | for (auto&& outputLayer : m_OptimizedNetwork->GetGraph().GetOutputLayers()) |
| 333 | { |
Narumol Prangnawarat | ac2770a | 2020-04-01 16:51:23 +0100 | [diff] [blame] | 334 | ARMNN_ASSERT_MSG(outputLayer->GetNumInputSlots() == 1, "Output layer should have exactly 1 input slot"); |
| 335 | ARMNN_ASSERT_MSG(outputLayer->GetInputSlot(0).GetConnection(), "Input slot on Output layer must be connected"); |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 336 | if (outputLayer->GetBindingId() == layerId) |
| 337 | { |
| 338 | return outputLayer->GetInputSlot(0).GetConnection()->GetTensorInfo(); |
| 339 | } |
| 340 | } |
| 341 | |
Colm Donelan | 5b5c222 | 2020-09-09 12:48:16 +0100 | [diff] [blame] | 342 | throw InvalidArgumentException(fmt::format("No output layer is associated with id {}", layerId)); |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 343 | } |
| 344 | |
surmeh01 | 3537c2c | 2018-05-18 16:31:43 +0100 | [diff] [blame] | 345 | const IWorkloadFactory& LoadedNetwork::GetWorkloadFactory(const Layer& layer) const |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 346 | { |
surmeh01 | 3537c2c | 2018-05-18 16:31:43 +0100 | [diff] [blame] | 347 | const IWorkloadFactory* workloadFactory = nullptr; |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 348 | |
David Beck | 29c75de | 2018-10-23 13:35:58 +0100 | [diff] [blame] | 349 | auto it = m_WorkloadFactories.find(layer.GetBackendId()); |
| 350 | if (it == m_WorkloadFactories.end()) |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 351 | { |
Colm Donelan | 5b5c222 | 2020-09-09 12:48:16 +0100 | [diff] [blame] | 352 | throw RuntimeException(fmt::format("No workload factory for {0} to be used for layer: {1}", |
| 353 | layer.GetBackendId().Get(), |
| 354 | layer.GetNameStr()), |
| 355 | CHECK_LOCATION()); |
David Beck | 33f0ae0 | 2018-10-18 15:13:56 +0100 | [diff] [blame] | 356 | } |
David Beck | 29c75de | 2018-10-23 13:35:58 +0100 | [diff] [blame] | 357 | |
Aron Virginas-Tar | 5caf907 | 2018-11-14 18:35:18 +0000 | [diff] [blame] | 358 | workloadFactory = it->second.first.get(); |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 359 | |
Narumol Prangnawarat | ac2770a | 2020-04-01 16:51:23 +0100 | [diff] [blame] | 360 | ARMNN_ASSERT_MSG(workloadFactory, "No workload factory"); |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 361 | |
| 362 | std::string reasonIfUnsupported; |
Sadik Armagan | 04a7297 | 2020-09-14 15:44:18 +0100 | [diff] [blame] | 363 | ARMNN_ASSERT_MSG(IWorkloadFactory::IsLayerSupported(layer, |
| 364 | {}, |
| 365 | reasonIfUnsupported, |
| 366 | m_OptimizedNetwork->GetModelOptions()), |
David Beck | 29c75de | 2018-10-23 13:35:58 +0100 | [diff] [blame] | 367 | "Factory does not support layer"); |
Jan Eilers | 8eb2560 | 2020-03-09 12:13:48 +0000 | [diff] [blame] | 368 | IgnoreUnused(reasonIfUnsupported); |
surmeh01 | 3537c2c | 2018-05-18 16:31:43 +0100 | [diff] [blame] | 369 | return *workloadFactory; |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 370 | } |
| 371 | |
| 372 | namespace { |
| 373 | |
| 374 | // Non-copyable class owning accelerator-specific tensor data. |
| 375 | class TensorPin |
| 376 | { |
| 377 | public: |
| 378 | TensorPin(std::unique_ptr<ITensorHandle> handle, const TensorInfo& info, LayerBindingId id) |
| 379 | : m_TensorHandle(std::move(handle)) |
| 380 | , m_TensorInfo(info) |
| 381 | , m_Id(id) |
| 382 | { |
| 383 | } |
| 384 | |
| 385 | ITensorHandle* GetTensorHandle() const { return m_TensorHandle.get(); } |
| 386 | const TensorInfo& GetTensorInfo() const { return m_TensorInfo; } |
| 387 | LayerBindingId GetBindingId() const { return m_Id; } |
| 388 | |
| 389 | private: |
| 390 | std::unique_ptr<ITensorHandle> m_TensorHandle; |
| 391 | TensorInfo m_TensorInfo; |
| 392 | LayerBindingId m_Id; |
| 393 | }; |
| 394 | |
| 395 | static const TensorPin& GetTensorPin(LayerBindingId id, |
| 396 | const std::vector<TensorPin>& pins, |
| 397 | char const* bindingPointDesc) |
| 398 | { |
| 399 | auto it = std::find_if(pins.begin(), pins.end(), |
| 400 | [id](const TensorPin& pin) |
| 401 | { |
| 402 | return pin.GetBindingId() == id; |
| 403 | }); |
| 404 | |
| 405 | if (it != pins.end()) |
| 406 | { |
| 407 | return *it; |
| 408 | } |
| 409 | else |
| 410 | { |
Colm Donelan | 5b5c222 | 2020-09-09 12:48:16 +0100 | [diff] [blame] | 411 | throw InvalidArgumentException(fmt::format("No tensor supplied for {0} {1}", bindingPointDesc, id)); |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 412 | } |
| 413 | } |
| 414 | |
| 415 | // Stores data that needs to be kept accessible for the entire execution of a workload. |
| 416 | class WorkloadData |
| 417 | { |
| 418 | public: |
| 419 | WorkloadData(const InputTensors& inputTensors, const OutputTensors& outputTensors) |
| 420 | { |
| 421 | m_InputTensorPins.reserve(inputTensors.size()); |
| 422 | m_OutputTensorPins.reserve(outputTensors.size()); |
| 423 | |
| 424 | for (auto inputTensorPair : inputTensors) |
| 425 | { |
| 426 | auto inputTensor = inputTensorPair.second; |
| 427 | |
| 428 | std::unique_ptr<ITensorHandle> tensorHandle = |
| 429 | std::make_unique<ConstPassthroughCpuTensorHandle>(inputTensor.GetInfo(),inputTensor.GetMemoryArea()); |
| 430 | LayerBindingId layerId = inputTensorPair.first; |
| 431 | |
| 432 | m_InputTensorPins.emplace_back(std::move(tensorHandle), inputTensor.GetInfo(), layerId); |
| 433 | } |
| 434 | |
| 435 | for (auto outputTensorPair : outputTensors) |
| 436 | { |
| 437 | auto outputTensor = outputTensorPair.second; |
| 438 | |
| 439 | std::unique_ptr<ITensorHandle> tensorHandle = |
| 440 | std::make_unique<PassthroughCpuTensorHandle>(outputTensor.GetInfo(), outputTensor.GetMemoryArea()); |
| 441 | LayerBindingId layerId = outputTensorPair.first; |
| 442 | |
| 443 | m_OutputTensorPins.emplace_back(std::move(tensorHandle), outputTensor.GetInfo(), layerId); |
| 444 | } |
| 445 | } |
| 446 | |
| 447 | const TensorPin& GetInputTensorPin(LayerBindingId id) const |
| 448 | { |
| 449 | return GetTensorPin(id, m_InputTensorPins, "input"); |
| 450 | } |
| 451 | |
| 452 | const TensorPin& GetOutputTensorPin(LayerBindingId id) const |
| 453 | { |
| 454 | return GetTensorPin(id, m_OutputTensorPins, "output"); |
| 455 | } |
| 456 | |
| 457 | private: |
| 458 | |
| 459 | std::vector<TensorPin> m_InputTensorPins; |
| 460 | std::vector<TensorPin> m_OutputTensorPins; |
| 461 | }; |
| 462 | |
| 463 | } |
| 464 | |
| 465 | Status LoadedNetwork::EnqueueWorkload(const InputTensors& inputTensors, |
surmeh01 | 3537c2c | 2018-05-18 16:31:43 +0100 | [diff] [blame] | 466 | const OutputTensors& outputTensors) |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 467 | { |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 468 | const Graph& graph = m_OptimizedNetwork->GetGraph(); |
| 469 | |
telsoa01 | c577f2c | 2018-08-31 09:22:23 +0100 | [diff] [blame] | 470 | // Walk graph to determine the order of execution. |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 471 | if (graph.GetNumLayers() < 2) |
| 472 | { |
Derek Lamberti | 0844697 | 2019-11-26 16:38:31 +0000 | [diff] [blame] | 473 | ARMNN_LOG(warning) << "IRuntime::EnqueueWorkload()::Less than two nodes in graph"; |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 474 | return Status::Failure; |
| 475 | } |
| 476 | |
telsoa01 | c577f2c | 2018-08-31 09:22:23 +0100 | [diff] [blame] | 477 | // Data that must be kept alive for the entire execution of the workload. |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 478 | WorkloadData workloadData(inputTensors, outputTensors); |
| 479 | |
| 480 | if (graph.GetNumInputs() != inputTensors.size()) |
| 481 | { |
| 482 | throw InvalidArgumentException("Number of inputs provided does not match network."); |
| 483 | } |
| 484 | |
telsoa01 | c577f2c | 2018-08-31 09:22:23 +0100 | [diff] [blame] | 485 | // For each input to the network, call EnqueueInput with the data passed by the user. |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 486 | { |
Derek Lamberti | a08d29b | 2020-06-19 14:33:05 +0100 | [diff] [blame] | 487 | ARMNN_SCOPED_PROFILING_EVENT(Compute::Undefined, "PrepareInputs"); |
| 488 | m_InputQueue.clear(); |
| 489 | m_InputQueue.reserve(graph.GetNumInputs()); |
| 490 | for (const BindableLayer* inputLayer : graph.GetInputLayers()) |
| 491 | { |
| 492 | const TensorPin& pin = workloadData.GetInputTensorPin(inputLayer->GetBindingId()); |
| 493 | EnqueueInput(*inputLayer, pin.GetTensorHandle(), pin.GetTensorInfo()); |
| 494 | } |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 495 | } |
| 496 | |
telsoa01 | c577f2c | 2018-08-31 09:22:23 +0100 | [diff] [blame] | 497 | // For each output to the network, call EnqueueOutput with the data passed by the user. |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 498 | { |
Derek Lamberti | a08d29b | 2020-06-19 14:33:05 +0100 | [diff] [blame] | 499 | ARMNN_SCOPED_PROFILING_EVENT(Compute::Undefined, "PrepareOutputs"); |
| 500 | m_OutputQueue.clear(); |
| 501 | m_OutputQueue.reserve(graph.GetNumOutputs()); |
| 502 | for (const BindableLayer* outputLayer : graph.GetOutputLayers()) |
| 503 | { |
| 504 | const TensorPin& pin = workloadData.GetOutputTensorPin(outputLayer->GetBindingId()); |
| 505 | EnqueueOutput(*outputLayer, pin.GetTensorHandle(), pin.GetTensorInfo()); |
| 506 | } |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 507 | } |
| 508 | |
Sadik Armagan | 3184c90 | 2020-03-18 10:57:30 +0000 | [diff] [blame] | 509 | std::unique_ptr<TimelineUtilityMethods> timelineUtils = |
| 510 | TimelineUtilityMethods::GetTimelineUtils(m_ProfilingService); |
| 511 | ProfilingGuid inferenceGuid = m_ProfilingService.GetNextGuid(); |
David Monahan | 6198fe0 | 2019-12-02 08:35:43 +0000 | [diff] [blame] | 512 | if (timelineUtils) |
| 513 | { |
| 514 | // Add inference timeline trace if profiling is enabled. |
| 515 | ProfilingGuid networkGuid = m_OptimizedNetwork->GetGuid(); |
| 516 | timelineUtils->CreateTypedEntity(inferenceGuid, LabelsAndEventClasses::INFERENCE_GUID); |
Jim Flynn | 6398a98 | 2020-05-27 17:05:21 +0100 | [diff] [blame] | 517 | timelineUtils->CreateRelationship(ProfilingRelationshipType::RetentionLink, |
| 518 | networkGuid, |
| 519 | inferenceGuid, |
| 520 | LabelsAndEventClasses::EXECUTION_OF_GUID); |
David Monahan | 6198fe0 | 2019-12-02 08:35:43 +0000 | [diff] [blame] | 521 | timelineUtils->RecordEvent(inferenceGuid, LabelsAndEventClasses::ARMNN_PROFILING_SOL_EVENT_CLASS); |
| 522 | } |
| 523 | |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 524 | bool executionSucceeded = true; |
| 525 | |
| 526 | { |
Sadik Armagan | 3184c90 | 2020-03-18 10:57:30 +0000 | [diff] [blame] | 527 | if (m_ProfilingService.IsProfilingEnabled()) |
Keith Davis | e394bd9 | 2019-12-02 15:12:19 +0000 | [diff] [blame] | 528 | { |
Sadik Armagan | 3184c90 | 2020-03-18 10:57:30 +0000 | [diff] [blame] | 529 | m_ProfilingService.IncrementCounterValue(armnn::profiling::INFERENCES_RUN); |
Keith Davis | e394bd9 | 2019-12-02 15:12:19 +0000 | [diff] [blame] | 530 | } |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 531 | ARMNN_SCOPED_PROFILING_EVENT(Compute::Undefined, "Execute"); |
surmeh01 | 3537c2c | 2018-05-18 16:31:43 +0100 | [diff] [blame] | 532 | ARMNN_SCOPED_HEAP_PROFILING("Executing"); |
David Monahan | 6198fe0 | 2019-12-02 08:35:43 +0000 | [diff] [blame] | 533 | executionSucceeded = Execute(timelineUtils, inferenceGuid); |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 534 | } |
| 535 | |
David Monahan | 6198fe0 | 2019-12-02 08:35:43 +0000 | [diff] [blame] | 536 | if (timelineUtils) |
| 537 | { |
| 538 | // Add end of life of the inference timeline if profiling is enabled. |
| 539 | timelineUtils->RecordEvent(inferenceGuid, LabelsAndEventClasses::ARMNN_PROFILING_EOL_EVENT_CLASS); |
| 540 | timelineUtils->Commit(); |
| 541 | } |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 542 | return executionSucceeded ? Status::Success : Status::Failure; |
| 543 | } |
| 544 | |
surmeh01 | 3537c2c | 2018-05-18 16:31:43 +0100 | [diff] [blame] | 545 | void LoadedNetwork::EnqueueInput(const BindableLayer& layer, ITensorHandle* tensorHandle, const TensorInfo& tensorInfo) |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 546 | { |
| 547 | if (layer.GetType() != LayerType::Input) |
| 548 | { |
| 549 | throw InvalidArgumentException("EnqueueInput: given layer not an InputLayer"); |
| 550 | } |
| 551 | |
| 552 | if (tensorHandle == nullptr) |
| 553 | { |
| 554 | throw InvalidArgumentException("EnqueueInput: tensorHandle must not be NULL"); |
| 555 | } |
| 556 | |
| 557 | InputQueueDescriptor inputQueueDescriptor; |
| 558 | WorkloadInfo info; |
| 559 | |
| 560 | inputQueueDescriptor.m_Inputs.push_back(tensorHandle); |
| 561 | info.m_InputTensorInfos.push_back(tensorInfo); |
| 562 | |
Narumol Prangnawarat | ac2770a | 2020-04-01 16:51:23 +0100 | [diff] [blame] | 563 | ARMNN_ASSERT_MSG(layer.GetNumOutputSlots() == 1, "Can only handle Input Layer with one output"); |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 564 | const OutputHandler& handler = layer.GetOutputHandler(); |
| 565 | const TensorInfo& outputTensorInfo = handler.GetTensorInfo(); |
| 566 | ITensorHandle* outputTensorHandle = handler.GetData(); |
Narumol Prangnawarat | ac2770a | 2020-04-01 16:51:23 +0100 | [diff] [blame] | 567 | ARMNN_ASSERT_MSG(outputTensorHandle != nullptr, |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 568 | "Data should have been allocated."); |
| 569 | inputQueueDescriptor.m_Outputs.push_back(outputTensorHandle); |
| 570 | info.m_OutputTensorInfos.push_back(outputTensorInfo); |
| 571 | |
Derek Lamberti | f674aa0 | 2019-08-01 15:56:25 +0100 | [diff] [blame] | 572 | MemorySourceFlags importFlags = outputTensorHandle->GetImportFlags(); |
Narumol Prangnawarat | 265e53e | 2020-10-30 16:06:55 +0000 | [diff] [blame^] | 573 | bool needMemCopy = true; |
Ferran Balaguer | 83239f9 | 2019-09-19 11:49:25 +0100 | [diff] [blame] | 574 | if (m_IsImportEnabled) // Try import the input tensor |
Derek Lamberti | f674aa0 | 2019-08-01 15:56:25 +0100 | [diff] [blame] | 575 | { |
Ferran Balaguer | 83239f9 | 2019-09-19 11:49:25 +0100 | [diff] [blame] | 576 | if(CheckFlag(importFlags, MemorySource::Malloc) ) |
Derek Lamberti | f674aa0 | 2019-08-01 15:56:25 +0100 | [diff] [blame] | 577 | { |
Narumol Prangnawarat | 265e53e | 2020-10-30 16:06:55 +0000 | [diff] [blame^] | 578 | needMemCopy = false; |
Ferran Balaguer | 83239f9 | 2019-09-19 11:49:25 +0100 | [diff] [blame] | 579 | // This assumes a CPU Tensor handle |
| 580 | void* mem = tensorHandle->Map(false); |
| 581 | if (outputTensorHandle->Import(mem, MemorySource::Malloc)) |
| 582 | { |
| 583 | tensorHandle->Unmap(); |
| 584 | return; // No need for a workload since the import has been done. |
| 585 | } |
Derek Lamberti | f674aa0 | 2019-08-01 15:56:25 +0100 | [diff] [blame] | 586 | tensorHandle->Unmap(); |
Ferran Balaguer | 83239f9 | 2019-09-19 11:49:25 +0100 | [diff] [blame] | 587 | throw MemoryImportException("EnqueueInput: Memory Import failed"); |
Derek Lamberti | f674aa0 | 2019-08-01 15:56:25 +0100 | [diff] [blame] | 588 | } |
Derek Lamberti | f674aa0 | 2019-08-01 15:56:25 +0100 | [diff] [blame] | 589 | } |
Narumol Prangnawarat | 265e53e | 2020-10-30 16:06:55 +0000 | [diff] [blame^] | 590 | if (needMemCopy) |
David Monahan | 4f1e8e4 | 2019-09-04 09:22:10 +0100 | [diff] [blame] | 591 | { |
| 592 | // Create a mem copy workload for input since we did not import |
Narumol Prangnawarat | aa68e01 | 2019-11-29 17:17:43 +0000 | [diff] [blame] | 593 | std::unique_ptr<IWorkload> inputWorkload = std::make_unique<CopyMemGenericWorkload>(inputQueueDescriptor, info); |
Derek Lamberti | f674aa0 | 2019-08-01 15:56:25 +0100 | [diff] [blame] | 594 | |
Narumol Prangnawarat | ac2770a | 2020-04-01 16:51:23 +0100 | [diff] [blame] | 595 | ARMNN_ASSERT_MSG(inputWorkload, "No input workload created"); |
Narumol Prangnawarat | aa68e01 | 2019-11-29 17:17:43 +0000 | [diff] [blame] | 596 | |
Sadik Armagan | 3184c90 | 2020-03-18 10:57:30 +0000 | [diff] [blame] | 597 | std::unique_ptr<TimelineUtilityMethods> timelineUtils = |
| 598 | TimelineUtilityMethods::GetTimelineUtils(m_ProfilingService); |
Narumol Prangnawarat | aa68e01 | 2019-11-29 17:17:43 +0000 | [diff] [blame] | 599 | if (timelineUtils) |
| 600 | { |
| 601 | // Add Input Workload to the post-optimisation network structure |
| 602 | AddWorkloadStructure(timelineUtils, inputWorkload, layer); |
| 603 | timelineUtils->Commit(); |
| 604 | } |
| 605 | |
David Monahan | 4f1e8e4 | 2019-09-04 09:22:10 +0100 | [diff] [blame] | 606 | m_InputQueue.push_back(move(inputWorkload)); |
| 607 | } |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 608 | } |
| 609 | |
surmeh01 | 3537c2c | 2018-05-18 16:31:43 +0100 | [diff] [blame] | 610 | void LoadedNetwork::EnqueueOutput(const BindableLayer& layer, ITensorHandle* tensorHandle, const TensorInfo& tensorInfo) |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 611 | { |
| 612 | if (layer.GetType() != LayerType::Output) |
| 613 | { |
| 614 | throw InvalidArgumentException("EnqueueOutput: given layer not an OutputLayer"); |
| 615 | } |
| 616 | |
| 617 | if (tensorHandle == nullptr) |
| 618 | { |
| 619 | throw InvalidArgumentException("EnqueueOutput: tensorHandle must not be NULL"); |
| 620 | } |
| 621 | |
| 622 | OutputQueueDescriptor outputQueueDescriptor; |
| 623 | WorkloadInfo info; |
| 624 | |
| 625 | outputQueueDescriptor.m_Outputs.push_back(tensorHandle); |
| 626 | info.m_OutputTensorInfos.push_back(tensorInfo); |
| 627 | |
Narumol Prangnawarat | ac2770a | 2020-04-01 16:51:23 +0100 | [diff] [blame] | 628 | ARMNN_ASSERT_MSG(layer.GetNumInputSlots() == 1, "Output Layer should have exactly one input."); |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 629 | |
telsoa01 | c577f2c | 2018-08-31 09:22:23 +0100 | [diff] [blame] | 630 | // Gets the output handler from the previous node. |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 631 | const OutputHandler& outputHandler = layer.GetInputSlots()[0].GetConnectedOutputSlot()->GetOutputHandler(); |
| 632 | |
| 633 | const TensorInfo& inputTensorInfo = outputHandler.GetTensorInfo(); |
| 634 | ITensorHandle* inputTensorHandle = outputHandler.GetData(); |
Narumol Prangnawarat | ac2770a | 2020-04-01 16:51:23 +0100 | [diff] [blame] | 635 | ARMNN_ASSERT_MSG(inputTensorHandle != nullptr, "Data should have been allocated."); |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 636 | |
Derek Lamberti | f674aa0 | 2019-08-01 15:56:25 +0100 | [diff] [blame] | 637 | // Try import the output tensor. |
| 638 | // Note: We can only import the output pointer if all of the following hold true: |
| 639 | // a) The imported pointer is aligned sufficiently |
| 640 | // b) The tensor has zero padding |
| 641 | // c) There is only one connection to the OutputSlot and it is to an OutputLayer. |
| 642 | // d) The output pointer is allocated via malloc. (Other types will be supported in a later release) |
Ferran Balaguer | 83239f9 | 2019-09-19 11:49:25 +0100 | [diff] [blame] | 643 | // e) m_IsExportEnabled must be set to true |
Narumol Prangnawarat | 265e53e | 2020-10-30 16:06:55 +0000 | [diff] [blame^] | 644 | bool needMemCopy = true; |
Ferran Balaguer | 83239f9 | 2019-09-19 11:49:25 +0100 | [diff] [blame] | 645 | if (m_IsExportEnabled && (layer.GetInputSlots()[0].GetConnectedOutputSlot()->GetNumConnections() == 1)) |
Derek Lamberti | f674aa0 | 2019-08-01 15:56:25 +0100 | [diff] [blame] | 646 | { |
Ferran Balaguer | 83239f9 | 2019-09-19 11:49:25 +0100 | [diff] [blame] | 647 | if(layer.GetInputSlots()[0].GetConnectedOutputSlot()->GetOwningLayer().GetType() != LayerType::Input) |
Derek Lamberti | f674aa0 | 2019-08-01 15:56:25 +0100 | [diff] [blame] | 648 | { |
Ferran Balaguer | bfeb271 | 2019-08-07 15:14:56 +0100 | [diff] [blame] | 649 | MemorySourceFlags importFlags = inputTensorHandle->GetImportFlags(); |
| 650 | if (CheckFlag(importFlags, MemorySource::Malloc)) |
Derek Lamberti | f674aa0 | 2019-08-01 15:56:25 +0100 | [diff] [blame] | 651 | { |
Narumol Prangnawarat | 265e53e | 2020-10-30 16:06:55 +0000 | [diff] [blame^] | 652 | needMemCopy = false; |
Ferran Balaguer | bfeb271 | 2019-08-07 15:14:56 +0100 | [diff] [blame] | 653 | void *mem = tensorHandle->Map(false); |
| 654 | bool importOk = inputTensorHandle->Import(mem, MemorySource::Malloc); |
| 655 | tensorHandle->Unmap(); |
Derek Lamberti | f674aa0 | 2019-08-01 15:56:25 +0100 | [diff] [blame] | 656 | |
Ferran Balaguer | bfeb271 | 2019-08-07 15:14:56 +0100 | [diff] [blame] | 657 | if (importOk) |
| 658 | { |
| 659 | // Insert synchronization workload |
| 660 | MemSyncQueueDescriptor syncDesc; |
| 661 | syncDesc.m_Inputs.push_back(inputTensorHandle); |
| 662 | info.m_InputTensorInfos.push_back(inputTensorInfo); |
| 663 | auto syncWorkload = std::make_unique<SyncMemGenericWorkload>(syncDesc, info); |
Narumol Prangnawarat | ac2770a | 2020-04-01 16:51:23 +0100 | [diff] [blame] | 664 | ARMNN_ASSERT_MSG(syncWorkload, "No sync workload created"); |
Ferran Balaguer | bfeb271 | 2019-08-07 15:14:56 +0100 | [diff] [blame] | 665 | m_OutputQueue.push_back(move(syncWorkload)); |
Ferran Balaguer | bfeb271 | 2019-08-07 15:14:56 +0100 | [diff] [blame] | 666 | } |
David Monahan | 4f1e8e4 | 2019-09-04 09:22:10 +0100 | [diff] [blame] | 667 | else |
| 668 | { |
| 669 | throw MemoryExportException("EnqueueOutput: Memory Export failed"); |
| 670 | } |
Derek Lamberti | f674aa0 | 2019-08-01 15:56:25 +0100 | [diff] [blame] | 671 | } |
| 672 | } |
| 673 | } |
Narumol Prangnawarat | 265e53e | 2020-10-30 16:06:55 +0000 | [diff] [blame^] | 674 | if (needMemCopy) |
David Monahan | 4f1e8e4 | 2019-09-04 09:22:10 +0100 | [diff] [blame] | 675 | { |
Sadik Armagan | 890bf65 | 2020-09-29 15:12:36 +0100 | [diff] [blame] | 676 | const Layer& connectedLayer = layer.GetInputSlots()[0].GetConnectedOutputSlot()->GetOwningLayer(); |
| 677 | // Do not add MemCopy Layer if OutputLayer is already connected the MemCopy Layer |
| 678 | if (connectedLayer.GetType() != LayerType::MemCopy) |
Narumol Prangnawarat | aa68e01 | 2019-11-29 17:17:43 +0000 | [diff] [blame] | 679 | { |
Sadik Armagan | 890bf65 | 2020-09-29 15:12:36 +0100 | [diff] [blame] | 680 | // If we got here then we didn't export the memory, so add an output workload which performs a memcopy. |
| 681 | outputQueueDescriptor.m_Inputs.push_back(inputTensorHandle); |
| 682 | info.m_InputTensorInfos.push_back(inputTensorInfo); |
Narumol Prangnawarat | aa68e01 | 2019-11-29 17:17:43 +0000 | [diff] [blame] | 683 | |
Sadik Armagan | 890bf65 | 2020-09-29 15:12:36 +0100 | [diff] [blame] | 684 | std::unique_ptr<IWorkload> outputWorkload = |
| 685 | std::make_unique<CopyMemGenericWorkload>(outputQueueDescriptor, info); |
| 686 | ARMNN_ASSERT_MSG(outputWorkload, "No output workload created"); |
| 687 | |
| 688 | std::unique_ptr<TimelineUtilityMethods> timelineUtils = |
| 689 | TimelineUtilityMethods::GetTimelineUtils(m_ProfilingService); |
| 690 | if (timelineUtils) |
| 691 | { |
| 692 | // Add Output Workload to the post-optimisation network structure |
| 693 | AddWorkloadStructure(timelineUtils, outputWorkload, layer); |
| 694 | timelineUtils->Commit(); |
| 695 | } |
| 696 | |
| 697 | m_OutputQueue.push_back(move(outputWorkload)); |
| 698 | } |
David Monahan | 4f1e8e4 | 2019-09-04 09:22:10 +0100 | [diff] [blame] | 699 | } |
Derek Lamberti | 03614f6 | 2018-10-02 15:52:46 +0100 | [diff] [blame] | 700 | } |
| 701 | |
Derek Lamberti | a08d29b | 2020-06-19 14:33:05 +0100 | [diff] [blame] | 702 | void LoadedNetwork::AllocateWorkingMemory(std::lock_guard<std::mutex>& lock) |
Derek Lamberti | 03614f6 | 2018-10-02 15:52:46 +0100 | [diff] [blame] | 703 | { |
Derek Lamberti | a08d29b | 2020-06-19 14:33:05 +0100 | [diff] [blame] | 704 | ARMNN_SCOPED_PROFILING_EVENT(Compute::Undefined, "Working Memory Allocation"); |
| 705 | |
| 706 | // this unused parameter makes sure we can only call this function with a valid lock |
| 707 | IgnoreUnused(lock); |
| 708 | |
Derek Lamberti | 03614f6 | 2018-10-02 15:52:46 +0100 | [diff] [blame] | 709 | if (m_IsWorkingMemAllocated) |
| 710 | { |
| 711 | return; |
| 712 | } |
David Beck | 29c75de | 2018-10-23 13:35:58 +0100 | [diff] [blame] | 713 | for (auto&& workloadFactory : m_WorkloadFactories) |
| 714 | { |
Aron Virginas-Tar | 5caf907 | 2018-11-14 18:35:18 +0000 | [diff] [blame] | 715 | IBackendInternal::IMemoryManagerSharedPtr memoryManager = workloadFactory.second.second; |
| 716 | if (memoryManager) |
| 717 | { |
| 718 | memoryManager->Acquire(); |
| 719 | } |
David Beck | 29c75de | 2018-10-23 13:35:58 +0100 | [diff] [blame] | 720 | } |
Narumol Prangnawarat | 11bd261 | 2019-08-13 10:26:53 +0100 | [diff] [blame] | 721 | m_TensorHandleFactoryRegistry.AquireMemory(); |
Derek Lamberti | 03614f6 | 2018-10-02 15:52:46 +0100 | [diff] [blame] | 722 | m_IsWorkingMemAllocated = true; |
| 723 | } |
| 724 | |
| 725 | void LoadedNetwork::FreeWorkingMemory() |
| 726 | { |
Matthew Bentham | 2a326b5 | 2019-03-19 10:11:01 +0000 | [diff] [blame] | 727 | std::lock_guard<std::mutex> lockGuard(m_WorkingMemMutex); |
Derek Lamberti | 03614f6 | 2018-10-02 15:52:46 +0100 | [diff] [blame] | 728 | if (!m_IsWorkingMemAllocated) |
| 729 | { |
| 730 | return; |
| 731 | } |
| 732 | // Informs the memory managers to release memory in it's respective memory group |
David Beck | 29c75de | 2018-10-23 13:35:58 +0100 | [diff] [blame] | 733 | for (auto&& workloadFactory : m_WorkloadFactories) |
| 734 | { |
Aron Virginas-Tar | 5caf907 | 2018-11-14 18:35:18 +0000 | [diff] [blame] | 735 | IBackendInternal::IMemoryManagerSharedPtr memoryManager = workloadFactory.second.second; |
| 736 | if (memoryManager) |
| 737 | { |
| 738 | memoryManager->Release(); |
| 739 | } |
David Beck | 29c75de | 2018-10-23 13:35:58 +0100 | [diff] [blame] | 740 | } |
Narumol Prangnawarat | 11bd261 | 2019-08-13 10:26:53 +0100 | [diff] [blame] | 741 | m_TensorHandleFactoryRegistry.ReleaseMemory(); |
Derek Lamberti | 03614f6 | 2018-10-02 15:52:46 +0100 | [diff] [blame] | 742 | m_IsWorkingMemAllocated = false; |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 743 | } |
| 744 | |
David Monahan | 6198fe0 | 2019-12-02 08:35:43 +0000 | [diff] [blame] | 745 | bool LoadedNetwork::Execute(std::unique_ptr<TimelineUtilityMethods>& timelineUtils, |
| 746 | profiling::ProfilingGuid inferenceGuid) |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 747 | { |
| 748 | bool success = true; |
| 749 | |
Aron Virginas-Tar | a8e06ed | 2018-10-19 16:46:15 +0100 | [diff] [blame] | 750 | auto Fail = [&](const std::exception& error) |
| 751 | { |
Derek Lamberti | 0844697 | 2019-11-26 16:38:31 +0000 | [diff] [blame] | 752 | ARMNN_LOG(error) << "An error occurred attempting to execute a workload: " << error.what(); |
Aron Virginas-Tar | a8e06ed | 2018-10-19 16:46:15 +0100 | [diff] [blame] | 753 | success = false; |
| 754 | }; |
| 755 | |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 756 | try |
| 757 | { |
Matthew Bentham | 2a326b5 | 2019-03-19 10:11:01 +0000 | [diff] [blame] | 758 | std::lock_guard<std::mutex> lockGuard(m_WorkingMemMutex); |
Derek Lamberti | a08d29b | 2020-06-19 14:33:05 +0100 | [diff] [blame] | 759 | AllocateWorkingMemory(lockGuard); |
Derek Lamberti | 03614f6 | 2018-10-02 15:52:46 +0100 | [diff] [blame] | 760 | |
David Monahan | 6198fe0 | 2019-12-02 08:35:43 +0000 | [diff] [blame] | 761 | ProfilingDynamicGuid workloadInferenceID(0); |
Derek Lamberti | a08d29b | 2020-06-19 14:33:05 +0100 | [diff] [blame] | 762 | auto ExecuteQueue = [&timelineUtils, &workloadInferenceID, &inferenceGuid](WorkloadQueue& queue) |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 763 | { |
Derek Lamberti | a08d29b | 2020-06-19 14:33:05 +0100 | [diff] [blame] | 764 | for (auto& workload : queue) |
David Monahan | 6198fe0 | 2019-12-02 08:35:43 +0000 | [diff] [blame] | 765 | { |
Derek Lamberti | a08d29b | 2020-06-19 14:33:05 +0100 | [diff] [blame] | 766 | if(timelineUtils) |
| 767 | { |
| 768 | workloadInferenceID = timelineUtils->RecordWorkloadInferenceAndStartOfLifeEvent(workload->GetGuid(), |
| 769 | inferenceGuid); |
| 770 | } |
| 771 | workload->Execute(); |
| 772 | if(timelineUtils) |
| 773 | { |
| 774 | timelineUtils->RecordEndOfLifeEvent(workloadInferenceID); |
| 775 | } |
David Monahan | 6198fe0 | 2019-12-02 08:35:43 +0000 | [diff] [blame] | 776 | } |
Derek Lamberti | a08d29b | 2020-06-19 14:33:05 +0100 | [diff] [blame] | 777 | }; |
Derek Lamberti | 03614f6 | 2018-10-02 15:52:46 +0100 | [diff] [blame] | 778 | |
Derek Lamberti | a08d29b | 2020-06-19 14:33:05 +0100 | [diff] [blame] | 779 | ExecuteQueue(m_InputQueue); |
| 780 | ExecuteQueue(m_WorkloadQueue); |
| 781 | ExecuteQueue(m_OutputQueue); |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 782 | } |
Aron Virginas-Tar | a8e06ed | 2018-10-19 16:46:15 +0100 | [diff] [blame] | 783 | catch (const RuntimeException& error) |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 784 | { |
Aron Virginas-Tar | a8e06ed | 2018-10-19 16:46:15 +0100 | [diff] [blame] | 785 | Fail(error); |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 786 | } |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 787 | catch (const std::runtime_error& error) |
| 788 | { |
Aron Virginas-Tar | a8e06ed | 2018-10-19 16:46:15 +0100 | [diff] [blame] | 789 | Fail(error); |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 790 | } |
| 791 | |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 792 | return success; |
| 793 | } |
| 794 | |
Nattapat Chaimanowong | 6e94820 | 2019-03-22 14:01:46 +0000 | [diff] [blame] | 795 | void LoadedNetwork::RegisterDebugCallback(const DebugCallbackFunction& func) |
| 796 | { |
| 797 | for (auto&& workloadPtr: m_WorkloadQueue) |
| 798 | { |
| 799 | workloadPtr.get()->RegisterDebugCallback(func); |
| 800 | } |
| 801 | } |
| 802 | |
telsoa01 | 4fcda01 | 2018-03-09 14:13:49 +0000 | [diff] [blame] | 803 | } |