blob: 7daae26bd4a6c9d6944a29216bb4af2a6b3af60a [file] [log] [blame]
telsoa014fcda012018-03-09 14:13:49 +00001//
Sadik Armagana9c2ce12020-07-14 10:02:22 +01002// Copyright © 2017 Arm Ltd and Contributors. All rights reserved.
David Beckecb56cd2018-09-05 12:52:57 +01003// SPDX-License-Identifier: MIT
telsoa014fcda012018-03-09 14:13:49 +00004//
Matteo Martincighc601aa62019-10-29 15:03:22 +00005
telsoa014fcda012018-03-09 14:13:49 +00006#pragma once
Matteo Martincighc601aa62019-10-29 15:03:22 +00007
David Beckf0b48452018-10-19 15:20:56 +01008#include <armnn/ArmNN.hpp>
alered01a7227ac2020-05-07 14:58:29 +01009#include <armnn/Logging.hpp>
10#include <armnn/utility/Timer.hpp>
Matteo Martincighc601aa62019-10-29 15:03:22 +000011#include <armnn/BackendRegistry.hpp>
Narumol Prangnawaratac2770a2020-04-01 16:51:23 +010012#include <armnn/utility/Assert.hpp>
Matthew Sloyan80c6b142020-09-08 12:00:32 +010013#include <armnn/utility/NumericCast.hpp>
telsoa01c577f2c2018-08-31 09:22:23 +010014
Aron Virginas-Tar64e4ccb2019-02-12 11:27:53 +000015#if defined(ARMNN_SERIALIZER)
Derek Lamberti0028d1b2019-02-20 13:57:42 +000016#include "armnnDeserializer/IDeserializer.hpp"
Aron Virginas-Tar64e4ccb2019-02-12 11:27:53 +000017#endif
telsoa01c577f2c2018-08-31 09:22:23 +010018#if defined(ARMNN_TF_LITE_PARSER)
David Beckf0b48452018-10-19 15:20:56 +010019#include <armnnTfLiteParser/ITfLiteParser.hpp>
telsoa01c577f2c2018-08-31 09:22:23 +010020#endif
telsoa01c577f2c2018-08-31 09:22:23 +010021#if defined(ARMNN_ONNX_PARSER)
David Beckf0b48452018-10-19 15:20:56 +010022#include <armnnOnnxParser/IOnnxParser.hpp>
telsoa01c577f2c2018-08-31 09:22:23 +010023#endif
telsoa014fcda012018-03-09 14:13:49 +000024
Francis Murtagh532a29d2020-06-29 11:50:01 +010025#include <Filesystem.hpp>
Aron Virginas-Tar64e4ccb2019-02-12 11:27:53 +000026#include <HeapProfiling.hpp>
Jim Flynn2fd61002019-05-03 12:54:26 +010027#include <TensorIOUtils.hpp>
Aron Virginas-Tar64e4ccb2019-02-12 11:27:53 +000028
David Monahana8837bf2020-04-16 10:01:56 +010029#include "armnn/utility/StringUtils.hpp"
James Wardc89829f2020-10-12 14:17:36 +010030#include <cxxopts/cxxopts.hpp>
31#include "CxxoptsUtils.hpp"
James Ward08f40162020-09-07 16:45:07 +010032#include <fmt/format.h>
James Ward6d9f5c52020-09-28 11:56:35 +010033#include <mapbox/variant.hpp>
telsoa014fcda012018-03-09 14:13:49 +000034
Aron Virginas-Tar339bcae2019-01-31 16:44:26 +000035#include <algorithm>
36#include <iterator>
Aron Virginas-Tar5cc8e562018-10-23 15:14:46 +010037#include <fstream>
telsoa014fcda012018-03-09 14:13:49 +000038#include <map>
39#include <string>
Aron Virginas-Tar7cf0eaa2019-01-24 17:05:36 +000040#include <vector>
telsoa01c577f2c2018-08-31 09:22:23 +010041#include <type_traits>
42
Aron Virginas-Tar5cc8e562018-10-23 15:14:46 +010043namespace
44{
45
46inline bool CheckRequestedBackendsAreValid(const std::vector<armnn::BackendId>& backendIds,
47 armnn::Optional<std::string&> invalidBackendIds = armnn::EmptyOptional())
48{
49 if (backendIds.empty())
50 {
51 return false;
52 }
53
54 armnn::BackendIdSet validBackendIds = armnn::BackendRegistryInstance().GetBackendIds();
55
56 bool allValid = true;
57 for (const auto& backendId : backendIds)
58 {
59 if (std::find(validBackendIds.begin(), validBackendIds.end(), backendId) == validBackendIds.end())
60 {
61 allValid = false;
62 if (invalidBackendIds)
63 {
64 if (!invalidBackendIds.value().empty())
65 {
66 invalidBackendIds.value() += ", ";
67 }
68 invalidBackendIds.value() += backendId;
69 }
70 }
71 }
72 return allValid;
73}
74
75} // anonymous namespace
76
telsoa01c577f2c2018-08-31 09:22:23 +010077namespace InferenceModelInternal
78{
Jim Flynnb4d7eae2019-05-01 14:44:27 +010079using BindingPointInfo = armnn::BindingPointInfo;
telsoa01c577f2c2018-08-31 09:22:23 +010080
81using QuantizationParams = std::pair<float,int32_t>;
82
83struct Params
84{
Aron Virginas-Tar7cf0eaa2019-01-24 17:05:36 +000085 std::string m_ModelPath;
86 std::vector<std::string> m_InputBindings;
87 std::vector<armnn::TensorShape> m_InputShapes;
88 std::vector<std::string> m_OutputBindings;
Aron Virginas-Tar339bcae2019-01-31 16:44:26 +000089 std::vector<armnn::BackendId> m_ComputeDevices;
Matteo Martincigh00dda4a2019-08-14 11:42:30 +010090 std::string m_DynamicBackendsPath;
Aron Virginas-Tar7cf0eaa2019-01-24 17:05:36 +000091 size_t m_SubgraphId;
92 bool m_IsModelBinary;
93 bool m_VisualizePostOptimizationModel;
94 bool m_EnableFp16TurboMode;
Narumol Prangnawaratd8cc8112020-03-24 13:54:05 +000095 bool m_EnableBf16TurboMode;
Matthew Jackson54658b92019-08-27 15:35:59 +010096 bool m_PrintIntermediateLayers;
Derek Lamberti132563c2019-12-02 16:06:40 +000097 bool m_ParseUnsupported;
Sadik Armagana9c2ce12020-07-14 10:02:22 +010098 bool m_InferOutputShape;
Sadik Armagana25886e2020-09-15 17:17:08 +010099 bool m_EnableFastMath;
Matthew Sloyan42432112021-01-08 10:30:51 +0000100 bool m_SaveCachedNetwork;
101 std::string m_CachedNetworkFilePath;
telsoa01c577f2c2018-08-31 09:22:23 +0100102
103 Params()
Matteo Martincigh00dda4a2019-08-14 11:42:30 +0100104 : m_ComputeDevices{}
telsoa01c577f2c2018-08-31 09:22:23 +0100105 , m_SubgraphId(0)
106 , m_IsModelBinary(true)
107 , m_VisualizePostOptimizationModel(false)
108 , m_EnableFp16TurboMode(false)
Narumol Prangnawaratd8cc8112020-03-24 13:54:05 +0000109 , m_EnableBf16TurboMode(false)
Matthew Jackson54658b92019-08-27 15:35:59 +0100110 , m_PrintIntermediateLayers(false)
Derek Lamberti132563c2019-12-02 16:06:40 +0000111 , m_ParseUnsupported(false)
Sadik Armagana9c2ce12020-07-14 10:02:22 +0100112 , m_InferOutputShape(false)
Sadik Armagana25886e2020-09-15 17:17:08 +0100113 , m_EnableFastMath(false)
Matthew Sloyan42432112021-01-08 10:30:51 +0000114 , m_SaveCachedNetwork(false)
115 , m_CachedNetworkFilePath("")
telsoa01c577f2c2018-08-31 09:22:23 +0100116 {}
117};
118
119} // namespace InferenceModelInternal
120
121template <typename IParser>
122struct CreateNetworkImpl
123{
124public:
125 using Params = InferenceModelInternal::Params;
telsoa01c577f2c2018-08-31 09:22:23 +0100126
127 static armnn::INetworkPtr Create(const Params& params,
Jim Flynnb4d7eae2019-05-01 14:44:27 +0100128 std::vector<armnn::BindingPointInfo>& inputBindings,
129 std::vector<armnn::BindingPointInfo>& outputBindings)
telsoa01c577f2c2018-08-31 09:22:23 +0100130 {
Aron Virginas-Tar7cf0eaa2019-01-24 17:05:36 +0000131 const std::string& modelPath = params.m_ModelPath;
telsoa01c577f2c2018-08-31 09:22:23 +0100132
Aron Virginas-Tar7cf0eaa2019-01-24 17:05:36 +0000133 // Create a network from a file on disk
134 auto parser(IParser::Create());
telsoa01c577f2c2018-08-31 09:22:23 +0100135
Aron Virginas-Tar7cf0eaa2019-01-24 17:05:36 +0000136 std::map<std::string, armnn::TensorShape> inputShapes;
137 if (!params.m_InputShapes.empty())
138 {
139 const size_t numInputShapes = params.m_InputShapes.size();
140 const size_t numInputBindings = params.m_InputBindings.size();
141 if (numInputShapes < numInputBindings)
142 {
James Ward08f40162020-09-07 16:45:07 +0100143 throw armnn::Exception(fmt::format(
144 "Not every input has its tensor shape specified: expected={0}, got={1}",
145 numInputBindings, numInputShapes));
Aron Virginas-Tar7cf0eaa2019-01-24 17:05:36 +0000146 }
telsoa01c577f2c2018-08-31 09:22:23 +0100147
Aron Virginas-Tar7cf0eaa2019-01-24 17:05:36 +0000148 for (size_t i = 0; i < numInputShapes; i++)
149 {
150 inputShapes[params.m_InputBindings[i]] = params.m_InputShapes[i];
151 }
152 }
telsoa01c577f2c2018-08-31 09:22:23 +0100153
Aron Virginas-Tar7cf0eaa2019-01-24 17:05:36 +0000154 std::vector<std::string> requestedOutputs = params.m_OutputBindings;
155 armnn::INetworkPtr network{nullptr, [](armnn::INetwork *){}};
156
157 {
158 ARMNN_SCOPED_HEAP_PROFILING("Parsing");
159 // Handle text and binary input differently by calling the corresponding parser function
160 network = (params.m_IsModelBinary ?
161 parser->CreateNetworkFromBinaryFile(modelPath.c_str(), inputShapes, requestedOutputs) :
162 parser->CreateNetworkFromTextFile(modelPath.c_str(), inputShapes, requestedOutputs));
163 }
164
165 for (const std::string& inputLayerName : params.m_InputBindings)
166 {
167 inputBindings.push_back(parser->GetNetworkInputBindingInfo(inputLayerName));
168 }
169
170 for (const std::string& outputLayerName : params.m_OutputBindings)
171 {
172 outputBindings.push_back(parser->GetNetworkOutputBindingInfo(outputLayerName));
173 }
174
175 return network;
telsoa01c577f2c2018-08-31 09:22:23 +0100176 }
177};
178
Aron Virginas-Tar64e4ccb2019-02-12 11:27:53 +0000179#if defined(ARMNN_SERIALIZER)
180template <>
Derek Lamberti0028d1b2019-02-20 13:57:42 +0000181struct CreateNetworkImpl<armnnDeserializer::IDeserializer>
Aron Virginas-Tar64e4ccb2019-02-12 11:27:53 +0000182{
183public:
Derek Lamberti0028d1b2019-02-20 13:57:42 +0000184 using IParser = armnnDeserializer::IDeserializer;
Aron Virginas-Tar64e4ccb2019-02-12 11:27:53 +0000185 using Params = InferenceModelInternal::Params;
Aron Virginas-Tar64e4ccb2019-02-12 11:27:53 +0000186
187 static armnn::INetworkPtr Create(const Params& params,
Jim Flynnb4d7eae2019-05-01 14:44:27 +0100188 std::vector<armnn::BindingPointInfo>& inputBindings,
189 std::vector<armnn::BindingPointInfo>& outputBindings)
Aron Virginas-Tar64e4ccb2019-02-12 11:27:53 +0000190 {
191 auto parser(IParser::Create());
Narumol Prangnawaratac2770a2020-04-01 16:51:23 +0100192 ARMNN_ASSERT(parser);
Aron Virginas-Tar64e4ccb2019-02-12 11:27:53 +0000193
194 armnn::INetworkPtr network{nullptr, [](armnn::INetwork *){}};
195
196 {
197 ARMNN_SCOPED_HEAP_PROFILING("Parsing");
Derek Lamberti2b183fb2019-02-18 16:36:57 +0000198
Francis Murtagh532a29d2020-06-29 11:50:01 +0100199 std::error_code errorCode;
200 fs::path pathToFile(params.m_ModelPath);
201 if (!fs::exists(pathToFile, errorCode))
Derek Lamberti2b183fb2019-02-18 16:36:57 +0000202 {
James Ward08f40162020-09-07 16:45:07 +0100203 throw armnn::FileNotFoundException(fmt::format("Cannot find the file ({0}) errorCode: {1} {2}",
204 params.m_ModelPath,
205 errorCode.message(),
Derek Lamberti2b183fb2019-02-18 16:36:57 +0000206 CHECK_LOCATION().AsString()));
207 }
208 std::ifstream file(params.m_ModelPath, std::ios::binary);
209
210 network = parser->CreateNetworkFromBinary(file);
Aron Virginas-Tar64e4ccb2019-02-12 11:27:53 +0000211 }
212
Matthew Sloyan80c6b142020-09-08 12:00:32 +0100213 unsigned int subgraphId = armnn::numeric_cast<unsigned int>(params.m_SubgraphId);
Aron Virginas-Tar64e4ccb2019-02-12 11:27:53 +0000214
215 for (const std::string& inputLayerName : params.m_InputBindings)
216 {
Derek Lamberti8ddae332019-02-21 16:29:43 +0000217 armnnDeserializer::BindingPointInfo inputBinding =
Derek Lambertiff05cc52019-04-26 13:05:17 +0100218 parser->GetNetworkInputBindingInfo(subgraphId, inputLayerName);
Derek Lamberti8ddae332019-02-21 16:29:43 +0000219 inputBindings.push_back(std::make_pair(inputBinding.m_BindingId, inputBinding.m_TensorInfo));
Aron Virginas-Tar64e4ccb2019-02-12 11:27:53 +0000220 }
221
222 for (const std::string& outputLayerName : params.m_OutputBindings)
223 {
Derek Lamberti8ddae332019-02-21 16:29:43 +0000224 armnnDeserializer::BindingPointInfo outputBinding =
Derek Lambertiff05cc52019-04-26 13:05:17 +0100225 parser->GetNetworkOutputBindingInfo(subgraphId, outputLayerName);
Derek Lamberti8ddae332019-02-21 16:29:43 +0000226 outputBindings.push_back(std::make_pair(outputBinding.m_BindingId, outputBinding.m_TensorInfo));
Aron Virginas-Tar64e4ccb2019-02-12 11:27:53 +0000227 }
228
229 return network;
230 }
231};
232#endif
233
telsoa01c577f2c2018-08-31 09:22:23 +0100234#if defined(ARMNN_TF_LITE_PARSER)
235template <>
236struct CreateNetworkImpl<armnnTfLiteParser::ITfLiteParser>
237{
238public:
239 using IParser = armnnTfLiteParser::ITfLiteParser;
240 using Params = InferenceModelInternal::Params;
telsoa01c577f2c2018-08-31 09:22:23 +0100241
242 static armnn::INetworkPtr Create(const Params& params,
Jim Flynnb4d7eae2019-05-01 14:44:27 +0100243 std::vector<armnn::BindingPointInfo>& inputBindings,
244 std::vector<armnn::BindingPointInfo>& outputBindings)
telsoa01c577f2c2018-08-31 09:22:23 +0100245 {
Aron Virginas-Tar7cf0eaa2019-01-24 17:05:36 +0000246 const std::string& modelPath = params.m_ModelPath;
telsoa01c577f2c2018-08-31 09:22:23 +0100247
Aron Virginas-Tar7cf0eaa2019-01-24 17:05:36 +0000248 // Create a network from a file on disk
Derek Lamberti132563c2019-12-02 16:06:40 +0000249 IParser::TfLiteParserOptions options;
250 options.m_StandInLayerForUnsupported = params.m_ParseUnsupported;
Sadik Armagana9c2ce12020-07-14 10:02:22 +0100251 options.m_InferAndValidate = params.m_InferOutputShape;
Derek Lamberti132563c2019-12-02 16:06:40 +0000252 auto parser(IParser::Create(options));
telsoa01c577f2c2018-08-31 09:22:23 +0100253
Aron Virginas-Tar7cf0eaa2019-01-24 17:05:36 +0000254 armnn::INetworkPtr network{nullptr, [](armnn::INetwork *){}};
telsoa01c577f2c2018-08-31 09:22:23 +0100255
Aron Virginas-Tar7cf0eaa2019-01-24 17:05:36 +0000256 {
257 ARMNN_SCOPED_HEAP_PROFILING("Parsing");
258 network = parser->CreateNetworkFromBinaryFile(modelPath.c_str());
259 }
telsoa01c577f2c2018-08-31 09:22:23 +0100260
Aron Virginas-Tar7cf0eaa2019-01-24 17:05:36 +0000261 for (const std::string& inputLayerName : params.m_InputBindings)
262 {
Jim Flynnb4d7eae2019-05-01 14:44:27 +0100263 armnn::BindingPointInfo inputBinding =
Aron Virginas-Tar7cf0eaa2019-01-24 17:05:36 +0000264 parser->GetNetworkInputBindingInfo(params.m_SubgraphId, inputLayerName);
265 inputBindings.push_back(inputBinding);
266 }
267
268 for (const std::string& outputLayerName : params.m_OutputBindings)
269 {
Jim Flynnb4d7eae2019-05-01 14:44:27 +0100270 armnn::BindingPointInfo outputBinding =
Aron Virginas-Tar7cf0eaa2019-01-24 17:05:36 +0000271 parser->GetNetworkOutputBindingInfo(params.m_SubgraphId, outputLayerName);
272 outputBindings.push_back(outputBinding);
273 }
274
275 return network;
telsoa01c577f2c2018-08-31 09:22:23 +0100276 }
277};
278#endif
279
280#if defined(ARMNN_ONNX_PARSER)
281template <>
282struct CreateNetworkImpl<armnnOnnxParser::IOnnxParser>
283{
284public:
285 using IParser = armnnOnnxParser::IOnnxParser;
286 using Params = InferenceModelInternal::Params;
287 using BindingPointInfo = InferenceModelInternal::BindingPointInfo;
288
289 static armnn::INetworkPtr Create(const Params& params,
Aron Virginas-Tar7cf0eaa2019-01-24 17:05:36 +0000290 std::vector<BindingPointInfo>& inputBindings,
291 std::vector<BindingPointInfo>& outputBindings)
telsoa01c577f2c2018-08-31 09:22:23 +0100292 {
Aron Virginas-Tar7cf0eaa2019-01-24 17:05:36 +0000293 const std::string& modelPath = params.m_ModelPath;
telsoa01c577f2c2018-08-31 09:22:23 +0100294
Aron Virginas-Tar7cf0eaa2019-01-24 17:05:36 +0000295 // Create a network from a file on disk
296 auto parser(IParser::Create());
telsoa01c577f2c2018-08-31 09:22:23 +0100297
Aron Virginas-Tar7cf0eaa2019-01-24 17:05:36 +0000298 armnn::INetworkPtr network{nullptr, [](armnn::INetwork *){}};
telsoa01c577f2c2018-08-31 09:22:23 +0100299
Aron Virginas-Tar7cf0eaa2019-01-24 17:05:36 +0000300 {
301 ARMNN_SCOPED_HEAP_PROFILING("Parsing");
302 network = (params.m_IsModelBinary ?
303 parser->CreateNetworkFromBinaryFile(modelPath.c_str()) :
304 parser->CreateNetworkFromTextFile(modelPath.c_str()));
305 }
telsoa01c577f2c2018-08-31 09:22:23 +0100306
Aron Virginas-Tar7cf0eaa2019-01-24 17:05:36 +0000307 for (const std::string& inputLayerName : params.m_InputBindings)
308 {
309 BindingPointInfo inputBinding = parser->GetNetworkInputBindingInfo(inputLayerName);
310 inputBindings.push_back(inputBinding);
311 }
312
313 for (const std::string& outputLayerName : params.m_OutputBindings)
314 {
315 BindingPointInfo outputBinding = parser->GetNetworkOutputBindingInfo(outputLayerName);
316 outputBindings.push_back(outputBinding);
317 }
318
319 return network;
telsoa01c577f2c2018-08-31 09:22:23 +0100320 }
321};
322#endif
telsoa014fcda012018-03-09 14:13:49 +0000323
Aron Virginas-Tar7cf0eaa2019-01-24 17:05:36 +0000324
telsoa014fcda012018-03-09 14:13:49 +0000325
326template <typename IParser, typename TDataType>
327class InferenceModel
328{
329public:
Aron Virginas-Tar7cf0eaa2019-01-24 17:05:36 +0000330 using DataType = TDataType;
331 using Params = InferenceModelInternal::Params;
Aron Virginas-Tar7cf0eaa2019-01-24 17:05:36 +0000332 using QuantizationParams = InferenceModelInternal::QuantizationParams;
James Ward6d9f5c52020-09-28 11:56:35 +0100333 using TContainer = mapbox::util::variant<std::vector<float>, std::vector<int>, std::vector<unsigned char>>;
telsoa014fcda012018-03-09 14:13:49 +0000334
335 struct CommandLineOptions
336 {
337 std::string m_ModelDir;
Aron Virginas-Tar339bcae2019-01-31 16:44:26 +0000338 std::vector<std::string> m_ComputeDevices;
Matteo Martincigh00dda4a2019-08-14 11:42:30 +0100339 std::string m_DynamicBackendsPath;
surmeh013537c2c2018-05-18 16:31:43 +0100340 bool m_VisualizePostOptimizationModel;
telsoa01c577f2c2018-08-31 09:22:23 +0100341 bool m_EnableFp16TurboMode;
Narumol Prangnawaratd8cc8112020-03-24 13:54:05 +0000342 bool m_EnableBf16TurboMode;
Pablo Tello507f39d2019-04-15 15:44:39 +0100343 std::string m_Labels;
Aron Virginas-Tar339bcae2019-01-31 16:44:26 +0000344
345 std::vector<armnn::BackendId> GetComputeDevicesAsBackendIds()
346 {
347 std::vector<armnn::BackendId> backendIds;
348 std::copy(m_ComputeDevices.begin(), m_ComputeDevices.end(), std::back_inserter(backendIds));
349 return backendIds;
350 }
telsoa014fcda012018-03-09 14:13:49 +0000351 };
352
James Wardc89829f2020-10-12 14:17:36 +0100353 static void AddCommandLineOptions(cxxopts::Options& options,
354 CommandLineOptions& cLineOptions, std::vector<std::string>& required)
telsoa014fcda012018-03-09 14:13:49 +0000355 {
Aron Virginas-Tar339bcae2019-01-31 16:44:26 +0000356 const std::vector<std::string> defaultComputes = { "CpuAcc", "CpuRef" };
David Beckf0b48452018-10-19 15:20:56 +0100357
Aron Virginas-Tar5cc8e562018-10-23 15:14:46 +0100358 const std::string backendsMessage = "Which device to run layers on by default. Possible choices: "
359 + armnn::BackendRegistryInstance().GetBackendIdsAsString();
360
James Wardc89829f2020-10-12 14:17:36 +0100361 options
362 .allow_unrecognised_options()
363 .add_options()
364 ("m,model-dir", "Path to directory containing model files (.caffemodel/.prototxt/.tflite)",
365 cxxopts::value<std::string>(cLineOptions.m_ModelDir))
366 ("c,compute", backendsMessage.c_str(),
367 cxxopts::value<std::vector<std::string>>(cLineOptions.m_ComputeDevices)->default_value("CpuRef"))
368 ("b,dynamic-backends-path",
369 "Path where to load any available dynamic backend from. "
370 "If left empty (the default), dynamic backends will not be used.",
371 cxxopts::value(cLineOptions.m_DynamicBackendsPath))
372 ("l,labels",
373 "Text file containing one image filename - correct label pair per line, "
374 "used to test the accuracy of the network.", cxxopts::value<std::string>(cLineOptions.m_Labels))
375 ("v,visualize-optimized-model",
376 "Produce a dot file useful for visualizing the graph post optimization."
377 "The file will have the same name as the model with the .dot extention.",
378 cxxopts::value<bool>(cLineOptions.m_VisualizePostOptimizationModel)->default_value("false"))
379 ("fp16-turbo-mode",
380 "If this option is enabled FP32 layers, weights and biases will be converted "
381 "to FP16 where the backend supports it.",
382 cxxopts::value<bool>(cLineOptions.m_EnableFp16TurboMode)->default_value("false"))
383 ("bf16-turbo-mode",
384 "If this option is enabled FP32 layers, weights and biases will be converted "
385 "to BF16 where the backend supports it.",
386 cxxopts::value<bool>(cLineOptions.m_EnableBf16TurboMode)->default_value("false"));
387
388 required.emplace_back("model-dir");
telsoa014fcda012018-03-09 14:13:49 +0000389 }
390
Matthew Bentham3e68b972019-04-09 13:10:46 +0100391 InferenceModel(const Params& params,
392 bool enableProfiling,
Matteo Martincigh00dda4a2019-08-14 11:42:30 +0100393 const std::string& dynamicBackendsPath,
Matthew Bentham3e68b972019-04-09 13:10:46 +0100394 const std::shared_ptr<armnn::IRuntime>& runtime = nullptr)
395 : m_EnableProfiling(enableProfiling)
Matteo Martincigh00dda4a2019-08-14 11:42:30 +0100396 , m_DynamicBackendsPath(dynamicBackendsPath)
telsoa014fcda012018-03-09 14:13:49 +0000397 {
telsoa01c577f2c2018-08-31 09:22:23 +0100398 if (runtime)
telsoa014fcda012018-03-09 14:13:49 +0000399 {
telsoa01c577f2c2018-08-31 09:22:23 +0100400 m_Runtime = runtime;
telsoa014fcda012018-03-09 14:13:49 +0000401 }
telsoa01c577f2c2018-08-31 09:22:23 +0100402 else
telsoa014fcda012018-03-09 14:13:49 +0000403 {
telsoa01c577f2c2018-08-31 09:22:23 +0100404 armnn::IRuntime::CreationOptions options;
Nina Drozd549ae372018-09-10 14:26:44 +0100405 options.m_EnableGpuProfiling = m_EnableProfiling;
Matteo Martincigh00dda4a2019-08-14 11:42:30 +0100406 options.m_DynamicBackendsPath = m_DynamicBackendsPath;
telsoa01c577f2c2018-08-31 09:22:23 +0100407 m_Runtime = std::move(armnn::IRuntime::Create(options));
surmeh013537c2c2018-05-18 16:31:43 +0100408 }
telsoa014fcda012018-03-09 14:13:49 +0000409
Aron Virginas-Tar5cc8e562018-10-23 15:14:46 +0100410 std::string invalidBackends;
Aron Virginas-Tar339bcae2019-01-31 16:44:26 +0000411 if (!CheckRequestedBackendsAreValid(params.m_ComputeDevices, armnn::Optional<std::string&>(invalidBackends)))
Aron Virginas-Tar5cc8e562018-10-23 15:14:46 +0100412 {
413 throw armnn::Exception("Some backend IDs are invalid: " + invalidBackends);
414 }
415
alered01a7227ac2020-05-07 14:58:29 +0100416 const auto parsing_start_time = armnn::GetTimeNow();
Matteo Martincigh00dda4a2019-08-14 11:42:30 +0100417 armnn::INetworkPtr network = CreateNetworkImpl<IParser>::Create(params, m_InputBindings, m_OutputBindings);
telsoa014fcda012018-03-09 14:13:49 +0000418
alered01a7227ac2020-05-07 14:58:29 +0100419 ARMNN_LOG(info) << "Network parsing time: " << std::setprecision(2)
420 << std::fixed << armnn::GetTimeDuration(parsing_start_time).count() << " ms\n";
421
Matteo Martincigh00dda4a2019-08-14 11:42:30 +0100422 armnn::IOptimizedNetworkPtr optNet{nullptr, [](armnn::IOptimizedNetwork*){}};
surmeh013537c2c2018-05-18 16:31:43 +0100423 {
424 ARMNN_SCOPED_HEAP_PROFILING("Optimizing");
telsoa01c577f2c2018-08-31 09:22:23 +0100425
426 armnn::OptimizerOptions options;
427 options.m_ReduceFp32ToFp16 = params.m_EnableFp16TurboMode;
Narumol Prangnawaratd8cc8112020-03-24 13:54:05 +0000428 options.m_ReduceFp32ToBf16 = params.m_EnableBf16TurboMode;
Matthew Jackson54658b92019-08-27 15:35:59 +0100429 options.m_Debug = params.m_PrintIntermediateLayers;
telsoa01c577f2c2018-08-31 09:22:23 +0100430
Sadik Armagana25886e2020-09-15 17:17:08 +0100431 armnn::BackendOptions gpuAcc("GpuAcc",
432 {
Matthew Sloyan42432112021-01-08 10:30:51 +0000433 { "FastMathEnabled", params.m_EnableFastMath },
434 { "SaveCachedNetwork", params.m_SaveCachedNetwork },
435 { "CachedNetworkFilePath", params.m_CachedNetworkFilePath }
Sadik Armagana25886e2020-09-15 17:17:08 +0100436 });
437 armnn::BackendOptions cpuAcc("CpuAcc",
438 {
439 { "FastMathEnabled", params.m_EnableFastMath }
440 });
441 options.m_ModelOptions.push_back(gpuAcc);
442 options.m_ModelOptions.push_back(cpuAcc);
443
alered01a7227ac2020-05-07 14:58:29 +0100444 const auto optimization_start_time = armnn::GetTimeNow();
Aron Virginas-Tar339bcae2019-01-31 16:44:26 +0000445 optNet = armnn::Optimize(*network, params.m_ComputeDevices, m_Runtime->GetDeviceSpec(), options);
alered01a7227ac2020-05-07 14:58:29 +0100446
447 ARMNN_LOG(info) << "Optimization time: " << std::setprecision(2)
448 << std::fixed << armnn::GetTimeDuration(optimization_start_time).count() << " ms\n";
449
telsoa01c577f2c2018-08-31 09:22:23 +0100450 if (!optNet)
451 {
452 throw armnn::Exception("Optimize returned nullptr");
453 }
surmeh013537c2c2018-05-18 16:31:43 +0100454 }
telsoa014fcda012018-03-09 14:13:49 +0000455
surmeh013537c2c2018-05-18 16:31:43 +0100456 if (params.m_VisualizePostOptimizationModel)
457 {
Francis Murtagh532a29d2020-06-29 11:50:01 +0100458 fs::path filename = params.m_ModelPath;
surmeh013537c2c2018-05-18 16:31:43 +0100459 filename.replace_extension("dot");
Rob Hughes9e10c2b2019-07-23 15:37:19 +0100460 std::fstream file(filename.c_str(), std::ios_base::out);
surmeh013537c2c2018-05-18 16:31:43 +0100461 optNet->SerializeToDot(file);
462 }
463
464 armnn::Status ret;
465 {
466 ARMNN_SCOPED_HEAP_PROFILING("LoadNetwork");
467 ret = m_Runtime->LoadNetwork(m_NetworkIdentifier, std::move(optNet));
468 }
469
telsoa014fcda012018-03-09 14:13:49 +0000470 if (ret == armnn::Status::Failure)
471 {
472 throw armnn::Exception("IRuntime::LoadNetwork failed");
473 }
474 }
475
Aron Virginas-Tar7cf0eaa2019-01-24 17:05:36 +0000476 void CheckInputIndexIsValid(unsigned int inputIndex) const
telsoa014fcda012018-03-09 14:13:49 +0000477 {
Aron Virginas-Tar7cf0eaa2019-01-24 17:05:36 +0000478 if (m_InputBindings.size() < inputIndex + 1)
479 {
James Ward08f40162020-09-07 16:45:07 +0100480 throw armnn::Exception(fmt::format("Input index out of range: {}", inputIndex));
Aron Virginas-Tar7cf0eaa2019-01-24 17:05:36 +0000481 }
telsoa014fcda012018-03-09 14:13:49 +0000482 }
483
Aron Virginas-Tar7cf0eaa2019-01-24 17:05:36 +0000484 void CheckOutputIndexIsValid(unsigned int outputIndex) const
telsoa014fcda012018-03-09 14:13:49 +0000485 {
Aron Virginas-Tar7cf0eaa2019-01-24 17:05:36 +0000486 if (m_OutputBindings.size() < outputIndex + 1)
487 {
James Ward08f40162020-09-07 16:45:07 +0100488 throw armnn::Exception(fmt::format("Output index out of range: {}", outputIndex));
Aron Virginas-Tar7cf0eaa2019-01-24 17:05:36 +0000489 }
490 }
491
Aron Virginas-Tarc82c8732019-10-24 17:07:43 +0100492 unsigned int GetInputSize(unsigned int inputIndex = 0u) const
493 {
494 CheckInputIndexIsValid(inputIndex);
495 return m_InputBindings[inputIndex].second.GetNumElements();
496 }
497
Aron Virginas-Tar7cf0eaa2019-01-24 17:05:36 +0000498 unsigned int GetOutputSize(unsigned int outputIndex = 0u) const
499 {
500 CheckOutputIndexIsValid(outputIndex);
501 return m_OutputBindings[outputIndex].second.GetNumElements();
502 }
503
James Conroy7b4886f2019-04-11 10:23:58 +0100504 std::chrono::duration<double, std::milli> Run(
505 const std::vector<TContainer>& inputContainers,
506 std::vector<TContainer>& outputContainers)
Aron Virginas-Tar7cf0eaa2019-01-24 17:05:36 +0000507 {
Ferran Balaguerc602f292019-02-08 17:09:55 +0000508 for (unsigned int i = 0; i < outputContainers.size(); ++i)
Aron Virginas-Tar7cf0eaa2019-01-24 17:05:36 +0000509 {
510 const unsigned int expectedOutputDataSize = GetOutputSize(i);
Ferran Balaguerc602f292019-02-08 17:09:55 +0000511
James Ward6d9f5c52020-09-28 11:56:35 +0100512 mapbox::util::apply_visitor([expectedOutputDataSize, i](auto&& value)
Aron Virginas-Tar7cf0eaa2019-01-24 17:05:36 +0000513 {
Matthew Sloyan80c6b142020-09-08 12:00:32 +0100514 const unsigned int actualOutputDataSize = armnn::numeric_cast<unsigned int>(value.size());
Ferran Balaguerc602f292019-02-08 17:09:55 +0000515 if (actualOutputDataSize < expectedOutputDataSize)
516 {
Matthew Sloyan80c6b142020-09-08 12:00:32 +0100517 unsigned int outputIndex = i;
Ferran Balaguerc602f292019-02-08 17:09:55 +0000518 throw armnn::Exception(
James Ward08f40162020-09-07 16:45:07 +0100519 fmt::format("Not enough data for output #{0}: expected "
520 "{1} elements, got {2}", outputIndex, expectedOutputDataSize, actualOutputDataSize));
Ferran Balaguerc602f292019-02-08 17:09:55 +0000521 }
522 },
523 outputContainers[i]);
Aron Virginas-Tar7cf0eaa2019-01-24 17:05:36 +0000524 }
telsoa01c577f2c2018-08-31 09:22:23 +0100525
526 std::shared_ptr<armnn::IProfiler> profiler = m_Runtime->GetProfiler(m_NetworkIdentifier);
527 if (profiler)
528 {
529 profiler->EnableProfiling(m_EnableProfiling);
530 }
531
James Conroy7b4886f2019-04-11 10:23:58 +0100532 // Start timer to record inference time in EnqueueWorkload (in milliseconds)
alered01a7227ac2020-05-07 14:58:29 +0100533 const auto start_time = armnn::GetTimeNow();
James Conroy7b4886f2019-04-11 10:23:58 +0100534
telsoa014fcda012018-03-09 14:13:49 +0000535 armnn::Status ret = m_Runtime->EnqueueWorkload(m_NetworkIdentifier,
Aron Virginas-Tar7cf0eaa2019-01-24 17:05:36 +0000536 MakeInputTensors(inputContainers),
537 MakeOutputTensors(outputContainers));
Sadik Armagan2b7a1582018-09-05 16:33:58 +0100538
alered01a7227ac2020-05-07 14:58:29 +0100539 const auto duration = armnn::GetTimeDuration(start_time);
James Conroy7b4886f2019-04-11 10:23:58 +0100540
Sadik Armagan2b7a1582018-09-05 16:33:58 +0100541 // if profiling is enabled print out the results
542 if (profiler && profiler->IsProfilingEnabled())
543 {
544 profiler->Print(std::cout);
545 }
546
telsoa014fcda012018-03-09 14:13:49 +0000547 if (ret == armnn::Status::Failure)
548 {
549 throw armnn::Exception("IRuntime::EnqueueWorkload failed");
550 }
James Conroy7b4886f2019-04-11 10:23:58 +0100551 else
552 {
alered01a7227ac2020-05-07 14:58:29 +0100553 return duration;
James Conroy7b4886f2019-04-11 10:23:58 +0100554 }
telsoa014fcda012018-03-09 14:13:49 +0000555 }
556
Jim Flynnb4d7eae2019-05-01 14:44:27 +0100557 const armnn::BindingPointInfo& GetInputBindingInfo(unsigned int inputIndex = 0u) const
telsoa01c577f2c2018-08-31 09:22:23 +0100558 {
Aron Virginas-Tar7cf0eaa2019-01-24 17:05:36 +0000559 CheckInputIndexIsValid(inputIndex);
560 return m_InputBindings[inputIndex];
telsoa01c577f2c2018-08-31 09:22:23 +0100561 }
562
Jim Flynnb4d7eae2019-05-01 14:44:27 +0100563 const std::vector<armnn::BindingPointInfo>& GetInputBindingInfos() const
telsoa01c577f2c2018-08-31 09:22:23 +0100564 {
Aron Virginas-Tar7cf0eaa2019-01-24 17:05:36 +0000565 return m_InputBindings;
telsoa01c577f2c2018-08-31 09:22:23 +0100566 }
567
Jim Flynnb4d7eae2019-05-01 14:44:27 +0100568 const armnn::BindingPointInfo& GetOutputBindingInfo(unsigned int outputIndex = 0u) const
telsoa01c577f2c2018-08-31 09:22:23 +0100569 {
Aron Virginas-Tar7cf0eaa2019-01-24 17:05:36 +0000570 CheckOutputIndexIsValid(outputIndex);
571 return m_OutputBindings[outputIndex];
572 }
573
Jim Flynnb4d7eae2019-05-01 14:44:27 +0100574 const std::vector<armnn::BindingPointInfo>& GetOutputBindingInfos() const
Aron Virginas-Tar7cf0eaa2019-01-24 17:05:36 +0000575 {
576 return m_OutputBindings;
577 }
578
579 QuantizationParams GetQuantizationParams(unsigned int outputIndex = 0u) const
580 {
581 CheckOutputIndexIsValid(outputIndex);
582 return std::make_pair(m_OutputBindings[outputIndex].second.GetQuantizationScale(),
583 m_OutputBindings[outputIndex].second.GetQuantizationOffset());
584 }
585
Narumol Prangnawarat4628d052019-02-25 17:26:05 +0000586 QuantizationParams GetInputQuantizationParams(unsigned int inputIndex = 0u) const
587 {
588 CheckInputIndexIsValid(inputIndex);
589 return std::make_pair(m_InputBindings[inputIndex].second.GetQuantizationScale(),
590 m_InputBindings[inputIndex].second.GetQuantizationOffset());
591 }
592
Aron Virginas-Tar7cf0eaa2019-01-24 17:05:36 +0000593 std::vector<QuantizationParams> GetAllQuantizationParams() const
594 {
595 std::vector<QuantizationParams> quantizationParams;
596 for (unsigned int i = 0u; i < m_OutputBindings.size(); i++)
597 {
598 quantizationParams.push_back(GetQuantizationParams(i));
599 }
600 return quantizationParams;
telsoa01c577f2c2018-08-31 09:22:23 +0100601 }
602
telsoa014fcda012018-03-09 14:13:49 +0000603private:
telsoa01c577f2c2018-08-31 09:22:23 +0100604 armnn::NetworkId m_NetworkIdentifier;
605 std::shared_ptr<armnn::IRuntime> m_Runtime;
606
Jim Flynnb4d7eae2019-05-01 14:44:27 +0100607 std::vector<armnn::BindingPointInfo> m_InputBindings;
608 std::vector<armnn::BindingPointInfo> m_OutputBindings;
telsoa01c577f2c2018-08-31 09:22:23 +0100609 bool m_EnableProfiling;
Matteo Martincigh00dda4a2019-08-14 11:42:30 +0100610 std::string m_DynamicBackendsPath;
telsoa01c577f2c2018-08-31 09:22:23 +0100611
telsoa014fcda012018-03-09 14:13:49 +0000612 template<typename TContainer>
Aron Virginas-Tar7cf0eaa2019-01-24 17:05:36 +0000613 armnn::InputTensors MakeInputTensors(const std::vector<TContainer>& inputDataContainers)
telsoa014fcda012018-03-09 14:13:49 +0000614 {
Jim Flynn2fd61002019-05-03 12:54:26 +0100615 return armnnUtils::MakeInputTensors(m_InputBindings, inputDataContainers);
telsoa014fcda012018-03-09 14:13:49 +0000616 }
617
618 template<typename TContainer>
Aron Virginas-Tar7cf0eaa2019-01-24 17:05:36 +0000619 armnn::OutputTensors MakeOutputTensors(std::vector<TContainer>& outputDataContainers)
telsoa014fcda012018-03-09 14:13:49 +0000620 {
Jim Flynn2fd61002019-05-03 12:54:26 +0100621 return armnnUtils::MakeOutputTensors(m_OutputBindings, outputDataContainers);
telsoa014fcda012018-03-09 14:13:49 +0000622 }
Ferran Balaguerc602f292019-02-08 17:09:55 +0000623};