Teresa Charlin | 83b4291 | 2022-07-07 14:24:59 +0100 | [diff] [blame] | 1 | // |
Mike Kelly | 5446a4d | 2023-01-20 15:51:05 +0000 | [diff] [blame] | 2 | // Copyright © 2022-2023 Arm Ltd and Contributors. All rights reserved. |
Teresa Charlin | 83b4291 | 2022-07-07 14:24:59 +0100 | [diff] [blame] | 3 | // SPDX-License-Identifier: MIT |
| 4 | // |
| 5 | |
| 6 | #include "TfliteExecutor.hpp" |
Colm Donelan | 3811a97 | 2023-01-25 21:19:49 +0000 | [diff] [blame] | 7 | #include "tensorflow/lite/kernels/kernel_util.h" |
Teresa Charlin | 83b4291 | 2022-07-07 14:24:59 +0100 | [diff] [blame] | 8 | |
Colm Donelan | 35a0689 | 2023-02-06 15:01:57 +0000 | [diff] [blame] | 9 | TfLiteExecutor::TfLiteExecutor(const ExecuteNetworkParams& params, armnn::IRuntime::CreationOptions runtimeOptions) |
| 10 | : m_Params(params) |
Teresa Charlin | 83b4291 | 2022-07-07 14:24:59 +0100 | [diff] [blame] | 11 | { |
Teresa Charlin | c814e80 | 2022-08-05 13:57:04 +0100 | [diff] [blame] | 12 | m_Model = tflite::FlatBufferModel::BuildFromFile(m_Params.m_ModelPath.c_str()); |
Colm Donelan | 18e6f04 | 2023-01-24 22:10:12 +0000 | [diff] [blame] | 13 | if (!m_Model) |
| 14 | { |
| 15 | LogAndThrow("Failed to load TfLite model from: " + m_Params.m_ModelPath); |
| 16 | } |
Teresa Charlin | 83b4291 | 2022-07-07 14:24:59 +0100 | [diff] [blame] | 17 | m_TfLiteInterpreter = std::make_unique<Interpreter>(); |
| 18 | tflite::ops::builtin::BuiltinOpResolver resolver; |
| 19 | |
Teresa Charlin | c814e80 | 2022-08-05 13:57:04 +0100 | [diff] [blame] | 20 | tflite::InterpreterBuilder builder(*m_Model, resolver); |
Colm Donelan | f0755de | 2023-04-04 21:41:29 +0100 | [diff] [blame^] | 21 | if (builder(&m_TfLiteInterpreter) != kTfLiteOk) |
| 22 | { |
| 23 | LogAndThrow("Error loading the model into the TfLiteInterpreter."); |
| 24 | } |
| 25 | if (m_TfLiteInterpreter->AllocateTensors() != kTfLiteOk) |
| 26 | { |
| 27 | LogAndThrow("Failed to allocate tensors in the TfLiteInterpreter."); |
| 28 | } |
Teresa Charlin | 83b4291 | 2022-07-07 14:24:59 +0100 | [diff] [blame] | 29 | if (m_Params.m_TfLiteExecutor == ExecuteNetworkParams::TfLiteExecutor::ArmNNTfLiteDelegate) |
| 30 | { |
| 31 | // Create the Armnn Delegate |
| 32 | // Populate a DelegateOptions from the ExecuteNetworkParams. |
| 33 | armnnDelegate::DelegateOptions delegateOptions = m_Params.ToDelegateOptions(); |
Colm Donelan | 35a0689 | 2023-02-06 15:01:57 +0000 | [diff] [blame] | 34 | delegateOptions.SetRuntimeOptions(runtimeOptions); |
Teresa Charlin | 83b4291 | 2022-07-07 14:24:59 +0100 | [diff] [blame] | 35 | std::unique_ptr<TfLiteDelegate, decltype(&armnnDelegate::TfLiteArmnnDelegateDelete)> |
| 36 | theArmnnDelegate(armnnDelegate::TfLiteArmnnDelegateCreate(delegateOptions), |
| 37 | armnnDelegate::TfLiteArmnnDelegateDelete); |
| 38 | // Register armnn_delegate to TfLiteInterpreter |
Colm Donelan | f0755de | 2023-04-04 21:41:29 +0100 | [diff] [blame^] | 39 | if (m_TfLiteInterpreter->ModifyGraphWithDelegate(std::move(theArmnnDelegate)) != kTfLiteOk) |
Teresa Charlin | 83b4291 | 2022-07-07 14:24:59 +0100 | [diff] [blame] | 40 | { |
Colm Donelan | f0755de | 2023-04-04 21:41:29 +0100 | [diff] [blame^] | 41 | LogAndThrow("Could not register ArmNN TfLite Delegate to TfLiteInterpreter."); |
Teresa Charlin | 83b4291 | 2022-07-07 14:24:59 +0100 | [diff] [blame] | 42 | } |
| 43 | } |
| 44 | else |
| 45 | { |
| 46 | std::cout << "Running on TfLite without ArmNN delegate\n"; |
| 47 | } |
| 48 | |
Teresa Charlin | f53b28f | 2022-11-11 11:14:50 +0000 | [diff] [blame] | 49 | const size_t numInputs = m_TfLiteInterpreter->inputs().size(); |
Teresa Charlin | 83b4291 | 2022-07-07 14:24:59 +0100 | [diff] [blame] | 50 | |
| 51 | for(unsigned int inputIndex = 0; inputIndex < numInputs; ++inputIndex) |
| 52 | { |
Cathal Corbett | aa21230 | 2022-08-04 17:58:09 +0100 | [diff] [blame] | 53 | armnn::Optional<std::string> dataFile = m_Params.m_GenerateTensorData |
| 54 | ? armnn::EmptyOptional() |
| 55 | : armnn::MakeOptional<std::string>(m_Params.m_InputTensorDataFilePaths[inputIndex]); |
| 56 | |
Teresa Charlin | 83b4291 | 2022-07-07 14:24:59 +0100 | [diff] [blame] | 57 | int input = m_TfLiteInterpreter->inputs()[inputIndex]; |
Cathal Corbett | aa21230 | 2022-08-04 17:58:09 +0100 | [diff] [blame] | 58 | const auto& inputName = m_TfLiteInterpreter->tensor(input)->name; |
Teresa Charlin | 83b4291 | 2022-07-07 14:24:59 +0100 | [diff] [blame] | 59 | |
Colm Donelan | 3811a97 | 2023-01-25 21:19:49 +0000 | [diff] [blame] | 60 | // Before we start, check if the tensor is constant. |
| 61 | if (!tflite::IsConstantTensor(m_TfLiteInterpreter->tensor(input))) |
Teresa Charlin | 83b4291 | 2022-07-07 14:24:59 +0100 | [diff] [blame] | 62 | { |
Colm Donelan | 3811a97 | 2023-01-25 21:19:49 +0000 | [diff] [blame] | 63 | TfLiteIntArray* inputDims = m_TfLiteInterpreter->tensor(input)->dims; |
| 64 | |
| 65 | unsigned int inputSize = 1; |
| 66 | for (unsigned int dim = 0; dim < static_cast<unsigned int>(inputDims->size); ++dim) |
Teresa Charlin | 83b4291 | 2022-07-07 14:24:59 +0100 | [diff] [blame] | 67 | { |
Colm Donelan | 3811a97 | 2023-01-25 21:19:49 +0000 | [diff] [blame] | 68 | inputSize *= inputDims->data[dim]; |
Teresa Charlin | 83b4291 | 2022-07-07 14:24:59 +0100 | [diff] [blame] | 69 | } |
Colm Donelan | 3811a97 | 2023-01-25 21:19:49 +0000 | [diff] [blame] | 70 | |
| 71 | const auto& dataType = m_TfLiteInterpreter->tensor(input)->type; |
| 72 | |
| 73 | switch (dataType) |
Teresa Charlin | 83b4291 | 2022-07-07 14:24:59 +0100 | [diff] [blame] | 74 | { |
Colm Donelan | 3811a97 | 2023-01-25 21:19:49 +0000 | [diff] [blame] | 75 | case kTfLiteFloat32: |
| 76 | { |
| 77 | auto inputData = m_TfLiteInterpreter->typed_tensor<float>(input); |
| 78 | PopulateTensorWithData<float>(inputData, inputSize, dataFile, inputName); |
| 79 | break; |
| 80 | } |
| 81 | case kTfLiteInt32: |
| 82 | { |
| 83 | auto inputData = m_TfLiteInterpreter->typed_tensor<int32_t>(input); |
| 84 | PopulateTensorWithData<int32_t>(inputData, inputSize, dataFile, inputName); |
| 85 | break; |
| 86 | } |
| 87 | case kTfLiteUInt8: |
| 88 | { |
| 89 | auto inputData = m_TfLiteInterpreter->typed_tensor<uint8_t>(input); |
| 90 | PopulateTensorWithData<uint8_t>(inputData, inputSize, dataFile, inputName); |
| 91 | break; |
| 92 | } |
| 93 | case kTfLiteInt16: |
| 94 | { |
| 95 | auto inputData = m_TfLiteInterpreter->typed_tensor<int16_t>(input); |
| 96 | PopulateTensorWithData<int16_t>(inputData, inputSize, dataFile, inputName); |
| 97 | break; |
| 98 | } |
| 99 | case kTfLiteInt8: |
| 100 | { |
| 101 | auto inputData = m_TfLiteInterpreter->typed_tensor<int8_t>(input); |
| 102 | PopulateTensorWithData<int8_t>(inputData, inputSize, dataFile, inputName); |
| 103 | break; |
| 104 | } |
| 105 | default: |
| 106 | { |
| 107 | LogAndThrow("Unsupported input tensor data type"); |
| 108 | } |
Teresa Charlin | 83b4291 | 2022-07-07 14:24:59 +0100 | [diff] [blame] | 109 | } |
Colm Donelan | 3811a97 | 2023-01-25 21:19:49 +0000 | [diff] [blame] | 110 | } |
| 111 | else |
| 112 | { |
| 113 | ARMNN_LOG(info) << "Input tensor \"" << inputName << "\" is constant and will not be populated with data."; |
Teresa Charlin | 83b4291 | 2022-07-07 14:24:59 +0100 | [diff] [blame] | 114 | } |
| 115 | } |
| 116 | } |
| 117 | |
| 118 | std::vector<const void *> TfLiteExecutor::Execute() |
| 119 | { |
| 120 | int status = 0; |
| 121 | std::vector<const void*> results; |
| 122 | for (size_t x = 0; x < m_Params.m_Iterations; x++) |
| 123 | { |
| 124 | // Start timer to record inference time in milliseconds. |
| 125 | const auto start_time = armnn::GetTimeNow(); |
| 126 | // Run the inference |
| 127 | status = m_TfLiteInterpreter->Invoke(); |
| 128 | const auto duration = armnn::GetTimeDuration(start_time); |
| 129 | |
Kevin May | 2fef6f6 | 2022-11-14 17:07:49 +0000 | [diff] [blame] | 130 | if (!m_Params.m_DontPrintOutputs) |
Teresa Charlin | 83b4291 | 2022-07-07 14:24:59 +0100 | [diff] [blame] | 131 | { |
Kevin May | 2fef6f6 | 2022-11-14 17:07:49 +0000 | [diff] [blame] | 132 | // Print out the output |
| 133 | for (unsigned int outputIndex = 0; outputIndex < m_TfLiteInterpreter->outputs().size(); ++outputIndex) |
Teresa Charlin | 83b4291 | 2022-07-07 14:24:59 +0100 | [diff] [blame] | 134 | { |
Kevin May | 2fef6f6 | 2022-11-14 17:07:49 +0000 | [diff] [blame] | 135 | auto tfLiteDelegateOutputId = m_TfLiteInterpreter->outputs()[outputIndex]; |
| 136 | TfLiteIntArray* outputDims = m_TfLiteInterpreter->tensor(tfLiteDelegateOutputId)->dims; |
| 137 | // If we've been asked to write to a file then set a file output stream. Otherwise use stdout. |
| 138 | FILE* outputTensorFile = stdout; |
| 139 | if (!m_Params.m_OutputTensorFiles.empty()) |
Teresa Charlin | 83b4291 | 2022-07-07 14:24:59 +0100 | [diff] [blame] | 140 | { |
Kevin May | 2fef6f6 | 2022-11-14 17:07:49 +0000 | [diff] [blame] | 141 | outputTensorFile = fopen(m_Params.m_OutputTensorFiles[outputIndex].c_str(), "w"); |
| 142 | if (outputTensorFile == NULL) |
| 143 | { |
| 144 | LogAndThrow("Specified output tensor file, \"" + m_Params.m_OutputTensorFiles[outputIndex] + |
| 145 | "\", cannot be created. Defaulting to stdout. Error was: " + std::strerror(errno)); |
| 146 | } |
| 147 | else |
| 148 | { |
| 149 | ARMNN_LOG(info) << "Writing output " << outputIndex << "' of iteration: " << x + 1 |
| 150 | << " to file: '" << m_Params.m_OutputTensorFiles[outputIndex] << "'"; |
| 151 | } |
Teresa Charlin | 83b4291 | 2022-07-07 14:24:59 +0100 | [diff] [blame] | 152 | } |
Kevin May | 2fef6f6 | 2022-11-14 17:07:49 +0000 | [diff] [blame] | 153 | long outputSize = 1; |
| 154 | for (unsigned int dim = 0; dim < static_cast<unsigned int>(outputDims->size); ++dim) |
Teresa Charlin | 83b4291 | 2022-07-07 14:24:59 +0100 | [diff] [blame] | 155 | { |
Kevin May | 2fef6f6 | 2022-11-14 17:07:49 +0000 | [diff] [blame] | 156 | outputSize *= outputDims->data[dim]; |
Teresa Charlin | 83b4291 | 2022-07-07 14:24:59 +0100 | [diff] [blame] | 157 | } |
Kevin May | 2fef6f6 | 2022-11-14 17:07:49 +0000 | [diff] [blame] | 158 | |
| 159 | std::cout << m_TfLiteInterpreter->tensor(tfLiteDelegateOutputId)->name << ": "; |
| 160 | results.push_back(m_TfLiteInterpreter->tensor(tfLiteDelegateOutputId)->allocation); |
| 161 | |
| 162 | switch (m_TfLiteInterpreter->tensor(tfLiteDelegateOutputId)->type) |
| 163 | { |
| 164 | |
| 165 | case kTfLiteFloat32: |
| 166 | { |
| 167 | auto tfLiteDelegateOutputData = m_TfLiteInterpreter->typed_tensor<float>( |
| 168 | tfLiteDelegateOutputId); |
| 169 | |
| 170 | for (int i = 0; i < outputSize; ++i) |
| 171 | { |
| 172 | fprintf(outputTensorFile, "%f ", tfLiteDelegateOutputData[i]); |
| 173 | } |
| 174 | break; |
| 175 | } |
| 176 | case kTfLiteInt32: |
| 177 | { |
| 178 | auto tfLiteDelegateOutputData = m_TfLiteInterpreter->typed_tensor<int32_t>( |
| 179 | tfLiteDelegateOutputId); |
| 180 | for (int i = 0; i < outputSize; ++i) |
| 181 | { |
| 182 | fprintf(outputTensorFile, "%d ", tfLiteDelegateOutputData[i]); |
| 183 | } |
| 184 | break; |
| 185 | } |
| 186 | case kTfLiteUInt8: |
| 187 | { |
| 188 | auto tfLiteDelegateOutputData = m_TfLiteInterpreter->typed_tensor<uint8_t>( |
| 189 | tfLiteDelegateOutputId); |
| 190 | for (int i = 0; i < outputSize; ++i) |
| 191 | { |
| 192 | fprintf(outputTensorFile, "%u ", tfLiteDelegateOutputData[i]); |
| 193 | } |
| 194 | break; |
| 195 | } |
| 196 | case kTfLiteInt8: |
| 197 | { |
| 198 | auto tfLiteDelegateOutputData = m_TfLiteInterpreter->typed_tensor<int8_t>( |
| 199 | tfLiteDelegateOutputId); |
| 200 | for (int i = 0; i < outputSize; ++i) |
| 201 | { |
| 202 | fprintf(outputTensorFile, "%d ", tfLiteDelegateOutputData[i]); |
| 203 | } |
| 204 | break; |
| 205 | } |
Ryan OShea | 39831da | 2023-01-26 17:43:45 +0000 | [diff] [blame] | 206 | case kTfLiteBool: |
| 207 | { |
| 208 | auto tfLiteDelegateOutputData = m_TfLiteInterpreter->typed_tensor<bool>( |
| 209 | tfLiteDelegateOutputId); |
| 210 | for (int i = 0; i < outputSize; ++i) { |
| 211 | fprintf(outputTensorFile, "%u ", tfLiteDelegateOutputData[i]); |
| 212 | } |
| 213 | break; |
| 214 | } |
Kevin May | 2fef6f6 | 2022-11-14 17:07:49 +0000 | [diff] [blame] | 215 | default: |
| 216 | { |
| 217 | LogAndThrow("Unsupported output type"); |
| 218 | } |
| 219 | } |
| 220 | |
| 221 | std::cout << std::endl; |
Teresa Charlin | 83b4291 | 2022-07-07 14:24:59 +0100 | [diff] [blame] | 222 | } |
Teresa Charlin | 83b4291 | 2022-07-07 14:24:59 +0100 | [diff] [blame] | 223 | } |
| 224 | CheckInferenceTimeThreshold(duration, m_Params.m_ThresholdTime); |
| 225 | } |
| 226 | |
| 227 | std::cout << status; |
| 228 | return results; |
| 229 | } |
| 230 | |
| 231 | void TfLiteExecutor::CompareAndPrintResult(std::vector<const void*> otherOutput) |
| 232 | { |
| 233 | for (unsigned int outputIndex = 0; outputIndex < m_TfLiteInterpreter->outputs().size(); ++outputIndex) |
| 234 | { |
| 235 | auto tfLiteDelegateOutputId = m_TfLiteInterpreter->outputs()[outputIndex]; |
Colm Donelan | d047262 | 2023-03-06 12:34:54 +0000 | [diff] [blame] | 236 | size_t size = m_TfLiteInterpreter->tensor(tfLiteDelegateOutputId)->bytes; |
| 237 | double result = ComputeByteLevelRMSE(m_TfLiteInterpreter->tensor(tfLiteDelegateOutputId)->allocation, |
| 238 | otherOutput[outputIndex], size); |
| 239 | std::cout << "Byte level root mean square error: " << result << "\n"; |
Teresa Charlin | 83b4291 | 2022-07-07 14:24:59 +0100 | [diff] [blame] | 240 | } |
| 241 | }; |