alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 1 | /* |
Kshitij Sisodia | 2ea4623 | 2022-12-19 16:37:33 +0000 | [diff] [blame^] | 2 | * SPDX-FileCopyrightText: Copyright 2021-2022 Arm Limited and/or its affiliates |
| 3 | * <open-source-office@arm.com> SPDX-License-Identifier: Apache-2.0 |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 4 | * |
| 5 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | * you may not use this file except in compliance with the License. |
| 7 | * You may obtain a copy of the License at |
| 8 | * |
| 9 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | * |
| 11 | * Unless required by applicable law or agreed to in writing, software |
| 12 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | * See the License for the specific language governing permissions and |
| 15 | * limitations under the License. |
| 16 | */ |
| 17 | #include "UseCaseHandler.hpp" |
| 18 | |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 19 | #include "AdMelSpectrogram.hpp" |
Kshitij Sisodia | 2ea4623 | 2022-12-19 16:37:33 +0000 | [diff] [blame^] | 20 | #include "AdModel.hpp" |
Richard Burton | 4e00279 | 2022-05-04 09:45:02 +0100 | [diff] [blame] | 21 | #include "AdProcessing.hpp" |
Kshitij Sisodia | 2ea4623 | 2022-12-19 16:37:33 +0000 | [diff] [blame^] | 22 | #include "AudioUtils.hpp" |
| 23 | #include "Classifier.hpp" |
| 24 | #include "ImageUtils.hpp" |
| 25 | #include "InputFiles.hpp" |
| 26 | #include "UseCaseCommonUtils.hpp" |
| 27 | #include "hal.h" |
| 28 | #include "log_macros.h" |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 29 | |
| 30 | namespace arm { |
| 31 | namespace app { |
| 32 | |
| 33 | /** |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 34 | * @brief Presents inference results using the data presentation |
| 35 | * object. |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 36 | * @param[in] result average sum of classification results |
Isabella Gottardi | 56ee620 | 2021-05-12 08:27:15 +0100 | [diff] [blame] | 37 | * @param[in] threshold if larger than this value we have an anomaly |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 38 | * @return true if successful, false otherwise |
| 39 | **/ |
Kshitij Sisodia | 68fdd11 | 2022-04-06 13:03:20 +0100 | [diff] [blame] | 40 | static bool PresentInferenceResult(float result, float threshold); |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 41 | |
Richard Burton | 4e00279 | 2022-05-04 09:45:02 +0100 | [diff] [blame] | 42 | /** @brief Given a wav file name return AD model output index. |
| 43 | * @param[in] wavFileName Audio WAV filename. |
| 44 | * File name should be in format anything_goes_XX_here.wav |
| 45 | * where XX is the machine ID e.g. 00, 02, 04 or 06 |
| 46 | * @return AD model output index as 8 bit integer. |
Kshitij Sisodia | 2ea4623 | 2022-12-19 16:37:33 +0000 | [diff] [blame^] | 47 | **/ |
Richard Burton | 4e00279 | 2022-05-04 09:45:02 +0100 | [diff] [blame] | 48 | static int8_t OutputIndexFromFileName(std::string wavFileName); |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 49 | |
Richard Burton | 4e00279 | 2022-05-04 09:45:02 +0100 | [diff] [blame] | 50 | /* Anomaly Detection inference handler */ |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 51 | bool ClassifyVibrationHandler(ApplicationContext& ctx, uint32_t clipIndex, bool runAll) |
| 52 | { |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 53 | constexpr uint32_t dataPsnTxtInfStartX = 20; |
| 54 | constexpr uint32_t dataPsnTxtInfStartY = 40; |
| 55 | |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 56 | auto& model = ctx.Get<Model&>("model"); |
| 57 | |
| 58 | /* If the request has a valid size, set the audio index */ |
| 59 | if (clipIndex < NUMBER_OF_FILES) { |
Kshitij Sisodia | 2ea4623 | 2022-12-19 16:37:33 +0000 | [diff] [blame^] | 60 | if (!SetAppCtxIfmIdx(ctx, clipIndex, "clipIndex")) { |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 61 | return false; |
| 62 | } |
| 63 | } |
| 64 | if (!model.IsInited()) { |
| 65 | printf_err("Model is not initialised! Terminating processing.\n"); |
| 66 | return false; |
| 67 | } |
| 68 | |
Kshitij Sisodia | 2ea4623 | 2022-12-19 16:37:33 +0000 | [diff] [blame^] | 69 | auto& profiler = ctx.Get<Profiler&>("profiler"); |
Richard Burton | 4e00279 | 2022-05-04 09:45:02 +0100 | [diff] [blame] | 70 | const auto melSpecFrameLength = ctx.Get<uint32_t>("frameLength"); |
| 71 | const auto melSpecFrameStride = ctx.Get<uint32_t>("frameStride"); |
Kshitij Sisodia | 2ea4623 | 2022-12-19 16:37:33 +0000 | [diff] [blame^] | 72 | const auto scoreThreshold = ctx.Get<float>("scoreThreshold"); |
| 73 | const auto trainingMean = ctx.Get<float>("trainingMean"); |
| 74 | auto startClipIdx = ctx.Get<uint32_t>("clipIndex"); |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 75 | |
| 76 | TfLiteTensor* outputTensor = model.GetOutputTensor(0); |
Kshitij Sisodia | 2ea4623 | 2022-12-19 16:37:33 +0000 | [diff] [blame^] | 77 | TfLiteTensor* inputTensor = model.GetInputTensor(0); |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 78 | |
| 79 | if (!inputTensor->dims) { |
| 80 | printf_err("Invalid input tensor dims\n"); |
| 81 | return false; |
| 82 | } |
| 83 | |
Kshitij Sisodia | 2ea4623 | 2022-12-19 16:37:33 +0000 | [diff] [blame^] | 84 | AdPreProcess preProcess{inputTensor, melSpecFrameLength, melSpecFrameStride, trainingMean}; |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 85 | |
Richard Burton | 4e00279 | 2022-05-04 09:45:02 +0100 | [diff] [blame] | 86 | AdPostProcess postProcess{outputTensor}; |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 87 | |
| 88 | do { |
Kshitij Sisodia | 68fdd11 | 2022-04-06 13:03:20 +0100 | [diff] [blame] | 89 | hal_lcd_clear(COLOR_BLACK); |
Richard Burton | 9b8d67a | 2021-12-10 12:32:51 +0000 | [diff] [blame] | 90 | |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 91 | auto currentIndex = ctx.Get<uint32_t>("clipIndex"); |
| 92 | |
| 93 | /* Get the output index to look at based on id in the filename. */ |
Kshitij Sisodia | 2ea4623 | 2022-12-19 16:37:33 +0000 | [diff] [blame^] | 94 | int8_t machineOutputIndex = OutputIndexFromFileName(GetFilename(currentIndex)); |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 95 | if (machineOutputIndex == -1) { |
| 96 | return false; |
| 97 | } |
| 98 | |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 99 | /* Creating a sliding window through the whole audio clip. */ |
Kshitij Sisodia | 2ea4623 | 2022-12-19 16:37:33 +0000 | [diff] [blame^] | 100 | auto audioDataSlider = |
| 101 | audio::SlidingWindow<const int16_t>(GetAudioArray(currentIndex), |
| 102 | GetAudioArraySize(currentIndex), |
| 103 | preProcess.GetAudioWindowSize(), |
| 104 | preProcess.GetAudioDataStride()); |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 105 | |
| 106 | /* Result is an averaged sum over inferences. */ |
| 107 | float result = 0; |
| 108 | |
| 109 | /* Display message on the LCD - inference running. */ |
| 110 | std::string str_inf{"Running inference... "}; |
Kshitij Sisodia | 68fdd11 | 2022-04-06 13:03:20 +0100 | [diff] [blame] | 111 | hal_lcd_display_text( |
Kshitij Sisodia | 2ea4623 | 2022-12-19 16:37:33 +0000 | [diff] [blame^] | 112 | str_inf.c_str(), str_inf.size(), dataPsnTxtInfStartX, dataPsnTxtInfStartY, 0); |
Richard Burton | 4e00279 | 2022-05-04 09:45:02 +0100 | [diff] [blame] | 113 | |
| 114 | info("Running inference on audio clip %" PRIu32 " => %s\n", |
Kshitij Sisodia | 2ea4623 | 2022-12-19 16:37:33 +0000 | [diff] [blame^] | 115 | currentIndex, |
| 116 | GetFilename(currentIndex)); |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 117 | |
| 118 | /* Start sliding through audio clip. */ |
| 119 | while (audioDataSlider.HasNext()) { |
Richard Burton | 4e00279 | 2022-05-04 09:45:02 +0100 | [diff] [blame] | 120 | const int16_t* inferenceWindow = audioDataSlider.Next(); |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 121 | |
Richard Burton | 4e00279 | 2022-05-04 09:45:02 +0100 | [diff] [blame] | 122 | preProcess.SetAudioWindowIndex(audioDataSlider.Index()); |
| 123 | preProcess.DoPreProcess(inferenceWindow, preProcess.GetAudioWindowSize()); |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 124 | |
Kshitij Sisodia | 2ea4623 | 2022-12-19 16:37:33 +0000 | [diff] [blame^] | 125 | info("Inference %zu/%zu\n", |
| 126 | audioDataSlider.Index() + 1, |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 127 | audioDataSlider.TotalStrides() + 1); |
| 128 | |
| 129 | /* Run inference over this audio clip sliding window */ |
alexander | 27b62d9 | 2021-05-04 20:46:08 +0100 | [diff] [blame] | 130 | if (!RunInference(model, profiler)) { |
| 131 | return false; |
| 132 | } |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 133 | |
Richard Burton | 4e00279 | 2022-05-04 09:45:02 +0100 | [diff] [blame] | 134 | postProcess.DoPostProcess(); |
| 135 | result += 0 - postProcess.GetOutputValue(machineOutputIndex); |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 136 | |
| 137 | #if VERIFY_TEST_OUTPUT |
Richard Burton | 4e00279 | 2022-05-04 09:45:02 +0100 | [diff] [blame] | 138 | DumpTensor(outputTensor); |
Kshitij Sisodia | 2ea4623 | 2022-12-19 16:37:33 +0000 | [diff] [blame^] | 139 | #endif /* VERIFY_TEST_OUTPUT */ |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 140 | } /* while (audioDataSlider.HasNext()) */ |
| 141 | |
| 142 | /* Use average over whole clip as final score. */ |
| 143 | result /= (audioDataSlider.TotalStrides() + 1); |
| 144 | |
| 145 | /* Erase. */ |
| 146 | str_inf = std::string(str_inf.size(), ' '); |
Kshitij Sisodia | 68fdd11 | 2022-04-06 13:03:20 +0100 | [diff] [blame] | 147 | hal_lcd_display_text( |
Kshitij Sisodia | 2ea4623 | 2022-12-19 16:37:33 +0000 | [diff] [blame^] | 148 | str_inf.c_str(), str_inf.size(), dataPsnTxtInfStartX, dataPsnTxtInfStartY, 0); |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 149 | |
| 150 | ctx.Set<float>("result", result); |
Kshitij Sisodia | 68fdd11 | 2022-04-06 13:03:20 +0100 | [diff] [blame] | 151 | if (!PresentInferenceResult(result, scoreThreshold)) { |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 152 | return false; |
| 153 | } |
| 154 | |
Isabella Gottardi | 8df12f3 | 2021-04-07 17:15:31 +0100 | [diff] [blame] | 155 | profiler.PrintProfilingResult(); |
| 156 | |
Kshitij Sisodia | 2ea4623 | 2022-12-19 16:37:33 +0000 | [diff] [blame^] | 157 | IncrementAppCtxIfmIdx(ctx, "clipIndex"); |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 158 | |
| 159 | } while (runAll && ctx.Get<uint32_t>("clipIndex") != startClipIdx); |
| 160 | |
| 161 | return true; |
| 162 | } |
| 163 | |
Kshitij Sisodia | 68fdd11 | 2022-04-06 13:03:20 +0100 | [diff] [blame] | 164 | static bool PresentInferenceResult(float result, float threshold) |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 165 | { |
| 166 | constexpr uint32_t dataPsnTxtStartX1 = 20; |
| 167 | constexpr uint32_t dataPsnTxtStartY1 = 30; |
| 168 | constexpr uint32_t dataPsnTxtYIncr = 16; /* Row index increment */ |
| 169 | |
Kshitij Sisodia | 68fdd11 | 2022-04-06 13:03:20 +0100 | [diff] [blame] | 170 | hal_lcd_set_text_color(COLOR_GREEN); |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 171 | |
| 172 | /* Display each result */ |
| 173 | uint32_t rowIdx1 = dataPsnTxtStartY1 + 2 * dataPsnTxtYIncr; |
| 174 | |
Kshitij Sisodia | 2ea4623 | 2022-12-19 16:37:33 +0000 | [diff] [blame^] | 175 | std::string anomalyScore = |
| 176 | std::string{"Average anomaly score is: "} + std::to_string(result); |
| 177 | std::string anomalyThreshold = |
| 178 | std::string("Anomaly threshold is: ") + std::to_string(threshold); |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 179 | |
George Gekov | 93e5951 | 2021-08-03 11:18:41 +0100 | [diff] [blame] | 180 | std::string anomalyResult; |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 181 | if (result > threshold) { |
George Gekov | 93e5951 | 2021-08-03 11:18:41 +0100 | [diff] [blame] | 182 | anomalyResult += std::string("Anomaly detected!"); |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 183 | } else { |
George Gekov | 93e5951 | 2021-08-03 11:18:41 +0100 | [diff] [blame] | 184 | anomalyResult += std::string("Everything fine, no anomaly detected!"); |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 185 | } |
| 186 | |
Kshitij Sisodia | 68fdd11 | 2022-04-06 13:03:20 +0100 | [diff] [blame] | 187 | hal_lcd_display_text( |
Kshitij Sisodia | 2ea4623 | 2022-12-19 16:37:33 +0000 | [diff] [blame^] | 188 | anomalyScore.c_str(), anomalyScore.size(), dataPsnTxtStartX1, rowIdx1, false); |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 189 | |
George Gekov | 93e5951 | 2021-08-03 11:18:41 +0100 | [diff] [blame] | 190 | info("%s\n", anomalyScore.c_str()); |
| 191 | info("%s\n", anomalyThreshold.c_str()); |
| 192 | info("%s\n", anomalyResult.c_str()); |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 193 | |
| 194 | return true; |
| 195 | } |
| 196 | |
Richard Burton | 4e00279 | 2022-05-04 09:45:02 +0100 | [diff] [blame] | 197 | static int8_t OutputIndexFromFileName(std::string wavFileName) |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 198 | { |
Richard Burton | 4e00279 | 2022-05-04 09:45:02 +0100 | [diff] [blame] | 199 | /* Filename is assumed in the form machine_id_00.wav */ |
Kshitij Sisodia | 2ea4623 | 2022-12-19 16:37:33 +0000 | [diff] [blame^] | 200 | std::string delimiter = "_"; /* First character used to split the file name up. */ |
Richard Burton | 4e00279 | 2022-05-04 09:45:02 +0100 | [diff] [blame] | 201 | size_t delimiterStart; |
| 202 | std::string subString; |
Kshitij Sisodia | 2ea4623 | 2022-12-19 16:37:33 +0000 | [diff] [blame^] | 203 | size_t machineIdxInString = |
| 204 | 3; /* Which part of the file name the machine id should be at. */ |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 205 | |
Richard Burton | 4e00279 | 2022-05-04 09:45:02 +0100 | [diff] [blame] | 206 | for (size_t i = 0; i < machineIdxInString; ++i) { |
| 207 | delimiterStart = wavFileName.find(delimiter); |
Kshitij Sisodia | 2ea4623 | 2022-12-19 16:37:33 +0000 | [diff] [blame^] | 208 | subString = wavFileName.substr(0, delimiterStart); |
Richard Burton | 4e00279 | 2022-05-04 09:45:02 +0100 | [diff] [blame] | 209 | wavFileName.erase(0, delimiterStart + delimiter.length()); |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 210 | } |
Richard Burton | 4e00279 | 2022-05-04 09:45:02 +0100 | [diff] [blame] | 211 | |
| 212 | /* At this point substring should be 00.wav */ |
Kshitij Sisodia | 2ea4623 | 2022-12-19 16:37:33 +0000 | [diff] [blame^] | 213 | delimiter = "."; /* Second character used to split the file name up. */ |
Richard Burton | 4e00279 | 2022-05-04 09:45:02 +0100 | [diff] [blame] | 214 | delimiterStart = subString.find(delimiter); |
Kshitij Sisodia | 2ea4623 | 2022-12-19 16:37:33 +0000 | [diff] [blame^] | 215 | subString = |
| 216 | (delimiterStart != std::string::npos) ? subString.substr(0, delimiterStart) : subString; |
Richard Burton | 4e00279 | 2022-05-04 09:45:02 +0100 | [diff] [blame] | 217 | |
Kshitij Sisodia | 2ea4623 | 2022-12-19 16:37:33 +0000 | [diff] [blame^] | 218 | auto is_number = [](const std::string& str) -> bool { |
Richard Burton | 4e00279 | 2022-05-04 09:45:02 +0100 | [diff] [blame] | 219 | std::string::const_iterator it = str.begin(); |
Kshitij Sisodia | 2ea4623 | 2022-12-19 16:37:33 +0000 | [diff] [blame^] | 220 | while (it != str.end() && std::isdigit(*it)) |
| 221 | ++it; |
Richard Burton | 4e00279 | 2022-05-04 09:45:02 +0100 | [diff] [blame] | 222 | return !str.empty() && it == str.end(); |
| 223 | }; |
| 224 | |
| 225 | const int8_t machineIdx = is_number(subString) ? std::stoi(subString) : -1; |
| 226 | |
| 227 | /* Return corresponding index in the output vector. */ |
| 228 | if (machineIdx == 0) { |
| 229 | return 0; |
| 230 | } else if (machineIdx == 2) { |
| 231 | return 1; |
| 232 | } else if (machineIdx == 4) { |
| 233 | return 2; |
| 234 | } else if (machineIdx == 6) { |
| 235 | return 3; |
| 236 | } else { |
| 237 | printf_err("%d is an invalid machine index \n", machineIdx); |
| 238 | return -1; |
| 239 | } |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 240 | } |
| 241 | |
| 242 | } /* namespace app */ |
| 243 | } /* namespace arm */ |