MLECO-3075: Add KWS use case API
Removed some of the templates for feature calculation that we are unlikely to ever use.
We might be able to refactor the feature caching and feature calculator code in the future
to better integrate it with with PreProcess API.
Signed-off-by: Richard Burton <richard.burton@arm.com>
Change-Id: Ic0c0c581c71e2553d41ff72cd1ed3b3efa64fa92
diff --git a/source/use_case/kws/include/KwsProcessing.hpp b/source/use_case/kws/include/KwsProcessing.hpp
new file mode 100644
index 0000000..abf20ab
--- /dev/null
+++ b/source/use_case/kws/include/KwsProcessing.hpp
@@ -0,0 +1,135 @@
+/*
+ * Copyright (c) 2022 Arm Limited. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef KWS_PROCESSING_HPP
+#define KWS_PROCESSING_HPP
+
+#include <AudioUtils.hpp>
+#include "BaseProcessing.hpp"
+#include "Model.hpp"
+#include "Classifier.hpp"
+#include "MicroNetKwsMfcc.hpp"
+
+#include <functional>
+
+namespace arm {
+namespace app {
+
+ /**
+ * @brief Pre-processing class for Keyword Spotting use case.
+ * Implements methods declared by BasePreProcess and anything else needed
+ * to populate input tensors ready for inference.
+ */
+ class KWSPreProcess : public BasePreProcess {
+
+ public:
+ /**
+ * @brief Constructor
+ * @param[in] model Pointer to the the KWS Model object.
+ * @param[in] numFeatures How many MFCC features to use.
+ * @param[in] mfccFrameLength Number of audio samples used to calculate one set of MFCC values when
+ * sliding a window through the audio sample.
+ * @param[in] mfccFrameStride Number of audio samples between consecutive windows.
+ **/
+ explicit KWSPreProcess(Model* model, size_t numFeatures, int mfccFrameLength, int mfccFrameStride);
+
+ /**
+ * @brief Should perform pre-processing of 'raw' input audio data and load it into
+ * TFLite Micro input tensors ready for inference.
+ * @param[in] input Pointer to the data that pre-processing will work on.
+ * @param[in] inputSize Size of the input data.
+ * @return true if successful, false otherwise.
+ **/
+ bool DoPreProcess(const void* input, size_t inputSize) override;
+
+ size_t m_audioWindowIndex = 0; /* Index of audio slider, used when caching features in longer clips. */
+ size_t m_audioDataWindowSize; /* Amount of audio needed for 1 inference. */
+ size_t m_audioDataStride; /* Amount of audio to stride across if doing >1 inference in longer clips. */
+
+ private:
+ const int m_mfccFrameLength;
+ const int m_mfccFrameStride;
+
+ audio::MicroNetKwsMFCC m_mfcc;
+ audio::SlidingWindow<const int16_t> m_mfccSlidingWindow;
+ size_t m_numMfccVectorsInAudioStride;
+ size_t m_numReusedMfccVectors;
+ std::function<void (std::vector<int16_t>&, int, bool, size_t)> m_mfccFeatureCalculator;
+
+ /**
+ * @brief Returns a function to perform feature calculation and populates input tensor data with
+ * MFCC data.
+ *
+ * Input tensor data type check is performed to choose correct MFCC feature data type.
+ * If tensor has an integer data type then original features are quantised.
+ *
+ * Warning: MFCC calculator provided as input must have the same life scope as returned function.
+ *
+ * @param[in] mfcc MFCC feature calculator.
+ * @param[in,out] inputTensor Input tensor pointer to store calculated features.
+ * @param[in] cacheSize Size of the feature vectors cache (number of feature vectors).
+ * @return Function to be called providing audio sample and sliding window index.
+ */
+ std::function<void (std::vector<int16_t>&, int, bool, size_t)>
+ GetFeatureCalculator(audio::MicroNetKwsMFCC& mfcc,
+ TfLiteTensor* inputTensor,
+ size_t cacheSize);
+
+ template<class T>
+ std::function<void (std::vector<int16_t>&, size_t, bool, size_t)>
+ FeatureCalc(TfLiteTensor* inputTensor, size_t cacheSize,
+ std::function<std::vector<T> (std::vector<int16_t>& )> compute);
+ };
+
+ /**
+ * @brief Post-processing class for Keyword Spotting use case.
+ * Implements methods declared by BasePostProcess and anything else needed
+ * to populate result vector.
+ */
+ class KWSPostProcess : public BasePostProcess {
+
+ private:
+ Classifier& m_kwsClassifier;
+ const std::vector<std::string>& m_labels;
+ std::vector<ClassificationResult>& m_results;
+
+ public:
+ const float m_scoreThreshold;
+ /**
+ * @brief Constructor
+ * @param[in] classifier Classifier object used to get top N results from classification.
+ * @param[in] model Pointer to the the Image classification Model object.
+ * @param[in] labels Vector of string labels to identify each output of the model.
+ * @param[in] results Vector of classification results to store decoded outputs.
+ * @param[in] scoreThreshold Predicted model score must be larger than this value to be accepted.
+ **/
+ KWSPostProcess(Classifier& classifier, Model* model,
+ const std::vector<std::string>& labels,
+ std::vector<ClassificationResult>& results,
+ float scoreThreshold);
+
+ /**
+ * @brief Should perform post-processing of the result of inference then populate
+ * populate KWS result data for any later use.
+ * @return true if successful, false otherwise.
+ **/
+ bool DoPostProcess() override;
+ };
+
+} /* namespace app */
+} /* namespace arm */
+
+#endif /* KWS_PROCESSING_HPP */
\ No newline at end of file