source/use_case/kws/include/KwsProcessing.hpp

/*
 * Copyright (c) 2022 Arm Limited. All rights reserved.
 * SPDX-License-Identifier: Apache-2.0
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
#ifndef KWS_PROCESSING_HPP
#define KWS_PROCESSING_HPP

#include <AudioUtils.hpp>
#include "BaseProcessing.hpp"
#include "Model.hpp"
#include "Classifier.hpp"
#include "MicroNetKwsMfcc.hpp"

#include <functional>

namespace arm {
namespace app {

    /**
     * @brief   Pre-processing class for Keyword Spotting use case.
     *          Implements methods declared by BasePreProcess and anything else needed
     *          to populate input tensors ready for inference.
     */
    class KWSPreProcess : public BasePreProcess {

    public:
        /**
         * @brief       Constructor
         * @param[in]   model             Pointer to the the KWS Model object.
         * @param[in]   numFeatures       How many MFCC features to use.
         * @param[in]   mfccFrameLength   Number of audio samples used to calculate one set of MFCC values when
         *                                sliding a window through the audio sample.
         * @param[in]   mfccFrameStride   Number of audio samples between consecutive windows.
         **/
        explicit KWSPreProcess(Model* model, size_t numFeatures, int mfccFrameLength, int mfccFrameStride);

        /**
         * @brief       Should perform pre-processing of 'raw' input audio data and load it into
         *              TFLite Micro input tensors ready for inference.
         * @param[in]   input      Pointer to the data that pre-processing will work on.
         * @param[in]   inputSize  Size of the input data.
         * @return      true if successful, false otherwise.
         **/
        bool DoPreProcess(const void* input, size_t inputSize) override;

        size_t m_audioWindowIndex = 0;  /* Index of audio slider, used when caching features in longer clips. */
        size_t m_audioDataWindowSize;   /* Amount of audio needed for 1 inference. */
        size_t m_audioDataStride;       /* Amount of audio to stride across if doing >1 inference in longer clips. */

    private:
        const int m_mfccFrameLength;
        const int m_mfccFrameStride;

        audio::MicroNetKwsMFCC m_mfcc;
        audio::SlidingWindow<const int16_t> m_mfccSlidingWindow;
        size_t m_numMfccVectorsInAudioStride;
        size_t m_numReusedMfccVectors;
        std::function<void (std::vector<int16_t>&, int, bool, size_t)> m_mfccFeatureCalculator;

        /**
         * @brief Returns a function to perform feature calculation and populates input tensor data with
         * MFCC data.
         *
         * Input tensor data type check is performed to choose correct MFCC feature data type.
         * If tensor has an integer data type then original features are quantised.
         *
         * Warning: MFCC calculator provided as input must have the same life scope as returned function.
         *
         * @param[in]       mfcc          MFCC feature calculator.
         * @param[in,out]   inputTensor   Input tensor pointer to store calculated features.
         * @param[in]       cacheSize     Size of the feature vectors cache (number of feature vectors).
         * @return          Function to be called providing audio sample and sliding window index.
         */
        std::function<void (std::vector<int16_t>&, int, bool, size_t)>
        GetFeatureCalculator(audio::MicroNetKwsMFCC&  mfcc,
                             TfLiteTensor*            inputTensor,
                             size_t                   cacheSize);

        template<class T>
        std::function<void (std::vector<int16_t>&, size_t, bool, size_t)>
        FeatureCalc(TfLiteTensor* inputTensor, size_t cacheSize,
                    std::function<std::vector<T> (std::vector<int16_t>& )> compute);
    };

    /**
     * @brief   Post-processing class for Keyword Spotting use case.
     *          Implements methods declared by BasePostProcess and anything else needed
     *          to populate result vector.
     */
    class KWSPostProcess : public BasePostProcess {

    private:
        Classifier& m_kwsClassifier;
        const std::vector<std::string>& m_labels;
        std::vector<ClassificationResult>& m_results;

    public:
        const float m_scoreThreshold;
        /**
         * @brief       Constructor
         * @param[in]   classifier       Classifier object used to get top N results from classification.
         * @param[in]   model            Pointer to the the Image classification Model object.
         * @param[in]   labels           Vector of string labels to identify each output of the model.
         * @param[in]   results          Vector of classification results to store decoded outputs.
         * @param[in]   scoreThreshold   Predicted model score must be larger than this value to be accepted.
         **/
        KWSPostProcess(Classifier& classifier, Model* model,
                       const std::vector<std::string>& labels,
                       std::vector<ClassificationResult>& results,
                       float scoreThreshold);

        /**
         * @brief       Should perform post-processing of the result of inference then populate
         *              populate KWS result data for any later use.
         * @return      true if successful, false otherwise.
         **/
        bool DoPostProcess() override;
    };

} /* namespace app */
} /* namespace arm */

#endif /* KWS_PROCESSING_HPP */