Opensource ML embedded evaluation kit
Change-Id: I12e807f19f5cacad7cef82572b6dd48252fd61fd
diff --git a/source/application/main/include/AppContext.hpp b/source/application/main/include/AppContext.hpp
new file mode 100644
index 0000000..588dfaa
--- /dev/null
+++ b/source/application/main/include/AppContext.hpp
@@ -0,0 +1,102 @@
+/*
+ * Copyright (c) 2021 Arm Limited. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef APP_CTX_HPP
+#define APP_CTX_HPP
+
+#include <string>
+#include <map>
+
+namespace arm {
+namespace app {
+
+ class IAttribute
+ {
+ public:
+ virtual ~IAttribute() = default;
+ };
+
+ template<typename T>
+ class Attribute : public IAttribute
+ {
+ public:
+ ~Attribute() override = default;
+
+ explicit Attribute(const T value): _m_value(value){}
+
+ T Get()
+ {
+ return _m_value;
+ }
+ private:
+ T _m_value;
+ };
+
+ /* Application context class */
+ class ApplicationContext {
+ public:
+
+ /**
+ * @brief Saves given value as a named attribute in the context.
+ * @tparam T value type.
+ * @param[in] name Context attribute name.
+ * @param[in] object Value to save in the context.
+ */
+ template<typename T>
+ void Set(const std::string &name, T object)
+ {
+ this->_m_attributes[name] = new Attribute<T>(object);
+ }
+
+ /**
+ * @brief Gets the saved attribute from the context by the given name.
+ * @tparam T value type.
+ * @param[in] name Context attribute name.
+ * @return Value saved in the context.
+ */
+ template<typename T>
+ T Get(const std::string &name)
+ {
+ auto a = (Attribute<T>*)_m_attributes[name];
+ return a->Get();
+ }
+
+ /**
+ * @brief Checks if an attribute for a given name exists in the context.
+ * @param[in] name Attribute name.
+ * @return true if attribute exists, false otherwise
+ */
+ bool Has(const std::string& name)
+ {
+ return _m_attributes.find(name) != _m_attributes.end();
+ }
+
+ ApplicationContext() = default;
+
+ ~ApplicationContext() {
+ for (auto& attribute : _m_attributes)
+ delete attribute.second;
+
+ this->_m_attributes.clear();
+ }
+ private:
+ std::map<std::string, IAttribute*> _m_attributes;
+ };
+
+} /* namespace app */
+} /* namespace arm */
+
+#endif /* APP_CTX_HPP */
diff --git a/source/application/main/include/AudioUtils.hpp b/source/application/main/include/AudioUtils.hpp
new file mode 100644
index 0000000..cba981d
--- /dev/null
+++ b/source/application/main/include/AudioUtils.hpp
@@ -0,0 +1,171 @@
+/*
+ * Copyright (c) 2021 Arm Limited. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef AUDIO_UTILS_HPP
+#define AUDIO_UTILS_HPP
+
+#include <cstddef>
+#include <cstdint>
+
+namespace arm {
+namespace app {
+namespace audio {
+
+ template<class T>
+ class SlidingWindow {
+ public:
+
+ /**
+ * @brief Creates the window slider through the given data.
+ *
+ * @param[in] data Pointer to the data to slide through.
+ * @param[in] dataSize Size in T type elements wise.
+ * @param[in] windowSize Sliding window size in T type wise elements.
+ * @param[in] stride Stride size in T type wise elements.
+ */
+ SlidingWindow(T *data, size_t dataSize,
+ size_t windowSize, size_t stride) {
+ m_start = data;
+ m_dataSize = dataSize;
+ m_size = windowSize;
+ m_stride = stride;
+ }
+
+ SlidingWindow() = default;
+
+ ~SlidingWindow() = default;
+
+ /**
+ * @brief Get the next data window.
+ * @return Pointer to the next window, if next window is not available nullptr is returned.
+ */
+ virtual T *Next() {
+ if (HasNext()) {
+ m_count++;
+ return m_start + Index() * m_stride;
+ } else {
+ return nullptr;
+ }
+ }
+
+ /**
+ * @brief Checks if the next data portion is available.
+ * @return true if next data portion is available.
+ */
+ virtual bool HasNext() {
+ return m_size + m_count * m_stride <= m_dataSize;
+ }
+
+ /**
+ * @brief Reset the slider to the initial position.
+ */
+ virtual void Reset() {
+ m_count = 0;
+ }
+
+ /**
+ * @brief Resets the slider to the start of the new data.
+ * New data size MUST be the same as the old one.
+ * @param[in] newStart Pointer to the new data to slide through.
+ */
+ virtual void Reset(T *newStart) {
+ m_start = newStart;
+ Reset();
+ }
+
+ /**
+ * @brief Gets current index of the sliding window.
+ * @return Current position of the sliding window in number of strides.
+ */
+ size_t Index() {
+ return m_count == 0? 0: m_count - 1;
+ }
+
+ /**
+ * @brief Gets the index from the start of the data where the next window will begin.
+ * While Index() returns the index of sliding window itself this function
+ * returns the index of the data element itself.
+ * @return Index from the start of the data where the next sliding window will begin.
+ */
+ virtual uint32_t NextWindowStartIndex() {
+ return m_count == 0? 0: ((m_count) * m_stride);
+ }
+
+ /**
+ * @brief Go to given sliding window index.
+ * @param[in] index New position of the sliding window. If index is invalid
+ * (greater than possible range of strides) then next call to Next() will return nullptr.
+ */
+ void FastForward(size_t index) {
+ m_count = index;
+ }
+
+ /**
+ * @brief Calculates whole number of times the window can stride through the given data.
+ * @return Maximum number of whole strides.
+ */
+ size_t TotalStrides() {
+ if (m_size > m_dataSize) {
+ return 0;
+ }
+ return ((m_dataSize - m_size)/m_stride);
+ }
+
+ /**
+ * @brief Calculates number of times the window can stride through the given data.
+ * May not be a whole number.
+ * @return Number of strides to cover all data.
+ */
+ float FractionalTotalStrides() {
+ if (this->m_dataSize < this->m_size) {
+ return 0;
+ } else {
+ return ((this->m_dataSize - this->m_size)/ static_cast<float>(this->m_stride));
+ }
+ }
+
+ protected:
+ T *m_start = nullptr;
+ size_t m_dataSize = 0;
+ size_t m_size = 0;
+ size_t m_stride = 0;
+ size_t m_count = 0;
+ };
+
+ /*
+ * Sliding window for ASR will cover the whole of the input, even if
+ * this means the last window is not a full window length.
+ */
+ template<class T>
+ class ASRSlidingWindow : public SlidingWindow<T> {
+ public:
+ using SlidingWindow<T>::SlidingWindow;
+
+ /**
+ * @brief Checks if the next data portion is available.
+ * @return true if next data portion is available.
+ */
+ bool HasNext() {
+ return this->m_count < 1 + this->FractionalTotalStrides() && (this->NextWindowStartIndex() < this->m_dataSize);
+ }
+ };
+
+
+} /* namespace audio */
+} /* namespace app */
+} /* namespace arm */
+
+#endif /* AUDIO_UTILS_HPP */
\ No newline at end of file
diff --git a/source/application/main/include/ClassificationResult.hpp b/source/application/main/include/ClassificationResult.hpp
new file mode 100644
index 0000000..eae28e4
--- /dev/null
+++ b/source/application/main/include/ClassificationResult.hpp
@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2021 Arm Limited. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef CLASSIFICATION_RESULT_HPP
+#define CLASSIFICATION_RESULT_HPP
+
+#include <string>
+
+namespace arm {
+namespace app {
+
+ /**
+ * @brief Class representing a single classification result.
+ */
+ class ClassificationResult {
+ public:
+ double m_normalisedVal = 0.0;
+ std::string m_label;
+ uint32_t m_labelIdx = 0;
+
+ ClassificationResult() = default;
+ ~ClassificationResult() = default;
+ };
+
+} /* namespace app */
+} /* namespace arm */
+
+#endif /* CLASSIFICATION_RESULT_HPP */
\ No newline at end of file
diff --git a/source/application/main/include/Classifier.hpp b/source/application/main/include/Classifier.hpp
new file mode 100644
index 0000000..510e6f9
--- /dev/null
+++ b/source/application/main/include/Classifier.hpp
@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2021 Arm Limited. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef CLASSIFIER_HPP
+#define CLASSIFIER_HPP
+
+#include "ClassificationResult.hpp"
+#include "TensorFlowLiteMicro.hpp"
+
+#include <vector>
+
+namespace arm {
+namespace app {
+
+ /**
+ * @brief Classifier - a helper class to get certain number of top
+ * results from the output vector from a classification NN.
+ **/
+ class Classifier{
+ public:
+ /** @brief Constructor. */
+ Classifier() = default;
+
+ /**
+ * @brief Gets the top N classification results from the
+ * output vector.
+ * @param[in] outputTensor Inference output tensor from an NN model.
+ * @param[out] vecResults A vector of classification results.
+ * populated by this function.
+ * @param[in] labels Labels vector to match classified classes.
+ * @param[in] topNCount Number of top classifications to pick. Default is 1.
+ * @return true if successful, false otherwise.
+ **/
+ virtual bool GetClassificationResults(
+ TfLiteTensor* outputTensor,
+ std::vector<ClassificationResult>& vecResults,
+ const std::vector <std::string>& labels, uint32_t topNCount);
+
+ private:
+ /**
+ * @brief Utility function that gets the top N classification results from the
+ * output vector.
+ * @tparam T value type
+ * @param[in] tensor Inference output tensor from an NN model.
+ * @param[out] vecResults A vector of classification results
+ * populated by this function.
+ * @param[in] topNCount Number of top classifications to pick.
+ * @param[in] labels Labels vector to match classified classes.
+ * @return true if successful, false otherwise.
+ **/
+ template<typename T>
+ bool _GetTopNResults(TfLiteTensor* tensor,
+ std::vector<ClassificationResult>& vecResults,
+ uint32_t topNCount,
+ const std::vector <std::string>& labels);
+ };
+
+} /* namespace app */
+} /* namespace arm */
+
+#endif /* CLASSIFIER_HPP */
diff --git a/source/application/main/include/DataStructures.hpp b/source/application/main/include/DataStructures.hpp
new file mode 100644
index 0000000..5cc8b5e
--- /dev/null
+++ b/source/application/main/include/DataStructures.hpp
@@ -0,0 +1,132 @@
+/*
+ * Copyright (c) 2021 Arm Limited. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef DATA_STRUCTURES_HPP
+#define DATA_STRUCTURES_HPP
+
+#include "hal.h"
+
+#include <iterator>
+
+namespace arm {
+namespace app {
+
+ /**
+ * Class Array2d is a data structure that represents a two dimensional array.
+ * The data is allocated in contiguous memory, arranged row-wise
+ * and individual elements can be accessed with the () operator.
+ * For example a two dimensional array D of size (M, N) can be accessed:
+ *
+ * _|<------------- col size = N -------->|
+ * | D(r=0, c=0) D(r=0, c=1)... D(r=0, c=N)
+ * | D(r=1, c=0) D(r=1, c=1)... D(r=1, c=N)
+ * | ...
+ * row size = M ...
+ * | ...
+ * _ D(r=M, c=0) D(r=M, c=1)... D(r=M, c=N)
+ *
+ */
+ template<typename T>
+ class Array2d {
+ public:
+ /**
+ * @brief Creates the array2d with the given sizes.
+ * @param[in] rows Number of rows.
+ * @param[in] cols Number of columns.
+ */
+ Array2d(unsigned rows, unsigned cols)
+ {
+ if (rows == 0 || cols == 0) {
+ printf_err("Array2d constructor has 0 size.\n");
+ _m_data = nullptr;
+ return;
+ }
+ _m_rows = rows;
+ _m_cols = cols;
+ _m_data = new T[rows * cols];
+ }
+
+ ~Array2d()
+ {
+ delete[] _m_data;
+ }
+
+ T& operator() (unsigned int row, unsigned int col)
+ {
+#if defined(DEBUG)
+ if (row >= _m_rows || col >= _m_cols || _m_data == nullptr) {
+ printf_err("Array2d subscript out of bounds.\n");
+ }
+#endif /* defined(DEBUG) */
+ return _m_data[_m_cols * row + col];
+ }
+
+ T operator() (unsigned int row, unsigned int col) const
+ {
+#if defined(DEBUG)
+ if (row >= _m_rows || col >= _m_cols || _m_data == nullptr) {
+ printf_err("const Array2d subscript out of bounds.\n");
+ }
+#endif /* defined(DEBUG) */
+ return _m_data[_m_cols * row + col];
+ }
+
+ /**
+ * @brief Gets rows number of the current array2d.
+ * @return Number of rows.
+ */
+ size_t size(size_t dim)
+ {
+ switch (dim)
+ {
+ case 0:
+ return _m_rows;
+ case 1:
+ return _m_cols;
+ default:
+ return 0;
+ }
+ }
+
+ /**
+ * @brief Gets the array2d total size.
+ */
+ size_t totalSize()
+ {
+ return _m_rows * _m_cols;
+ }
+
+ /**
+ * array2d iterator.
+ */
+ using iterator=T*;
+ using const_iterator=T const*;
+
+ iterator begin() { return _m_data; }
+ iterator end() { return _m_data + totalSize(); }
+ const_iterator begin() const { return _m_data; }
+ const_iterator end() const { return _m_data + totalSize(); };
+
+ private:
+ size_t _m_rows;
+ size_t _m_cols;
+ T* _m_data;
+ };
+
+} /* namespace app */
+} /* namespace arm */
+
+#endif /* DATA_STRUCTURES_HPP */
\ No newline at end of file
diff --git a/source/application/main/include/Mfcc.hpp b/source/application/main/include/Mfcc.hpp
new file mode 100644
index 0000000..6379fab
--- /dev/null
+++ b/source/application/main/include/Mfcc.hpp
@@ -0,0 +1,255 @@
+/*
+ * Copyright (c) 2021 Arm Limited. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef MFCC_HPP
+#define MFCC_HPP
+
+#include "PlatformMath.hpp"
+
+#include <vector>
+#include <cstdint>
+#include <cmath>
+#include <limits>
+#include <string>
+
+namespace arm {
+namespace app {
+namespace audio {
+
+ /* MFCC's consolidated parameters. */
+ class MfccParams {
+ public:
+ float m_samplingFreq;
+ uint32_t m_numFbankBins;
+ float m_melLoFreq;
+ float m_melHiFreq;
+ uint32_t m_numMfccFeatures;
+ uint32_t m_frameLen;
+ uint32_t m_frameLenPadded;
+ bool m_useHtkMethod;
+
+ /** @brief Constructor */
+ MfccParams(float samplingFreq, uint32_t numFbankBins,
+ float melLoFreq, float melHiFreq,
+ uint32_t numMfccFeats, uint32_t frameLen,
+ bool useHtkMethod);
+
+ MfccParams() = delete;
+
+ ~MfccParams() = default;
+
+ /** @brief String representation of parameters */
+ std::string Str();
+ };
+
+ /**
+ * @brief Class for MFCC feature extraction.
+ * Based on https://github.com/ARM-software/ML-KWS-for-MCU/blob/master/Deployment/Source/MFCC/mfcc.cpp
+ * This class is designed to be generic and self-sufficient but
+ * certain calculation routines can be overridden to accommodate
+ * use-case specific requirements.
+ */
+ class MFCC {
+ public:
+ /**
+ * @brief Constructor
+ * @param[in] params MFCC parameters
+ */
+ explicit MFCC(const MfccParams& params);
+
+ MFCC() = delete;
+
+ ~MFCC() = default;
+
+ /**
+ * @brief Extract MFCC features for one single small frame of
+ * audio data e.g. 640 samples.
+ * @param[in] audioData Vector of audio samples to calculate
+ * features for.
+ * @return Vector of extracted MFCC features.
+ **/
+ std::vector<float> MfccCompute(const std::vector<int16_t>& audioData);
+
+ /** @brief Initialise. */
+ void Init();
+
+ /**
+ * @brief Extract MFCC features and quantise for one single small
+ * frame of audio data e.g. 640 samples.
+ * @param[in] audioData Vector of audio samples to calculate
+ * features for.
+ * @param[in] quantScale Quantisation scale.
+ * @param[in] quantOffset Quantisation offset.
+ * @return Vector of extracted quantised MFCC features.
+ **/
+ template<typename T>
+ std::vector<T> MfccComputeQuant(const std::vector<int16_t>& audioData,
+ const float quantScale,
+ const int quantOffset)
+ {
+ this->_MfccComputePreFeature(audioData);
+ float minVal = std::numeric_limits<T>::min();
+ float maxVal = std::numeric_limits<T>::max();
+
+ std::vector<T> mfccOut(this->_m_params.m_numMfccFeatures);
+ const size_t numFbankBins = this->_m_params.m_numFbankBins;
+
+ /* Take DCT. Uses matrix mul. */
+ for (size_t i = 0, j = 0; i < mfccOut.size(); ++i, j += numFbankBins) {
+ float sum = 0;
+ for (size_t k = 0; k < numFbankBins; ++k) {
+ sum += this->_m_dctMatrix[j + k] * this->_m_melEnergies[k];
+ }
+ /* Quantize to T. */
+ sum = std::round((sum / quantScale) + quantOffset);
+ mfccOut[i] = static_cast<T>(std::min<float>(std::max<float>(sum, minVal), maxVal));
+ }
+
+ return mfccOut;
+ }
+
+ /* Constants */
+ static constexpr float ms_logStep = /*logf(6.4)*/ 1.8562979903656 / 27.0;
+ static constexpr float ms_freqStep = 200.0 / 3;
+ static constexpr float ms_minLogHz = 1000.0;
+ static constexpr float ms_minLogMel = ms_minLogHz / ms_freqStep;
+
+ protected:
+ /**
+ * @brief Project input frequency to Mel Scale.
+ * @param[in] freq Input frequency in floating point.
+ * @param[in] useHTKmethod bool to signal if HTK method is to be
+ * used for calculation.
+ * @return Mel transformed frequency in floating point.
+ **/
+ static float MelScale(float freq,
+ bool useHTKMethod = true);
+
+ /**
+ * @brief Inverse Mel transform - convert MEL warped frequency
+ * back to normal frequency.
+ * @param[in] freq Mel frequency in floating point.
+ * @param[in] useHTKmethod bool to signal if HTK method is to be
+ * used for calculation.
+ * @return Real world frequency in floating point.
+ **/
+ static float InverseMelScale(float melFreq,
+ bool useHTKMethod = true);
+
+ /**
+ * @brief Populates MEL energies after applying the MEL filter
+ * bank weights and adding them up to be placed into
+ * bins, according to the filter bank's first and last
+ * indices (pre-computed for each filter bank element
+ * by _CreateMelFilterBank function).
+ * @param[in] fftVec Vector populated with FFT magnitudes.
+ * @param[in] melFilterBank 2D Vector with filter bank weights.
+ * @param[in] filterBankFilterFirst Vector containing the first indices of filter bank
+ * to be used for each bin.
+ * @param[in] filterBankFilterLast Vector containing the last indices of filter bank
+ * to be used for each bin.
+ * @param[out] melEnergies Pre-allocated vector of MEL energies to be
+ * populated.
+ * @return true if successful, false otherwise.
+ */
+ virtual bool ApplyMelFilterBank(
+ std::vector<float>& fftVec,
+ std::vector<std::vector<float>>& melFilterBank,
+ std::vector<int32_t>& filterBankFilterFirst,
+ std::vector<int32_t>& filterBankFilterLast,
+ std::vector<float>& melEnergies);
+
+ /**
+ * @brief Converts the Mel energies for logarithmic scale.
+ * @param[in,out] melEnergies 1D vector of Mel energies.
+ **/
+ virtual void ConvertToLogarithmicScale(std::vector<float>& melEnergies);
+
+ /**
+ * @brief Create a matrix used to calculate Discrete Cosine
+ * Transform.
+ * @param[in] inputLength Input length of the buffer on which
+ * DCT will be performed.
+ * @param[in] coefficientCount Total coefficients per input length.
+ * @return 1D vector with inputLength x coefficientCount elements
+ * populated with DCT coefficients.
+ */
+ virtual std::vector<float> CreateDCTMatrix(
+ int32_t inputLength,
+ int32_t coefficientCount);
+
+ /**
+ * @brief Given the low and high Mel values, get the normaliser
+ * for weights to be applied when populating the filter
+ * bank.
+ * @param[in] leftMel Low Mel frequency value.
+ * @param[in] rightMel High Mel frequency value.
+ * @param[in] useHTKMethod bool to signal if HTK method is to be
+ * used for calculation.
+ * @return Value to use for normalizing.
+ */
+ virtual float GetMelFilterBankNormaliser(
+ const float& leftMel,
+ const float& rightMel,
+ bool useHTKMethod);
+
+ private:
+ MfccParams _m_params;
+ std::vector<float> _m_frame;
+ std::vector<float> _m_buffer;
+ std::vector<float> _m_melEnergies;
+ std::vector<float> _m_windowFunc;
+ std::vector<std::vector<float>> _m_melFilterBank;
+ std::vector<float> _m_dctMatrix;
+ std::vector<int32_t> _m_filterBankFilterFirst;
+ std::vector<int32_t> _m_filterBankFilterLast;
+ bool _m_filterBankInitialised;
+ arm::app::math::FftInstance _m_fftInstance;
+
+ /**
+ * @brief Initialises the filter banks and the DCT matrix. **/
+ void _InitMelFilterBank();
+
+ /**
+ * @brief Signals whether the instance of MFCC has had its
+ * required buffers initialised.
+ * @return true if initialised, false otherwise.
+ **/
+ bool _IsMelFilterBankInited();
+
+ /**
+ * @brief Create mel filter banks for MFCC calculation.
+ * @return 2D vector of floats.
+ **/
+ std::vector<std::vector<float>> _CreateMelFilterBank();
+
+ /**
+ * @brief Computes and populates internal memeber buffers used
+ * in MFCC feature calculation
+ * @param[in] audioData 1D vector of 16-bit audio data.
+ */
+ void _MfccComputePreFeature(const std::vector<int16_t>& audioData);
+
+ /** @brief Computes the magnitude from an interleaved complex array. */
+ void _ConvertToPowerSpectrum();
+
+ };
+
+} /* namespace audio */
+} /* namespace app */
+} /* namespace arm */
+
+#endif /* MFCC_HPP */
\ No newline at end of file
diff --git a/source/application/main/include/PlatformMath.hpp b/source/application/main/include/PlatformMath.hpp
new file mode 100644
index 0000000..45e6a9e
--- /dev/null
+++ b/source/application/main/include/PlatformMath.hpp
@@ -0,0 +1,151 @@
+/*
+ * Copyright (c) 2021 Arm Limited. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef PLATFORM_MATH_HPP
+#define PLATFORM_MATH_HPP
+
+#include "hal.h"
+
+/* See if ARM DSP functions can be used. */
+#if PLATFORM_HAL == PLATFORM_CORTEX_M_BAREMETAL
+ #if defined(__DSP_PRESENT) && (__DSP_PRESENT == 1U)
+
+ #define ARM_DSP_AVAILABLE (1U)
+ #include "arm_math.h"
+ #define M_PI (PI)
+
+ #endif /* defined(__DSP_PRESENT) && (__DSP_PRESENT == 1U) */
+#endif /* PLATFORM_HAL == PLATFORM_CORTEX_M_BAREMETAL */
+
+#include <vector>
+
+namespace arm {
+namespace app {
+namespace math {
+
+ struct FftInstance {
+#if ARM_DSP_AVAILABLE
+ arm_rfft_fast_instance_f32 instance;
+#endif
+ bool initialised = false;
+ };
+
+ /* Class to provide Math functions like FFT, mean, stddev etc.
+ * This will allow other classes, functions to be independent of
+ * #if definition checks and provide a cleaner API. Also, it will
+ * consolidate all arm math functions used in one place and make
+ * them easier to test. */
+ class MathUtils {
+
+ public:
+ /**
+ * @brief Get the cosine value of the argument in floating point.
+ * @param[in] radians Angle in radians.
+ * @return Cosine value (floating point).
+ */
+ static float CosineF32(float radians);
+
+ /**
+ * @brief Get the square root of the argument in floating point.
+ * @param[in] input Value to compute square root of.
+ * @return Square root (floating point) value.
+ */
+ static float SqrtF32(float input);
+
+ /**
+ * @brief Gets the mean of a floating point array of elements.
+ * @param[in] ptrSrc Pointer to the first element.
+ * @param[in] srcLen Number of elements in the array/vector.
+ * @return Average value.
+ */
+ static float MeanF32(float* ptrSrc, uint32_t srcLen);
+
+ /**
+ * @brief Gets the standard deviation of a floating point array
+ * of elements.
+ * @param[in] ptrSrc Pointer to the first element.
+ * @param[in] srcLen Number of elements in the array/vector.
+ * @param[in] mean Pre-computed mean value.
+ * @return Standard deviation value.
+ */
+ static float StdDevF32(float* ptrSrc, uint32_t srcLen,
+ float mean);
+
+ /**
+ * @brief Initialises the internal FFT structures (if available
+ * for the platform). This function should be called
+ * prior to Fft32 function call if built with ARM DSP functions.
+ * @param[in] fftLen Requested length of the FFT.
+ * @param[in] fftInstance FFT instance struct to use.
+ * @return true if successful, false otherwise.
+ */
+ static bool FftInitF32(const uint16_t fftLen, arm::app::math::FftInstance& fftInstance);
+
+ /**
+ * @brief Computes the FFT for the input vector.
+ * @param[in] input Floating point vector of input elements
+ * @param[out] fftOutput Output buffer to be populated by computed FFTs.
+ * @param[in] fftInstance FFT instance struct to use.
+ */
+ static void FftF32(std::vector<float>& input,
+ std::vector<float>& fftOutput,
+ arm::app::math::FftInstance& fftInstance);
+
+ /**
+ * @brief Computes the natural logarithms of input floating point
+ * vector
+ * @param[in] input Floating point input vector
+ * @param[out] output Pre-allocated buffer to be populated with
+ * natural log values of each input element.
+ */
+ static void VecLogarithmF32(std::vector <float>& input,
+ std::vector <float>& output);
+
+ /**
+ * @brief Computes the dot product of two 1D floating point
+ * vectors.
+ * result = sum(srcA[0]*srcB[0] + srcA[1]*srcB[1] + ..)
+ * @param[in] srcPtrA Pointer to the first element of first
+ * array.
+ * @param[in] srcPtrB Pointer to the first element of second
+ * array.
+ * @param[in] srcLen Number of elements in the array/vector.
+ * @return Dot product.
+ */
+ static float DotProductF32(float* srcPtrA, float* srcPtrB,
+ const uint32_t srcLen);
+
+ /**
+ * @brief Computes the squared magnitude of floating point
+ * complex number array.
+ * @param[in] ptrSrc Pointer to the first element of input
+ * array.
+ * @param[in] srcLen Number of elements in the array/vector.
+ * @param[out] ptrDst Output buffer to be populated.
+ * @param[in] dstLen Output buffer len (for sanity check only).
+ * @return true if successful, false otherwise.
+ */
+ static bool ComplexMagnitudeSquaredF32(float* ptrSrc,
+ const uint32_t srcLen,
+ float* ptrDst,
+ const uint32_t dstLen);
+
+ };
+} /* namespace math */
+} /* namespace app */
+} /* namespace arm */
+
+#endif /* PLATFORM_MATH_HPP */
\ No newline at end of file
diff --git a/source/application/main/include/Profiler.hpp b/source/application/main/include/Profiler.hpp
new file mode 100644
index 0000000..b16a63b
--- /dev/null
+++ b/source/application/main/include/Profiler.hpp
@@ -0,0 +1,110 @@
+/*
+ * Copyright (c) 2021 Arm Limited. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef APP_PROFILER_HPP
+#define APP_PROFILER_HPP
+
+#include "hal.h"
+
+#include <string>
+#include <map>
+#include <vector>
+
+namespace arm {
+namespace app {
+
+ /** A single profiling unit definition. */
+ struct ProfilingUnit {
+ uint64_t npuCycles = 0;
+ uint64_t activeNpuCycles = 0;
+ uint64_t cpuCycles = 0;
+ time_t time = 0;
+ };
+
+ /* A collection of profiling units. */
+ using ProfilingSeries = std::vector<arm::app::ProfilingUnit>;
+
+ /* A map for string identifiable profiling series. */
+ using ProfilingMap = std::map<std::string, ProfilingSeries>;
+
+ /**
+ * @brief A very simple profiler example using the platform timer
+ * implementation.
+ */
+ class Profiler {
+ public:
+ /**
+ * @brief Constructor for profiler.
+ * @param[in] platform Pointer to a valid, initialised hal platform.
+ * @param[in] name A friendly name for this profiler.
+ **/
+ Profiler(hal_platform* platform, const char* name);
+
+ /** Block the default constructor. */
+ Profiler() = delete;
+
+ /** Default destructor. */
+ ~Profiler() = default;
+
+ /** @brief Start profiling => get starting time-stamp. */
+ bool StartProfiling(const char* name = nullptr);
+
+ /** @brief Stop profiling => get the ending time-stamp. */
+ bool StopProfiling();
+
+ /** @brief Stops the profiling and internally resets the
+ * platform timers. */
+ bool StopProfilingAndReset();
+
+ /** @brief Reset the platform timers. */
+ void Reset();
+
+ /**
+ * @brief Gets the results as string and resets the profiler.
+ * @returns Result string.
+ **/
+ std::string GetResultsAndReset();
+
+ /** @brief Set the profiler name. */
+ void SetName(const char* str);
+
+ private:
+ ProfilingMap _m_series; /* Profiling series map. */
+ time_counter _m_tstampSt; /* Container for a current starting timestamp. */
+ time_counter _m_tstampEnd; /* Container for a current ending timestamp. */
+ hal_platform * _m_pPlatform = nullptr; /* Platform pointer - to get the timer. */
+
+ bool _m_started = false; /* Indicates profiler has been started. */
+
+ std::string _m_name; /* Name given to this profiler. */
+
+ /**
+ * @brief Appends the profiling unit computed by the "start" and
+ * "end" timestamps to the profiling series identified by
+ * the name provided.
+ * @param[in] start Starting time-stamp.
+ * @param[in] end Ending time-stamp.
+ * @param[in] name Name for the profiling unit series to be
+ * appended to.
+ **/
+ void _AddProfilingUnit(time_counter start, time_counter end,
+ const std::string& name);
+ };
+
+} /* namespace app */
+} /* namespace arm */
+
+#endif /* APP_PROFILER_HPP */
diff --git a/source/application/main/include/UseCaseCommonUtils.hpp b/source/application/main/include/UseCaseCommonUtils.hpp
new file mode 100644
index 0000000..02200e8
--- /dev/null
+++ b/source/application/main/include/UseCaseCommonUtils.hpp
@@ -0,0 +1,76 @@
+/*
+ * Copyright (c) 2021 Arm Limited. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef USECASE_COMMON_UTILS_HPP
+#define USECASE_COMMON_UTILS_HPP
+
+#include "hal.h"
+#include "Model.hpp"
+#include "AppContext.hpp"
+#include "Profiler.hpp"
+
+/* Helper macro to convert RGB888 to RGB565 format. */
+#define RGB888_TO_RGB565(R8,G8,B8) ((((R8>>3) & 0x1F) << 11) | \
+ (((G8>>2) & 0x3F) << 5) | \
+ ((B8>>3) & 0x1F))
+
+constexpr uint16_t COLOR_BLACK = 0;
+constexpr uint16_t COLOR_GREEN = RGB888_TO_RGB565( 0, 255, 0); // 2016;
+constexpr uint16_t COLOR_YELLOW = RGB888_TO_RGB565(255, 255, 0); // 65504;
+
+namespace arm {
+namespace app {
+
+ /**
+ * @brief Run inference using given model
+ * object. If profiling is enabled, it will log the
+ * statistics too.
+ * @param[in] platform Reference to the hal platform object.
+ * @param[in] model Reference to the initialised model.
+ * @return true if inference succeeds, false otherwise.
+ **/
+ bool RunInference(hal_platform& platform, arm::app::Model& model);
+
+ /**
+ * @brief Read input and return as an integer.
+ * @param[in] platform Reference to the hal platform object.
+ * @param[in] model Reference to the initialised model.
+ * @return Integer value corresponding to the user input.
+ **/
+ int ReadUserInputAsInt(hal_platform& platform);
+
+#if VERIFY_TEST_OUTPUT
+ /**
+ * @brief Helper function to dump a tensor to stdout
+ * @param[in] tensor tensor to be dumped
+ * @param[in] lineBreakForNumElements number of elements
+ * after which line break will be added.
+ **/
+ void DumpTensor(TfLiteTensor* tensor,
+ const size_t lineBreakForNumElements = 16);
+#endif /* VERIFY_TEST_OUTPUT */
+
+ /**
+ * @brief List the files baked in the application.
+ * @param[in] ctx Reference to the application context.
+ * @return true or false based on event being handled.
+ **/
+ bool ListFilesHandler(ApplicationContext& ctx);
+
+} /* namespace app */
+} /* namespace arm */
+
+#endif /* USECASE_COMMON_UTILS_HPP */
\ No newline at end of file