source/use_case/asr/src/UseCaseHandler.cc - ml/ethos-u/ml-embedded-evaluation-kit - Gitiles

 /*
  * Copyright (c) 2021 Arm Limited. All rights reserved.
  * SPDX-License-Identifier: Apache-2.0
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
 #include "UseCaseHandler.hpp"

 #include "InputFiles.hpp"
 #include "AsrClassifier.hpp"
 #include "Wav2LetterModel.hpp"
 #include "hal.h"
 #include "Wav2LetterMfcc.hpp"
 #include "AudioUtils.hpp"
 #include "UseCaseCommonUtils.hpp"
 #include "AsrResult.hpp"
 #include "Wav2LetterPreprocess.hpp"
 #include "Wav2LetterPostprocess.hpp"
 #include "OutputDecode.hpp"

 namespace arm {
 namespace app {

     /**
     * @brief           Helper function to increment current audio clip index.
     * @param[in,out]   ctx   Pointer to the application context object.
     **/
     static void _IncrementAppCtxClipIdx(ApplicationContext& ctx);

     /**
      * @brief           Helper function to set the audio clip index.
      * @param[in,out]   ctx   Pointer to the application context object.
      * @param[in]       idx   Value to be set.
      * @return          true if index is set, false otherwise.
      **/
     static bool _SetAppCtxClipIdx(ApplicationContext& ctx, uint32_t idx);

     /**
      * @brief           Presents inference results using the data presentation
      *                  object.
      * @param[in]       platform    Reference to the hal platform object.
      * @param[in]       results     Vector of classification results to be displayed.
      * @param[in]       infTimeMs   Inference time in milliseconds, if available
      *                              otherwise, this can be passed in as 0.
      * @return          true if successful, false otherwise.
      **/
     static bool _PresentInferenceResult(
                     hal_platform& platform,
                     const std::vector<arm::app::asr::AsrResult>& results);

     /* Audio inference classification handler. */
     bool ClassifyAudioHandler(ApplicationContext& ctx, uint32_t clipIndex, bool runAll)
     {
         constexpr uint32_t dataPsnTxtInfStartX = 20;
         constexpr uint32_t dataPsnTxtInfStartY = 40;

         auto& platform = ctx.Get<hal_platform&>("platform");
         platform.data_psn->clear(COLOR_BLACK);

         /* If the request has a valid size, set the audio index. */
         if (clipIndex < NUMBER_OF_FILES) {
             if (!_SetAppCtxClipIdx(ctx, clipIndex)) {
                 return false;
             }
         }

         /* Get model reference. */
         auto& model = ctx.Get<Model&>("model");
         if (!model.IsInited()) {
             printf_err("Model is not initialised! Terminating processing.\n");
             return false;
         }

         /* Get score threshold to be applied for the classifier (post-inference). */
         auto scoreThreshold = ctx.Get<float>("scoreThreshold");

         /* Get tensors. Dimensions of the tensor should have been verified by
          * the callee. */
         TfLiteTensor* inputTensor = model.GetInputTensor(0);
         TfLiteTensor* outputTensor = model.GetOutputTensor(0);
         const uint32_t inputRows = inputTensor->dims->data[arm::app::Wav2LetterModel::ms_inputRowsIdx];

         /* Populate MFCC related parameters. */
         auto mfccParamsWinLen = ctx.Get<uint32_t>("frameLength");
         auto mfccParamsWinStride = ctx.Get<uint32_t>("frameStride");

         /* Populate ASR inference context and inner lengths for input. */
         auto inputCtxLen = ctx.Get<uint32_t>("ctxLen");
         const uint32_t inputInnerLen = inputRows - (2 * inputCtxLen);

         /* Audio data stride corresponds to inputInnerLen feature vectors. */
         const uint32_t audioParamsWinLen = (inputRows - 1) * mfccParamsWinStride + (mfccParamsWinLen);
         const uint32_t audioParamsWinStride = inputInnerLen * mfccParamsWinStride;
         const float audioParamsSecondsPerSample = (1.0/audio::Wav2LetterMFCC::ms_defaultSamplingFreq);

         /* Get pre/post-processing objects. */
         auto& prep = ctx.Get<audio::asr::Preprocess&>("preprocess");
         auto& postp = ctx.Get<audio::asr::Postprocess&>("postprocess");

         /* Set default reduction axis for post-processing. */
         const uint32_t reductionAxis = arm::app::Wav2LetterModel::ms_outputRowsIdx;

         /* Audio clip start index. */
         auto startClipIdx = ctx.Get<uint32_t>("clipIndex");

         /* Loop to process audio clips. */
         do {
             /* Get current audio clip index. */
             auto currentIndex = ctx.Get<uint32_t>("clipIndex");

             /* Get the current audio buffer and respective size. */
             const int16_t* audioArr = get_audio_array(currentIndex);
             const uint32_t audioArrSize = get_audio_array_size(currentIndex);

             if (!audioArr) {
                 printf_err("Invalid audio array pointer\n");
                 return false;
             }

             /* Audio clip must have enough samples to produce 1 MFCC feature. */
             if (audioArrSize < mfccParamsWinLen) {
                 printf_err("Not enough audio samples, minimum needed is %u\n", mfccParamsWinLen);
                 return false;
             }

             /* Initialise an audio slider. */
             auto audioDataSlider = audio::ASRSlidingWindow<const int16_t>(
                                         audioArr,
                                         audioArrSize,
                                         audioParamsWinLen,
                                         audioParamsWinStride);

             /* Declare a container for results. */
             std::vector<arm::app::asr::AsrResult> results;

             /* Display message on the LCD - inference running. */
             std::string str_inf{"Running inference... "};
             platform.data_psn->present_data_text(
                                 str_inf.c_str(), str_inf.size(),
                                 dataPsnTxtInfStartX, dataPsnTxtInfStartY, 0);

             info("Running inference on audio clip %u => %s\n", currentIndex,
                  get_filename(currentIndex));

             size_t inferenceWindowLen = audioParamsWinLen;

             /* Start sliding through audio clip. */
             while (audioDataSlider.HasNext()) {

                 /* If not enough audio see how much can be sent for processing. */
                 size_t nextStartIndex = audioDataSlider.NextWindowStartIndex();
                 if (nextStartIndex + audioParamsWinLen > audioArrSize) {
                     inferenceWindowLen = audioArrSize - nextStartIndex;
                 }

                 const int16_t* inferenceWindow = audioDataSlider.Next();

                 info("Inference %zu/%zu\n", audioDataSlider.Index() + 1,
                      static_cast<size_t>(ceilf(audioDataSlider.FractionalTotalStrides() + 1)));

                 Profiler prepProfiler{&platform, "pre-processing"};
                 prepProfiler.StartProfiling();

                 /* Calculate MFCCs, deltas and populate the input tensor. */
                 prep.Invoke(inferenceWindow, inferenceWindowLen, inputTensor);

                 prepProfiler.StopProfiling();
                 std::string prepProfileResults = prepProfiler.GetResultsAndReset();
                 info("%s\n", prepProfileResults.c_str());

                 /* Run inference over this audio clip sliding window. */
                 arm::app::RunInference(platform, model);

                 /* Post-process. */
                 postp.Invoke(outputTensor, reductionAxis, !audioDataSlider.HasNext());

                 /* Get results. */
                 std::vector<ClassificationResult> classificationResult;
                 auto& classifier = ctx.Get<AsrClassifier&>("classifier");
                 classifier.GetClassificationResults(
                             outputTensor, classificationResult,
                             ctx.Get<std::vector<std::string>&>("labels"), 1);

                 results.emplace_back(asr::AsrResult(classificationResult,
                                                     (audioDataSlider.Index() *
                                                     audioParamsSecondsPerSample *
                                                     audioParamsWinStride),
                                                     audioDataSlider.Index(), scoreThreshold));

 #if VERIFY_TEST_OUTPUT
                 arm::app::DumpTensor(outputTensor,
                     outputTensor->dims->data[arm::app::Wav2LetterModel::ms_outputColsIdx]);
 #endif /* VERIFY_TEST_OUTPUT */

             }

             /* Erase. */
             str_inf = std::string(str_inf.size(), ' ');
             platform.data_psn->present_data_text(
                                 str_inf.c_str(), str_inf.size(),
                                 dataPsnTxtInfStartX, dataPsnTxtInfStartY, 0);

             ctx.Set<std::vector<arm::app::asr::AsrResult>>("results", results);

             if (!_PresentInferenceResult(platform, results)) {
                 return false;
             }

             _IncrementAppCtxClipIdx(ctx);

         } while (runAll && ctx.Get<uint32_t>("clipIndex") != startClipIdx);

         return true;
     }

     static void _IncrementAppCtxClipIdx(ApplicationContext& ctx)
     {
         auto curAudioIdx = ctx.Get<uint32_t>("clipIndex");

         if (curAudioIdx + 1 >= NUMBER_OF_FILES) {
             ctx.Set<uint32_t>("clipIndex", 0);
             return;
         }
         ++curAudioIdx;
         ctx.Set<uint32_t>("clipIndex", curAudioIdx);
     }

     static bool _SetAppCtxClipIdx(ApplicationContext& ctx, const uint32_t idx)
     {
         if (idx >= NUMBER_OF_FILES) {
             printf_err("Invalid idx %u (expected less than %u)\n",
                        idx, NUMBER_OF_FILES);
             return false;
         }

         ctx.Set<uint32_t>("clipIndex", idx);
         return true;
     }

     static bool _PresentInferenceResult(hal_platform& platform,
                                         const std::vector<arm::app::asr::AsrResult>& results)
     {
         constexpr uint32_t dataPsnTxtStartX1 = 20;
         constexpr uint32_t dataPsnTxtStartY1 = 60;
         constexpr bool allow_multiple_lines = true;

         platform.data_psn->set_text_color(COLOR_GREEN);

         /* Results from multiple inferences should be combined before processing. */
         std::vector<arm::app::ClassificationResult> combinedResults;
         for (auto& result : results) {
             combinedResults.insert(combinedResults.end(),
                                    result.m_resultVec.begin(),
                                    result.m_resultVec.end());
         }

         /* Get each inference result string using the decoder. */
         for (const auto & result : results) {
             std::string infResultStr = audio::asr::DecodeOutput(result.m_resultVec);

             info("Result for inf %u: %s\n", result.m_inferenceNumber,
                                             infResultStr.c_str());
         }

         /* Get the decoded result for the combined result. */
         std::string finalResultStr = audio::asr::DecodeOutput(combinedResults);

         platform.data_psn->present_data_text(
                             finalResultStr.c_str(), finalResultStr.size(),
                             dataPsnTxtStartX1, dataPsnTxtStartY1,
                             allow_multiple_lines);

         info("Final result: %s\n", finalResultStr.c_str());
         return true;
     }

 } /* namespace app */
 } /* namespace arm */
	/*
	* Copyright (c) 2021 Arm Limited. All rights reserved.
	* SPDX-License-Identifier: Apache-2.0
	*
	* Licensed under the Apache License, Version 2.0 (the "License");
	* you may not use this file except in compliance with the License.
	* You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/
	#include "UseCaseHandler.hpp"

	#include "InputFiles.hpp"
	#include "AsrClassifier.hpp"
	#include "Wav2LetterModel.hpp"
	#include "hal.h"
	#include "Wav2LetterMfcc.hpp"
	#include "AudioUtils.hpp"
	#include "UseCaseCommonUtils.hpp"
	#include "AsrResult.hpp"
	#include "Wav2LetterPreprocess.hpp"
	#include "Wav2LetterPostprocess.hpp"
	#include "OutputDecode.hpp"

	namespace arm {
	namespace app {

	/**
	* @brief Helper function to increment current audio clip index.
	* @param[in,out] ctx Pointer to the application context object.
	**/
	static void _IncrementAppCtxClipIdx(ApplicationContext& ctx);

	/**
	* @brief Helper function to set the audio clip index.
	* @param[in,out] ctx Pointer to the application context object.
	* @param[in] idx Value to be set.
	* @return true if index is set, false otherwise.
	**/
	static bool _SetAppCtxClipIdx(ApplicationContext& ctx, uint32_t idx);

	/**
	* @brief Presents inference results using the data presentation
	* object.
	* @param[in] platform Reference to the hal platform object.
	* @param[in] results Vector of classification results to be displayed.
	* @param[in] infTimeMs Inference time in milliseconds, if available
	* otherwise, this can be passed in as 0.
	* @return true if successful, false otherwise.
	**/
	static bool _PresentInferenceResult(
	hal_platform& platform,
	const std::vector<arm::app::asr::AsrResult>& results);

	/* Audio inference classification handler. */
	bool ClassifyAudioHandler(ApplicationContext& ctx, uint32_t clipIndex, bool runAll)
	{
	constexpr uint32_t dataPsnTxtInfStartX = 20;
	constexpr uint32_t dataPsnTxtInfStartY = 40;

	auto& platform = ctx.Get<hal_platform&>("platform");
	platform.data_psn->clear(COLOR_BLACK);

	/* If the request has a valid size, set the audio index. */
	if (clipIndex < NUMBER_OF_FILES) {
	if (!_SetAppCtxClipIdx(ctx, clipIndex)) {
	return false;
	}
	}

	/* Get model reference. */
	auto& model = ctx.Get<Model&>("model");
	if (!model.IsInited()) {
	printf_err("Model is not initialised! Terminating processing.\n");
	return false;
	}

	/* Get score threshold to be applied for the classifier (post-inference). */
	auto scoreThreshold = ctx.Get<float>("scoreThreshold");

	/* Get tensors. Dimensions of the tensor should have been verified by
	* the callee. */
	TfLiteTensor* inputTensor = model.GetInputTensor(0);
	TfLiteTensor* outputTensor = model.GetOutputTensor(0);
	const uint32_t inputRows = inputTensor->dims->data[arm::app::Wav2LetterModel::ms_inputRowsIdx];

	/* Populate MFCC related parameters. */
	auto mfccParamsWinLen = ctx.Get<uint32_t>("frameLength");
	auto mfccParamsWinStride = ctx.Get<uint32_t>("frameStride");

	/* Populate ASR inference context and inner lengths for input. */
	auto inputCtxLen = ctx.Get<uint32_t>("ctxLen");
	const uint32_t inputInnerLen = inputRows - (2 * inputCtxLen);

	/* Audio data stride corresponds to inputInnerLen feature vectors. */
	const uint32_t audioParamsWinLen = (inputRows - 1) * mfccParamsWinStride + (mfccParamsWinLen);
	const uint32_t audioParamsWinStride = inputInnerLen * mfccParamsWinStride;
	const float audioParamsSecondsPerSample = (1.0/audio::Wav2LetterMFCC::ms_defaultSamplingFreq);

	/* Get pre/post-processing objects. */
	auto& prep = ctx.Get<audio::asr::Preprocess&>("preprocess");
	auto& postp = ctx.Get<audio::asr::Postprocess&>("postprocess");

	/* Set default reduction axis for post-processing. */
	const uint32_t reductionAxis = arm::app::Wav2LetterModel::ms_outputRowsIdx;

	/* Audio clip start index. */
	auto startClipIdx = ctx.Get<uint32_t>("clipIndex");

	/* Loop to process audio clips. */
	do {
	/* Get current audio clip index. */
	auto currentIndex = ctx.Get<uint32_t>("clipIndex");

	/* Get the current audio buffer and respective size. */
	const int16_t* audioArr = get_audio_array(currentIndex);
	const uint32_t audioArrSize = get_audio_array_size(currentIndex);

	if (!audioArr) {
	printf_err("Invalid audio array pointer\n");
	return false;
	}

	/* Audio clip must have enough samples to produce 1 MFCC feature. */
	if (audioArrSize < mfccParamsWinLen) {
	printf_err("Not enough audio samples, minimum needed is %u\n", mfccParamsWinLen);
	return false;
	}

	/* Initialise an audio slider. */
	auto audioDataSlider = audio::ASRSlidingWindow<const int16_t>(
	audioArr,
	audioArrSize,
	audioParamsWinLen,
	audioParamsWinStride);

	/* Declare a container for results. */
	std::vector<arm::app::asr::AsrResult> results;

	/* Display message on the LCD - inference running. */
	std::string str_inf{"Running inference... "};
	platform.data_psn->present_data_text(
	str_inf.c_str(), str_inf.size(),
	dataPsnTxtInfStartX, dataPsnTxtInfStartY, 0);

	info("Running inference on audio clip %u => %s\n", currentIndex,
	get_filename(currentIndex));

	size_t inferenceWindowLen = audioParamsWinLen;

	/* Start sliding through audio clip. */
	while (audioDataSlider.HasNext()) {

	/* If not enough audio see how much can be sent for processing. */
	size_t nextStartIndex = audioDataSlider.NextWindowStartIndex();
	if (nextStartIndex + audioParamsWinLen > audioArrSize) {
	inferenceWindowLen = audioArrSize - nextStartIndex;
	}

	const int16_t* inferenceWindow = audioDataSlider.Next();

	info("Inference %zu/%zu\n", audioDataSlider.Index() + 1,
	static_cast<size_t>(ceilf(audioDataSlider.FractionalTotalStrides() + 1)));

	Profiler prepProfiler{&platform, "pre-processing"};
	prepProfiler.StartProfiling();

	/* Calculate MFCCs, deltas and populate the input tensor. */
	prep.Invoke(inferenceWindow, inferenceWindowLen, inputTensor);

	prepProfiler.StopProfiling();
	std::string prepProfileResults = prepProfiler.GetResultsAndReset();
	info("%s\n", prepProfileResults.c_str());

	/* Run inference over this audio clip sliding window. */
	arm::app::RunInference(platform, model);

	/* Post-process. */
	postp.Invoke(outputTensor, reductionAxis, !audioDataSlider.HasNext());

	/* Get results. */
	std::vector<ClassificationResult> classificationResult;
	auto& classifier = ctx.Get<AsrClassifier&>("classifier");
	classifier.GetClassificationResults(
	outputTensor, classificationResult,
	ctx.Get<std::vector<std::string>&>("labels"), 1);

	results.emplace_back(asr::AsrResult(classificationResult,
	(audioDataSlider.Index() *
	audioParamsSecondsPerSample *
	audioParamsWinStride),
	audioDataSlider.Index(), scoreThreshold));

	#if VERIFY_TEST_OUTPUT
	arm::app::DumpTensor(outputTensor,
	outputTensor->dims->data[arm::app::Wav2LetterModel::ms_outputColsIdx]);
	#endif /* VERIFY_TEST_OUTPUT */

	}

	/* Erase. */
	str_inf = std::string(str_inf.size(), ' ');
	platform.data_psn->present_data_text(
	str_inf.c_str(), str_inf.size(),
	dataPsnTxtInfStartX, dataPsnTxtInfStartY, 0);

	ctx.Set<std::vector<arm::app::asr::AsrResult>>("results", results);

	if (!_PresentInferenceResult(platform, results)) {
	return false;
	}

	_IncrementAppCtxClipIdx(ctx);

	} while (runAll && ctx.Get<uint32_t>("clipIndex") != startClipIdx);

	return true;
	}

	static void _IncrementAppCtxClipIdx(ApplicationContext& ctx)
	{
	auto curAudioIdx = ctx.Get<uint32_t>("clipIndex");

	if (curAudioIdx + 1 >= NUMBER_OF_FILES) {
	ctx.Set<uint32_t>("clipIndex", 0);
	return;
	}
	++curAudioIdx;
	ctx.Set<uint32_t>("clipIndex", curAudioIdx);
	}

	static bool _SetAppCtxClipIdx(ApplicationContext& ctx, const uint32_t idx)
	{
	if (idx >= NUMBER_OF_FILES) {
	printf_err("Invalid idx %u (expected less than %u)\n",
	idx, NUMBER_OF_FILES);
	return false;
	}

	ctx.Set<uint32_t>("clipIndex", idx);
	return true;
	}

	static bool _PresentInferenceResult(hal_platform& platform,
	const std::vector<arm::app::asr::AsrResult>& results)
	{
	constexpr uint32_t dataPsnTxtStartX1 = 20;
	constexpr uint32_t dataPsnTxtStartY1 = 60;
	constexpr bool allow_multiple_lines = true;

	platform.data_psn->set_text_color(COLOR_GREEN);

	/* Results from multiple inferences should be combined before processing. */
	std::vector<arm::app::ClassificationResult> combinedResults;
	for (auto& result : results) {
	combinedResults.insert(combinedResults.end(),
	result.m_resultVec.begin(),
	result.m_resultVec.end());
	}

	/* Get each inference result string using the decoder. */
	for (const auto & result : results) {
	std::string infResultStr = audio::asr::DecodeOutput(result.m_resultVec);

	info("Result for inf %u: %s\n", result.m_inferenceNumber,
	infResultStr.c_str());
	}

	/* Get the decoded result for the combined result. */
	std::string finalResultStr = audio::asr::DecodeOutput(combinedResults);

	platform.data_psn->present_data_text(
	finalResultStr.c_str(), finalResultStr.size(),
	dataPsnTxtStartX1, dataPsnTxtStartY1,
	allow_multiple_lines);

	info("Final result: %s\n", finalResultStr.c_str());
	return true;
	}

	} /* namespace app */
	} /* namespace arm */