MLECO-3183: Refactoring application sources
Platform agnostic application sources are moved into application
api module with their own independent CMake projects.
Changes for MLECO-3080 also included - they create CMake projects
individial API's (again, platform agnostic) that dependent on the
common logic. The API for KWS_API "joint" API has been removed and
now the use case relies on individual KWS, and ASR API libraries.
Change-Id: I1f7748dc767abb3904634a04e0991b74ac7b756d
Signed-off-by: Kshitij Sisodia <kshitij.sisodia@arm.com>
diff --git a/source/use_case/vww/include/VisualWakeWordModel.hpp b/source/use_case/vww/include/VisualWakeWordModel.hpp
deleted file mode 100644
index 1ed9202..0000000
--- a/source/use_case/vww/include/VisualWakeWordModel.hpp
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Copyright (c) 2021 - 2022 Arm Limited. All rights reserved.
- * SPDX-License-Identifier: Apache-2.0
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#ifndef VISUAL_WAKE_WORD_MODEL_HPP
-#define VISUAL_WAKE_WORD_MODEL_HPP
-
-#include "Model.hpp"
-
-namespace arm {
-namespace app {
-
- class VisualWakeWordModel : public Model {
-
- public:
- /* Indices for the expected model - based on input tensor shape */
- static constexpr uint32_t ms_inputRowsIdx = 1;
- static constexpr uint32_t ms_inputColsIdx = 2;
- static constexpr uint32_t ms_inputChannelsIdx = 3;
-
- protected:
- /** @brief Gets the reference to op resolver interface class. */
- const tflite::MicroOpResolver& GetOpResolver() override;
-
- /** @brief Adds operations to the op resolver instance. */
- bool EnlistOperations() override;
-
- const uint8_t* ModelPointer() override;
-
- size_t ModelSize() override;
- private:
- /* Maximum number of individual operations that can be enlisted. */
- static constexpr int ms_maxOpCnt = 7;
-
- /* A mutable op resolver instance. */
- tflite::MicroMutableOpResolver<ms_maxOpCnt> m_opResolver;
- };
-
-} /* namespace app */
-} /* namespace arm */
-
-#endif /* VISUAL_WAKE_WORD_MODEL_HPP */
diff --git a/source/use_case/vww/include/VisualWakeWordProcessing.hpp b/source/use_case/vww/include/VisualWakeWordProcessing.hpp
deleted file mode 100644
index f9f9d72..0000000
--- a/source/use_case/vww/include/VisualWakeWordProcessing.hpp
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Copyright (c) 2022 Arm Limited. All rights reserved.
- * SPDX-License-Identifier: Apache-2.0
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#ifndef VWW_PROCESSING_HPP
-#define VWW_PROCESSING_HPP
-
-#include "BaseProcessing.hpp"
-#include "Model.hpp"
-#include "Classifier.hpp"
-
-namespace arm {
-namespace app {
-
- /**
- * @brief Pre-processing class for Visual Wake Word use case.
- * Implements methods declared by BasePreProcess and anything else needed
- * to populate input tensors ready for inference.
- */
- class VisualWakeWordPreProcess : public BasePreProcess {
-
- public:
- /**
- * @brief Constructor
- * @param[in] inputTensor Pointer to the TFLite Micro input Tensor.
- * @param[in] rgb2Gray Convert image from 3 channel RGB to 1 channel grayscale.
- **/
- explicit VisualWakeWordPreProcess(TfLiteTensor* inputTensor, bool rgb2Gray=true);
-
- /**
- * @brief Should perform pre-processing of 'raw' input image data and load it into
- * TFLite Micro input tensors ready for inference
- * @param[in] input Pointer to the data that pre-processing will work on.
- * @param[in] inputSize Size of the input data.
- * @return true if successful, false otherwise.
- **/
- bool DoPreProcess(const void* input, size_t inputSize) override;
-
- private:
- TfLiteTensor* m_inputTensor;
- bool m_rgb2Gray;
- };
-
- /**
- * @brief Post-processing class for Visual Wake Word use case.
- * Implements methods declared by BasePostProcess and anything else needed
- * to populate result vector.
- */
- class VisualWakeWordPostProcess : public BasePostProcess {
-
- private:
- TfLiteTensor* m_outputTensor;
- Classifier& m_vwwClassifier;
- const std::vector<std::string>& m_labels;
- std::vector<ClassificationResult>& m_results;
-
- public:
- /**
- * @brief Constructor
- * @param[in] outputTensor Pointer to the TFLite Micro output Tensor.
- * @param[in] classifier Classifier object used to get top N results from classification.
- * @param[in] model Pointer to the VWW classification Model object.
- * @param[in] labels Vector of string labels to identify each output of the model.
- * @param[out] results Vector of classification results to store decoded outputs.
- **/
- VisualWakeWordPostProcess(TfLiteTensor* outputTensor, Classifier& classifier,
- const std::vector<std::string>& labels,
- std::vector<ClassificationResult>& results);
-
- /**
- * @brief Should perform post-processing of the result of inference then
- * populate classification result data for any later use.
- * @return true if successful, false otherwise.
- **/
- bool DoPostProcess() override;
- };
-
-} /* namespace app */
-} /* namespace arm */
-
-#endif /* VWW_PROCESSING_HPP */
\ No newline at end of file
diff --git a/source/use_case/vww/src/MainLoop.cc b/source/use_case/vww/src/MainLoop.cc
index 041ea18..2161b0a 100644
--- a/source/use_case/vww/src/MainLoop.cc
+++ b/source/use_case/vww/src/MainLoop.cc
@@ -21,7 +21,17 @@
#include "VisualWakeWordModel.hpp" /* Model class for running inference. */
#include "UseCaseHandler.hpp" /* Handlers for different user options. */
#include "UseCaseCommonUtils.hpp" /* Utils functions. */
-#include "log_macros.h"
+#include "log_macros.h" /* Logging functions */
+#include "BufAttributes.hpp" /* Buffer attributes to be applied */
+
+namespace arm {
+ namespace app {
+ static uint8_t tensorArena[ACTIVATION_BUF_SZ] ACTIVATION_BUF_ATTRIBUTE;
+ } /* namespace app */
+} /* namespace arm */
+
+extern uint8_t* GetModelPointer();
+extern size_t GetModelLen();
using ViusalWakeWordClassifier = arm::app::Classifier;
@@ -30,11 +40,22 @@
arm::app::VisualWakeWordModel model; /* Model wrapper object. */
/* Load the model. */
- if (!model.Init()) {
+ if (!model.Init(arm::app::tensorArena,
+ sizeof(arm::app::tensorArena),
+ GetModelPointer(),
+ GetModelLen())) {
printf_err("Failed to initialise model\n");
return;
}
+#if !defined(ARM_NPU)
+ /* If it is not a NPU build check if the model contains a NPU operator */
+ if (model.ContainsEthosUOperator()) {
+ printf_err("No driver support for Ethos-U operator found in the model.\n");
+ return;
+ }
+#endif /* ARM_NPU */
+
/* Instantiate application context. */
arm::app::ApplicationContext caseContext;
@@ -55,7 +76,7 @@
constexpr bool bUseMenu = NUMBER_OF_FILES > 1 ? true : false;
do {
int menuOption = common::MENU_OPT_RUN_INF_NEXT;
- if (bUseMenu) {
+ if (bUseMenu) {
DisplayCommonMenu();
menuOption = arm::app::ReadUserInputAsInt();
printf("\n");
diff --git a/source/use_case/vww/src/VisualWakeWordModel.cc b/source/use_case/vww/src/VisualWakeWordModel.cc
deleted file mode 100644
index 59beccc..0000000
--- a/source/use_case/vww/src/VisualWakeWordModel.cc
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Copyright (c) 2021 Arm Limited. All rights reserved.
- * SPDX-License-Identifier: Apache-2.0
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include "VisualWakeWordModel.hpp"
-#include "log_macros.h"
-
-const tflite::MicroOpResolver& arm::app::VisualWakeWordModel::GetOpResolver()
-{
- return this->m_opResolver;
-}
-
-bool arm::app::VisualWakeWordModel::EnlistOperations()
-{
- this->m_opResolver.AddDepthwiseConv2D();
- this->m_opResolver.AddConv2D();
- this->m_opResolver.AddAveragePool2D();
- this->m_opResolver.AddReshape();
- this->m_opResolver.AddPad();
- this->m_opResolver.AddAdd();
-
-#if defined(ARM_NPU)
- if (kTfLiteOk == this->m_opResolver.AddEthosU()) {
- info("Added %s support to op resolver\n",
- tflite::GetString_ETHOSU());
- } else {
- printf_err("Failed to add Arm NPU support to op resolver.");
- return false;
- }
-#endif /* ARM_NPU */
- return true;
-}
-
-extern uint8_t* GetModelPointer();
-const uint8_t* arm::app::VisualWakeWordModel::ModelPointer()
-{
- return GetModelPointer();
-}
-
-extern size_t GetModelLen();
-size_t arm::app::VisualWakeWordModel::ModelSize()
-{
- return GetModelLen();
-}
\ No newline at end of file
diff --git a/source/use_case/vww/src/VisualWakeWordProcessing.cc b/source/use_case/vww/src/VisualWakeWordProcessing.cc
deleted file mode 100644
index 4ae8a54..0000000
--- a/source/use_case/vww/src/VisualWakeWordProcessing.cc
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Copyright (c) 2022 Arm Limited. All rights reserved.
- * SPDX-License-Identifier: Apache-2.0
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include "VisualWakeWordProcessing.hpp"
-
-#include "ImageUtils.hpp"
-#include "VisualWakeWordModel.hpp"
-#include "log_macros.h"
-
-namespace arm {
-namespace app {
-
- VisualWakeWordPreProcess::VisualWakeWordPreProcess(TfLiteTensor* inputTensor, bool rgb2Gray)
- :m_inputTensor{inputTensor},
- m_rgb2Gray{rgb2Gray}
- {}
-
- bool VisualWakeWordPreProcess::DoPreProcess(const void* data, size_t inputSize)
- {
- if (data == nullptr) {
- printf_err("Data pointer is null");
- }
-
- auto input = static_cast<const uint8_t*>(data);
-
- uint8_t* unsignedDstPtr = this->m_inputTensor->data.uint8;
-
- if (this->m_rgb2Gray) {
- image::RgbToGrayscale(input, unsignedDstPtr, inputSize);
- } else {
- std::memcpy(unsignedDstPtr, input, inputSize);
- }
-
- /* VWW model pre-processing is image conversion from uint8 to [0,1] float values,
- * then quantize them with input quantization info. */
- QuantParams inQuantParams = GetTensorQuantParams(this->m_inputTensor);
-
- int8_t* signedDstPtr = this->m_inputTensor->data.int8;
- for (size_t i = 0; i < this->m_inputTensor->bytes; i++) {
- auto i_data_int8 = static_cast<int8_t>(
- ((static_cast<float>(unsignedDstPtr[i]) / 255.0f) / inQuantParams.scale) + inQuantParams.offset
- );
- signedDstPtr[i] = std::min<int8_t>(INT8_MAX, std::max<int8_t>(i_data_int8, INT8_MIN));
- }
-
- debug("Input tensor populated \n");
-
- return true;
- }
-
- VisualWakeWordPostProcess::VisualWakeWordPostProcess(TfLiteTensor* outputTensor, Classifier& classifier,
- const std::vector<std::string>& labels, std::vector<ClassificationResult>& results)
- :m_outputTensor{outputTensor},
- m_vwwClassifier{classifier},
- m_labels{labels},
- m_results{results}
- {}
-
- bool VisualWakeWordPostProcess::DoPostProcess()
- {
- return this->m_vwwClassifier.GetClassificationResults(
- this->m_outputTensor, this->m_results,
- this->m_labels, 1, true);
- }
-
-} /* namespace app */
-} /* namespace arm */
\ No newline at end of file
diff --git a/source/use_case/vww/usecase.cmake b/source/use_case/vww/usecase.cmake
index 8bf55fc..f6a3efe 100644
--- a/source/use_case/vww/usecase.cmake
+++ b/source/use_case/vww/usecase.cmake
@@ -1,3 +1,4 @@
+#----------------------------------------------------------------------------
# Copyright (c) 2021 Arm Limited. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
@@ -12,7 +13,10 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+#----------------------------------------------------------------------------
+# Append the API to use for this use case
+list(APPEND ${use_case}_API_LIST "vww")
USER_OPTION(${use_case}_FILE_PATH "Directory with custom image files, or path to a single image file, to use in the evaluation application"
${CMAKE_CURRENT_SOURCE_DIR}/resources/${use_case}/samples/