IVGCVSW-5752 Add missing runtime parameters to TfLite delegate.
* Adding Runtime parameter: dynamic-backends-path
* Add profiling parameters:
gpu-enable-profiling,
enable-internal-profiling, internal-profiling-detail,
enable-external-profiling, timeline-profiling, outgoing-capture-file,
incoming-capture-file, file-only-external-profiling,
counter-capture-period, profiling-file-format
* Adding utility parameter "serialize-to-dot"
Signed-off-by: Colm Donelan <Colm.Donelan@arm.com>
Change-Id: Ibff4b9a85ff0f0da5d70e8aa0bb6cba96aaabbc3
diff --git a/delegate/include/DelegateOptions.hpp b/delegate/include/DelegateOptions.hpp
index ace0859..b884744 100644
--- a/delegate/include/DelegateOptions.hpp
+++ b/delegate/include/DelegateOptions.hpp
@@ -41,6 +41,12 @@
void SetBackends(const std::vector<armnn::BackendId>& backends) { m_Backends = backends; }
+ void SetDynamicBackendsPath(const std::string& dynamicBackendsPath) { m_DynamicBackendsPath = dynamicBackendsPath; }
+ const std::string& GetDynamicBackendsPath() const { return m_DynamicBackendsPath; }
+
+ void SetGpuProfilingState(bool gpuProfilingState) { m_EnableGpuProfiling = gpuProfilingState; }
+ bool GetGpuProfilingState() { return m_EnableGpuProfiling; }
+
const std::vector<armnn::BackendOptions>& GetBackendOptions() const { return m_BackendOptions; }
/// Appends a backend option to the list of backend options
@@ -62,6 +68,23 @@
const armnn::Optional<armnn::DebugCallbackFunction>& GetDebugCallbackFunction() const
{ return m_DebugCallbackFunc; }
+ void SetInternalProfilingParams(bool internalProfilingState,
+ const armnn::ProfilingDetailsMethod& internalProfilingDetail)
+ { m_InternalProfilingEnabled = internalProfilingState; m_InternalProfilingDetail = internalProfilingDetail; }
+
+ bool GetInternalProfilingState() const { return m_InternalProfilingEnabled; }
+ const armnn::ProfilingDetailsMethod& GetInternalProfilingDetail() const { return m_InternalProfilingDetail; }
+
+ void SetExternalProfilingParams(
+ const armnn::IRuntime::CreationOptions::ExternalProfilingOptions& externalProfilingParams)
+ { m_ProfilingOptions = externalProfilingParams; }
+
+ const armnn::IRuntime::CreationOptions::ExternalProfilingOptions& GetExternalProfilingParams() const
+ { return m_ProfilingOptions; }
+
+ void SetSerializeToDot(const std::string& serializeToDotFile) { m_SerializeToDot = serializeToDotFile; }
+ const std::string& GetSerializeToDot() const { return m_SerializeToDot; }
+
private:
/// Which backend to run Delegate on.
/// Examples of possible values are: CpuRef, CpuAcc, GpuAcc.
@@ -87,6 +110,13 @@
/// "KernelProfilingEnabled" : bool [true | false]
std::vector<armnn::BackendOptions> m_BackendOptions;
+ /// Dynamic backend path.
+ /// This is the directory that will be searched for any dynamic backends.
+ std::string m_DynamicBackendsPath = "";
+
+ /// Enable Gpu Profiling.
+ bool m_EnableGpuProfiling = false;
+
/// OptimizerOptions
/// Reduce Fp32 data to Fp16 for faster processing
/// bool m_ReduceFp32ToFp16;
@@ -100,13 +130,39 @@
/// ModelOptions m_ModelOptions;
armnn::OptimizerOptions m_OptimizerOptions;
+ /// External profiling options.
+ /// Indicates whether external profiling is enabled or not.
+ /// bool m_EnableProfiling
+ /// Indicates whether external timeline profiling is enabled or not.
+ /// bool m_TimelineEnabled
+ /// Path to a file in which outgoing timeline profiling messages will be stored.
+ /// std::string m_OutgoingCaptureFile
+ /// Path to a file in which incoming timeline profiling messages will be stored.
+ /// std::string m_IncomingCaptureFile
+ /// Enable profiling output to file only.
+ /// bool m_FileOnly
+ /// The duration at which captured profiling messages will be flushed.
+ /// uint32_t m_CapturePeriod
+ /// The format of the file used for outputting profiling data.
+ /// std::string m_FileFormat
+ armnn::IRuntime::CreationOptions::ExternalProfilingOptions m_ProfilingOptions;
+
+ /// Internal profiling options.
+ /// Indicates whether internal profiling is enabled or not.
+ bool m_InternalProfilingEnabled = false;
+ /// Sets the level of detail output by the profiling. Options are DetailsWithEvents = 1 and DetailsOnly = 2
+ armnn::ProfilingDetailsMethod m_InternalProfilingDetail = armnn::ProfilingDetailsMethod::DetailsWithEvents;
+
/// Severity level for logging within ArmNN that will be used on creation of the delegate
armnn::Optional<armnn::LogSeverity> m_LoggingSeverity;
/// A callback function to debug layers performing custom computations on intermediate tensors.
/// If a function is not registered, and debug is enabled in OptimizerOptions,
/// debug will print information of the intermediate tensors.
- armnn::Optional<armnn::DebugCallbackFunction> m_DebugCallbackFunc;
+ armnn::Optional<armnn::DebugCallbackFunction> m_DebugCallbackFunc;
+
+ /// If not empty then the optimized model will be serialized to a file with this file name in "dot" format.
+ std::string m_SerializeToDot = "";
};
} // namespace armnnDelegate
diff --git a/delegate/src/armnn_delegate.cpp b/delegate/src/armnn_delegate.cpp
index f13bb57..2ede23c 100644
--- a/delegate/src/armnn_delegate.cpp
+++ b/delegate/src/armnn_delegate.cpp
@@ -39,6 +39,7 @@
#include "UnidirectionalSequenceLstm.hpp"
#include "Unpack.hpp"
+#include <armnnUtils/Filesystem.hpp>
#include <flatbuffers/flatbuffers.h>
#include <tensorflow/lite/context_util.h>
@@ -137,6 +138,9 @@
// Create ArmNN Runtime
armnn::IRuntime::CreationOptions runtimeOptions;
+ runtimeOptions.m_DynamicBackendsPath = m_Options.GetDynamicBackendsPath();
+ runtimeOptions.m_EnableGpuProfiling = m_Options.GetGpuProfilingState();
+ runtimeOptions.m_ProfilingOptions = m_Options.GetExternalProfilingParams();
auto backendOptions = m_Options.GetBackendOptions();
if (!backendOptions.empty())
@@ -363,27 +367,34 @@
throw armnn::Exception("TfLiteArmnnDelegate: Unable to optimize the network!");
}
+ // If set, we will serialize the optimized model into a dot file.
+ const std::string serializeToDotFile = delegate->m_Options.GetSerializeToDot();
+ if (!serializeToDotFile.empty())
+ {
+ fs::path filename = serializeToDotFile;
+ std::fstream file(filename.c_str(), std::ios_base::out);
+ optNet->SerializeToDot(file);
+ }
+
try
{
// Load graph into runtime
std::string errorMessage;
armnn::Status loadingStatus;
+ armnn::MemorySource memorySource = armnn::MemorySource::Undefined;
if (delegate->m_Options.GetOptimizerOptions().m_ImportEnabled)
{
- armnn::INetworkProperties networkProperties(false,
- armnn::MemorySource::Malloc,
- armnn::MemorySource::Malloc);
- loadingStatus = delegate->m_Runtime->LoadNetwork(networkId,
- std::move(optNet),
- errorMessage,
- networkProperties);
+ memorySource = armnn::MemorySource::Malloc;
}
- else
- {
- loadingStatus = delegate->m_Runtime->LoadNetwork(networkId,
- std::move(optNet),
- errorMessage);
- }
+ armnn::INetworkProperties networkProperties(false,
+ memorySource,
+ memorySource,
+ delegate->m_Options.GetInternalProfilingState(),
+ delegate->m_Options.GetInternalProfilingDetail());
+ loadingStatus = delegate->m_Runtime->LoadNetwork(networkId,
+ std::move(optNet),
+ errorMessage,
+ networkProperties);
if (loadingStatus != armnn::Status::Success)
{
// Optimize failed
diff --git a/delegate/src/armnn_external_delegate.cpp b/delegate/src/armnn_external_delegate.cpp
index 27eaf64..fb327de 100644
--- a/delegate/src/armnn_external_delegate.cpp
+++ b/delegate/src/armnn_external_delegate.cpp
@@ -43,6 +43,10 @@
* back to next backend in list if previous doesn't
* provide support for operation. e.g. "GpuAcc,CpuAcc"
*
+ * Option key: "dynamic-backends-path" \n
+ * Possible values: [filenameString] \n
+ * Descriptions: This is the directory that will be searched for any dynamic backends.
+ *
* Option key: "logging-severity" \n
* Possible values: ["trace"/"debug"/"info"/"warning"/"error"/"fatal"] \n
* Description: Sets the logging severity level for ArmNN. Logging
@@ -63,6 +67,10 @@
* Possible values: [filenameString] \n
* Description: File name for the tuning file.
*
+ * Option key: "gpu-enable-profiling" \n
+ * Possible values: ["true"/"false"] \n
+ * Description: Enables GPU profiling
+ *
* Option key: "gpu-kernel-profiling-enabled" \n
* Possible values: ["true"/"false"] \n
* Description: Enables GPU kernel profiling
@@ -105,6 +113,45 @@
* Possible values: ["true"/"false"] \n
* Description: Enable memory import
*
+ * Option key: "enable-internal-profiling" \n
+ * Possible values: ["true"/"false"] \n
+ * Description: Enable the internal profiling feature.
+ *
+ * Option key: "internal-profiling-detail" \n
+ * Possible values: [1/2] \n
+ * Description: Set the detail on the internal profiling. 1 = DetailsWithEvents, 2 = DetailsOnly.
+ *
+ * Option key: "enable-external-profiling" \n
+ * Possible values: ["true"/"false"] \n
+ * Description: Enable the external profiling feature.
+ *
+ * Option key: "timeline-profiling" \n
+ * Possible values: ["true"/"false"] \n
+ * Description: Indicates whether external timeline profiling is enabled or not.
+ *
+ * Option key: "outgoing-capture-file" \n
+ * Possible values: [filenameString] \n
+ * Description: Path to a file in which outgoing timeline profiling messages will be stored.
+ *
+ * Option key: "incoming-capture-file" \n
+ * Possible values: [filenameString] \n
+ * Description: Path to a file in which incoming timeline profiling messages will be stored.
+ *
+ * Option key: "file-only-external-profiling" \n
+ * Possible values: ["true"/"false"] \n
+ * Description: Enable profiling output to file only.
+ *
+ * Option key: "counter-capture-period" \n
+ * Possible values: Integer, Default is 10000u
+ * Description: Value in microseconds of the profiling capture period. \n
+ *
+ * Option key: "profiling-file-format" \n
+ * Possible values: String of ["binary"] \n
+ * Description: The format of the file used for outputting profiling data. Currently on "binary" is supported.
+ *
+ * Option key: "serialize-to-dot" \n
+ * Possible values: [filenameString] \n
+ * Description: Serialize the optimized network to the file specified in "dot" format.
*
* @param[in] option_keys Delegate option names
* @param[in] options_values Delegate option values
@@ -125,6 +172,9 @@
// (Initializes with CpuRef backend)
armnnDelegate::DelegateOptions options = armnnDelegate::TfLiteArmnnDelegateOptionsDefault();
armnn::OptimizerOptions optimizerOptions;
+ bool internalProfilingState = false;
+ armnn::ProfilingDetailsMethod internalProfilingDetail = armnn::ProfilingDetailsMethod::DetailsWithEvents;
+ armnn::IRuntime::CreationOptions::ExternalProfilingOptions extProfilingParams;
for (size_t i = 0; i < num_options; ++i)
{
// Process backends
@@ -141,6 +191,11 @@
}
options.SetBackends(backends);
}
+ // Process dynamic-backends-path
+ else if (std::string(options_keys[i]) == std::string("dynamic-backends-path"))
+ {
+ options.SetDynamicBackendsPath(std::string(options_values[i]));
+ }
// Process logging level
else if (std::string(options_keys[i]) == std::string("logging-severity"))
{
@@ -162,6 +217,10 @@
armnn::BackendOptions option("GpuAcc", {{"TuningFile", std::string(options_values[i])}});
options.AddBackendOption(option);
}
+ else if (std::string(options_keys[i]) == std::string("gpu-enable-profiling"))
+ {
+ options.SetGpuProfilingState(*options_values[i] != '0');
+ }
else if (std::string(options_keys[i]) == std::string("gpu-kernel-profiling-enabled"))
{
armnn::BackendOptions option("GpuAcc", {{"KernelProfilingEnabled", (*options_values[i] != '0')}});
@@ -213,12 +272,76 @@
{
optimizerOptions.m_ImportEnabled = *options_values[i] != '0';
}
+ // Process enable-internal-profiling
+ else if (std::string(options_keys[i]) == std::string("enable-internal-profiling"))
+ {
+ internalProfilingState = *options_values[i] != '0';
+ }
+ // Process internal-profiling-detail
+ else if (std::string(options_keys[i]) == std::string("internal-profiling-detail"))
+ {
+ uint32_t detailLevel = static_cast<uint32_t>(std::stoul(options_values[i]));
+ switch (detailLevel)
+ {
+ case 1:
+ internalProfilingDetail = armnn::ProfilingDetailsMethod::DetailsWithEvents;
+ break;
+ case 2:
+ internalProfilingDetail = armnn::ProfilingDetailsMethod::DetailsOnly;
+ break;
+ default:
+ internalProfilingDetail = armnn::ProfilingDetailsMethod::Undefined;
+ break;
+ }
+ }
+ // Process enable-external-profiling
+ else if (std::string(options_keys[i]) == std::string("enable-external-profiling"))
+ {
+ extProfilingParams.m_EnableProfiling = *options_values[i] != '0';
+ }
+ // Process timeline-profiling
+ else if (std::string(options_keys[i]) == std::string("timeline-profiling"))
+ {
+ extProfilingParams.m_TimelineEnabled = *options_values[i] != '0';
+ }
+ // Process outgoing-capture-file
+ else if (std::string(options_keys[i]) == std::string("outgoing-capture-file"))
+ {
+ extProfilingParams.m_OutgoingCaptureFile = options_values[i];
+ }
+ // Process incoming-capture-file
+ else if (std::string(options_keys[i]) == std::string("incoming-capture-file"))
+ {
+ extProfilingParams.m_IncomingCaptureFile = options_values[i];
+ }
+ // Process file-only-external-profiling
+ else if (std::string(options_keys[i]) == std::string("file-only-external-profiling"))
+ {
+ extProfilingParams.m_FileOnly = *options_values[i] != '0';
+ }
+ // Process counter-capture-period
+ else if (std::string(options_keys[i]) == std::string("counter-capture-period"))
+ {
+ extProfilingParams.m_CapturePeriod = static_cast<uint32_t>(std::stoul(options_values[i]));
+ }
+ // Process profiling-file-format
+ else if (std::string(options_keys[i]) == std::string("profiling-file-format"))
+ {
+ extProfilingParams.m_FileFormat = options_values[i];
+ }
+ // Process serialize-to-dot
+ else if (std::string(options_keys[i]) == std::string("serialize-to-dot"))
+ {
+ options.SetSerializeToDot(options_values[i]);
+ }
else
{
throw armnn::Exception("Unknown option for the ArmNN Delegate given: " + std::string(options_keys[i]));
}
}
options.SetOptimizerOptions(optimizerOptions);
+ options.SetInternalProfilingParams(internalProfilingState, internalProfilingDetail);
+ options.SetExternalProfilingParams(extProfilingParams);
delegate = TfLiteArmnnDelegateCreate(options);
}
catch (const std::exception& ex)
diff --git a/delegate/src/test/DelegateOptionsTest.cpp b/delegate/src/test/DelegateOptionsTest.cpp
index c7231fb..3387725 100644
--- a/delegate/src/test/DelegateOptionsTest.cpp
+++ b/delegate/src/test/DelegateOptionsTest.cpp
@@ -5,6 +5,7 @@
#include "DelegateOptionsTestHelper.hpp"
#include <common/include/ProfilingGuid.hpp>
+#include <armnnUtils/Filesystem.hpp>
namespace armnnDelegate
{
@@ -185,6 +186,44 @@
delegateOptions);
}
+TEST_CASE ("ArmnnDelegateSerializeToDot")
+{
+ const fs::path filename(fs::temp_directory_path() / "ArmnnDelegateSerializeToDot.dot");
+ if ( fs::exists(filename) )
+ {
+ fs::remove(filename);
+ }
+ std::stringstream ss;
+ {
+ StreamRedirector redirect(std::cout, ss.rdbuf());
+
+ std::vector<armnn::BackendId> backends = { armnn::Compute::CpuRef };
+ std::vector<int32_t> tensorShape { 1, 2, 2, 1 };
+ std::vector<float> inputData = { 1, 2, 3, 4 };
+ std::vector<float> divData = { 2, 2, 3, 4 };
+ std::vector<float> expectedResult = { 1, 2, 2, 2 };
+
+ armnn::OptimizerOptions optimizerOptions(false, false, false, false);
+ armnnDelegate::DelegateOptions delegateOptions(backends, optimizerOptions);
+ // Enable serialize to dot by specifying the target file name.
+ delegateOptions.SetSerializeToDot(filename);
+ DelegateOptionTest<float>(::tflite::TensorType_FLOAT32,
+ backends,
+ tensorShape,
+ inputData,
+ inputData,
+ divData,
+ expectedResult,
+ delegateOptions);
+ }
+ CHECK(fs::exists(filename));
+ // The file should have a size greater than 0 bytes.
+ CHECK(fs::file_size(filename) > 0);
+ // Clean up.
+ fs::remove(filename);
+}
+
+
}
} // namespace armnnDelegate