IVGCVSW-6896 Fix pre-import when using sync execute.
* Refactor backend capability checks in LoadedNetwork.
* ImportInputs should check the number of tensors does not exceed the
number of inputs.
* In EnqueueWorkload the check for for the count of input tensors
was ignoring pre-imported inputs.
* Added checks to verify ImportInputs/ImportOutputs worked as expected
in EndToEndTestImpl.
* Improve documentation on ImportInputs/ImportOutputs in IRuntime.hpp.
* Disabled import tests in CL and Neon EndToEndTests that cannot work.
Signed-off-by: Colm Donelan <colm.donelan@arm.com>
Change-Id: Iae4b2644a1c9f01ee72bce1afb211661cc9ae2e3
diff --git a/src/armnn/LoadedNetwork.cpp b/src/armnn/LoadedNetwork.cpp
index 5dd7b6c..d55b99e 100644
--- a/src/armnn/LoadedNetwork.cpp
+++ b/src/armnn/LoadedNetwork.cpp
@@ -252,27 +252,23 @@
IBackendInternal* backend = it.first->second.get();
- if (networkProperties.m_AsyncEnabled &&
- !HasCapability(BackendOptions::BackendOption{"AsyncExecution", true}, backend->GetCapabilities()))
+ // If we're doing async execution verify that the backend supports it and ExternallyManagedMemory.
+ if (networkProperties.m_AsyncEnabled)
{
- std::string er = backend->GetId();
- er += " does not support AsyncExecution";
- throw BackendCapabilityException(er);
- }
-
- if (networkProperties.m_AsyncEnabled &&
- !HasCapability(BackendOptions::BackendOption{"ExternallyManagedMemory", true},
+ if (!HasCapability(BackendOptions::BackendOption{"AsyncExecution", true}, backend->GetCapabilities()))
+ {
+ std::string er = backend->GetId();
+ er += " does not support AsyncExecution";
+ throw BackendCapabilityException(er);
+ }
+ if (!HasCapability(BackendOptions::BackendOption{"ExternallyManagedMemory", true},
backend->GetCapabilities()))
- {
- std::string er = backend->GetId();
- er += " does not support ExternallyManagedMemory\n";
- er += "AsyncEnabled networks require all backends to support ExternallyManagedMemory";
- throw BackendCapabilityException(er);
- }
-
- if (HasCapability(BackendOptions::BackendOption{"ExternallyManagedMemory", true},backend->GetCapabilities())
- && (m_NetworkProperties.m_ExternalMemoryManagementEnabled || m_NetworkProperties.m_AsyncEnabled))
- {
+ {
+ std::string er = backend->GetId();
+ er += " does not support ExternallyManagedMemory\n";
+ er += "AsyncEnabled networks require all backends to support ExternallyManagedMemory";
+ throw BackendCapabilityException(er);
+ }
m_SupportsExternallyManagedMemory[backend->GetId()] = true;
useExternalMemoryManager = true;
}
@@ -864,7 +860,9 @@
// Data that must be kept alive for the entire execution of the workload.
WorkloadData workloadData(inputTensors, outputTensors);
- if (graph.GetNumInputs() != inputTensors.size())
+ // Input tensors can be provided as parameters or pre imported. Either way the number of
+ // tensors should match the number of inputs.
+ if (graph.GetNumInputs() != (inputTensors.size() + preImportedInputIds.size()))
{
throw InvalidArgumentException("Number of inputs provided does not match network.");
}
@@ -875,11 +873,6 @@
m_InputQueue.clear();
m_InputQueue.reserve(graph.GetNumInputs());
- if (preImportedInputIds.size() > graph.GetNumInputs())
- {
- throw InvalidArgumentException("Invalid number of preImportedInputIds");
- }
-
unsigned int inputIndex = 0;
unsigned int importedInputIdIndex = 0;
std::sort(preImportedInputIds.begin(), preImportedInputIds.end());
@@ -1437,9 +1430,10 @@
{
throw MemoryImportException("ImportInputs: Memory Import failed, NetworkProperties.m_ImportEnabled");
}
- if (inputTensors.size() != m_OptimizedNetwork->pOptimizedNetworkImpl->GetGraph().GetNumInputs())
+ // The number of pre imported tensors should not exceed the number of inputs.
+ if (inputTensors.size() > m_OptimizedNetwork->pOptimizedNetworkImpl->GetGraph().GetNumInputs())
{
- throw MemoryImportException("ImportInputs: Force Import failed, incorrect number of tensors");
+ throw MemoryImportException("ImportInputs: The number of tensors provided exceeds the number of inputs.");
}
std::vector<ImportedInputId> importedInputs;
diff --git a/src/armnn/test/RuntimeTests.cpp b/src/armnn/test/RuntimeTests.cpp
index 59f6554..e0d3a22 100644
--- a/src/armnn/test/RuntimeTests.cpp
+++ b/src/armnn/test/RuntimeTests.cpp
@@ -1466,4 +1466,70 @@
CHECK(er.find("However, it was enabled when this network was optimized") != -1);
}
+TEST_CASE("SyncExecutePreImportInputsHappyPath")
+{
+ // In this test case we'll mix "Pre Import" and pass by reference tensors as input.
+ //
+ // * Create a small network that takes two inputs.
+ // * Optimize it specifying that the inputs and outputs will not be imported or exported.
+ // * Create some malloc input and output tensors.
+ // * Use ImportInputs to import only one of the two inputs.
+ // * Call EnqueueWorkload passing one input tensor and one reference to a pre-imported tensor.
+
+ armnn::IRuntime::CreationOptions options;
+ armnn::IRuntimePtr runtime(armnn::IRuntime::Create(options));
+ armnn::NetworkId networkId = 1;
+ armnn::INetworkPtr testNetwork(armnn::INetwork::Create());
+
+ auto inputLayer1 = testNetwork->AddInputLayer(0, "input 1 layer");
+ auto inputLayer2 = testNetwork->AddInputLayer(1, "input 2 layer");
+ auto addLayer = testNetwork->AddAdditionLayer("add layer");
+ auto outputLayer = testNetwork->AddOutputLayer(2, "output layer");
+
+ TensorInfo tensorInfo{ { 4 }, armnn::DataType::Signed32 };
+
+ inputLayer1->GetOutputSlot(0).Connect(addLayer->GetInputSlot(0));
+ inputLayer1->GetOutputSlot(0).SetTensorInfo(tensorInfo);
+
+ inputLayer2->GetOutputSlot(0).Connect(addLayer->GetInputSlot(1));
+ inputLayer2->GetOutputSlot(0).SetTensorInfo(tensorInfo);
+
+ addLayer->GetOutputSlot(0).Connect(outputLayer->GetInputSlot(0));
+ addLayer->GetOutputSlot(0).SetTensorInfo(tensorInfo);
+
+ std::vector<armnn::BackendId> backends = { armnn::Compute::CpuRef };
+
+ std::string er;
+ armnn::INetworkProperties networkProperties(false, MemorySource::Undefined, MemorySource::Undefined);
+ runtime->LoadNetwork(networkId, Optimize(*testNetwork, backends, runtime->GetDeviceSpec()), er, networkProperties);
+
+ std::vector<int> inputData1(4, 10);
+ std::vector<int> inputData2(4, 20);
+ std::vector<int> output(4);
+
+ ConstTensor inputTensor1({ { 4 }, armnn::DataType::Signed32, 0.0f, 0, true }, inputData1.data());
+ ConstTensor inputTensor2({ { 4 }, armnn::DataType::Signed32, 0.0f, 0, true }, inputData2.data());
+ Tensor outputTensor({ { 4 }, armnn::DataType::Signed32 }, output.data());
+
+ // An extra check here: the number of inputs provided to ImportInputs should not exceed the number of inputs
+ // to the network.
+ CHECK_THROWS_AS(runtime->ImportInputs(networkId, { { 0, inputTensor1 }, { 0, inputTensor1 }, { 0, inputTensor1 } },
+ MemorySource::Malloc),
+ armnn::MemoryImportException);
+
+ // Pre Import one of the two input tensors.
+ std::vector<ImportedOutputId> importedInputVec =
+ runtime->ImportInputs(networkId, { { 0, inputTensor1 } }, MemorySource::Malloc);
+ CHECK(importedInputVec.size() == 1);
+ CHECK(importedInputVec[0] == 0);
+
+ // We've pre-imported tensor 1 and we'll pass tensor 2 by reference.
+ InputTensors inputTensors{ { 1, inputTensor2 } };
+ OutputTensors outputTensors{ { 2, outputTensor } };
+
+ // Do the inference
+ auto ret = runtime->EnqueueWorkload(networkId, inputTensors, outputTensors, importedInputVec,
+ std::vector<ImportedOutputId>());
+ REQUIRE(ret == Status::Success);
+}
}
diff --git a/src/backends/backendsCommon/test/EndToEndTestImpl.hpp b/src/backends/backendsCommon/test/EndToEndTestImpl.hpp
index cc5aa23..44ae2be 100644
--- a/src/backends/backendsCommon/test/EndToEndTestImpl.hpp
+++ b/src/backends/backendsCommon/test/EndToEndTestImpl.hpp
@@ -887,10 +887,12 @@
runtime->GetProfiler(netId)->EnableProfiling(true);
std::vector<ImportedInputId> importedInputIds =
runtime->ImportInputs(netId, inputTensors, MemorySource::Malloc);
+ CHECK(importedInputIds.size() == 1);
std::vector<ImportedOutputId> importedOutputIds =
runtime->ImportOutputs(netId, outputTensors, MemorySource::Malloc);
+ CHECK(importedOutputIds.size() == 1);
// Do the inference and force the import as the memory is aligned.
- runtime->EnqueueWorkload(netId, inputTensors, outputTensors, importedInputIds, importedOutputIds);
+ runtime->EnqueueWorkload(netId, InputTensors(), OutputTensors(), importedInputIds, importedOutputIds);
// Retrieve the Profiler.Print() output to get the workload execution
ProfilerManager& profilerManager = armnn::ProfilerManager::GetInstance();
@@ -997,11 +999,14 @@
runtime->GetProfiler(netId)->EnableProfiling(true);
std::vector<ImportedInputId> importedInputIds =
runtime->ImportInputs(netId, inputTensors, MemorySource::Malloc);
+ // We expect the import to have failed.
+ CHECK(importedInputIds.size() == 0);
std::vector<ImportedOutputId> importedOutputIds =
runtime->ImportOutputs(netId, outputTensors, MemorySource::Malloc);
+ CHECK(importedOutputIds.size() == 1);
// Do the inference and force the import as the memory is misaligned.
- runtime->EnqueueWorkload(netId, inputTensors, outputTensors, importedInputIds, importedOutputIds);
+ runtime->EnqueueWorkload(netId, inputTensors, OutputTensors(), importedInputIds, importedOutputIds);
// Retrieve the Profiler.Print() output to get the workload execution
ProfilerManager& profilerManager = armnn::ProfilerManager::GetInstance();
@@ -1113,11 +1118,14 @@
runtime->GetProfiler(netId)->EnableProfiling(true);
std::vector<ImportedInputId> importedInputIds =
runtime->ImportInputs(netId, inputTensors, MemorySource::Malloc);
+ CHECK(importedInputIds.size() == 1);
+ // We expect this to fail.
std::vector<ImportedOutputId> importedOutputIds =
runtime->ImportOutputs(netId, outputTensors, MemorySource::Malloc);
+ CHECK(importedOutputIds.size() == 0);
- // Do the inference and force the import as the memory is misaligned.
- runtime->EnqueueWorkload(netId, inputTensors, outputTensors, importedInputIds, importedOutputIds);
+ // Even if importing the output failed we still expect to be able to get it to work.
+ runtime->EnqueueWorkload(netId, InputTensors(), outputTensors, importedInputIds, importedOutputIds);
// Retrieve the Profiler.Print() output to get the workload execution
ProfilerManager& profilerManager = armnn::ProfilerManager::GetInstance();
@@ -1233,8 +1241,12 @@
runtime->GetProfiler(netId)->EnableProfiling(true);
std::vector<ImportedInputId> importedInputIds =
runtime->ImportInputs(netId, inputTensors, MemorySource::Malloc);
+ // Import should have failed.
+ CHECK(importedInputIds.size() == 0);
std::vector<ImportedOutputId> importedOutputIds =
runtime->ImportOutputs(netId, outputTensors, MemorySource::Malloc);
+ // Import should have failed.
+ CHECK(importedOutputIds.size() == 0);
// Do the inference and force the import as the memory is misaligned.
runtime->EnqueueWorkload(netId, inputTensors, outputTensors, importedInputIds, importedOutputIds);
@@ -1339,10 +1351,12 @@
runtime->GetProfiler(netId)->EnableProfiling(true);
std::vector<ImportedInputId> importedInputIds =
runtime->ImportInputs(netId, inputTensors, MemorySource::Malloc);
+ CHECK(importedInputIds.size() == 1);
std::vector<ImportedOutputId> importedOutputIds =
runtime->ImportOutputs(netId, outputTensors, MemorySource::Malloc);
+ CHECK(importedOutputIds.size() == 1);
// Do the inference and force the import as the memory is aligned.
- runtime->EnqueueWorkload(netId, inputTensors, outputTensors, importedInputIds, importedOutputIds);
+ runtime->EnqueueWorkload(netId, InputTensors(), OutputTensors(), importedInputIds, importedOutputIds);
// Retrieve the Profiler.AnalyzeEventsAndWriteResults() output to get the workload execution
ProfilerManager& profilerManager = armnn::ProfilerManager::GetInstance();
@@ -1408,7 +1422,11 @@
{0,armnn::Tensor(runtime->GetOutputTensorInfo(netId, 0), misalignedOutputPtr)}
};
importedInputIds = runtime->ImportInputs(netId, inputTensorsMisaligned, MemorySource::Malloc);
+ // Import should fail.
+ CHECK(importedInputIds.size() == 0);
importedOutputIds = runtime->ImportOutputs(netId, outputTensorsMisaligned, MemorySource::Malloc);
+ // Import should fail.
+ CHECK(importedOutputIds.size() == 0);
// Do the inference and force the import as the memory is misaligned.
runtime->EnqueueWorkload(netId,
@@ -1527,8 +1545,12 @@
runtime->GetProfiler(netId)->EnableProfiling(true);
std::vector<ImportedInputId> importedInputIds =
runtime->ImportInputs(netId, inputTensorsMisaligned, MemorySource::Malloc);
+ // Import should fail.
+ CHECK(importedInputIds.size() == 0);
std::vector<ImportedOutputId> importedOutputIds =
runtime->ImportOutputs(netId, outputTensorsMisaligned, MemorySource::Malloc);
+ // Import should fail.
+ CHECK(importedOutputIds.size() == 0);
// Do the inference and force the import as the memory is misaligned.
runtime->EnqueueWorkload(netId,
@@ -1593,9 +1615,11 @@
};
importedInputIds = runtime->ImportInputs(netId, inputTensors, MemorySource::Malloc);
+ CHECK(importedInputIds.size() == 1);
importedOutputIds = runtime->ImportOutputs(netId, outputTensors, MemorySource::Malloc);
+ CHECK(importedOutputIds.size() == 1);
// Do the inference and force the import as the memory is aligned.
- runtime->EnqueueWorkload(netId, inputTensors, outputTensors, importedInputIds, importedOutputIds);
+ runtime->EnqueueWorkload(netId, InputTensors(), OutputTensors(), importedInputIds, importedOutputIds);
// Retrieve the Profiler.AnalyzeEventsAndWriteResults() output to get the workload execution
// We need to use AnalyzeEventsAndWriteResults here to make sure the second inference has been profiled
diff --git a/src/backends/cl/test/ClEndToEndTests.cpp b/src/backends/cl/test/ClEndToEndTests.cpp
index fa6e027..f28679c 100644
--- a/src/backends/cl/test/ClEndToEndTests.cpp
+++ b/src/backends/cl/test/ClEndToEndTests.cpp
@@ -514,12 +514,18 @@
QLstmEndToEnd(clDefaultBackends);
}
-TEST_CASE("ClForceImportWithMisalignedInputBuffersEndToEndTest")
+TEST_CASE("ClForceImportWithMisalignedInputBuffersEndToEndTest"
+ // Currently, the CL workload for activation does not support tensor handle replacement so this test case
+ // will always fail.
+ * doctest::skip(true))
{
ForceImportWithMisalignedInputBuffersEndToEndTest(clDefaultBackends);
}
-TEST_CASE("ClForceImportWithMisalignedOutputBuffersEndToEndTest")
+TEST_CASE("ClForceImportWithMisalignedOutputBuffersEndToEndTest"
+ // Currently, the CL workload for activation does not support tensor handle replacement so this test case
+ // will always fail.
+ * doctest::skip(true))
{
ForceImportWithMisalignedOutputBuffersEndToEndTest(clDefaultBackends);
}
diff --git a/src/backends/cl/test/ClImportTensorHandleTests.cpp b/src/backends/cl/test/ClImportTensorHandleTests.cpp
index 9a075d2..1198cad 100644
--- a/src/backends/cl/test/ClImportTensorHandleTests.cpp
+++ b/src/backends/cl/test/ClImportTensorHandleTests.cpp
@@ -397,11 +397,14 @@
INFO("Run ImportInputs");
std::vector<ImportedInputId> importedInputIds =
runtime->ImportInputs(netId, inputTensors, MemorySource::Malloc);
+ // We expect the import to have succeeded.
+ CHECK(importedInputIds.size() == 1);
std::vector<ImportedOutputId> importedOutputIds =
runtime->ImportOutputs(netId, outputTensors, MemorySource::Malloc);
-
+ // We expect the import to have succeeded.
+ CHECK(importedOutputIds.size() == 1);
// Do the inference
- runtime->EnqueueWorkload(netId, inputTensors, outputTensors, importedInputIds, importedOutputIds);
+ runtime->EnqueueWorkload(netId, InputTensors(), OutputTensors(), importedInputIds, importedOutputIds);
// Retrieve the Profiler.Print() output to get the workload execution
ProfilerManager& profilerManager = armnn::ProfilerManager::GetInstance();
@@ -536,11 +539,15 @@
INFO("Run ImportInputs");
std::vector<ImportedInputId> importedInputIds =
runtime->ImportInputs(netId, inputTensors, MemorySource::Malloc);
+ // We expect the import to have succeeded.
+ CHECK(importedInputIds.size() == 1);
std::vector<ImportedOutputId> importedOutputIds =
runtime->ImportOutputs(netId, outputTensors, MemorySource::Malloc);
+ // We expect the import to have succeeded.
+ CHECK(importedOutputIds.size() == 1);
// Do the inference
- runtime->EnqueueWorkload(netId, inputTensors, outputTensors, importedInputIds, importedOutputIds);
+ runtime->EnqueueWorkload(netId, InputTensors(), OutputTensors(), importedInputIds, importedOutputIds);
// Retrieve the Profiler.Print() output to get the workload execution
ProfilerManager& profilerManager = armnn::ProfilerManager::GetInstance();
@@ -680,11 +687,15 @@
INFO("Run ImportInputs");
std::vector<ImportedInputId> importedInputIds =
runtime->ImportInputs(netId, inputTensors, MemorySource::Malloc);
+ // We expect the import to have succeeded.
+ CHECK(importedInputIds.size() == 1);
std::vector<ImportedOutputId> importedOutputIds =
runtime->ImportOutputs(netId, outputTensors, MemorySource::Malloc);
+ // We expect the import to have succeeded.
+ CHECK(importedOutputIds.size() == 1);
// Do the inference
- runtime->EnqueueWorkload(netId, inputTensors, outputTensors, importedInputIds, importedOutputIds);
+ runtime->EnqueueWorkload(netId, InputTensors(), OutputTensors(), importedInputIds, importedOutputIds);
// Retrieve the Profiler.Print() output to get the workload execution
ProfilerManager& profilerManager = armnn::ProfilerManager::GetInstance();
@@ -798,11 +809,13 @@
INFO("Run ImportInputs");
std::vector<ImportedInputId> importedInputIds =
runtime->ImportInputs(netId, inputTensors, MemorySource::Malloc);
+ CHECK(importedInputIds.size() == 1);
std::vector<ImportedOutputId> importedOutputIds =
runtime->ImportOutputs(netId, outputTensors, MemorySource::Malloc);
+ CHECK(importedOutputIds.size() == 1);
// Do the inference
- runtime->EnqueueWorkload(netId, inputTensors, outputTensors, importedInputIds, importedOutputIds);
+ runtime->EnqueueWorkload(netId, InputTensors(), OutputTensors(), importedInputIds, importedOutputIds);
// Retrieve the Profiler.Print() output to get the workload execution
ProfilerManager& profilerManager = armnn::ProfilerManager::GetInstance();
@@ -838,7 +851,7 @@
/*
* This is a test to check the functionality of the Forced Import functionality when using repeated inferences that
* require switching from importing to copy. For the first inference we create aligned Pointers and check they are
- * imported correctly. For the second we use similar pointers but don't use PreImporting to force fall back to copy.
+ * imported correctly. For the second we use similar pointers but don't use PreImporting.
*/
// Create runtime in which test will run
IRuntime::CreationOptions options;
@@ -959,11 +972,15 @@
INFO("Run ImportInputs");
std::vector<ImportedInputId> importedInputIds =
runtime->ImportInputs(netId, inputTensors, MemorySource::Malloc);
+ // We expect the import to have succeeded.
+ CHECK(importedInputIds.size() == 1);
std::vector<ImportedOutputId> importedOutputIds =
runtime->ImportOutputs(netId, outputTensors, MemorySource::Malloc);
+ // We expect the import to have succeeded.
+ CHECK(importedOutputIds.size() == 1);
// Do the inference
- runtime->EnqueueWorkload(netId, inputTensors, outputTensors, importedInputIds, importedOutputIds);
+ runtime->EnqueueWorkload(netId, InputTensors(), OutputTensors(), importedInputIds, importedOutputIds);
// Retrieve the Profiler.AnalyzeEventsAndWriteResults() output to get the workload execution
ProfilerManager& profilerManager = armnn::ProfilerManager::GetInstance();
@@ -1246,11 +1263,13 @@
INFO("Run ImportInputs");
std::vector<ImportedInputId> importedInputIds =
runtime->ImportInputs(netId, inputTensorsImport, MemorySource::Malloc);
+ CHECK(importedInputIds.size() == 1);
std::vector<ImportedOutputId> importedOutputIds =
runtime->ImportOutputs(netId, outputTensorsImport, MemorySource::Malloc);
+ CHECK(importedOutputIds.size() == 1);
// Do the inference with pre-imported inputs/outputs
- runtime->EnqueueWorkload(netId, inputTensorsImport, outputTensorsImport, importedInputIds, importedOutputIds);
+ runtime->EnqueueWorkload(netId, InputTensors(), OutputTensors(), importedInputIds, importedOutputIds);
// Sync the outputs so we can read the data
arm_compute::CLScheduler::get().sync();
diff --git a/src/backends/neon/test/NeonEndToEndTests.cpp b/src/backends/neon/test/NeonEndToEndTests.cpp
index ff13fb0..d680e6d 100644
--- a/src/backends/neon/test/NeonEndToEndTests.cpp
+++ b/src/backends/neon/test/NeonEndToEndTests.cpp
@@ -568,17 +568,26 @@
StridedSliceInvalidSliceEndToEndTest(neonDefaultBackends);
}
-TEST_CASE("NeonForceImportWithAlignedBuffersEndToEndTest")
+TEST_CASE("NeonForceImportWithAlignedBuffersEndToEndTest"
+ // Currently, the Neon workload for activation does not support tensor handle replacement so this test case
+ // will always fail.
+ * doctest::skip(true))
{
ForceImportWithAlignedBuffersEndToEndTest(neonDefaultBackends);
}
-TEST_CASE("NeonForceImportWithMisalignedInputBuffersEndToEndTest")
+TEST_CASE("NeonForceImportWithMisalignedInputBuffersEndToEndTest"
+ // Currently, the Neon workload for activation does not support tensor handle replacement so this test case
+ // will always fail.
+ * doctest::skip(true))
{
ForceImportWithMisalignedInputBuffersEndToEndTest(neonDefaultBackends);
}
-TEST_CASE("NeonForceImportWithMisalignedOutputBuffersEndToEndTest")
+TEST_CASE("NeonForceImportWithMisalignedOutputBuffersEndToEndTest"
+ // Currently, the Neon workload for activation does not support tensor handle replacement so this test case
+ // will always fail.
+ * doctest::skip(true))
{
ForceImportWithMisalignedOutputBuffersEndToEndTest(neonDefaultBackends);
}