IVGCVSW-3277 Refactor TensorHandle factory API
* Added backend support for multiple types of TensorHandle factories
* Refactored the backend API to enable new tensor strategies
* Added mechanism to determine memory strategies during optimization
* Perform mem-copy only when Direct access is not found
* Explicitly deleted the copy-constructor from OutputSlot to prevent
accidental local copies that would cause the DisconnectAll to be
called by the destructor
Change-Id: I7e812c8e5e6c1c20db1c5932749ac70fd93db7f8
Signed-off-by: Derek Lamberti <derek.lamberti@arm.com>
Signed-off-by: Matteo Martincigh <matteo.martincigh@arm.com>
diff --git a/src/backends/aclCommon/test/CreateWorkloadClNeon.hpp b/src/backends/aclCommon/test/CreateWorkloadClNeon.hpp
index f544c12..03bcf32 100644
--- a/src/backends/aclCommon/test/CreateWorkloadClNeon.hpp
+++ b/src/backends/aclCommon/test/CreateWorkloadClNeon.hpp
@@ -62,6 +62,7 @@
template<typename IComputeTensorHandle>
void CreateMemCopyWorkloads(IWorkloadFactory& factory)
{
+ TensorHandleFactoryRegistry registry;
Graph graph;
RefWorkloadFactory refFactory;
@@ -79,10 +80,10 @@
Connect(layer1, layer2, tensorInfo);
Connect(layer2, output, tensorInfo);
- input->CreateTensorHandles(graph, refFactory);
- layer1->CreateTensorHandles(graph, factory);
- layer2->CreateTensorHandles(graph, refFactory);
- output->CreateTensorHandles(graph, refFactory);
+ input->CreateTensorHandles(registry, refFactory);
+ layer1->CreateTensorHandles(registry, factory);
+ layer2->CreateTensorHandles(registry, refFactory);
+ output->CreateTensorHandles(registry, refFactory);
// make the workloads and check them
auto workload1 = MakeAndCheckWorkload<CopyMemGenericWorkload>(*layer1, graph, factory);
diff --git a/src/backends/backendsCommon/CMakeLists.txt b/src/backends/backendsCommon/CMakeLists.txt
index e1e387b..bc1c15b 100644
--- a/src/backends/backendsCommon/CMakeLists.txt
+++ b/src/backends/backendsCommon/CMakeLists.txt
@@ -11,6 +11,8 @@
CpuTensorHandle.hpp
IBackendInternal.hpp
IBackendContext.hpp
+ ITensorHandleFactory.cpp
+ ITensorHandleFactory.hpp
LayerSupportBase.cpp
LayerSupportBase.hpp
IMemoryManager.hpp
@@ -22,6 +24,8 @@
OptimizationViews.hpp
OutputHandler.cpp
OutputHandler.hpp
+ TensorHandleFactoryRegistry.cpp
+ TensorHandleFactoryRegistry.hpp
WorkloadDataCollector.hpp
WorkloadData.cpp
WorkloadDataFwd.hpp
diff --git a/src/backends/backendsCommon/IBackendInternal.hpp b/src/backends/backendsCommon/IBackendInternal.hpp
index fe9d620..a0d6569 100644
--- a/src/backends/backendsCommon/IBackendInternal.hpp
+++ b/src/backends/backendsCommon/IBackendInternal.hpp
@@ -10,7 +10,11 @@
#include <ISubgraphViewConverter.hpp>
#include <SubgraphView.hpp>
+#include <optimizations/Optimization.hpp>
+#include "IBackendContext.hpp"
+#include "IMemoryManager.hpp"
+#include "ITensorHandleFactory.hpp"
#include "OptimizationViews.hpp"
#include <vector>
@@ -18,9 +22,7 @@
namespace armnn
{
class IWorkloadFactory;
-class IBackendContext;
class IMemoryManager;
-class Optimization;
class ILayerSupport;
class IBackendInternal : public IBackend
@@ -60,7 +62,10 @@
}
ARMNN_DEPRECATED_MSG("Use \"OptimizationViews OptimizeSubgraphView(const SubgraphView&)\" instead")
- virtual Optimizations GetOptimizations() const = 0;
+ virtual Optimizations GetOptimizations() const
+ {
+ return Optimizations{};
+ }
ARMNN_DEPRECATED_MSG("Use \"OptimizationViews OptimizeSubgraphView(const SubgraphView&)\" instead")
virtual SubGraphUniquePtr OptimizeSubGraph(const SubGraph& subGraph, bool& optimizationAttempted) const
@@ -70,12 +75,19 @@
}
ARMNN_NO_DEPRECATE_WARN_END
- virtual IMemoryManagerUniquePtr CreateMemoryManager() const = 0;
+
+ virtual IMemoryManagerUniquePtr CreateMemoryManager() const
+ {
+ return IMemoryManagerUniquePtr();
+ };
virtual IWorkloadFactoryPtr CreateWorkloadFactory(
const IMemoryManagerSharedPtr& memoryManager = nullptr) const = 0;
- virtual IBackendContextPtr CreateBackendContext(const IRuntime::CreationOptions&) const = 0;
+ virtual IBackendContextPtr CreateBackendContext(const IRuntime::CreationOptions&) const
+ {
+ return IBackendContextPtr{};
+ }
virtual ILayerSupportSharedPtr GetLayerSupport() const = 0;
@@ -107,6 +119,29 @@
}
return result;
}
+
+ bool SupportsTensorAllocatorAPI() const { return GetHandleFactoryPreferences().empty() == false; }
+
+ ITensorHandleFactory::FactoryId GetBackwardCompatibleFavoriteHandleFactory()
+ {
+ auto favorites = GetHandleFactoryPreferences();
+ if (favorites.empty())
+ {
+ return ITensorHandleFactory::LegacyFactoryId;
+ }
+ return favorites[0];
+ }
+
+ /// (Optional) Returns a vector of supported TensorHandleFactory ids in preference order.
+ virtual std::vector<ITensorHandleFactory::FactoryId> GetHandleFactoryPreferences() const
+ {
+ return std::vector<ITensorHandleFactory::FactoryId>();
+ }
+
+ /// (Optional) Register TensorHandleFactories
+ /// Either this method or CreateMemoryManager() and
+ /// IWorkloadFactory::CreateTensor()/IWorkloadFactory::CreateSubtensor() methods must be implemented.
+ virtual void RegisterTensorHandleFactories(class TensorHandleFactoryRegistry& registry) {}
};
using IBackendInternalUniquePtr = std::unique_ptr<IBackendInternal>;
diff --git a/src/backends/backendsCommon/ITensorHandleFactory.cpp b/src/backends/backendsCommon/ITensorHandleFactory.cpp
new file mode 100644
index 0000000..91f5692
--- /dev/null
+++ b/src/backends/backendsCommon/ITensorHandleFactory.cpp
@@ -0,0 +1,14 @@
+//
+// Copyright © 2017 Arm Ltd. All rights reserved.
+// SPDX-License-Identifier: MIT
+//
+
+#include "ITensorHandleFactory.hpp"
+
+namespace armnn
+{
+
+const ITensorHandleFactory::FactoryId ITensorHandleFactory::LegacyFactoryId = "armnn_legacy_factory";
+const ITensorHandleFactory::FactoryId ITensorHandleFactory::DeferredFactoryId = "armnn_deferred_factory";
+
+} // namespace armnn
diff --git a/src/backends/backendsCommon/ITensorHandleFactory.hpp b/src/backends/backendsCommon/ITensorHandleFactory.hpp
new file mode 100644
index 0000000..7685061
--- /dev/null
+++ b/src/backends/backendsCommon/ITensorHandleFactory.hpp
@@ -0,0 +1,49 @@
+//
+// Copyright © 2017 Arm Ltd. All rights reserved.
+// SPDX-License-Identifier: MIT
+//
+
+#pragma once
+
+#include <armnn/Types.hpp>
+#include <armnn/IRuntime.hpp>
+
+namespace armnn
+{
+
+class ITensorHandleFactory
+{
+public:
+ using FactoryId = std::string;
+ static const FactoryId LegacyFactoryId; // Use the workload factory to create the tensor handle
+ static const FactoryId DeferredFactoryId; // Some TensorHandleFactory decisions are deferred to run-time
+
+ virtual ~ITensorHandleFactory() {}
+
+
+ virtual std::unique_ptr<ITensorHandle> CreateSubTensorHandle(ITensorHandle& parent,
+ TensorShape const& subTensorShape,
+ unsigned int const* subTensorOrigin) const = 0;
+
+ virtual std::unique_ptr<ITensorHandle> CreateTensorHandle(const TensorInfo& tensorInfo) const = 0;
+
+ virtual const FactoryId GetId() const = 0;
+
+ virtual bool SupportsSubTensors() const = 0;
+
+ virtual bool SupportsMapUnmap() const final { return true; }
+
+ virtual bool SupportsExport() const final { return false; }
+
+ virtual bool SupportsImport() const final { return false; }
+};
+
+enum class MemoryStrategy
+{
+ Undefined,
+ DirectCompatibility, // Only allocate the tensorhandle using the assigned factory
+ CopyToTarget, // Default + Insert MemCopy node before target
+ ExportToTarget, // Default + Insert Import node
+};
+
+} //namespace armnn
diff --git a/src/backends/backendsCommon/OutputHandler.cpp b/src/backends/backendsCommon/OutputHandler.cpp
index 2df2fb5..8f4942d 100644
--- a/src/backends/backendsCommon/OutputHandler.cpp
+++ b/src/backends/backendsCommon/OutputHandler.cpp
@@ -27,9 +27,9 @@
m_TensorHandle = factory.CreateTensorHandle(m_TensorInfo);
}
-void OutputHandler::CreateTensorHandles(const IWorkloadFactory& factory, DataLayout dataLayout)
+void OutputHandler::CreateTensorHandles(const ITensorHandleFactory& factory)
{
- m_TensorHandle = factory.CreateTensorHandle(m_TensorInfo, dataLayout);
+ m_TensorHandle = factory.CreateTensorHandle(m_TensorInfo);
}
void OutputHandler::CollectWorkloadOutputs(WorkloadDataCollector& dataCollector) const
diff --git a/src/backends/backendsCommon/OutputHandler.hpp b/src/backends/backendsCommon/OutputHandler.hpp
index 240b369..01e255d 100644
--- a/src/backends/backendsCommon/OutputHandler.hpp
+++ b/src/backends/backendsCommon/OutputHandler.hpp
@@ -5,6 +5,7 @@
#pragma once
#include "ITensorHandle.hpp"
+#include "ITensorHandleFactory.hpp"
#include <armnn/Descriptors.hpp>
#include <armnn/INetwork.hpp>
@@ -35,14 +36,10 @@
/// @param tensorInfo - TensorInfo for the output.
void SetTensorInfo(const TensorInfo& tensorInfo);
- /// @brief - Creates tensor handlers used by the intermediate tensors. Does not allocate memory.
+ /// @brief - Creates tensor handles used by the intermediate tensors. Does not allocate memory.
/// @param factory - Factory to be used for handler creation.
void CreateTensorHandles(const IWorkloadFactory& factory);
-
- /// @brief - Creates tensor handlers used by the intermediate tensors. Does not allocate memory.
- /// @param factory - Factory to be used for handler creation.
- /// @param dataLayout - Data Layout to be used for handler creation.
- void CreateTensorHandles(const IWorkloadFactory& factory, DataLayout dataLayout);
+ void CreateTensorHandles(const ITensorHandleFactory& factory);
/// @brief - Gets the matching TensorInfo for the output.
/// @return - References to the output TensorInfo.
diff --git a/src/backends/backendsCommon/TensorHandleFactoryRegistry.cpp b/src/backends/backendsCommon/TensorHandleFactoryRegistry.cpp
new file mode 100644
index 0000000..4692b9f
--- /dev/null
+++ b/src/backends/backendsCommon/TensorHandleFactoryRegistry.cpp
@@ -0,0 +1,69 @@
+//
+// Copyright © 2017 Arm Ltd. All rights reserved.
+// SPDX-License-Identifier: MIT
+//
+
+#include "TensorHandleFactoryRegistry.hpp"
+#include "IMemoryManager.hpp"
+
+
+namespace armnn
+{
+
+void TensorHandleFactoryRegistry::RegisterFactory(std::unique_ptr <ITensorHandleFactory> newFactory)
+{
+ if (!newFactory)
+ {
+ return;
+ }
+
+ ITensorHandleFactory::FactoryId id = newFactory->GetId();
+
+ // Don't register duplicates
+ for (auto& registeredFactory : m_Factories)
+ {
+ if (id == registeredFactory->GetId())
+ {
+ return;
+ }
+ }
+
+ // Take ownership of the new allocator
+ m_Factories.push_back(std::move(newFactory));
+}
+
+void TensorHandleFactoryRegistry::RegisterMemoryManager(std::shared_ptr<armnn::IMemoryManager> memoryManger)
+{
+ m_MemoryManagers.push_back(memoryManger);
+}
+
+ITensorHandleFactory* TensorHandleFactoryRegistry::GetFactory(ITensorHandleFactory::FactoryId id) const
+{
+ for (auto& factory : m_Factories)
+ {
+ if (factory->GetId() == id)
+ {
+ return factory.get();
+ }
+ }
+
+ return nullptr;
+}
+
+void TensorHandleFactoryRegistry::AquireMemory()
+{
+ for (auto& mgr : m_MemoryManagers)
+ {
+ mgr->Acquire();
+ }
+}
+
+void TensorHandleFactoryRegistry::ReleaseMemory()
+{
+ for (auto& mgr : m_MemoryManagers)
+ {
+ mgr->Release();
+ }
+}
+
+} // namespace armnn
diff --git a/src/backends/backendsCommon/TensorHandleFactoryRegistry.hpp b/src/backends/backendsCommon/TensorHandleFactoryRegistry.hpp
new file mode 100644
index 0000000..9e02985
--- /dev/null
+++ b/src/backends/backendsCommon/TensorHandleFactoryRegistry.hpp
@@ -0,0 +1,49 @@
+//
+// Copyright © 2017 Arm Ltd. All rights reserved.
+// SPDX-License-Identifier: MIT
+//
+
+#pragma once
+
+#include "ITensorHandleFactory.hpp"
+
+#include <memory>
+#include <vector>
+
+namespace armnn
+{
+
+//Forward
+class IMemoryManager;
+
+///
+class TensorHandleFactoryRegistry
+{
+public:
+ TensorHandleFactoryRegistry() = default;
+
+ TensorHandleFactoryRegistry(const TensorHandleFactoryRegistry& other) = delete;
+ TensorHandleFactoryRegistry(TensorHandleFactoryRegistry&& other) = delete;
+
+ /// Register a TensorHandleFactory and transfer ownership
+ void RegisterFactory(std::unique_ptr<ITensorHandleFactory> allocator);
+
+ /// Register a memory manager with shared ownership
+ void RegisterMemoryManager(std::shared_ptr<IMemoryManager> memoryManger);
+
+ /// Find a TensorHandleFactory by Id
+ /// Returns nullptr if not found
+ ITensorHandleFactory* GetFactory(ITensorHandleFactory::FactoryId id) const;
+
+ /// Aquire memory required for inference
+ void AquireMemory();
+
+ /// Release memory required for inference
+ void ReleaseMemory();
+
+private:
+ std::vector<std::unique_ptr<ITensorHandleFactory>> m_Factories;
+ std::vector<std::shared_ptr<IMemoryManager>> m_MemoryManagers;
+};
+
+} // namespace armnn
diff --git a/src/backends/backendsCommon/common.mk b/src/backends/backendsCommon/common.mk
index 90d3d16..8df5ab9 100644
--- a/src/backends/backendsCommon/common.mk
+++ b/src/backends/backendsCommon/common.mk
@@ -10,10 +10,12 @@
COMMON_SOURCES := \
BackendRegistry.cpp \
CpuTensorHandle.cpp \
+ ITensorHandleFactory.cpp \
LayerSupportBase.cpp \
MemCopyWorkload.cpp \
OptimizationViews.cpp \
OutputHandler.cpp \
+ TensorHandleFactoryRegistry.cpp \
WorkloadData.cpp \
WorkloadFactory.cpp \
WorkloadUtils.cpp