diff --git a/Android.mk b/Android.mk
index 87fec7b..25ed834 100644
--- a/Android.mk
+++ b/Android.mk
@@ -121,13 +121,7 @@
         src/armnn/Tensor.cpp \
         src/armnn/Utils.cpp \
         src/armnn/LayerSupport.cpp \
-        src/armnn/Observable.cpp \
-        src/armnn/memory/BaseMemoryManager.cpp \
-        src/armnn/memory/BlobLifetimeManager.cpp \
-        src/armnn/memory/BlobMemoryPool.cpp \
-        src/armnn/memory/OffsetLifetimeManager.cpp \
-        src/armnn/memory/OffsetMemoryPool.cpp \
-        src/armnn/memory/PoolManager.cpp
+        src/armnn/Observable.cpp
 
 LOCAL_STATIC_LIBRARIES := \
 	armnn-arm_compute \
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 1956643..22eeedd 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -277,23 +277,6 @@
     src/armnn/Observable.cpp
     )
 
-if(ARMCOMPUTENEON OR ARMCOMPUTECL)
-    list(APPEND armnn_sources
-        src/armnn/memory/IMemoryPool.hpp
-        src/armnn/memory/BlobMemoryPool.cpp
-        src/armnn/memory/BlobMemoryPool.hpp
-        src/armnn/memory/BlobLifetimeManager.cpp
-        src/armnn/memory/BlobLifetimeManager.hpp
-        src/armnn/memory/PoolManager.cpp
-        src/armnn/memory/PoolManager.hpp
-        src/armnn/memory/BaseMemoryManager.hpp
-        src/armnn/memory/BaseMemoryManager.cpp
-        src/armnn/memory/OffsetMemoryPool.cpp
-        src/armnn/memory/OffsetMemoryPool.hpp
-        src/armnn/memory/OffsetLifetimeManager.cpp
-        src/armnn/memory/OffsetLifetimeManager.hpp)
-endif()
-
 # Files used for Streamline-based profiling backend
 if(PROFILING_BACKEND_STREAMLINE)
     list(APPEND armnn_sources
diff --git a/src/backends/aclCommon/CMakeLists.txt b/src/backends/aclCommon/CMakeLists.txt
index 6f99f4f..d99b90b 100644
--- a/src/backends/aclCommon/CMakeLists.txt
+++ b/src/backends/aclCommon/CMakeLists.txt
@@ -7,6 +7,20 @@
     ArmComputeTensorUtils.hpp
     ArmComputeTensorUtils.cpp
     ArmComputeUtils.hpp
+    memory/BaseMemoryManager.cpp
+    memory/BaseMemoryManager.hpp
+    memory/BlobLifetimeManager.cpp
+    memory/BlobLifetimeManager.hpp
+    memory/BlobMemoryPool.cpp
+    memory/BlobMemoryPool.hpp
+    memory/IMemoryPool.hpp
+    memory/IPoolManager.hpp
+    memory/OffsetLifetimeManager.cpp
+    memory/OffsetLifetimeManager.hpp
+    memory/OffsetMemoryPool.cpp
+    memory/OffsetMemoryPool.hpp
+    memory/PoolManager.cpp
+    memory/PoolManager.hpp
 )
 
 add_subdirectory(test)
diff --git a/src/backends/aclCommon/common.mk b/src/backends/aclCommon/common.mk
index b371bfa..9854f6f 100644
--- a/src/backends/aclCommon/common.mk
+++ b/src/backends/aclCommon/common.mk
@@ -7,4 +7,11 @@
 # in the Android build and it is picked up by the Android.mk
 # file in the root of ArmNN
 
-COMMON_SOURCES := ArmComputeTensorUtils.cpp
+COMMON_SOURCES := \
+    ArmComputeTensorUtils.cpp \
+    memory/BaseMemoryManager.cpp \
+    memory/BlobLifetimeManager.cpp \
+    memory/BlobMemoryPool.cpp \
+    memory/OffsetLifetimeManager.cpp \
+    memory/OffsetMemoryPool.cpp \
+    memory/PoolManager.cpp
\ No newline at end of file
diff --git a/src/armnn/memory/BaseMemoryManager.cpp b/src/backends/aclCommon/memory/BaseMemoryManager.cpp
similarity index 96%
rename from src/armnn/memory/BaseMemoryManager.cpp
rename to src/backends/aclCommon/memory/BaseMemoryManager.cpp
index 041c042..532692b 100644
--- a/src/armnn/memory/BaseMemoryManager.cpp
+++ b/src/backends/aclCommon/memory/BaseMemoryManager.cpp
@@ -5,9 +5,9 @@
 #include "BaseMemoryManager.hpp"
 
 #if defined(ARMCOMPUTENEON_ENABLED) || defined(ARMCOMPUTECL_ENABLED)
-#include "memory/BlobLifetimeManager.hpp"
-#include "memory/PoolManager.hpp"
-#include "memory/OffsetLifetimeManager.hpp"
+#include "BlobLifetimeManager.hpp"
+#include "PoolManager.hpp"
+#include "OffsetLifetimeManager.hpp"
 #endif
 
 #include <boost/polymorphic_cast.hpp>
diff --git a/src/armnn/memory/BaseMemoryManager.hpp b/src/backends/aclCommon/memory/BaseMemoryManager.hpp
similarity index 99%
rename from src/armnn/memory/BaseMemoryManager.hpp
rename to src/backends/aclCommon/memory/BaseMemoryManager.hpp
index c82eca6..2afc1cb 100644
--- a/src/armnn/memory/BaseMemoryManager.hpp
+++ b/src/backends/aclCommon/memory/BaseMemoryManager.hpp
@@ -101,4 +101,4 @@
 #endif
 };
 
-} //namespace armnn
\ No newline at end of file
+} //namespace armnn
diff --git a/src/armnn/memory/BlobLifetimeManager.cpp b/src/backends/aclCommon/memory/BlobLifetimeManager.cpp
similarity index 97%
rename from src/armnn/memory/BlobLifetimeManager.cpp
rename to src/backends/aclCommon/memory/BlobLifetimeManager.cpp
index fb8080c..41100e9 100644
--- a/src/armnn/memory/BlobLifetimeManager.cpp
+++ b/src/backends/aclCommon/memory/BlobLifetimeManager.cpp
@@ -5,7 +5,7 @@
 #include "BlobLifetimeManager.hpp"
 #include "BlobMemoryPool.hpp"
 
-#include "arm_compute/runtime/IMemoryGroup.h"
+#include <arm_compute/runtime/IMemoryGroup.h>
 
 #include "boost/assert.hpp"
 
diff --git a/src/armnn/memory/BlobLifetimeManager.hpp b/src/backends/aclCommon/memory/BlobLifetimeManager.hpp
similarity index 92%
rename from src/armnn/memory/BlobLifetimeManager.hpp
rename to src/backends/aclCommon/memory/BlobLifetimeManager.hpp
index 7c93985..d777ba5 100644
--- a/src/armnn/memory/BlobLifetimeManager.hpp
+++ b/src/backends/aclCommon/memory/BlobLifetimeManager.hpp
@@ -4,7 +4,7 @@
 //
 #pragma once
 
-#include "arm_compute/runtime/ISimpleLifetimeManager.h"
+#include <arm_compute/runtime/ISimpleLifetimeManager.h>
 
 namespace armnn
 {
diff --git a/src/armnn/memory/BlobMemoryPool.cpp b/src/backends/aclCommon/memory/BlobMemoryPool.cpp
similarity index 100%
rename from src/armnn/memory/BlobMemoryPool.cpp
rename to src/backends/aclCommon/memory/BlobMemoryPool.cpp
diff --git a/src/armnn/memory/BlobMemoryPool.hpp b/src/backends/aclCommon/memory/BlobMemoryPool.hpp
similarity index 93%
rename from src/armnn/memory/BlobMemoryPool.hpp
rename to src/backends/aclCommon/memory/BlobMemoryPool.hpp
index 26f2ccb..4d42e6e 100644
--- a/src/armnn/memory/BlobMemoryPool.hpp
+++ b/src/backends/aclCommon/memory/BlobMemoryPool.hpp
@@ -6,8 +6,8 @@
 
 #include "IMemoryPool.hpp"
 
-#include "arm_compute/runtime/IAllocator.h"
-#include "arm_compute/runtime/Types.h"
+#include <arm_compute/runtime/IAllocator.h>
+#include <arm_compute/runtime/Types.h>
 
 namespace armnn
 {
diff --git a/src/armnn/memory/IMemoryPool.hpp b/src/backends/aclCommon/memory/IMemoryPool.hpp
similarity index 89%
rename from src/armnn/memory/IMemoryPool.hpp
rename to src/backends/aclCommon/memory/IMemoryPool.hpp
index e55c915..9913069 100644
--- a/src/armnn/memory/IMemoryPool.hpp
+++ b/src/backends/aclCommon/memory/IMemoryPool.hpp
@@ -4,7 +4,7 @@
 //
 #pragma once
 
-#include "arm_compute/runtime/IMemoryPool.h"
+#include <arm_compute/runtime/IMemoryPool.h>
 
 namespace armnn
 {
diff --git a/src/armnn/memory/IPoolManager.hpp b/src/backends/aclCommon/memory/IPoolManager.hpp
similarity index 89%
rename from src/armnn/memory/IPoolManager.hpp
rename to src/backends/aclCommon/memory/IPoolManager.hpp
index 4fbca77..be15fdd 100644
--- a/src/armnn/memory/IPoolManager.hpp
+++ b/src/backends/aclCommon/memory/IPoolManager.hpp
@@ -4,7 +4,7 @@
 //
 #pragma once
 
-#include "arm_compute/runtime/IPoolManager.h"
+#include <arm_compute/runtime/IPoolManager.h>
 
 namespace armnn
 {
diff --git a/src/armnn/memory/OffsetLifetimeManager.cpp b/src/backends/aclCommon/memory/OffsetLifetimeManager.cpp
similarity index 95%
rename from src/armnn/memory/OffsetLifetimeManager.cpp
rename to src/backends/aclCommon/memory/OffsetLifetimeManager.cpp
index 6d47972..d0174f8 100644
--- a/src/armnn/memory/OffsetLifetimeManager.cpp
+++ b/src/backends/aclCommon/memory/OffsetLifetimeManager.cpp
@@ -5,11 +5,11 @@
 #include "OffsetLifetimeManager.hpp"
 #include "OffsetMemoryPool.hpp"
 
-#include "arm_compute/runtime/IMemoryGroup.h"
+#include <arm_compute/runtime/IMemoryGroup.h>
 
 #include <numeric>
 
-#include "boost/assert.hpp"
+#include <boost/assert.hpp>
 
 namespace armnn
 {
diff --git a/src/armnn/memory/OffsetLifetimeManager.hpp b/src/backends/aclCommon/memory/OffsetLifetimeManager.hpp
similarity index 93%
rename from src/armnn/memory/OffsetLifetimeManager.hpp
rename to src/backends/aclCommon/memory/OffsetLifetimeManager.hpp
index ab4de77..1283193 100644
--- a/src/armnn/memory/OffsetLifetimeManager.hpp
+++ b/src/backends/aclCommon/memory/OffsetLifetimeManager.hpp
@@ -4,7 +4,7 @@
 //
 #pragma once
 
-#include "arm_compute/runtime/ISimpleLifetimeManager.h"
+#include <arm_compute/runtime/ISimpleLifetimeManager.h>
 
 namespace armnn
 {
diff --git a/src/armnn/memory/OffsetMemoryPool.cpp b/src/backends/aclCommon/memory/OffsetMemoryPool.cpp
similarity index 97%
rename from src/armnn/memory/OffsetMemoryPool.cpp
rename to src/backends/aclCommon/memory/OffsetMemoryPool.cpp
index e5587cf..48bea5e 100644
--- a/src/armnn/memory/OffsetMemoryPool.cpp
+++ b/src/backends/aclCommon/memory/OffsetMemoryPool.cpp
@@ -4,7 +4,7 @@
 //
 #include "OffsetMemoryPool.hpp"
 
-#include "boost/assert.hpp"
+#include <boost/assert.hpp>
 
 #include <algorithm>
 
diff --git a/src/armnn/memory/OffsetMemoryPool.hpp b/src/backends/aclCommon/memory/OffsetMemoryPool.hpp
similarity index 93%
rename from src/armnn/memory/OffsetMemoryPool.hpp
rename to src/backends/aclCommon/memory/OffsetMemoryPool.hpp
index 6843744..25cf8cd 100644
--- a/src/armnn/memory/OffsetMemoryPool.hpp
+++ b/src/backends/aclCommon/memory/OffsetMemoryPool.hpp
@@ -6,8 +6,8 @@
 
 #include "IMemoryPool.hpp"
 
-#include "arm_compute/runtime/IAllocator.h"
-#include "arm_compute/runtime/Types.h"
+#include <arm_compute/runtime/IAllocator.h>
+#include <arm_compute/runtime/Types.h>
 
 namespace armnn
 {
diff --git a/src/armnn/memory/PoolManager.cpp b/src/backends/aclCommon/memory/PoolManager.cpp
similarity index 97%
rename from src/armnn/memory/PoolManager.cpp
rename to src/backends/aclCommon/memory/PoolManager.cpp
index 172c46b..363b459 100644
--- a/src/armnn/memory/PoolManager.cpp
+++ b/src/backends/aclCommon/memory/PoolManager.cpp
@@ -5,8 +5,8 @@
 #include "IMemoryPool.hpp"
 #include "PoolManager.hpp"
 
-#include "boost/assert.hpp"
-#include "boost/polymorphic_cast.hpp"
+#include <boost/assert.hpp>
+#include <boost/polymorphic_cast.hpp>
 
 #include <algorithm>
 
diff --git a/src/armnn/memory/PoolManager.hpp b/src/backends/aclCommon/memory/PoolManager.hpp
similarity index 89%
rename from src/armnn/memory/PoolManager.hpp
rename to src/backends/aclCommon/memory/PoolManager.hpp
index bb84440..39bc665 100644
--- a/src/armnn/memory/PoolManager.hpp
+++ b/src/backends/aclCommon/memory/PoolManager.hpp
@@ -6,10 +6,11 @@
 
 #include "IPoolManager.hpp"
 
-#include "arm_compute/runtime/IMemoryPool.h"
-#include "arm_compute/core/Error.h"
-#include "support/Mutex.h"
-#include "support/Semaphore.h"
+#include <arm_compute/runtime/IMemoryPool.h>
+#include <arm_compute/core/Error.h>
+
+#include <support/Mutex.h>
+#include <support/Semaphore.h>
 
 #include <cstddef>
 #include <list>
diff --git a/src/backends/cl/ClWorkloadFactory.cpp b/src/backends/cl/ClWorkloadFactory.cpp
index 499d13d..68d3713 100644
--- a/src/backends/cl/ClWorkloadFactory.cpp
+++ b/src/backends/cl/ClWorkloadFactory.cpp
@@ -21,7 +21,7 @@
 #include <backends/MemCopyWorkload.hpp>
 #include <backends/cl/ClTensorHandle.hpp>
 
-#include <memory/IPoolManager.hpp>
+#include <backends/aclCommon/memory/IPoolManager.hpp>
 #endif
 
 #include <backends/MakeWorkloadHelper.hpp>
diff --git a/src/backends/cl/ClWorkloadFactory.hpp b/src/backends/cl/ClWorkloadFactory.hpp
index 59ae3b3..9f8ec62 100644
--- a/src/backends/cl/ClWorkloadFactory.hpp
+++ b/src/backends/cl/ClWorkloadFactory.hpp
@@ -4,12 +4,12 @@
 //
 #pragma once
 
-#include <backends/OutputHandler.hpp>
-
 #include <armnn/IRuntime.hpp>
-#include <boost/optional.hpp>
 
-#include "memory/BaseMemoryManager.hpp"
+#include <backends/OutputHandler.hpp>
+#include <backends/aclCommon/memory/BaseMemoryManager.hpp>
+
+#include <boost/optional.hpp>
 
 namespace armnn
 {
diff --git a/src/backends/cl/OpenClTimer.hpp b/src/backends/cl/OpenClTimer.hpp
index a7ae138..0dc9ff7 100644
--- a/src/backends/cl/OpenClTimer.hpp
+++ b/src/backends/cl/OpenClTimer.hpp
@@ -5,7 +5,7 @@
 
 #pragma once
 
-#include "Instrument.hpp"
+#include <armnn/Instrument.hpp>
 
 #include <arm_compute/runtime/CL/CLScheduler.h>
 #include <arm_compute/core/CL/OpenCL.h>
diff --git a/src/backends/neon/NeonWorkloadFactory.cpp b/src/backends/neon/NeonWorkloadFactory.cpp
index 4742ae9..1e8ab1a 100644
--- a/src/backends/neon/NeonWorkloadFactory.cpp
+++ b/src/backends/neon/NeonWorkloadFactory.cpp
@@ -15,7 +15,7 @@
 #include "workloads/NeonWorkloadUtils.hpp"
 #include "workloads/NeonWorkloads.hpp"
 
-#include <memory/IPoolManager.hpp>
+#include <backends/aclCommon/memory/IPoolManager.hpp>
 #endif
 
 #include <backends/MakeWorkloadHelper.hpp>
diff --git a/src/backends/neon/NeonWorkloadFactory.hpp b/src/backends/neon/NeonWorkloadFactory.hpp
index 440bba6..6495161 100644
--- a/src/backends/neon/NeonWorkloadFactory.hpp
+++ b/src/backends/neon/NeonWorkloadFactory.hpp
@@ -5,8 +5,7 @@
 #pragma once
 
 #include <backends/OutputHandler.hpp>
-
-#include <memory/BaseMemoryManager.hpp>
+#include <backends/aclCommon/memory/BaseMemoryManager.hpp>
 
 #include <boost/core/ignore_unused.hpp>
 #include <boost/optional.hpp>