IVGCVSW-1709: Clean up memory manager bodge

 * Added quick workaround for the memory management errors in the
   CL/Neon unit tests

Change-Id: I56250db462cdbdc1acc0a5824807d288fb0c1d11
diff --git a/src/backends/aclCommon/CMakeLists.txt b/src/backends/aclCommon/CMakeLists.txt
index 83c7c87..933d7ab 100644
--- a/src/backends/aclCommon/CMakeLists.txt
+++ b/src/backends/aclCommon/CMakeLists.txt
@@ -7,20 +7,8 @@
     ArmComputeTensorUtils.hpp
     ArmComputeTensorUtils.cpp
     ArmComputeUtils.hpp
-    memory/BaseMemoryManager.cpp
-    memory/BaseMemoryManager.hpp
-    memory/BlobLifetimeManager.cpp
-    memory/BlobLifetimeManager.hpp
-    memory/BlobMemoryPool.cpp
-    memory/BlobMemoryPool.hpp
-    memory/IMemoryPool.hpp
-    memory/IPoolManager.hpp
-    memory/OffsetLifetimeManager.cpp
-    memory/OffsetLifetimeManager.hpp
-    memory/OffsetMemoryPool.cpp
-    memory/OffsetMemoryPool.hpp
-    memory/PoolManager.cpp
-    memory/PoolManager.hpp
+    BaseMemoryManager.cpp
+    BaseMemoryManager.hpp
 )
 
 if(BUILD_UNIT_TESTS)