IVGCVSW-1709: Clean up memory manager bodge

 * Added quick workaround for the memory management errors in the
   CL/Neon unit tests

Change-Id: I56250db462cdbdc1acc0a5824807d288fb0c1d11
diff --git a/src/backends/aclCommon/common.mk b/src/backends/aclCommon/common.mk
index 67ba01b..cebceaf 100644
--- a/src/backends/aclCommon/common.mk
+++ b/src/backends/aclCommon/common.mk
@@ -9,12 +9,7 @@
 
 COMMON_SOURCES := \
     ArmComputeTensorUtils.cpp \
-    memory/BaseMemoryManager.cpp \
-    memory/BlobLifetimeManager.cpp \
-    memory/BlobMemoryPool.cpp \
-    memory/OffsetLifetimeManager.cpp \
-    memory/OffsetMemoryPool.cpp \
-    memory/PoolManager.cpp
+    BaseMemoryManager.cpp
 
 # COMMON_TEST_SOURCES contains the list of files to be included
 # in the Android unit test build (armnn-tests) and it is picked