IVGCVSW-5221 Fix dynamic backend build failure

Signed-off-by: Narumol Prangnawarat <narumol.prangnawarat@arm.com>
Change-Id: I0c007db23cd703f7c37c120ef177d67476882c34
diff --git a/src/backends/neon/test/CMakeLists.txt b/src/backends/neon/test/CMakeLists.txt
index dd13b63..a821df7 100644
--- a/src/backends/neon/test/CMakeLists.txt
+++ b/src/backends/neon/test/CMakeLists.txt
@@ -6,7 +6,6 @@
 list(APPEND armnnNeonBackendUnitTests_sources
     NeonCreateWorkloadTests.cpp
     NeonEndToEndTests.cpp
-    NeonFallbackTests.cpp
     NeonJsonPrinterTests.cpp
     NeonLayerSupportTests.cpp
     NeonLayerTests.cpp
@@ -19,6 +18,7 @@
 
 if (ARMNNREF)
     list(APPEND armnnNeonBackendUnitTests_sources
+        NeonFallbackTests.cpp
         NeonMemCopyTests.cpp
         )
 endif()