diff --git a/src/backends/neon/test/CMakeLists.txt b/src/backends/neon/test/CMakeLists.txt
index 19512f9..87929fb 100644
--- a/src/backends/neon/test/CMakeLists.txt
+++ b/src/backends/neon/test/CMakeLists.txt
@@ -9,13 +9,18 @@
     NeonJsonPrinterTests.cpp
     NeonLayerSupportTests.cpp
     NeonLayerTests.cpp
-    NeonMemCopyTests.cpp
     NeonOptimizedNetworkTests.cpp
     NeonRuntimeTests.cpp
     NeonTimerTest.cpp
     NeonWorkloadFactoryHelper.hpp
 )
 
+if (ARMCOMPUTEREF)
+    list(APPEND armnnNeonBackendUnitTests_sources
+        NeonMemCopyTests.cpp
+        )
+endif()
+
 add_library(armnnNeonBackendUnitTests OBJECT ${armnnNeonBackendUnitTests_sources})
 target_include_directories(armnnNeonBackendUnitTests PRIVATE ${PROJECT_SOURCE_DIR}/src/armnn)
 target_include_directories(armnnNeonBackendUnitTests PRIVATE ${PROJECT_SOURCE_DIR}/src/armnnUtils)
diff --git a/src/backends/neon/test/NeonCreateWorkloadTests.cpp b/src/backends/neon/test/NeonCreateWorkloadTests.cpp
index 056bfb2..6317ac2 100644
--- a/src/backends/neon/test/NeonCreateWorkloadTests.cpp
+++ b/src/backends/neon/test/NeonCreateWorkloadTests.cpp
@@ -710,6 +710,10 @@
     BOOST_TEST(validDataPointers);
 }
 
+#if defined(ARMCOMPUTEREF_ENABLED)
+
+// This test unit needs the reference backend, it's not available if the reference backend is not built
+
 BOOST_AUTO_TEST_CASE(CreateMemCopyWorkloadsNeon)
 {
     NeonWorkloadFactory factory =
@@ -717,6 +721,8 @@
     CreateMemCopyWorkloads<IAclTensorHandle>(factory);
 }
 
+#endif
+
 template <typename L2NormalizationWorkloadType, typename armnn::DataType DataType>
 static void NeonCreateL2NormalizationWorkloadTest(DataLayout dataLayout)
 {
diff --git a/src/backends/neon/test/NeonEndToEndTests.cpp b/src/backends/neon/test/NeonEndToEndTests.cpp
index 9f94151..bee5b7e 100644
--- a/src/backends/neon/test/NeonEndToEndTests.cpp
+++ b/src/backends/neon/test/NeonEndToEndTests.cpp
@@ -25,6 +25,10 @@
     BOOST_TEST(ConstantUsageFloat32Test(defaultBackends));
 }
 
+#if defined(ARMCOMPUTEREF_ENABLED)
+
+// This test unit needs the reference backend, it's not available if the reference backend is not built
+
 BOOST_AUTO_TEST_CASE(FallbackToCpuRef)
 {
     using namespace armnn;
@@ -59,6 +63,8 @@
     BOOST_TEST(runtime->LoadNetwork(netId, std::move(optNet)) == Status::Success);
 }
 
+#endif
+
 BOOST_AUTO_TEST_CASE(NeonGreaterSimpleEndToEndTest)
 {
     const std::vector<uint8_t> expectedOutput({ 0, 0, 0, 0,  1, 1, 1, 1,
diff --git a/src/backends/neon/test/NeonLayerTests.cpp b/src/backends/neon/test/NeonLayerTests.cpp
index e795174..27f1dd2 100644
--- a/src/backends/neon/test/NeonLayerTests.cpp
+++ b/src/backends/neon/test/NeonLayerTests.cpp
@@ -848,6 +848,11 @@
                      MultiChannelTransposeConvolution2dTest<DataType::QuantisedAsymm8, DataType::Signed32>,
                      DataLayout::NHWC)
 
+#if defined(ARMCOMPUTEREF_ENABLED)
+
+// The ARMNN_COMPARE_REF_AUTO_TEST_CASE and the ARMNN_COMPARE_REF_FIXTURE_TEST_CASE test units are not available
+// if the reference backend is not built
+
 // ============================================================================
 // COMPARE tests
 
@@ -934,4 +939,7 @@
 
 ARMNN_COMPARE_REF_FIXTURE_TEST_CASE(CompareSquareActivationWithReference, ActivationFixture,
                                     CompareActivationTest, ActivationFunction::Square, 5u)
+
+#endif
+
 BOOST_AUTO_TEST_SUITE_END()