diff --git a/src/armnn/Runtime.cpp b/src/armnn/Runtime.cpp
index 824a2b0..bbcbb9f 100644
--- a/src/armnn/Runtime.cpp
+++ b/src/armnn/Runtime.cpp
@@ -265,7 +265,6 @@
 {
     const auto start_time = armnn::GetTimeNow();
     ARMNN_LOG(info) << "ArmNN v" << ARMNN_VERSION << "\n";
-
     if ( options.m_ProfilingOptions.m_TimelineEnabled && !options.m_ProfilingOptions.m_EnableProfiling )
     {
         throw RuntimeException(
@@ -282,13 +281,18 @@
         // Store backend contexts for the supported ones
         try {
             auto factoryFun = BackendRegistryInstance().GetFactory(id);
+            ARMNN_ASSERT(factoryFun != nullptr);
             auto backend = factoryFun();
+            ARMNN_ASSERT(backend != nullptr);
             ARMNN_ASSERT(backend.get() != nullptr);
 
             auto customAllocatorMapIterator = options.m_CustomAllocatorMap.find(id);
             if (customAllocatorMapIterator != options.m_CustomAllocatorMap.end() &&
                 customAllocatorMapIterator->second == nullptr)
             {
+                // We need to manually clean up  the dynamic backends before throwing an exception.
+                DynamicBackendUtils::DeregisterDynamicBackends(m_DeviceSpec.GetDynamicBackends());
+                m_DeviceSpec.ClearDynamicBackends();
                 throw armnn::Exception("Allocator associated with id " + id.Get() + " is null");
             }
 
diff --git a/src/armnn/test/OptimizerTests.cpp b/src/armnn/test/OptimizerTests.cpp
index 38aef67..66da3ad 100644
--- a/src/armnn/test/OptimizerTests.cpp
+++ b/src/armnn/test/OptimizerTests.cpp
@@ -804,6 +804,9 @@
     {
         (*it)->Accept(visitor);
     }
+    // Clean up the registry for the next test.
+    backendRegistry.Deregister("MockBackend");
+    backendRegistry.Deregister("CustomBackend");
 }
 
 // Tests that OptimizeForExclusiveConnections works, fusing when needed, using BatchNorm fusing as example
diff --git a/src/backends/backendsCommon/test/BackendRegistryTests.cpp b/src/backends/backendsCommon/test/BackendRegistryTests.cpp
index ba407d2..ba21b33 100644
--- a/src/backends/backendsCommon/test/BackendRegistryTests.cpp
+++ b/src/backends/backendsCommon/test/BackendRegistryTests.cpp
@@ -141,6 +141,8 @@
         // Caught
         CHECK_EQ(e.what(), exceptionMessage);
     }
+    // Clean up the registry for the next test.
+    BackendRegistryInstance().Deregister(mockBackendId);
 }
 
 }
diff --git a/src/backends/cl/test/ClCustomAllocatorTests.cpp b/src/backends/cl/test/ClCustomAllocatorTests.cpp
index a9b9e41..e614f4c 100644
--- a/src/backends/cl/test/ClCustomAllocatorTests.cpp
+++ b/src/backends/cl/test/ClCustomAllocatorTests.cpp
@@ -181,7 +181,6 @@
     auto customAllocator = std::make_shared<SampleClBackendCustomAllocator>();
     options.m_CustomAllocatorMap = {{"CpuAcc", std::move(customAllocator)}};
     IRuntimePtr run = IRuntime::Create(options);
-
     TensorInfo inputTensorInfo(TensorShape({1, 1}), DataType::Float32);
     INetworkPtr myNetwork = CreateTestNetwork(inputTensorInfo);
 
@@ -191,16 +190,9 @@
     IOptimizedNetworkPtr optNet(nullptr, nullptr);
     std::vector<std::string> errMessages;
 
-    try
-    {
-        optNet = Optimize(*myNetwork, {"CpuAcc"}, run->GetDeviceSpec(), optOptions, errMessages);
-        FAIL("Should have thrown an exception as GetAvailablePreferredBackends() should be empty in Optimize().");
-    }
-    catch (const armnn::InvalidArgumentException& e)
-    {
-        // Different exceptions are thrown on different backends
-    }
-    CHECK(errMessages.size() > 0);
+    CHECK_THROWS_AS_MESSAGE(Optimize(*myNetwork, {"CpuAcc"}, run->GetDeviceSpec(), optOptions, errMessages),
+                            armnn::InvalidArgumentException,
+                            "Expected an exception as GetAvailablePreferredBackends() should be empty in Optimize().");
 
     auto& backendRegistry = armnn::BackendRegistryInstance();
     backendRegistry.DeregisterAllocator(NeonBackend::GetIdStatic());
@@ -208,4 +200,18 @@
 
 #endif
 
+TEST_CASE("ClCustomAllocatorGpuAccNullptrTest")
+{
+    using namespace armnn;
+
+    // Create ArmNN runtime
+    IRuntime::CreationOptions options; // default options
+    auto customAllocator = std::make_shared<SampleClBackendCustomAllocator>();
+    options.m_CustomAllocatorMap = {{"GpuAcc", nullptr}};
+
+    CHECK_THROWS_AS_MESSAGE(IRuntimePtr run = IRuntime::Create(options),
+                            armnn::Exception,
+                            "Expected exception in RuntimeImpl::RuntimeImpl() as allocator was nullptr.");
+}
+
 } // test suite ClCustomAllocatorTests
\ No newline at end of file