diff --git a/src/ethosu_driver.c b/src/ethosu_driver.c
index 77b8727..d21e0a7 100644
--- a/src/ethosu_driver.c
+++ b/src/ethosu_driver.c
@@ -337,14 +337,13 @@
 
         if (base_addr_size != NULL && base_addr_size[FAST_MEMORY_BASE_ADDR_INDEX] > ethosu_drv.fast_memory_size)
         {
-            LOG_WARN("Fast memory area too small. fast_memory_size=%u, base_addr_size=%u\n",
-                     ethosu_drv.fast_memory_size,
-                     base_addr_size[FAST_MEMORY_BASE_ADDR_INDEX]);
+            LOG_ERR("Fast memory area too small. fast_memory_size=%u, base_addr_size=%u\n",
+                    ethosu_drv.fast_memory_size,
+                    base_addr_size[FAST_MEMORY_BASE_ADDR_INDEX]);
+            return -1;
         }
-        else
-        {
-            *fast_memory = ethosu_drv.fast_memory;
-        }
+
+        *fast_memory = ethosu_drv.fast_memory;
     }
 
     if (ethosu_drv.dev.reset != ethosu_read_reg(&ethosu_drv.dev, NPU_REG_PROT))