Avoid casting to integer of different size

Avoid casting pointers to integer types larger than the pointer itself,
and let assignement handle the conversion.

Change-Id: Icfd2401617f2cd0fc96c2a6a9ad8d2d8bec94c55
diff --git a/src/ethosu_driver.c b/src/ethosu_driver.c
index a5f0bfe..c640f25 100644
--- a/src/ethosu_driver.c
+++ b/src/ethosu_driver.c
@@ -235,7 +235,7 @@
              fast_memory,
              fast_memory_size);
 
-    ethosu_drv.fast_memory      = (uint64_t)fast_memory;
+    ethosu_drv.fast_memory      = (uint32_t)fast_memory;
     ethosu_drv.fast_memory_size = fast_memory_size;
 
     if (ETHOSU_SUCCESS != ethosu_dev_init(&ethosu_drv.dev, base_address))