Split DMA memory and buffer setup in kernel driver
To allow the NPU kernel driver to allocate and use DMA memory internally
without creating a buffer instance, the DMA memory management has been
split out from the buffer code.
Change-Id: I46fdeee51b5ef786a54b8e7c866d137d91222724
Signed-off-by: Mikael Olsson <mikael.olsson@arm.com>
diff --git a/kernel/Kbuild b/kernel/Kbuild
index a83400f..98bece3 100644
--- a/kernel/Kbuild
+++ b/kernel/Kbuild
@@ -23,6 +23,7 @@
ethosu-objs := ethosu_driver.o \
ethosu_buffer.o \
+ ethosu_dma_mem.o \
ethosu_device.o \
ethosu_inference.o \
ethosu_mailbox.o \
diff --git a/kernel/ethosu_buffer.c b/kernel/ethosu_buffer.c
index bf7d745..69b5007 100644
--- a/kernel/ethosu_buffer.c
+++ b/kernel/ethosu_buffer.c
@@ -24,6 +24,7 @@
#include "ethosu_buffer.h"
#include "ethosu_device.h"
+#include "ethosu_dma_mem.h"
#include "uapi/ethosu.h"
#include <linux/anon_inodes.h>
@@ -71,9 +72,7 @@
dev_dbg(dev, "Buffer destroy. buf=0x%pK", buf);
- memset(buf->cpu_addr, 0, buf->size);
- dma_free_coherent(dev, buf->size, buf->cpu_addr,
- buf->dma_addr);
+ ethosu_dma_mem_free(&buf->dma_mem);
memset(buf, 0, sizeof(*buf));
devm_kfree(dev, buf);
@@ -103,8 +102,8 @@
dev_dbg(dev, "Buffer mmap. file=0x%pK, buf=0x%pK\n",
file, buf);
- ret = dma_mmap_coherent(dev, vma, buf->cpu_addr,
- buf->dma_addr, buf->size);
+ ret = dma_mmap_coherent(dev, vma, buf->dma_mem->cpu_addr,
+ buf->dma_mem->dma_addr, buf->dma_mem->size);
return ret;
}
@@ -126,7 +125,7 @@
*/
switch (whence) {
case SEEK_END:
- return buf->size;
+ return buf->dma_mem->size;
case SEEK_SET:
return 0;
default:
@@ -148,13 +147,13 @@
return -ENOMEM;
buf->dev = dev;
- buf->size = size;
kref_init(&buf->kref);
- buf->cpu_addr = dma_alloc_coherent(dev, size, &buf->dma_addr,
- GFP_KERNEL);
- if (!buf->cpu_addr)
+ buf->dma_mem = ethosu_dma_mem_alloc(dev, size);
+ if (IS_ERR(buf->dma_mem)) {
+ ret = PTR_ERR(buf->dma_mem);
goto free_buf;
+ }
ret = anon_inode_getfd("ethosu-buffer", ðosu_buffer_fops, buf,
O_RDWR | O_CLOEXEC);
@@ -168,14 +167,13 @@
dev_dbg(dev,
"Buffer create. file=0x%pK, fd=%d, buf=0x%pK, size=%zu, cpu_addr=0x%pK, dma_addr=0x%llx, phys_addr=0x%llx\n",
- buf->file, ret, buf, size, buf->cpu_addr, buf->dma_addr,
- virt_to_phys(buf->cpu_addr));
+ buf->file, ret, buf, size, buf->dma_mem->cpu_addr,
+ buf->dma_mem->dma_addr, virt_to_phys(buf->dma_mem->cpu_addr));
return ret;
free_dma:
- dma_free_coherent(dev, buf->size, buf->cpu_addr,
- buf->dma_addr);
+ ethosu_dma_mem_free(&buf->dma_mem);
free_buf:
memset(buf, 0, sizeof(*buf));
diff --git a/kernel/ethosu_buffer.h b/kernel/ethosu_buffer.h
index 1829fbe..8bef2d2 100644
--- a/kernel/ethosu_buffer.h
+++ b/kernel/ethosu_buffer.h
@@ -31,25 +31,22 @@
* Types
****************************************************************************/
+struct ethosu_dma_mem;
struct ethosu_device;
struct device;
/**
- * struct ethosu_buffer - Buffer
- * @dev: Device
- * @file: File
- * @kref: Reference counting
- * @size: Size of the buffer
- * @cpu_addr: Kernel mapped address
- * @dma_addr: DMA address
+ * struct ethosu_buffer - User data buffer
+ * @dev: Device
+ * @file: File
+ * @kref: Reference counting
+ * @dma_mem: DMA memory allocated for the buffer
*/
struct ethosu_buffer {
- struct device *dev;
- struct file *file;
- struct kref kref;
- size_t size;
- void *cpu_addr;
- dma_addr_t dma_addr;
+ struct device *dev;
+ struct file *file;
+ struct kref kref;
+ struct ethosu_dma_mem *dma_mem;
};
/****************************************************************************
diff --git a/kernel/ethosu_dma_mem.c b/kernel/ethosu_dma_mem.c
new file mode 100644
index 0000000..cb63081
--- /dev/null
+++ b/kernel/ethosu_dma_mem.c
@@ -0,0 +1,77 @@
+/*
+ * SPDX-FileCopyrightText: Copyright 2023 Arm Limited and/or its affiliates <open-source-office@arm.com>
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, you can access it online at
+ * http://www.gnu.org/licenses/gpl-2.0.html.
+ */
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include "ethosu_dma_mem.h"
+
+#include <linux/err.h>
+#include <linux/dma-mapping.h>
+
+/****************************************************************************
+ * Functions
+ ****************************************************************************/
+
+struct ethosu_dma_mem *ethosu_dma_mem_alloc(struct device *dev,
+ size_t size)
+{
+ struct ethosu_dma_mem *dma_mem;
+
+ if (!size)
+ return ERR_PTR(-EINVAL);
+
+ dma_mem = devm_kzalloc(dev, sizeof(*dma_mem), GFP_KERNEL);
+ if (!dma_mem)
+ return ERR_PTR(-ENOMEM);
+
+ dma_mem->dev = dev;
+ dma_mem->size = size;
+ dma_mem->cpu_addr = dma_alloc_coherent(dev, size, &dma_mem->dma_addr,
+ GFP_KERNEL);
+ if (!dma_mem->cpu_addr) {
+ memset(dma_mem, 0, sizeof(*dma_mem));
+ devm_kfree(dev, dma_mem);
+
+ return ERR_PTR(-ENOMEM);
+ }
+
+ return dma_mem;
+}
+
+void ethosu_dma_mem_free(struct ethosu_dma_mem **dma_mem)
+{
+ struct device *dev;
+ struct ethosu_dma_mem *mem;
+
+ if (!dma_mem || !*dma_mem)
+ return;
+
+ mem = *dma_mem;
+ dev = mem->dev;
+
+ memset(mem->cpu_addr, 0, mem->size);
+ dma_free_coherent(dev, mem->size, mem->cpu_addr, mem->dma_addr);
+
+ memset(mem, 0, sizeof(*mem));
+ devm_kfree(dev, mem);
+
+ *dma_mem = NULL;
+}
diff --git a/kernel/ethosu_dma_mem.h b/kernel/ethosu_dma_mem.h
new file mode 100644
index 0000000..11cdd39
--- /dev/null
+++ b/kernel/ethosu_dma_mem.h
@@ -0,0 +1,58 @@
+/*
+ * SPDX-FileCopyrightText: Copyright 2023 Arm Limited and/or its affiliates <open-source-office@arm.com>
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, you can access it online at
+ * http://www.gnu.org/licenses/gpl-2.0.html.
+ */
+
+#ifndef ETHOSU_DMA_MEM_H
+#define ETHOSU_DMA_MEM_H
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include <linux/types.h>
+
+/****************************************************************************
+ * Types
+ ****************************************************************************/
+
+struct device;
+
+/**
+ * struct ethosu_dma_mem - DMA memory allocation
+ * @dev: Device
+ * @size: Size of the allocation
+ * @cpu_addr: Kernel mapped address
+ * @dma_addr: DMA address
+ */
+struct ethosu_dma_mem {
+ struct device *dev;
+ size_t size;
+ void *cpu_addr;
+ dma_addr_t dma_addr;
+};
+
+/****************************************************************************
+ * Functions
+ ****************************************************************************/
+
+struct ethosu_dma_mem *ethosu_dma_mem_alloc(struct device *dev,
+ size_t size);
+
+void ethosu_dma_mem_free(struct ethosu_dma_mem **dma_mem);
+
+#endif /* ETHOSU_DMA_MEM_H */
diff --git a/kernel/ethosu_mailbox.c b/kernel/ethosu_mailbox.c
index e499860..9b9cd18 100644
--- a/kernel/ethosu_mailbox.c
+++ b/kernel/ethosu_mailbox.c
@@ -26,6 +26,8 @@
#include "ethosu_buffer.h"
#include "ethosu_core_rpmsg.h"
#include "ethosu_device.h"
+#include "ethosu_dma_mem.h"
+#include "ethosu_network.h"
#include <linux/atomic.h>
#include <linux/jiffies.h>
@@ -121,11 +123,11 @@
return ret;
}
-static void ethosu_core_set_size(struct ethosu_buffer *buf,
- struct ethosu_core_buffer *cbuf)
+static void ethosu_core_buffer_dma_mem_set(struct ethosu_dma_mem *dma_mem,
+ struct ethosu_core_buffer *cbuf)
{
- cbuf->ptr = (uint32_t)buf->dma_addr;
- cbuf->size = (uint32_t)buf->size;
+ cbuf->ptr = (uint32_t)dma_mem->dma_addr;
+ cbuf->size = (uint32_t)dma_mem->size;
}
int ethosu_mailbox_register(struct ethosu_mailbox *mbox,
@@ -267,17 +269,20 @@
inf_req->pmu_cycle_counter_enable = pmu_cycle_counter_enable;
for (i = 0; i < ifm_count; i++)
- ethosu_core_set_size(ifm[i], &inf_req->ifm[i]);
+ ethosu_core_buffer_dma_mem_set(ifm[i]->dma_mem,
+ &inf_req->ifm[i]);
for (i = 0; i < ofm_count; i++)
- ethosu_core_set_size(ofm[i], &inf_req->ofm[i]);
+ ethosu_core_buffer_dma_mem_set(ofm[i]->dma_mem,
+ &inf_req->ofm[i]);
for (i = 0; i < ETHOSU_CORE_PMU_MAX; i++)
inf_req->pmu_event_config[i] = pmu_event_config[i];
if (network != NULL) {
inf_req->network.type = ETHOSU_CORE_NETWORK_BUFFER;
- ethosu_core_set_size(network, &inf_req->network.buffer);
+ ethosu_core_buffer_dma_mem_set(network->dma_mem,
+ &inf_req->network.buffer);
} else {
inf_req->network.type = ETHOSU_CORE_NETWORK_INDEX;
inf_req->network.index = network_index;
@@ -305,7 +310,8 @@
if (network != NULL) {
info_req->network.type = ETHOSU_CORE_NETWORK_BUFFER;
- ethosu_core_set_size(network, &info_req->network.buffer);
+ ethosu_core_buffer_dma_mem_set(network->dma_mem,
+ &info_req->network.buffer);
} else {
info_req->network.type = ETHOSU_CORE_NETWORK_INDEX;
info_req->network.index = network_index;