blob: 836f22d8d45b80f80ed1f4fa055bbd1a009efa92 [file] [log] [blame]
Kristofer Jonsson49bdee82020-04-06 13:21:21 +02001/*
Jonny Svärdc9644242023-01-09 15:54:24 +01002 * SPDX-FileCopyrightText: Copyright 2019-2023 Arm Limited and/or its affiliates <open-source-office@arm.com>
Kristofer Jonsson49bdee82020-04-06 13:21:21 +02003 *
4 * SPDX-License-Identifier: Apache-2.0
5 *
6 * Licensed under the Apache License, Version 2.0 (the License); you may
7 * not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
9 *
10 * www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an AS IS BASIS, WITHOUT
14 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18
Kristofer Jonsson2b201c32020-09-02 16:42:43 +020019/******************************************************************************
20 * Includes
21 ******************************************************************************/
22
Kristofer Jonsson49bdee82020-04-06 13:21:21 +020023#include "ethosu_driver.h"
Kristofer Jonsson49bdee82020-04-06 13:21:21 +020024#include "ethosu_device.h"
Anton Moberg6eab40b2021-07-07 11:43:51 +020025#include "ethosu_log.h"
Per Åstrand25d78c02020-04-21 14:19:44 +020026
Kristofer Jonsson1c0e7ae2022-05-12 11:35:33 +020027#ifdef ETHOSU55
28#include "ethosu_config_u55.h"
29#else
30#include "ethosu_config_u65.h"
31#endif
32
Kristofer Jonsson49bdee82020-04-06 13:21:21 +020033#include <assert.h>
Per Åstrand25d78c02020-04-21 14:19:44 +020034#include <cmsis_compiler.h>
Per Åstrand14ccfee2020-09-25 10:40:20 +020035#include <inttypes.h>
Kristofer Jonsson49bdee82020-04-06 13:21:21 +020036#include <stdbool.h>
Bhavik Patelbf7ae632020-06-11 21:00:16 +020037#include <stddef.h>
Kristofer Jonsson49bdee82020-04-06 13:21:21 +020038#include <stdio.h>
39#include <stdlib.h>
40
Kristofer Jonsson2b201c32020-09-02 16:42:43 +020041/******************************************************************************
42 * Defines
43 ******************************************************************************/
44
Jonny Svärd136810f2021-10-13 16:04:26 +020045#define UNUSED(x) ((void)x)
46
Kristofer Jonsson2b201c32020-09-02 16:42:43 +020047#define BYTES_IN_32_BITS 4
Kristofer Jonsson2b201c32020-09-02 16:42:43 +020048#define MASK_16_BYTE_ALIGN (0xF)
Jonny Svärd136810f2021-10-13 16:04:26 +020049#define OPTIMIZER_CONFIG_LENGTH_32_BIT_WORD 2
50#define DRIVER_ACTION_LENGTH_32_BIT_WORD 1
51#define ETHOSU_FOURCC ('1' << 24 | 'P' << 16 | 'O' << 8 | 'C') // "Custom Operator Payload 1"
52
Kristofer Jonsson2b201c32020-09-02 16:42:43 +020053#define FAST_MEMORY_BASE_ADDR_INDEX 2
54
55/******************************************************************************
56 * Types
57 ******************************************************************************/
58
59// Driver actions
60enum DRIVER_ACTION_e
61{
62 RESERVED = 0,
63 OPTIMIZER_CONFIG = 1,
64 COMMAND_STREAM = 2,
Kristofer Jonsson2b201c32020-09-02 16:42:43 +020065 NOP = 5,
66};
67
Jonny Svärd136810f2021-10-13 16:04:26 +020068// Custom operator payload data struct
69struct cop_data_s
Kristofer Jonsson2b201c32020-09-02 16:42:43 +020070{
71 union
72 {
73 // Driver action data
74 struct
75 {
Jonny Svärd136810f2021-10-13 16:04:26 +020076 uint8_t driver_action_command; // (valid values in DRIVER_ACTION_e)
Kristofer Jonsson2b201c32020-09-02 16:42:43 +020077 uint8_t reserved;
78
79 // Driver action data
80 union
81 {
82 // DA_CMD_OPT_CFG
83 struct
84 {
85 uint16_t rel_nbr : 4;
86 uint16_t patch_nbr : 4;
87 uint16_t opt_cfg_reserved : 8;
88 };
89
90 // DA_CMD_CMSTRM
91 struct
92 {
93 uint16_t length;
94 };
95
Kristofer Jonsson2b201c32020-09-02 16:42:43 +020096 uint16_t driver_action_data;
97 };
98 };
99
100 uint32_t word;
101 };
102};
103
104// optimizer config struct
105struct opt_cfg_s
106{
Jonny Svärd136810f2021-10-13 16:04:26 +0200107 struct cop_data_s da_data;
108 uint32_t cfg;
109 uint32_t id;
Kristofer Jonsson2b201c32020-09-02 16:42:43 +0200110};
111
112/******************************************************************************
Jonny Svärda830f172021-06-07 16:57:00 +0200113 * Variables
Kristofer Jonsson2b201c32020-09-02 16:42:43 +0200114 ******************************************************************************/
115
Anton Moberg61da4d32020-12-22 16:00:31 +0100116// Registered drivers linked list HEAD
117static struct ethosu_driver *registered_drivers = NULL;
118
Jonny Svärda830f172021-06-07 16:57:00 +0200119/******************************************************************************
120 * Weak functions - Cache
121 *
122 * Default NOP operations. Override if available on the targeted device.
123 ******************************************************************************/
Anton Mobergdfed5fd2021-03-11 14:41:11 +0100124
Jonny Svärda830f172021-06-07 16:57:00 +0200125/*
126 * Flush/clean the data cache by address and size. Passing NULL as p argument
127 * expects the whole cache to be flushed.
128 */
129void __attribute__((weak)) ethosu_flush_dcache(uint32_t *p, size_t bytes)
130{
131 UNUSED(p);
132 UNUSED(bytes);
133}
134
135/*
136 * Invalidate the data cache by address and size. Passing NULL as p argument
137 * expects the whole cache to be invalidated.
138 */
139void __attribute__((weak)) ethosu_invalidate_dcache(uint32_t *p, size_t bytes)
140{
141 UNUSED(p);
142 UNUSED(bytes);
143}
144
145/******************************************************************************
146 * Weak functions - Semaphore/Mutex for multi NPU
147 *
148 * Following section handles the minimal sempahore and mutex implementation in
149 * case of baremetal applications. Weak symbols will be overridden by RTOS
150 * definitions and implement true thread-safety (in application layer).
151 ******************************************************************************/
152
Anton Mobergdfed5fd2021-03-11 14:41:11 +0100153struct ethosu_semaphore_t
154{
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100155 uint8_t count;
Anton Mobergdfed5fd2021-03-11 14:41:11 +0100156};
157
Anton Mobergdfed5fd2021-03-11 14:41:11 +0100158static void *ethosu_mutex;
159static void *ethosu_semaphore;
160
Anton Moberg9f346ab2021-05-21 17:20:21 +0200161void *__attribute__((weak)) ethosu_mutex_create(void)
162{
Jonny Svärda2732ec2023-12-18 17:19:15 +0100163 static uint8_t mutex_placeholder;
164 return &mutex_placeholder;
Anton Moberg9f346ab2021-05-21 17:20:21 +0200165}
Anton Mobergdfed5fd2021-03-11 14:41:11 +0100166
Jonny Svärd136810f2021-10-13 16:04:26 +0200167void __attribute__((weak)) ethosu_mutex_destroy(void *mutex)
168{
169 UNUSED(mutex);
170}
171
Ledion Dajac6505f32022-04-20 09:55:21 +0200172int __attribute__((weak)) ethosu_mutex_lock(void *mutex)
Anton Moberg61ec36b2021-04-30 17:10:48 +0200173{
174 UNUSED(mutex);
Ledion Dajac6505f32022-04-20 09:55:21 +0200175 return 0;
Anton Moberg61ec36b2021-04-30 17:10:48 +0200176}
Anton Mobergdfed5fd2021-03-11 14:41:11 +0100177
Ledion Dajac6505f32022-04-20 09:55:21 +0200178int __attribute__((weak)) ethosu_mutex_unlock(void *mutex)
Anton Moberg61ec36b2021-04-30 17:10:48 +0200179{
180 UNUSED(mutex);
Ledion Dajac6505f32022-04-20 09:55:21 +0200181 return 0;
Anton Moberg61ec36b2021-04-30 17:10:48 +0200182}
Anton Mobergdfed5fd2021-03-11 14:41:11 +0100183
184// Baremetal implementation of creating a semaphore
185void *__attribute__((weak)) ethosu_semaphore_create(void)
186{
187 struct ethosu_semaphore_t *sem = malloc(sizeof(*sem));
Ledion Dajaecdce6c2023-01-16 15:39:29 +0100188 if (sem != NULL)
189 {
190 sem->count = 0;
191 }
Anton Mobergdfed5fd2021-03-11 14:41:11 +0100192 return sem;
193}
194
Jonny Svärd136810f2021-10-13 16:04:26 +0200195void __attribute__((weak)) ethosu_semaphore_destroy(void *sem)
196{
197 free((struct ethosu_semaphore_t *)sem);
198}
199
Anton Mobergdfed5fd2021-03-11 14:41:11 +0100200// Baremetal simulation of waiting/sleeping for and then taking a semaphore using intrisics
Jonny Svärda2732ec2023-12-18 17:19:15 +0100201int __attribute__((weak)) ethosu_semaphore_take(void *sem, uint64_t timeout)
Anton Mobergdfed5fd2021-03-11 14:41:11 +0100202{
Jonny Svärda2732ec2023-12-18 17:19:15 +0100203 UNUSED(timeout);
204 // Baremetal pseudo-example on how to trigger a timeout:
205 // if (timeout != ETHOSU_SEMAPHORE_WAIT_FOREVER) {
206 // setup_a_timer_to_call_SEV_after_time(timeout);
207 // }
Anton Mobergdfed5fd2021-03-11 14:41:11 +0100208 struct ethosu_semaphore_t *s = sem;
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100209 while (s->count == 0)
Anton Mobergdfed5fd2021-03-11 14:41:11 +0100210 {
211 __WFE();
Jonny Svärda2732ec2023-12-18 17:19:15 +0100212 // Baremetal pseudo-example check if timeout triggered:
213 // if (SEV_timer_triggered()) {
214 // return -1;
215 // }
Anton Mobergdfed5fd2021-03-11 14:41:11 +0100216 }
Jonny Svärdc9644242023-01-09 15:54:24 +0100217 s->count--;
Ledion Dajac6505f32022-04-20 09:55:21 +0200218 return 0;
Anton Mobergdfed5fd2021-03-11 14:41:11 +0100219}
220
221// Baremetal simulation of giving a semaphore and waking up processes using intrinsics
Ledion Dajac6505f32022-04-20 09:55:21 +0200222int __attribute__((weak)) ethosu_semaphore_give(void *sem)
Anton Mobergdfed5fd2021-03-11 14:41:11 +0100223{
224 struct ethosu_semaphore_t *s = sem;
Jonny Svärdc9644242023-01-09 15:54:24 +0100225 s->count++;
Anton Mobergdfed5fd2021-03-11 14:41:11 +0100226 __SEV();
Ledion Dajac6505f32022-04-20 09:55:21 +0200227 return 0;
Anton Mobergdfed5fd2021-03-11 14:41:11 +0100228}
Anton Mobergdfed5fd2021-03-11 14:41:11 +0100229
Jonny Svärda830f172021-06-07 16:57:00 +0200230/******************************************************************************
231 * Weak functions - Inference begin/end callbacks
232 ******************************************************************************/
Anton Moberg61da4d32020-12-22 16:00:31 +0100233
Kristofer Jonssonb3cde3c2022-01-27 17:30:15 +0100234void __attribute__((weak)) ethosu_inference_begin(struct ethosu_driver *drv, void *user_arg)
Kristofer Jonsson49bdee82020-04-06 13:21:21 +0200235{
Kristofer Jonssonb3cde3c2022-01-27 17:30:15 +0100236 UNUSED(user_arg);
Jonny Svärda830f172021-06-07 16:57:00 +0200237 UNUSED(drv);
Kristofer Jonsson49bdee82020-04-06 13:21:21 +0200238}
239
Kristofer Jonssonb3cde3c2022-01-27 17:30:15 +0100240void __attribute__((weak)) ethosu_inference_end(struct ethosu_driver *drv, void *user_arg)
Jonny Svärda830f172021-06-07 16:57:00 +0200241{
Kristofer Jonssonb3cde3c2022-01-27 17:30:15 +0100242 UNUSED(user_arg);
Jonny Svärda830f172021-06-07 16:57:00 +0200243 UNUSED(drv);
244}
245
246/******************************************************************************
247 * Static functions
248 ******************************************************************************/
Jonny Svärda830f172021-06-07 16:57:00 +0200249static void ethosu_register_driver(struct ethosu_driver *drv)
Jens Elofsson04961a42021-04-08 18:51:38 +0200250{
Jonny Svärdc9644242023-01-09 15:54:24 +0100251 ethosu_mutex_lock(ethosu_mutex);
Jonny Svärda830f172021-06-07 16:57:00 +0200252 drv->next = registered_drivers;
253 registered_drivers = drv;
Jonny Svärdc9644242023-01-09 15:54:24 +0100254 ethosu_mutex_unlock(ethosu_mutex);
255
256 ethosu_semaphore_give(ethosu_semaphore);
Jonny Svärda830f172021-06-07 16:57:00 +0200257
Kristofer Jonsson089a3472021-11-12 12:52:07 +0100258 LOG_INFO("New NPU driver registered (handle: 0x%p, NPU: 0x%p)", drv, drv->dev->reg);
Jens Elofsson04961a42021-04-08 18:51:38 +0200259}
260
Jonny Svärda830f172021-06-07 16:57:00 +0200261static int ethosu_deregister_driver(struct ethosu_driver *drv)
262{
Jonny Svärdc9644242023-01-09 15:54:24 +0100263 struct ethosu_driver *curr;
264 struct ethosu_driver **prev;
Jonny Svärda830f172021-06-07 16:57:00 +0200265
Jonny Svärdc9644242023-01-09 15:54:24 +0100266 ethosu_mutex_lock(ethosu_mutex);
267 curr = registered_drivers;
268 prev = &registered_drivers;
269
270 while (curr != NULL)
Jonny Svärda830f172021-06-07 16:57:00 +0200271 {
Jonny Svärdc9644242023-01-09 15:54:24 +0100272 if (curr == drv)
Jonny Svärda830f172021-06-07 16:57:00 +0200273 {
Jonny Svärdc9644242023-01-09 15:54:24 +0100274 *prev = curr->next;
Kristofer Jonsson089a3472021-11-12 12:52:07 +0100275 LOG_INFO("NPU driver handle %p deregistered.", drv);
Jonny Svärda2732ec2023-12-18 17:19:15 +0100276 ethosu_semaphore_take(ethosu_semaphore, ETHOSU_SEMAPHORE_WAIT_FOREVER);
Jonny Svärdc9644242023-01-09 15:54:24 +0100277 break;
Jonny Svärda830f172021-06-07 16:57:00 +0200278 }
279
Jonny Svärdc9644242023-01-09 15:54:24 +0100280 prev = &curr->next;
281 curr = curr->next;
Jonny Svärda830f172021-06-07 16:57:00 +0200282 }
283
Jonny Svärdc9644242023-01-09 15:54:24 +0100284 ethosu_mutex_unlock(ethosu_mutex);
Jonny Svärda830f172021-06-07 16:57:00 +0200285
Jonny Svärdc9644242023-01-09 15:54:24 +0100286 if (curr == NULL)
Jonny Svärda830f172021-06-07 16:57:00 +0200287 {
Jonny Svärdc9644242023-01-09 15:54:24 +0100288 LOG_ERR("No NPU driver handle registered at address %p.", drv);
289 return -1;
Jonny Svärda830f172021-06-07 16:57:00 +0200290 }
291
Jonny Svärdc9644242023-01-09 15:54:24 +0100292 return 0;
Jonny Svärda830f172021-06-07 16:57:00 +0200293}
294
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100295static void ethosu_reset_job(struct ethosu_driver *drv)
296{
297 memset(&drv->job, 0, sizeof(struct ethosu_job));
298}
299
Ledion Dajaecdce6c2023-01-16 15:39:29 +0100300static int handle_optimizer_config(struct ethosu_driver *drv, struct opt_cfg_s const *opt_cfg_p)
Jonny Svärda830f172021-06-07 16:57:00 +0200301{
Ledion Dajaecdce6c2023-01-16 15:39:29 +0100302 LOG_INFO("Optimizer release nbr: %u patch: %u", opt_cfg_p->da_data.rel_nbr, opt_cfg_p->da_data.patch_nbr);
Jonny Svärda830f172021-06-07 16:57:00 +0200303
Jonny Svärd136810f2021-10-13 16:04:26 +0200304 if (ethosu_dev_verify_optimizer_config(drv->dev, opt_cfg_p->cfg, opt_cfg_p->id) != true)
Jonny Svärda830f172021-06-07 16:57:00 +0200305 {
306 return -1;
307 }
308
Jonny Svärda830f172021-06-07 16:57:00 +0200309 return 0;
310}
311
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100312static int handle_command_stream(struct ethosu_driver *drv, const uint8_t *cmd_stream, const int cms_length)
Jonny Svärda830f172021-06-07 16:57:00 +0200313{
Jonny Svärda830f172021-06-07 16:57:00 +0200314 uint32_t cms_bytes = cms_length * BYTES_IN_32_BITS;
315 ptrdiff_t cmd_stream_ptr = (ptrdiff_t)cmd_stream;
316
Kristofer Jonsson089a3472021-11-12 12:52:07 +0100317 LOG_INFO("handle_command_stream: cmd_stream=%p, cms_length %d", cmd_stream, cms_length);
Jonny Svärda830f172021-06-07 16:57:00 +0200318
319 if (0 != ((ptrdiff_t)cmd_stream & MASK_16_BYTE_ALIGN))
320 {
Kristofer Jonsson089a3472021-11-12 12:52:07 +0100321 LOG_ERR("Command stream addr %p not aligned to 16 bytes", cmd_stream);
Jonny Svärda830f172021-06-07 16:57:00 +0200322 return -1;
323 }
324
Jonny Svärd136810f2021-10-13 16:04:26 +0200325 // Verify 16 byte alignment for base address'
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100326 for (int i = 0; i < drv->job.num_base_addr; i++)
Jonny Svärda830f172021-06-07 16:57:00 +0200327 {
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100328 if (0 != (drv->job.base_addr[i] & MASK_16_BYTE_ALIGN))
Jonny Svärda830f172021-06-07 16:57:00 +0200329 {
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100330 LOG_ERR("Base addr %d: 0x%llx not aligned to 16 bytes", i, drv->job.base_addr[i]);
Jonny Svärd136810f2021-10-13 16:04:26 +0200331 return -1;
Jonny Svärda830f172021-06-07 16:57:00 +0200332 }
333 }
334
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100335 // Flush the cache if available on CPU.
336 // The upcasting to uin32_t* is ok since the pointer never is dereferenced.
337 // The base_addr_size is null if invoking from prior to invoke_V2, in that case
338 // the whole cache is being flushed.
339
340 if (drv->job.base_addr_size != NULL)
Jonny Svärda830f172021-06-07 16:57:00 +0200341 {
342 ethosu_flush_dcache((uint32_t *)cmd_stream_ptr, cms_bytes);
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100343 for (int i = 0; i < drv->job.num_base_addr; i++)
Jonny Svärda830f172021-06-07 16:57:00 +0200344 {
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100345 ethosu_flush_dcache((uint32_t *)(uintptr_t)drv->job.base_addr[i], drv->job.base_addr_size[i]);
Jonny Svärda830f172021-06-07 16:57:00 +0200346 }
347 }
348 else
349 {
350 ethosu_flush_dcache(NULL, 0);
351 }
352
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100353 // Request power gating disabled during inference run
Kristofer Jonsson68b5c912022-11-14 10:52:38 +0100354 if (ethosu_request_power(drv))
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100355 {
Jonny Svärd301399d2022-04-26 18:31:24 +0200356 LOG_ERR("Failed to request power");
357 return -1;
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100358 }
359
Jonny Svärd301399d2022-04-26 18:31:24 +0200360 drv->job.state = ETHOSU_JOB_RUNNING;
361
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100362 // Inference begin callback
363 ethosu_inference_begin(drv, drv->job.user_arg);
364
Jonny Svärd136810f2021-10-13 16:04:26 +0200365 // Execute the command stream
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100366 ethosu_dev_run_command_stream(drv->dev, cmd_stream, cms_bytes, drv->job.base_addr, drv->job.num_base_addr);
Jonny Svärda830f172021-06-07 16:57:00 +0200367
Jonny Svärda830f172021-06-07 16:57:00 +0200368 return 0;
369}
370
371/******************************************************************************
372 * Weak functions - Interrupt handler
373 ******************************************************************************/
374void __attribute__((weak)) ethosu_irq_handler(struct ethosu_driver *drv)
375{
Kristofer Jonsson089a3472021-11-12 12:52:07 +0100376 LOG_DEBUG("Got interrupt from Ethos-U");
Jonny Svärda830f172021-06-07 16:57:00 +0200377
Jonny Svärda2732ec2023-12-18 17:19:15 +0100378 // Prevent race condition where interrupt triggered after a timeout waiting
379 // for semaphore, but before NPU is reset.
380 if (drv->job.result == ETHOSU_JOB_RESULT_TIMEOUT)
Jonny Svärda830f172021-06-07 16:57:00 +0200381 {
Jonny Svärda2732ec2023-12-18 17:19:15 +0100382 return;
Jonny Svärda830f172021-06-07 16:57:00 +0200383 }
Jonny Svärda2732ec2023-12-18 17:19:15 +0100384
385 drv->job.state = ETHOSU_JOB_DONE;
386 drv->job.result = ethosu_dev_handle_interrupt(drv->dev) ? ETHOSU_JOB_RESULT_OK : ETHOSU_JOB_RESULT_ERROR;
Jonny Svärda830f172021-06-07 16:57:00 +0200387 ethosu_semaphore_give(drv->semaphore);
388}
389
390/******************************************************************************
391 * Functions API
392 ******************************************************************************/
Kristofer Jonsson49bdee82020-04-06 13:21:21 +0200393
Anton Mobergeffc7aa2021-05-03 09:25:06 +0200394int ethosu_init(struct ethosu_driver *drv,
Ledion Dajaecdce6c2023-01-16 15:39:29 +0100395 void *const base_address,
Anton Mobergeffc7aa2021-05-03 09:25:06 +0200396 const void *fast_memory,
397 const size_t fast_memory_size,
398 uint32_t secure_enable,
399 uint32_t privilege_enable)
Kristofer Jonsson49bdee82020-04-06 13:21:21 +0200400{
Jonny Svärda830f172021-06-07 16:57:00 +0200401 LOG_INFO("Initializing NPU: base_address=%p, fast_memory=%p, fast_memory_size=%zu, secure=%" PRIu32
Kristofer Jonsson089a3472021-11-12 12:52:07 +0100402 ", privileged=%" PRIu32,
Kristofer Jonsson2b201c32020-09-02 16:42:43 +0200403 base_address,
404 fast_memory,
Per Åstrande6498f02020-11-09 15:33:12 +0100405 fast_memory_size,
406 secure_enable,
407 privilege_enable);
Kristofer Jonsson2b201c32020-09-02 16:42:43 +0200408
Anton Mobergdfed5fd2021-03-11 14:41:11 +0100409 if (!ethosu_mutex)
410 {
411 ethosu_mutex = ethosu_mutex_create();
Jonny Svärda2732ec2023-12-18 17:19:15 +0100412 if (!ethosu_mutex)
413 {
414 LOG_ERR("Failed to create global driver mutex");
415 return -1;
416 }
Anton Mobergdfed5fd2021-03-11 14:41:11 +0100417 }
418
419 if (!ethosu_semaphore)
420 {
421 ethosu_semaphore = ethosu_semaphore_create();
Jonny Svärdc9644242023-01-09 15:54:24 +0100422 if (!ethosu_semaphore)
423 {
424 LOG_ERR("Failed to create global driver semaphore");
425 return -1;
426 }
Anton Mobergdfed5fd2021-03-11 14:41:11 +0100427 }
428
Jonny Svärd301399d2022-04-26 18:31:24 +0200429 drv->fast_memory = (uint32_t)fast_memory;
430 drv->fast_memory_size = fast_memory_size;
431 drv->power_request_counter = 0;
Anton Moberg61da4d32020-12-22 16:00:31 +0100432
Jonny Svärd136810f2021-10-13 16:04:26 +0200433 // Initialize the device and set requested security state and privilege mode
434 drv->dev = ethosu_dev_init(base_address, secure_enable, privilege_enable);
435
436 if (drv->dev == NULL)
Bhavik Pateldae5be02020-06-18 15:25:15 +0200437 {
Kristofer Jonsson089a3472021-11-12 12:52:07 +0100438 LOG_ERR("Failed to initialize Ethos-U device");
Bhavik Pateldae5be02020-06-18 15:25:15 +0200439 return -1;
440 }
441
Jonny Svärdc9644242023-01-09 15:54:24 +0100442 drv->semaphore = ethosu_semaphore_create();
443 if (!drv->semaphore)
444 {
445 LOG_ERR("Failed to create driver semaphore");
446 ethosu_dev_deinit(drv->dev);
447 drv->dev = NULL;
448 return -1;
449 }
450
Jonny Svärd301399d2022-04-26 18:31:24 +0200451 ethosu_reset_job(drv);
Jonny Svärda830f172021-06-07 16:57:00 +0200452 ethosu_register_driver(drv);
453
Jonny Svärd136810f2021-10-13 16:04:26 +0200454 return 0;
Kristofer Jonsson49bdee82020-04-06 13:21:21 +0200455}
456
Anton Moberg547ca532021-06-14 09:43:53 +0200457void ethosu_deinit(struct ethosu_driver *drv)
458{
459 ethosu_deregister_driver(drv);
Jonny Svärd136810f2021-10-13 16:04:26 +0200460 ethosu_semaphore_destroy(drv->semaphore);
461 ethosu_dev_deinit(drv->dev);
462 drv->dev = NULL;
Anton Moberg547ca532021-06-14 09:43:53 +0200463}
464
Kristofer Jonsson68b5c912022-11-14 10:52:38 +0100465int ethosu_soft_reset(struct ethosu_driver *drv)
Jonny Svärd301399d2022-04-26 18:31:24 +0200466{
467 // Soft reset the NPU
468 if (ethosu_dev_soft_reset(drv->dev) != ETHOSU_SUCCESS)
469 {
470 LOG_ERR("Failed to soft-reset NPU");
Kristofer Jonsson68b5c912022-11-14 10:52:38 +0100471 return -1;
Jonny Svärd301399d2022-04-26 18:31:24 +0200472 }
473
474 // Update power and clock gating after the soft reset
475 ethosu_dev_set_clock_and_power(drv->dev,
476 drv->power_request_counter > 0 ? ETHOSU_CLOCK_Q_DISABLE : ETHOSU_CLOCK_Q_ENABLE,
477 drv->power_request_counter > 0 ? ETHOSU_POWER_Q_DISABLE : ETHOSU_POWER_Q_ENABLE);
478
Kristofer Jonsson68b5c912022-11-14 10:52:38 +0100479 return 0;
Jonny Svärd301399d2022-04-26 18:31:24 +0200480}
481
Kristofer Jonsson68b5c912022-11-14 10:52:38 +0100482int ethosu_request_power(struct ethosu_driver *drv)
Jonny Svärd301399d2022-04-26 18:31:24 +0200483{
484 // Check if this is the first power request, increase counter
485 if (drv->power_request_counter++ == 0)
486 {
487 // Always reset to a known state. Changes to requested
488 // security state/privilege mode if necessary.
Kristofer Jonsson68b5c912022-11-14 10:52:38 +0100489 if (ethosu_soft_reset(drv))
Jonny Svärd301399d2022-04-26 18:31:24 +0200490 {
491 LOG_ERR("Failed to request power for Ethos-U");
492 drv->power_request_counter--;
Kristofer Jonsson68b5c912022-11-14 10:52:38 +0100493 return -1;
Jonny Svärd301399d2022-04-26 18:31:24 +0200494 }
495 }
Kristofer Jonsson68b5c912022-11-14 10:52:38 +0100496 return 0;
Jonny Svärd301399d2022-04-26 18:31:24 +0200497}
498
499void ethosu_release_power(struct ethosu_driver *drv)
500{
501 if (drv->power_request_counter == 0)
502 {
503 LOG_WARN("No power request left to release, reference counter is 0");
504 }
505 else
506 {
507 // Decrement ref counter and enable power gating if no requests remain
508 if (--drv->power_request_counter == 0)
509 {
510 ethosu_dev_set_clock_and_power(drv->dev, ETHOSU_CLOCK_Q_ENABLE, ETHOSU_POWER_Q_ENABLE);
511 }
512 }
513}
514
Jonny Svärda830f172021-06-07 16:57:00 +0200515void ethosu_get_driver_version(struct ethosu_driver_version *ver)
Kristofer Jonsson49bdee82020-04-06 13:21:21 +0200516{
Jonny Svärda830f172021-06-07 16:57:00 +0200517 assert(ver != NULL);
518 ver->major = ETHOSU_DRIVER_VERSION_MAJOR;
519 ver->minor = ETHOSU_DRIVER_VERSION_MINOR;
520 ver->patch = ETHOSU_DRIVER_VERSION_PATCH;
521}
Kristofer Jonsson49bdee82020-04-06 13:21:21 +0200522
Jonny Svärda830f172021-06-07 16:57:00 +0200523void ethosu_get_hw_info(struct ethosu_driver *drv, struct ethosu_hw_info *hw)
524{
525 assert(hw != NULL);
Jonny Svärd136810f2021-10-13 16:04:26 +0200526 ethosu_dev_get_hw_info(drv->dev, hw);
Kristofer Jonsson49bdee82020-04-06 13:21:21 +0200527}
528
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100529int ethosu_wait(struct ethosu_driver *drv, bool block)
Kristofer Jonsson49bdee82020-04-06 13:21:21 +0200530{
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100531 int ret = 0;
532
533 switch (drv->job.state)
534 {
535 case ETHOSU_JOB_IDLE:
536 LOG_ERR("Inference job not running...");
537 ret = -2;
538 break;
539 case ETHOSU_JOB_RUNNING:
540 if (!block)
541 {
542 // Inference still running, do not block
543 ret = 1;
544 break;
545 }
546 // fall through
547 case ETHOSU_JOB_DONE:
548 // Wait for interrupt in blocking mode. In non-blocking mode
549 // the interrupt has already triggered
Jonny Svärda2732ec2023-12-18 17:19:15 +0100550 ret = ethosu_semaphore_take(drv->semaphore, ETHOSU_SEMAPHORE_WAIT_INFERENCE);
551 if (ret < 0)
552 {
553 drv->job.result = ETHOSU_JOB_RESULT_TIMEOUT;
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100554
Jonny Svärda2732ec2023-12-18 17:19:15 +0100555 // There's a race where the NPU interrupt can have fired between semaphore
556 // timing out and setting the result above (checked in interrupt handler).
557 // By checking if the job state has been changed (only set to DONE by interrupt
558 // handler), we know if the interrupt handler has run, if so decrement the
559 // semaphore count by one (given in interrupt handler).
560 if (drv->job.state == ETHOSU_JOB_DONE)
561 {
562 drv->job.result = ETHOSU_JOB_RESULT_TIMEOUT; // Reset back to timeout
563 ethosu_semaphore_take(drv->semaphore, ETHOSU_SEMAPHORE_WAIT_INFERENCE);
564 }
565 }
566
567 // Inference done callback - always called even in case of timeout
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100568 ethosu_inference_end(drv, drv->job.user_arg);
569
Jonny Svärda2732ec2023-12-18 17:19:15 +0100570 // Release power gating disabled requirement
Jonny Svärd301399d2022-04-26 18:31:24 +0200571 ethosu_release_power(drv);
572
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100573 // Check NPU and interrupt status
Jonny Svärda2732ec2023-12-18 17:19:15 +0100574 if (drv->job.result)
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100575 {
Jonny Svärda2732ec2023-12-18 17:19:15 +0100576 if (drv->job.result == ETHOSU_JOB_RESULT_ERROR)
577 {
578 LOG_ERR("NPU error(s) occured during inference.");
579 ethosu_dev_print_err_status(drv->dev);
580 }
581 else
582 {
583 LOG_ERR("NPU inference timed out.");
584 }
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100585
586 // Reset the NPU
Jonny Svärd301399d2022-04-26 18:31:24 +0200587 (void)ethosu_soft_reset(drv);
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100588
589 ret = -1;
590 }
Jonny Svärda2732ec2023-12-18 17:19:15 +0100591 else
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100592 {
593 // Invalidate cache
594 if (drv->job.base_addr_size != NULL)
595 {
596 for (int i = 0; i < drv->job.num_base_addr; i++)
597 {
598 ethosu_invalidate_dcache((uint32_t *)(uintptr_t)drv->job.base_addr[i], drv->job.base_addr_size[i]);
599 }
600 }
601 else
602 {
603 ethosu_invalidate_dcache(NULL, 0);
604 }
605
606 LOG_DEBUG("Inference finished successfully...");
Jonny Svärda2732ec2023-12-18 17:19:15 +0100607 ret = 0;
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100608 }
609
610 // Reset internal job (state resets to IDLE)
611 ethosu_reset_job(drv);
612 break;
613
614 default:
615 LOG_ERR("Unexpected job state");
616 ethosu_reset_job(drv);
617 ret = -1;
618 break;
619 }
620
621 // Return inference job status
622 return ret;
623}
624
625int ethosu_invoke_async(struct ethosu_driver *drv,
626 const void *custom_data_ptr,
627 const int custom_data_size,
Ledion Dajaecdce6c2023-01-16 15:39:29 +0100628 uint64_t *const base_addr,
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100629 const size_t *base_addr_size,
630 const int num_base_addr,
631 void *user_arg)
632{
633
Jonny Svärd136810f2021-10-13 16:04:26 +0200634 const struct cop_data_s *data_ptr = custom_data_ptr;
Kristofer Jonsson24455ee2022-02-08 13:33:22 +0100635 const struct cop_data_s *data_end = (struct cop_data_s *)((ptrdiff_t)custom_data_ptr + custom_data_size);
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100636
637 // Make sure an inference is not already running
638 if (drv->job.state != ETHOSU_JOB_IDLE)
639 {
640 LOG_ERR("Inference already running, or waiting to be cleared...");
641 return -1;
642 }
643
644 drv->job.state = ETHOSU_JOB_IDLE;
645 drv->job.custom_data_ptr = custom_data_ptr;
646 drv->job.custom_data_size = custom_data_size;
647 drv->job.base_addr = base_addr;
648 drv->job.base_addr_size = base_addr_size;
649 drv->job.num_base_addr = num_base_addr;
650 drv->job.user_arg = user_arg;
Kristofer Jonsson49bdee82020-04-06 13:21:21 +0200651
Kristofer Jonsson49bdee82020-04-06 13:21:21 +0200652 // First word in custom_data_ptr should contain "Custom Operator Payload 1"
Kristofer Jonsson2b201c32020-09-02 16:42:43 +0200653 if (data_ptr->word != ETHOSU_FOURCC)
Kristofer Jonsson49bdee82020-04-06 13:21:21 +0200654 {
Kristofer Jonsson089a3472021-11-12 12:52:07 +0100655 LOG_ERR("Custom Operator Payload: %" PRIu32 " is not correct, expected %x", data_ptr->word, ETHOSU_FOURCC);
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100656 goto err;
Kristofer Jonsson49bdee82020-04-06 13:21:21 +0200657 }
Kristofer Jonsson49bdee82020-04-06 13:21:21 +0200658
Kristofer Jonsson2b201c32020-09-02 16:42:43 +0200659 // Custom data length must be a multiple of 32 bits
Kristofer Jonsson49bdee82020-04-06 13:21:21 +0200660 if ((custom_data_size % BYTES_IN_32_BITS) != 0)
661 {
Ledion Dajaecdce6c2023-01-16 15:39:29 +0100662 LOG_ERR("custom_data_size=0x%x not a multiple of 4", (unsigned)custom_data_size);
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100663 goto err;
Kristofer Jonsson49bdee82020-04-06 13:21:21 +0200664 }
Kristofer Jonsson2b201c32020-09-02 16:42:43 +0200665
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100666 data_ptr++;
Kristofer Jonsson2b201c32020-09-02 16:42:43 +0200667
668 // Adjust base address to fast memory area
Anton Moberg61da4d32020-12-22 16:00:31 +0100669 if (drv->fast_memory != 0 && num_base_addr >= FAST_MEMORY_BASE_ADDR_INDEX)
Kristofer Jonsson2b201c32020-09-02 16:42:43 +0200670 {
Kristofer Jonsson2b201c32020-09-02 16:42:43 +0200671
Anton Moberg61da4d32020-12-22 16:00:31 +0100672 if (base_addr_size != NULL && base_addr_size[FAST_MEMORY_BASE_ADDR_INDEX] > drv->fast_memory_size)
Kristofer Jonsson2b201c32020-09-02 16:42:43 +0200673 {
Kristofer Jonsson089a3472021-11-12 12:52:07 +0100674 LOG_ERR("Fast memory area too small. fast_memory_size=%u, base_addr_size=%u",
Anton Moberg61da4d32020-12-22 16:00:31 +0100675 drv->fast_memory_size,
Kristofer Jonsson4c94b302020-11-06 10:33:21 +0100676 base_addr_size[FAST_MEMORY_BASE_ADDR_INDEX]);
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100677 goto err;
Kristofer Jonsson2b201c32020-09-02 16:42:43 +0200678 }
Kristofer Jonsson4c94b302020-11-06 10:33:21 +0100679
Ledion Dajaecdce6c2023-01-16 15:39:29 +0100680 base_addr[FAST_MEMORY_BASE_ADDR_INDEX] = drv->fast_memory;
Kristofer Jonsson2b201c32020-09-02 16:42:43 +0200681 }
Kristofer Jonsson49bdee82020-04-06 13:21:21 +0200682
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100683 // Parse Custom Operator Payload data
Kristofer Jonsson2b201c32020-09-02 16:42:43 +0200684 while (data_ptr < data_end)
Kristofer Jonsson49bdee82020-04-06 13:21:21 +0200685 {
686 switch (data_ptr->driver_action_command)
687 {
688 case OPTIMIZER_CONFIG:
Kristofer Jonsson089a3472021-11-12 12:52:07 +0100689 LOG_DEBUG("OPTIMIZER_CONFIG");
Ledion Dajaecdce6c2023-01-16 15:39:29 +0100690 struct opt_cfg_s const *opt_cfg_p = (const struct opt_cfg_s *)data_ptr;
Kristofer Jonsson49bdee82020-04-06 13:21:21 +0200691
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100692 if (handle_optimizer_config(drv, opt_cfg_p) < 0)
693 {
694 goto err;
695 }
Kristofer Jonsson49bdee82020-04-06 13:21:21 +0200696 data_ptr += DRIVER_ACTION_LENGTH_32_BIT_WORD + OPTIMIZER_CONFIG_LENGTH_32_BIT_WORD;
697 break;
698 case COMMAND_STREAM:
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100699 // Vela only supports putting one COMMAND_STREAM per op
Kristofer Jonsson089a3472021-11-12 12:52:07 +0100700 LOG_DEBUG("COMMAND_STREAM");
Ledion Dajaecdce6c2023-01-16 15:39:29 +0100701 const uint8_t *command_stream = (const uint8_t *)(data_ptr + 1);
702 int cms_length = (data_ptr->reserved << 16) | data_ptr->length;
Kristofer Jonsson49bdee82020-04-06 13:21:21 +0200703
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100704 if (handle_command_stream(drv, command_stream, cms_length) < 0)
Kristofer Jonsson49bdee82020-04-06 13:21:21 +0200705 {
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100706 goto err;
Kristofer Jonsson49bdee82020-04-06 13:21:21 +0200707 }
Kristofer Jonsson49bdee82020-04-06 13:21:21 +0200708 data_ptr += DRIVER_ACTION_LENGTH_32_BIT_WORD + cms_length;
709 break;
Kristofer Jonsson49bdee82020-04-06 13:21:21 +0200710 case NOP:
Kristofer Jonsson089a3472021-11-12 12:52:07 +0100711 LOG_DEBUG("NOP");
Kristofer Jonsson49bdee82020-04-06 13:21:21 +0200712 data_ptr += DRIVER_ACTION_LENGTH_32_BIT_WORD;
713 break;
714 default:
Ledion Dajaecdce6c2023-01-16 15:39:29 +0100715 LOG_ERR("UNSUPPORTED driver_action_command: %u", data_ptr->driver_action_command);
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100716 goto err;
Bhavik Patele645fed2020-06-12 14:46:47 +0200717 break;
Kristofer Jonsson49bdee82020-04-06 13:21:21 +0200718 }
719 }
Jonny Svärd136810f2021-10-13 16:04:26 +0200720
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100721 return 0;
722err:
723 LOG_ERR("Failed to invoke inference.");
724 ethosu_reset_job(drv);
725 return -1;
726}
Kristofer Jonsson2b201c32020-09-02 16:42:43 +0200727
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100728int ethosu_invoke_v3(struct ethosu_driver *drv,
729 const void *custom_data_ptr,
730 const int custom_data_size,
Ledion Dajaecdce6c2023-01-16 15:39:29 +0100731 uint64_t *const base_addr,
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100732 const size_t *base_addr_size,
733 const int num_base_addr,
734 void *user_arg)
735{
736 if (ethosu_invoke_async(
737 drv, custom_data_ptr, custom_data_size, base_addr, base_addr_size, num_base_addr, user_arg) < 0)
Bhavik Patel5f8dad12020-09-30 09:06:52 +0200738 {
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100739 return -1;
Bhavik Patel5f8dad12020-09-30 09:06:52 +0200740 }
Kristofer Jonsson2b201c32020-09-02 16:42:43 +0200741
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100742 return ethosu_wait(drv, true);
Kristofer Jonsson49bdee82020-04-06 13:21:21 +0200743}
744
Anton Moberg61da4d32020-12-22 16:00:31 +0100745struct ethosu_driver *ethosu_reserve_driver(void)
746{
Anton Mobergdf386e02021-02-02 11:26:48 +0100747 struct ethosu_driver *drv = NULL;
748
Jonny Svärdc9644242023-01-09 15:54:24 +0100749 LOG_INFO("Acquiring NPU driver handle");
Jonny Svärda2732ec2023-12-18 17:19:15 +0100750 ethosu_semaphore_take(ethosu_semaphore, ETHOSU_SEMAPHORE_WAIT_FOREVER); // This is meant to block until available
Anton Mobergdf386e02021-02-02 11:26:48 +0100751
Jonny Svärdc9644242023-01-09 15:54:24 +0100752 ethosu_mutex_lock(ethosu_mutex);
753 drv = registered_drivers;
754
755 while (drv != NULL)
756 {
757 if (!drv->reserved)
Anton Mobergdf386e02021-02-02 11:26:48 +0100758 {
Jonny Svärdc9644242023-01-09 15:54:24 +0100759 drv->reserved = true;
760 LOG_DEBUG("NPU driver handle %p reserved", drv);
Anton Mobergdf386e02021-02-02 11:26:48 +0100761 break;
762 }
Jonny Svärdc9644242023-01-09 15:54:24 +0100763 drv = drv->next;
764 }
765 ethosu_mutex_unlock(ethosu_mutex);
Anton Mobergdf386e02021-02-02 11:26:48 +0100766
Jonny Svärdc9644242023-01-09 15:54:24 +0100767 if (!drv)
768 {
769 LOG_ERR("No NPU driver handle available, but semaphore taken");
770 }
Anton Mobergdf386e02021-02-02 11:26:48 +0100771
772 return drv;
773}
774
Anton Moberg61da4d32020-12-22 16:00:31 +0100775void ethosu_release_driver(struct ethosu_driver *drv)
776{
Anton Mobergdfed5fd2021-03-11 14:41:11 +0100777 ethosu_mutex_lock(ethosu_mutex);
Anton Moberg61da4d32020-12-22 16:00:31 +0100778 if (drv != NULL && drv->reserved)
779 {
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100780 if (drv->job.state == ETHOSU_JOB_RUNNING || drv->job.state == ETHOSU_JOB_DONE)
781 {
782 // Give the inference one shot to complete or force kill the job
783 if (ethosu_wait(drv, false) == 1)
784 {
785 // Still running, soft reset the NPU and reset driver
Jonny Svärd301399d2022-04-26 18:31:24 +0200786 drv->power_request_counter = 0;
787 ethosu_soft_reset(drv);
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100788 ethosu_reset_job(drv);
Jonny Svärd1a3bb922022-02-25 16:28:21 +0100789 }
790 }
791
Anton Moberg61da4d32020-12-22 16:00:31 +0100792 drv->reserved = false;
Kristofer Jonsson089a3472021-11-12 12:52:07 +0100793 LOG_DEBUG("NPU driver handle %p released", drv);
Anton Mobergdfed5fd2021-03-11 14:41:11 +0100794 ethosu_semaphore_give(ethosu_semaphore);
Anton Moberg8d65b6f2020-12-21 09:37:18 +0100795 }
Anton Mobergdfed5fd2021-03-11 14:41:11 +0100796 ethosu_mutex_unlock(ethosu_mutex);
Anton Moberg8d65b6f2020-12-21 09:37:18 +0100797}