blob: 16673257cc7ccfd9ce971d30a93345bdea329dfe [file] [log] [blame]
/*
* Copyright (c) 2019-2021 Arm Limited. All rights reserved.
*
* SPDX-License-Identifier: Apache-2.0
*
* Licensed under the Apache License, Version 2.0 (the License); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an AS IS BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "ethosu_device.h"
#include "ethosu_common.h"
#include "ethosu_config.h"
#include <assert.h>
#include <stddef.h>
#include <stdio.h>
#define BASEP_OFFSET 4
#define REG_OFFSET 4
#define BYTES_1KB 1024
#define ADDRESS_BITS 48
#define ADDRESS_MASK ((1ull << ADDRESS_BITS) - 1)
#if defined(ARM_NPU_STUB)
static uint32_t stream_length = 0;
#endif
enum ethosu_error_codes ethosu_dev_init(struct ethosu_device *dev,
const void *base_address,
uint32_t secure_enable,
uint32_t privilege_enable)
{
#if !defined(ARM_NPU_STUB)
dev->base_address = (volatile uint32_t *)base_address;
dev->secure = secure_enable;
dev->privileged = privilege_enable;
ethosu_save_pmu_config(dev);
#else
UNUSED(dev);
UNUSED(base_address);
#endif
return ETHOSU_SUCCESS;
}
enum ethosu_error_codes ethosu_get_id(struct ethosu_device *dev, struct ethosu_id *id)
{
struct id_r _id;
#if !defined(ARM_NPU_STUB)
_id.word = ethosu_read_reg(dev, NPU_REG_ID);
#else
UNUSED(dev);
_id.word = 0;
_id.arch_patch_rev = NNX_ARCH_VERSION_PATCH;
_id.arch_minor_rev = NNX_ARCH_VERSION_MINOR;
_id.arch_major_rev = NNX_ARCH_VERSION_MAJOR;
#endif
id->version_status = _id.version_status;
id->version_minor = _id.version_minor;
id->version_major = _id.version_major;
id->product_major = _id.product_major;
id->arch_patch_rev = _id.arch_patch_rev;
id->arch_minor_rev = _id.arch_minor_rev;
id->arch_major_rev = _id.arch_major_rev;
return ETHOSU_SUCCESS;
}
enum ethosu_error_codes ethosu_get_config(struct ethosu_device *dev, struct ethosu_config *config)
{
struct config_r cfg = {.word = 0};
#if !defined(ARM_NPU_STUB)
cfg.word = ethosu_read_reg(dev, NPU_REG_CONFIG);
#else
UNUSED(dev);
#endif
config->macs_per_cc = cfg.macs_per_cc;
config->cmd_stream_version = cfg.cmd_stream_version;
config->shram_size = cfg.shram_size;
config->custom_dma = cfg.custom_dma;
return ETHOSU_SUCCESS;
}
enum ethosu_error_codes ethosu_run_command_stream(struct ethosu_device *dev,
const uint8_t *cmd_stream_ptr,
uint32_t cms_length,
const uint64_t *base_addr,
int num_base_addr)
{
enum ethosu_error_codes ret_code = ETHOSU_SUCCESS;
#if !defined(ARM_NPU_STUB)
ASSERT(num_base_addr <= ETHOSU_DRIVER_BASEP_INDEXES);
uint64_t qbase = (uintptr_t)cmd_stream_ptr + BASE_POINTER_OFFSET;
ASSERT(qbase <= ADDRESS_MASK);
LOG_DEBUG("QBASE=0x%016llx, QSIZE=%u, base_pointer_offset=0x%08x\n", qbase, cms_length, BASE_POINTER_OFFSET);
ethosu_write_reg(dev, NPU_REG_QBASE0, qbase & 0xffffffff);
ethosu_write_reg(dev, NPU_REG_QBASE1, qbase >> 32);
ethosu_write_reg(dev, NPU_REG_QSIZE, cms_length);
for (int i = 0; i < num_base_addr; i++)
{
uint64_t addr = base_addr[i] + BASE_POINTER_OFFSET;
ASSERT(addr <= ADDRESS_MASK);
LOG_DEBUG("BASEP%d=0x%016llx\n", i, addr);
ethosu_write_reg(dev, NPU_REG_BASEP0 + (2 * i) * BASEP_OFFSET, addr & 0xffffffff);
ethosu_write_reg(dev, NPU_REG_BASEP0 + (2 * i + 1) * BASEP_OFFSET, addr >> 32);
}
ret_code = ethosu_set_command_run(dev);
#else
// NPU stubbed
UNUSED(dev);
stream_length = cms_length;
UNUSED(cmd_stream_ptr);
UNUSED(base_addr);
ASSERT(num_base_addr < ETHOSU_DRIVER_BASEP_INDEXES);
#if defined(NDEBUG)
UNUSED(num_base_addr);
#endif
#endif
return ret_code;
}
enum ethosu_error_codes ethosu_is_irq_raised(struct ethosu_device *dev, uint8_t *irq_raised)
{
#if !defined(ARM_NPU_STUB)
struct status_r status;
status.word = ethosu_read_reg(dev, NPU_REG_STATUS);
if (status.irq_raised == 1)
{
*irq_raised = 1;
}
else
{
*irq_raised = 0;
}
#else
UNUSED(dev);
*irq_raised = 1;
#endif
return ETHOSU_SUCCESS;
}
enum ethosu_error_codes ethosu_clear_irq_status(struct ethosu_device *dev)
{
#if !defined(ARM_NPU_STUB)
struct cmd_r oldcmd;
oldcmd.word = ethosu_read_reg(dev, NPU_REG_CMD);
struct cmd_r cmd;
cmd.word = 0;
cmd.clear_irq = 1;
cmd.clock_q_enable = oldcmd.clock_q_enable;
cmd.power_q_enable = oldcmd.power_q_enable;
ethosu_write_reg(dev, NPU_REG_CMD, cmd.word);
LOG_DEBUG("CMD=0x%08x\n", cmd.word);
#else
UNUSED(dev);
#endif
return ETHOSU_SUCCESS;
}
enum ethosu_error_codes ethosu_soft_reset(struct ethosu_device *dev)
{
enum ethosu_error_codes return_code = ETHOSU_SUCCESS;
#if !defined(ARM_NPU_STUB)
struct reset_r reset;
struct prot_r prot;
reset.word = 0;
reset.pending_CPL = dev->privileged ? PRIVILEGE_LEVEL_PRIVILEGED : PRIVILEGE_LEVEL_USER;
reset.pending_CSL = dev->secure ? SECURITY_LEVEL_SECURE : SECURITY_LEVEL_NON_SECURE;
// Reset and set security level
LOG_INFO("Soft reset NPU\n");
ethosu_write_reg(dev, NPU_REG_RESET, reset.word);
// Wait for reset to complete
return_code = ethosu_wait_for_reset(dev);
if (return_code != ETHOSU_SUCCESS)
{
LOG_ERR("Soft reset timed out\n");
return return_code;
}
// Verify that NPU has switched security state and privilege level
prot.word = ethosu_read_reg(dev, NPU_REG_PROT);
if (prot.active_CPL != reset.pending_CPL || prot.active_CSL != reset.pending_CSL)
{
LOG_ERR("Failed to switch security state and privilege level\n");
// Register access not permitted
return ETHOSU_GENERIC_FAILURE;
}
// Save the prot register
dev->proto = ethosu_read_reg(dev, NPU_REG_PROT);
// Soft reset will clear the PMU configuration and counters. The shadow PMU counters
// are cleared by saving the PMU counters to ram, which will read back zeros.
// The PMU configuration will be restored in the invoke function after power save
// has been disabled.
ethosu_save_pmu_counters(dev);
#else
UNUSED(dev);
#endif
return return_code;
}
enum ethosu_error_codes ethosu_wait_for_reset(struct ethosu_device *dev)
{
#if !defined(ARM_NPU_STUB)
struct status_r status;
// Wait until reset status indicates that reset has been completed
for (int i = 0; i < 100000; i++)
{
status.word = ethosu_read_reg(dev, NPU_REG_STATUS);
if (0 == status.reset_status)
{
break;
}
}
if (1 == status.reset_status)
{
return ETHOSU_GENERIC_FAILURE;
}
#else
UNUSED(dev);
#endif
return ETHOSU_SUCCESS;
}
enum ethosu_error_codes ethosu_read_apb_reg(struct ethosu_device *dev,
uint32_t start_address,
uint16_t num_reg,
uint32_t *reg)
{
#if !defined(ARM_NPU_STUB)
uint32_t address = start_address;
ASSERT((start_address + num_reg) < ID_REGISTERS_SIZE);
for (int i = 0; i < num_reg; i++)
{
reg[i] = ethosu_read_reg(dev, address);
address += REG_OFFSET;
}
#else
// NPU stubbed
UNUSED(dev);
UNUSED(start_address);
UNUSED(num_reg);
UNUSED(reg);
#endif
return ETHOSU_SUCCESS;
}
enum ethosu_error_codes ethosu_set_qconfig(struct ethosu_device *dev, enum ethosu_memory_type memory_type)
{
if (memory_type > ETHOSU_AXI1_OUTSTANDING_COUNTER3)
{
return ETHOSU_INVALID_PARAM;
}
#if !defined(ARM_NPU_STUB)
ethosu_write_reg(dev, NPU_REG_QCONFIG, memory_type);
LOG_DEBUG("QCONFIG=0x%08x\n", memory_type);
#else
// NPU stubbed
UNUSED(dev);
UNUSED(memory_type);
#endif
return ETHOSU_SUCCESS;
}
enum ethosu_error_codes ethosu_set_regioncfg(struct ethosu_device *dev,
uint8_t region,
enum ethosu_memory_type memory_type)
{
if (region > 7)
{
return ETHOSU_INVALID_PARAM;
}
#if !defined(ARM_NPU_STUB)
struct regioncfg_r regioncfg;
regioncfg.word = ethosu_read_reg(dev, NPU_REG_REGIONCFG);
regioncfg.word &= ~(0x3 << (2 * region));
regioncfg.word |= (memory_type & 0x3) << (2 * region);
ethosu_write_reg(dev, NPU_REG_REGIONCFG, regioncfg.word);
LOG_DEBUG("REGIONCFG%u=0x%08x\n", region, regioncfg.word);
#else
// NPU stubbed
UNUSED(dev);
UNUSED(region);
UNUSED(memory_type);
#endif
return ETHOSU_SUCCESS;
}
enum ethosu_error_codes ethosu_set_axi_limit0(struct ethosu_device *dev,
enum ethosu_axi_limit_beats max_beats,
enum ethosu_axi_limit_mem_type memtype,
uint8_t max_reads,
uint8_t max_writes)
{
#if !defined(ARM_NPU_STUB)
struct axi_limit0_r axi_limit0;
axi_limit0.word = 0;
axi_limit0.max_beats = max_beats;
axi_limit0.memtype = memtype;
axi_limit0.max_outstanding_read_m1 = max_reads - 1;
axi_limit0.max_outstanding_write_m1 = max_writes - 1;
ethosu_write_reg(dev, NPU_REG_AXI_LIMIT0, axi_limit0.word);
LOG_DEBUG("AXI_LIMIT0=0x%08x\n", axi_limit0.word);
#else
// NPU stubbed
UNUSED(dev);
UNUSED(max_beats);
UNUSED(memtype);
UNUSED(max_reads);
UNUSED(max_writes);
#endif
return ETHOSU_SUCCESS;
}
enum ethosu_error_codes ethosu_set_axi_limit1(struct ethosu_device *dev,
enum ethosu_axi_limit_beats max_beats,
enum ethosu_axi_limit_mem_type memtype,
uint8_t max_reads,
uint8_t max_writes)
{
#if !defined(ARM_NPU_STUB)
struct axi_limit1_r axi_limit1;
axi_limit1.word = 0;
axi_limit1.max_beats = max_beats;
axi_limit1.memtype = memtype;
axi_limit1.max_outstanding_read_m1 = max_reads - 1;
axi_limit1.max_outstanding_write_m1 = max_writes - 1;
ethosu_write_reg(dev, NPU_REG_AXI_LIMIT1, axi_limit1.word);
LOG_DEBUG("AXI_LIMIT1=0x%08x\n", axi_limit1.word);
#else
// NPU stubbed
UNUSED(dev);
UNUSED(max_beats);
UNUSED(memtype);
UNUSED(max_reads);
UNUSED(max_writes);
#endif
return ETHOSU_SUCCESS;
}
enum ethosu_error_codes ethosu_set_axi_limit2(struct ethosu_device *dev,
enum ethosu_axi_limit_beats max_beats,
enum ethosu_axi_limit_mem_type memtype,
uint8_t max_reads,
uint8_t max_writes)
{
#if !defined(ARM_NPU_STUB)
struct axi_limit2_r axi_limit2;
axi_limit2.word = 0;
axi_limit2.max_beats = max_beats;
axi_limit2.memtype = memtype;
axi_limit2.max_outstanding_read_m1 = max_reads - 1;
axi_limit2.max_outstanding_write_m1 = max_writes - 1;
ethosu_write_reg(dev, NPU_REG_AXI_LIMIT2, axi_limit2.word);
LOG_DEBUG("AXI_LIMIT2=0x%08x\n", axi_limit2.word);
#else
// NPU stubbed
UNUSED(dev);
UNUSED(max_beats);
UNUSED(memtype);
UNUSED(max_reads);
UNUSED(max_writes);
#endif
return ETHOSU_SUCCESS;
}
enum ethosu_error_codes ethosu_set_axi_limit3(struct ethosu_device *dev,
enum ethosu_axi_limit_beats max_beats,
enum ethosu_axi_limit_mem_type memtype,
uint8_t max_reads,
uint8_t max_writes)
{
#if !defined(ARM_NPU_STUB)
struct axi_limit3_r axi_limit3;
axi_limit3.word = 0;
axi_limit3.max_beats = max_beats;
axi_limit3.memtype = memtype;
axi_limit3.max_outstanding_read_m1 = max_reads - 1;
axi_limit3.max_outstanding_write_m1 = max_writes - 1;
ethosu_write_reg(dev, NPU_REG_AXI_LIMIT3, axi_limit3.word);
LOG_DEBUG("AXI_LIMIT3=0x%08x\n", axi_limit3.word);
#else
// NPU stubbed
UNUSED(dev);
UNUSED(max_beats);
UNUSED(memtype);
UNUSED(max_reads);
UNUSED(max_writes);
#endif
return ETHOSU_SUCCESS;
}
enum ethosu_error_codes ethosu_get_revision(struct ethosu_device *dev, uint32_t *revision)
{
#if !defined(ARM_NPU_STUB)
*revision = ethosu_read_reg(dev, NPU_REG_REVISION);
#else
UNUSED(dev);
*revision = 0xDEADC0DE;
#endif
return ETHOSU_SUCCESS;
}
enum ethosu_error_codes ethosu_get_qread(struct ethosu_device *dev, uint32_t *qread)
{
#if !defined(ARM_NPU_STUB)
*qread = ethosu_read_reg(dev, NPU_REG_QREAD);
#else
UNUSED(dev);
*qread = stream_length;
#endif
return ETHOSU_SUCCESS;
}
enum ethosu_error_codes ethosu_get_status_mask(struct ethosu_device *dev, uint16_t *status_mask)
{
#if !defined(ARM_NPU_STUB)
struct status_r status;
status.word = ethosu_read_reg(dev, NPU_REG_STATUS);
*status_mask = status.word & 0xFFFF;
#else
UNUSED(dev);
*status_mask = 0x0000;
#endif
return ETHOSU_SUCCESS;
}
enum ethosu_error_codes ethosu_get_irq_history_mask(struct ethosu_device *dev, uint16_t *irq_history_mask)
{
#if !defined(ARM_NPU_STUB)
struct status_r status;
status.word = ethosu_read_reg(dev, NPU_REG_STATUS);
*irq_history_mask = status.irq_history_mask;
#else
UNUSED(dev);
*irq_history_mask = 0xffff;
#endif
return ETHOSU_SUCCESS;
}
enum ethosu_error_codes ethosu_clear_irq_history_mask(struct ethosu_device *dev, uint16_t irq_history_clear_mask)
{
#if !defined(ARM_NPU_STUB)
struct cmd_r oldcmd;
oldcmd.word = ethosu_read_reg(dev, NPU_REG_CMD);
struct cmd_r cmd;
cmd.word = 0;
cmd.clock_q_enable = oldcmd.clock_q_enable;
cmd.power_q_enable = oldcmd.power_q_enable;
cmd.clear_irq_history = irq_history_clear_mask;
ethosu_write_reg(dev, NPU_REG_CMD, cmd.word);
LOG_DEBUG("CMD=0x%08x\n", cmd.word);
#else
UNUSED(dev);
UNUSED(irq_history_clear_mask);
#endif
return ETHOSU_SUCCESS;
}
enum ethosu_error_codes ethosu_set_command_run(struct ethosu_device *dev)
{
#if !defined(ARM_NPU_STUB)
struct cmd_r oldcmd;
oldcmd.word = ethosu_read_reg(dev, NPU_REG_CMD);
struct cmd_r cmd;
cmd.word = 0;
cmd.transition_to_running_state = 1;
cmd.clock_q_enable = oldcmd.clock_q_enable;
cmd.power_q_enable = oldcmd.power_q_enable;
ethosu_write_reg(dev, NPU_REG_CMD, cmd.word);
LOG_DEBUG("CMD=0x%08x\n", cmd.word);
#else
UNUSED(dev);
#endif
return ETHOSU_SUCCESS;
}
enum ethosu_error_codes ethosu_get_shram_data(struct ethosu_device *dev, int section, uint32_t *shram_p)
{
#if !defined(ARM_NPU_STUB)
int i = 0;
uint32_t address = NPU_REG_SHARED_BUFFER0;
ethosu_write_reg(dev, NPU_REG_DEBUG_ADDRESS, section * BYTES_1KB);
while (address <= NPU_REG_SHARED_BUFFER255)
{
shram_p[i] = ethosu_read_reg(dev, address);
address += REG_OFFSET;
i++;
}
#else
// NPU stubbed
UNUSED(dev);
UNUSED(section);
UNUSED(shram_p);
#endif
return ETHOSU_SUCCESS;
}
enum ethosu_error_codes ethosu_set_clock_and_power(struct ethosu_device *dev,
enum ethosu_clock_q_request clock_q,
enum ethosu_power_q_request power_q)
{
#if !defined(ARM_NPU_STUB)
struct cmd_r cmd;
cmd.word = 0;
cmd.clock_q_enable = clock_q;
cmd.power_q_enable = power_q;
ethosu_write_reg(dev, NPU_REG_CMD, cmd.word);
LOG_DEBUG("CMD=0x%08x\n", cmd.word);
#else
UNUSED(dev);
UNUSED(clock_q);
UNUSED(power_q);
#endif
return ETHOSU_SUCCESS;
}
uint32_t ethosu_read_reg(struct ethosu_device *dev, uint32_t address)
{
#if !defined(ARM_NPU_STUB)
ASSERT(dev->base_address != 0);
ASSERT(address % 4 == 0);
volatile uint32_t *reg = dev->base_address + address / sizeof(uint32_t);
return *reg;
#else
UNUSED(dev);
UNUSED(address);
return 0;
#endif
}
void ethosu_write_reg(struct ethosu_device *dev, uint32_t address, uint32_t value)
{
#if !defined(ARM_NPU_STUB)
ASSERT(dev->base_address != 0);
ASSERT(address % 4 == 0);
volatile uint32_t *reg = dev->base_address + address / sizeof(uint32_t);
*reg = value;
#else
UNUSED(dev);
UNUSED(address);
UNUSED(value);
#endif
}
void ethosu_write_reg_shadow(struct ethosu_device *dev, uint32_t address, uint32_t value, uint32_t *shadow)
{
ethosu_write_reg(dev, address, value);
*shadow = ethosu_read_reg(dev, address);
}
enum ethosu_error_codes ethosu_save_pmu_config(struct ethosu_device *dev)
{
#if !defined(ARM_NPU_STUB)
// Save the PMU control register
dev->pmcr = ethosu_read_reg(dev, NPU_REG_PMCR);
// Save IRQ control
dev->pmint = ethosu_read_reg(dev, NPU_REG_PMINTSET);
// Save the enabled events mask
dev->pmcnten = ethosu_read_reg(dev, NPU_REG_PMCNTENSET);
// Save start and stop event
dev->pmccntr_cfg = ethosu_read_reg(dev, NPU_REG_PMCCNTR_CFG);
// Save the event settings and counters
for (uint32_t i = 0; i < ETHOSU_PMU_NCOUNTERS; i++)
{
dev->pmu_evtypr[i] = ethosu_read_reg(dev, NPU_REG_PMEVTYPER0 + i * sizeof(uint32_t));
}
#else
UNUSED(dev);
#endif
return ETHOSU_SUCCESS;
}
enum ethosu_error_codes ethosu_restore_pmu_config(struct ethosu_device *dev)
{
#if !defined(ARM_NPU_STUB)
// Restore PMU control register
ethosu_write_reg(dev, NPU_REG_PMCR, dev->pmcr);
// Restore IRQ control
ethosu_write_reg(dev, NPU_REG_PMINTSET, dev->pmint);
// Restore enabled event mask
ethosu_write_reg(dev, NPU_REG_PMCNTENSET, dev->pmcnten);
// Restore start and stop event
ethosu_write_reg(dev, NPU_REG_PMCCNTR_CFG, dev->pmccntr_cfg);
// Save the event settings and counters
for (uint32_t i = 0; i < ETHOSU_PMU_NCOUNTERS; i++)
{
ethosu_write_reg(dev, NPU_REG_PMEVTYPER0 + i * sizeof(uint32_t), dev->pmu_evtypr[i]);
}
#else
UNUSED(dev);
#endif
return ETHOSU_SUCCESS;
}
enum ethosu_error_codes ethosu_save_pmu_counters(struct ethosu_device *dev)
{
#if !defined(ARM_NPU_STUB)
// Save the cycle counter
dev->pmccntr[0] = ethosu_read_reg(dev, NPU_REG_PMCCNTR_LO);
dev->pmccntr[1] = ethosu_read_reg(dev, NPU_REG_PMCCNTR_HI);
// Save the event settings and counters
for (uint32_t i = 0; i < ETHOSU_PMU_NCOUNTERS; i++)
{
dev->pmu_evcntr[i] = ethosu_read_reg(dev, NPU_REG_PMEVCNTR0 + i * sizeof(uint32_t));
}
#else
UNUSED(dev);
#endif
return ETHOSU_SUCCESS;
}
bool ethosu_status_has_error(struct ethosu_device *dev)
{
bool status_error = false;
#if !defined(ARM_NPU_STUB)
struct status_r status;
status.word = ethosu_read_reg(dev, NPU_REG_STATUS);
status_error = ((1 == status.bus_status) || (1 == status.cmd_parse_error) || (1 == status.wd_fault) ||
(1 == status.ecc_fault));
#else
UNUSED(dev);
#endif
return status_error;
}