Kernel watchdog timeout
Implement kernel watchdog that detects when firmware becomes
unresponsive.
Change-Id: I5c5b58a56a2ce629e1fd7cabae83b61823239ea6
diff --git a/kernel/Kbuild b/kernel/Kbuild
index a6664ac..0b92c12 100644
--- a/kernel/Kbuild
+++ b/kernel/Kbuild
@@ -26,4 +26,5 @@
ethosu_inference.o \
ethosu_mailbox.o \
ethosu_network.o \
- ethosu_network_info.o
+ ethosu_network_info.o \
+ ethosu_watchdog.o
diff --git a/kernel/ethosu_device.c b/kernel/ethosu_device.c
index dfdaa58..f440331 100644
--- a/kernel/ethosu_device.c
+++ b/kernel/ethosu_device.c
@@ -252,6 +252,18 @@
return ret;
}
+static void ethosu_watchdog_callback(struct ethosu_watchdog *wdog)
+{
+ struct ethosu_device *edev =
+ container_of(wdog, struct ethosu_device, watchdog);
+
+ mutex_lock(&edev->mutex);
+
+ dev_warn(edev->dev, "Device watchdog timeout");
+
+ mutex_unlock(&edev->mutex);
+}
+
static int ethosu_open(struct inode *inode,
struct file *file)
{
@@ -442,11 +454,16 @@
dma_set_mask_and_coherent(edev->dev, DMA_BIT_MASK(DMA_ADDR_BITS));
- ret = ethosu_mailbox_init(&edev->mailbox, dev, in_queue, out_queue,
- ethosu_mbox_rx, edev);
+ ret = ethosu_watchdog_init(&edev->watchdog, dev,
+ ethosu_watchdog_callback);
if (ret)
goto release_reserved_mem;
+ ret = ethosu_mailbox_init(&edev->mailbox, dev, in_queue, out_queue,
+ ethosu_mbox_rx, edev, &edev->watchdog);
+ if (ret)
+ goto deinit_watchdog;
+
cdev_init(&edev->cdev, &fops);
edev->cdev.owner = THIS_MODULE;
@@ -476,6 +493,9 @@
deinit_mailbox:
ethosu_mailbox_deinit(&edev->mailbox);
+deinit_watchdog:
+ ethosu_watchdog_deinit(&edev->watchdog);
+
release_reserved_mem:
of_reserved_mem_device_release(edev->dev);
@@ -485,6 +505,7 @@
void ethosu_dev_deinit(struct ethosu_device *edev)
{
ethosu_mailbox_deinit(&edev->mailbox);
+ ethosu_watchdog_deinit(&edev->watchdog);
device_destroy(edev->class, edev->cdev.dev);
cdev_del(&edev->cdev);
of_reserved_mem_device_release(edev->dev);
diff --git a/kernel/ethosu_device.h b/kernel/ethosu_device.h
index d096e23..7d8791a 100644
--- a/kernel/ethosu_device.h
+++ b/kernel/ethosu_device.h
@@ -27,6 +27,7 @@
#include "uapi/ethosu.h"
#include "ethosu_mailbox.h"
+#include "ethosu_watchdog.h"
#include <linux/device.h>
#include <linux/cdev.h>
@@ -43,15 +44,16 @@
* struct ethosu_device - Device structure
*/
struct ethosu_device {
- struct device *dev;
- struct cdev cdev;
- struct class *class;
- dev_t devt;
- struct mutex mutex;
- struct ethosu_mailbox mailbox;
- struct list_head capabilities_list;
- struct list_head inference_list;
- struct list_head network_info_list;
+ struct device *dev;
+ struct cdev cdev;
+ struct class *class;
+ dev_t devt;
+ struct mutex mutex;
+ struct ethosu_mailbox mailbox;
+ struct ethosu_watchdog watchdog;
+ struct list_head capabilities_list;
+ struct list_head inference_list;
+ struct list_head network_info_list;
};
/**
diff --git a/kernel/ethosu_mailbox.c b/kernel/ethosu_mailbox.c
index e3d31fe..7753baa 100644
--- a/kernel/ethosu_mailbox.c
+++ b/kernel/ethosu_mailbox.c
@@ -23,6 +23,7 @@
****************************************************************************/
#include "ethosu_mailbox.h"
+#include "ethosu_watchdog.h"
#include "ethosu_buffer.h"
#include "ethosu_core_interface.h"
@@ -35,6 +36,38 @@
* Functions
****************************************************************************/
+static void ethosu_wd_inc(struct ethosu_mailbox *mbox,
+ enum ethosu_core_msg_type type)
+{
+ switch (type) {
+ case ETHOSU_CORE_MSG_PING:
+ case ETHOSU_CORE_MSG_INFERENCE_REQ:
+ case ETHOSU_CORE_MSG_VERSION_REQ:
+ case ETHOSU_CORE_MSG_CAPABILITIES_REQ:
+ case ETHOSU_CORE_MSG_NETWORK_INFO_REQ:
+ ethosu_watchdog_inc(mbox->wdog);
+ break;
+ default:
+ break;
+ }
+}
+
+static void ethosu_wd_dec(struct ethosu_mailbox *mbox,
+ enum ethosu_core_msg_type type)
+{
+ switch (type) {
+ case ETHOSU_CORE_MSG_PONG:
+ case ETHOSU_CORE_MSG_INFERENCE_RSP:
+ case ETHOSU_CORE_MSG_VERSION_RSP:
+ case ETHOSU_CORE_MSG_CAPABILITIES_RSP:
+ case ETHOSU_CORE_MSG_NETWORK_INFO_RSP:
+ ethosu_watchdog_dec(mbox->wdog);
+ break;
+ default:
+ break;
+ }
+}
+
static void ethosu_core_set_size(struct ethosu_buffer *buf,
struct ethosu_core_buffer *cbuf)
{
@@ -113,8 +146,15 @@
{ &msg, sizeof(msg) },
{ data, length }
};
+ int ret;
- return ethosu_queue_write(mbox, vec, 2);
+ ret = ethosu_queue_write(mbox, vec, 2);
+ if (ret)
+ return ret;
+
+ ethosu_wd_inc(mbox, type);
+
+ return 0;
}
static int ethosu_queue_read(struct ethosu_mailbox *mbox,
@@ -196,6 +236,8 @@
return -EBADMSG;
}
+ ethosu_wd_dec(mbox, header->type);
+
return 0;
}
@@ -329,13 +371,15 @@
struct resource *in_queue,
struct resource *out_queue,
ethosu_mailbox_cb callback,
- void *user_arg)
+ void *user_arg,
+ struct ethosu_watchdog *wdog)
{
int ret;
mbox->dev = dev;
mbox->callback = callback;
mbox->user_arg = user_arg;
+ mbox->wdog = wdog;
mbox->client.dev = dev;
mbox->client.rx_callback = ethosu_mailbox_rx_callback;
diff --git a/kernel/ethosu_mailbox.h b/kernel/ethosu_mailbox.h
index f004f92..956685b 100644
--- a/kernel/ethosu_mailbox.h
+++ b/kernel/ethosu_mailbox.h
@@ -39,6 +39,7 @@
struct ethosu_device;
struct ethosu_core_msg;
struct ethosu_core_queue;
+struct ethosu_watchdog;
struct resource;
typedef void (*ethosu_mailbox_cb)(void *user_arg);
@@ -54,6 +55,7 @@
struct mbox_chan *tx;
ethosu_mailbox_cb callback;
void *user_arg;
+ struct ethosu_watchdog *wdog;
};
/****************************************************************************
@@ -70,7 +72,8 @@
struct resource *in_queue,
struct resource *out_queue,
ethosu_mailbox_cb callback,
- void *user_arg);
+ void *user_arg,
+ struct ethosu_watchdog *wdog);
/**
* ethosu_mailbox_deinit() - Deinitialize mailbox
diff --git a/kernel/ethosu_network_info.c b/kernel/ethosu_network_info.c
index 2b1f841..24be677 100644
--- a/kernel/ethosu_network_info.c
+++ b/kernel/ethosu_network_info.c
@@ -47,6 +47,24 @@
devm_kfree(info->edev->dev, info);
}
+static int ethosu_network_info_send(struct ethosu_network_info *info)
+{
+ int ret;
+
+ /* Send network info request to firmware */
+ ret = ethosu_mailbox_network_info_request(&info->edev->mailbox,
+ info,
+ info->net->buf,
+ info->net->index);
+ if (ret)
+ return ret;
+
+ /* Increase reference count for the pending network info response */
+ ethosu_network_info_get(info);
+
+ return 0;
+}
+
struct ethosu_network_info *ethosu_network_info_create(
struct ethosu_device *edev,
struct ethosu_network *net,
@@ -71,17 +89,10 @@
/* Get reference to network */
ethosu_network_get(net);
- /* Send network info request to firmware */
- ret = ethosu_mailbox_network_info_request(&info->edev->mailbox,
- info,
- info->net->buf,
- info->net->index);
+ ret = ethosu_network_info_send(info);
if (ret)
goto put_info;
- /* Increase reference count for the pending network info response */
- ethosu_network_info_get(info);
-
dev_info(edev->dev, "Network info create. handle=%p\n", info);
return info;
diff --git a/kernel/ethosu_watchdog.c b/kernel/ethosu_watchdog.c
new file mode 100644
index 0000000..bde0803
--- /dev/null
+++ b/kernel/ethosu_watchdog.c
@@ -0,0 +1,130 @@
+/*
+ * Copyright (c) 2022 Arm Limited.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, you can access it online at
+ * http://www.gnu.org/licenses/gpl-2.0.html.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ */
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include "ethosu_watchdog.h"
+
+#include <linux/device.h>
+#include <linux/jiffies.h>
+#include <linux/moduleparam.h>
+#include <linux/version.h>
+
+/****************************************************************************
+ * Variables
+ ****************************************************************************/
+
+static unsigned long watchdog_timeout_ms = 3000;
+module_param(watchdog_timeout_ms, ulong, 0664);
+MODULE_PARM_DESC(watchdog_timeout_ms,
+ "Watchdog timeout in milliseconds for unresponsive firmware.");
+
+/****************************************************************************
+ * Functions
+ ****************************************************************************/
+
+static void ethosu_watchdog_update(struct ethosu_watchdog *wdog)
+{
+ int ret;
+
+ ret = mod_timer(&wdog->timer,
+ jiffies + msecs_to_jiffies(watchdog_timeout_ms));
+
+ dev_info(wdog->dev,
+ "Wdog: Update watchdog timeout. ret=%d, timeout_ms=%lu, refcount=%u", ret,
+ watchdog_timeout_ms, atomic_read(&wdog->refcount));
+}
+
+static void ethosu_watchdog_work(struct work_struct *work)
+{
+ struct ethosu_watchdog *wdog =
+ container_of(work, struct ethosu_watchdog, work);
+
+ dev_info(wdog->dev, "Wdog: Watchdog timeout. refcount=%u",
+ atomic_read(&wdog->refcount));
+
+ wdog->callback(wdog);
+}
+
+static void ethosu_watchdog_timeout(struct timer_list *timer)
+{
+ struct ethosu_watchdog *wdog =
+ container_of(timer, struct ethosu_watchdog, timer);
+
+ queue_work(system_unbound_wq, &wdog->work);
+}
+
+#if KERNEL_VERSION(4, 14, 0) > LINUX_VERSION_CODE
+static void ethosu_watchdog_timeout_legacy(unsigned long data)
+{
+ ethosu_watchdog_timeout((struct timer_list *)data);
+}
+
+#endif
+
+int ethosu_watchdog_init(struct ethosu_watchdog *wdog,
+ struct device *dev,
+ ethosu_watchdog_cb callback)
+{
+ wdog->dev = dev;
+ wdog->callback = callback;
+ atomic_set(&wdog->refcount, 0);
+ INIT_WORK(&wdog->work, ethosu_watchdog_work);
+
+#if KERNEL_VERSION(4, 14, 0) <= LINUX_VERSION_CODE
+ timer_setup(&wdog->timer, ethosu_watchdog_timeout, 0);
+#else
+ setup_timer(&wdog->timer, ethosu_watchdog_timeout_legacy,
+ (unsigned long)&wdog->timer);
+#endif
+
+ return 0;
+}
+
+void ethosu_watchdog_deinit(struct ethosu_watchdog *wdog)
+{
+ del_timer(&wdog->timer);
+}
+
+int ethosu_watchdog_reset(struct ethosu_watchdog *wdog)
+{
+ del_timer(&wdog->timer);
+ atomic_set(&wdog->refcount, 0);
+
+ return 0;
+}
+
+void ethosu_watchdog_inc(struct ethosu_watchdog *wdog)
+{
+ atomic_inc(&wdog->refcount);
+ ethosu_watchdog_update(wdog);
+}
+
+void ethosu_watchdog_dec(struct ethosu_watchdog *wdog)
+{
+ if (atomic_dec_and_test(&wdog->refcount)) {
+ dev_info(wdog->dev, "Wdog: Cancel watchdog timeout");
+ del_timer(&wdog->timer);
+ } else {
+ ethosu_watchdog_update(wdog);
+ }
+}
diff --git a/kernel/ethosu_watchdog.h b/kernel/ethosu_watchdog.h
new file mode 100644
index 0000000..d288af4
--- /dev/null
+++ b/kernel/ethosu_watchdog.h
@@ -0,0 +1,82 @@
+/*
+ * Copyright (c) 2022 Arm Limited.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, you can access it online at
+ * http://www.gnu.org/licenses/gpl-2.0.html.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ */
+
+#ifndef ETHOSU_WATCHDOG_H
+#define ETHOSU_WATCHDOG_H
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include <linux/types.h>
+#include <linux/timer.h>
+#include <linux/workqueue.h>
+
+/****************************************************************************
+ * Types
+ ****************************************************************************/
+
+struct device;
+struct ethosu_watchdog;
+
+typedef void (*ethosu_watchdog_cb)(struct ethosu_watchdog *wdog);
+
+struct ethosu_watchdog {
+ struct device *dev;
+ ethosu_watchdog_cb callback;
+ struct timer_list timer;
+ struct work_struct work;
+ atomic_t refcount;
+};
+
+/****************************************************************************
+ * Functions
+ ****************************************************************************/
+
+/**
+ * ethosu_watchdog_init() - Initialize watchdog
+ *
+ * Return: 0 on success, else error code.
+ */
+int ethosu_watchdog_init(struct ethosu_watchdog *wdog,
+ struct device *dev,
+ ethosu_watchdog_cb callback);
+
+/**
+ * ethosu_watchdog_deinit() - Deinitialize watchdog
+ */
+void ethosu_watchdog_deinit(struct ethosu_watchdog *wdog);
+
+/**
+ * ethosu_watchdog_reset() - Reset watchdog
+ */
+int ethosu_watchdog_reset(struct ethosu_watchdog *wdog);
+
+/**
+ * ethosu_watchdog_inc() - Increment reference count
+ */
+void ethosu_watchdog_inc(struct ethosu_watchdog *wdog);
+
+/**
+ * ethosu_watchdog_dec() - Decrement reference count
+ */
+void ethosu_watchdog_dec(struct ethosu_watchdog *wdog);
+
+#endif /* ETHOSU_WATCHDOG_H */