From 1a6163e69cf20465ab4e22314abbd73db19f802f Mon Sep 17 00:00:00 2001 From: Priyanka G Pai Date: Thu, 8 Aug 2024 19:30:59 +0530 Subject: [PATCH] msm: npu: Add NPU driver support for kernel 6.1 NPU driver snapshot from msm-5.15 branch commit f5053f87777d ("msm: npu: Fix OOB issue in IPC between driver and firmware"). Change-Id: Iea68b912dd7efd9a979969a91fb38d3611e3ff8c Signed-off-by: Priyanka G Pai --- drivers/media/platform/msm/npu/Kconfig | 11 + drivers/media/platform/msm/npu/Makefile | 11 + drivers/media/platform/msm/npu/npu_common.h | 277 ++ drivers/media/platform/msm/npu/npu_dbg.c | 33 + drivers/media/platform/msm/npu/npu_debugfs.c | 184 ++ drivers/media/platform/msm/npu/npu_dev.c | 2428 +++++++++++++++++ drivers/media/platform/msm/npu/npu_firmware.h | 176 ++ drivers/media/platform/msm/npu/npu_host_ipc.c | 438 +++ drivers/media/platform/msm/npu/npu_host_ipc.h | 464 ++++ drivers/media/platform/msm/npu/npu_hw.h | 53 + .../media/platform/msm/npu/npu_hw_access.c | 485 ++++ .../media/platform/msm/npu/npu_hw_access.h | 87 + drivers/media/platform/msm/npu/npu_mgr.c | 2112 ++++++++++++++ drivers/media/platform/msm/npu/npu_mgr.h | 147 + include/soc/qcom/subsystem_restart.h | 279 ++ 15 files changed, 7185 insertions(+) create mode 100644 drivers/media/platform/msm/npu/npu_common.h create mode 100644 drivers/media/platform/msm/npu/npu_dbg.c create mode 100644 drivers/media/platform/msm/npu/npu_debugfs.c create mode 100644 drivers/media/platform/msm/npu/npu_dev.c create mode 100644 drivers/media/platform/msm/npu/npu_firmware.h create mode 100644 drivers/media/platform/msm/npu/npu_host_ipc.c create mode 100644 drivers/media/platform/msm/npu/npu_host_ipc.h create mode 100644 drivers/media/platform/msm/npu/npu_hw.h create mode 100644 drivers/media/platform/msm/npu/npu_hw_access.c create mode 100644 drivers/media/platform/msm/npu/npu_hw_access.h create mode 100644 drivers/media/platform/msm/npu/npu_mgr.c create mode 100644 drivers/media/platform/msm/npu/npu_mgr.h create mode 100644 include/soc/qcom/subsystem_restart.h diff --git a/drivers/media/platform/msm/npu/Kconfig b/drivers/media/platform/msm/npu/Kconfig index 985ff9e1986d..5b4f10968a1c 100644 --- a/drivers/media/platform/msm/npu/Kconfig +++ b/drivers/media/platform/msm/npu/Kconfig @@ -7,3 +7,14 @@ config VIRTIO_NPU which provides acceleration for neural network processing. This driver is based on virtio. Say Y if you want to support virtual NPU. + +config MSM_NPU + tristate "QTI MSM Neural Processing Unit support" + depends on ARCH_QCOM + help + Enable support for Neural Processing Unit + for specific QTI chipsets. + This module serves as the common driver + for npu which provides acceleration for neural + network processing. + diff --git a/drivers/media/platform/msm/npu/Makefile b/drivers/media/platform/msm/npu/Makefile index 48f0b6ecc329..6899941e7ed0 100644 --- a/drivers/media/platform/msm/npu/Makefile +++ b/drivers/media/platform/msm/npu/Makefile @@ -1,3 +1,14 @@ # SPDX-License-Identifier: GPL-2.0-only +ifneq ($(CONFIG_VIRTIO_NPU),) obj-$(CONFIG_VIRTIO_NPU) := virtio_npu.o +else +msm_npu-objs := npu_dbg.o \ + npu_dev.o \ + npu_debugfs.o \ + npu_host_ipc.o \ + npu_hw_access.o \ + npu_mgr.o + +obj-$(CONFIG_MSM_NPU) := msm_npu.o +endif diff --git a/drivers/media/platform/msm/npu/npu_common.h b/drivers/media/platform/msm/npu/npu_common.h new file mode 100644 index 000000000000..5e64ea23ccdc --- /dev/null +++ b/drivers/media/platform/msm/npu/npu_common.h @@ -0,0 +1,277 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2017-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2023-2024, Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#ifndef _NPU_COMMON_H +#define _NPU_COMMON_H + +/* ------------------------------------------------------------------------- + * Includes + * ------------------------------------------------------------------------- + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "npu_mgr.h" + +/* ------------------------------------------------------------------------- + * Defines + * ------------------------------------------------------------------------- + */ +#define NPU_MAX_MBOX_NUM 2 +#define NPU_MBOX_LOW_PRI 0 +#define NPU_MBOX_HIGH_PRI 1 + +#define DEFAULT_REG_DUMP_NUM 64 +#define ROW_BYTES 16 +#define GROUP_BYTES 4 + +#define NUM_MAX_CLK_NUM 24 +#define NPU_MAX_REGULATOR_NUM 2 +#define NPU_MAX_DT_NAME_LEN 21 +#define NPU_MAX_PWRLEVELS 8 +#define NPU_MAX_STATS_BUF_SIZE 16384 +#define NPU_MAX_PATCH_NUM 160 + +#define PERF_MODE_DEFAULT 0 + +enum npu_power_level { + NPU_PWRLEVEL_MINSVS = 0, + NPU_PWRLEVEL_LOWSVS, + NPU_PWRLEVEL_SVS, + NPU_PWRLEVEL_SVS_L1, + NPU_PWRLEVEL_NOM, + NPU_PWRLEVEL_NOM_L1, + NPU_PWRLEVEL_TURBO, + NPU_PWRLEVEL_TURBO_L1, + NPU_PWRLEVEL_OFF = 0xFFFFFFFF, +}; + +/* ------------------------------------------------------------------------- + * Data Structures + * ------------------------------------------------------------------------- + */ +struct npu_smmu_ctx { + int domain; + struct dma_iommu_mapping *mmu_mapping; + struct reg_bus_client *reg_bus_clt; + int32_t attach_cnt; +}; + +struct npu_ion_buf { + int fd; + struct dma_buf *dma_buf; + struct dma_buf_attachment *attachment; + struct sg_table *table; + dma_addr_t iova; + uint32_t size; + void *phys_addr; + void *buf; + struct list_head list; +}; + +struct npu_clk { + struct clk *clk; + char clk_name[NPU_MAX_DT_NAME_LEN]; +}; + +struct npu_regulator { + struct regulator *regulator; + char regulator_name[NPU_MAX_DT_NAME_LEN]; +}; + +struct npu_debugfs_ctx { + struct dentry *root; + uint32_t reg_off; + uint32_t reg_cnt; +}; + +struct npu_debugfs_reg_ctx { + char *buf; + size_t buf_len; + struct npu_device *npu_dev; +}; + +struct npu_mbox { + struct mbox_client client; + struct mbox_chan *chan; + struct npu_device *npu_dev; + uint32_t id; +}; + +/** + * struct npu_pwrlevel - Struct holding different pwrlevel info obtained + * from dtsi file + * @pwr_level: NPU power level + * @freq[]: NPU frequency vote in Hz + */ +struct npu_pwrlevel { + uint32_t pwr_level; + long clk_freq[NUM_MAX_CLK_NUM]; +}; + +/* + * struct npu_reg - Struct holding npu register information + * @ off - register offset + * @ val - register value + * @ valid - if register value is valid + */ +struct npu_reg { + uint32_t off; + uint32_t val; + bool valid; +}; + +/** + * struct npu_pwrctrl - Power control settings for a NPU device + * @pwr_vote_num - voting information for power enable + * @pwrlevels - List of supported power levels + * @active_pwrlevel - The currently active power level + * @default_pwrlevel - device wake up power level + * @max_pwrlevel - maximum allowable powerlevel per the user + * @min_pwrlevel - minimum allowable powerlevel per the user + * @num_pwrlevels - number of available power levels + * @cdsprm_pwrlevel - maximum power level from cdsprm + * @fmax_pwrlevel - maximum power level from qfprom fmax setting + * @uc_pwrlevel - power level from user driver setting + * @perf_mode_override - perf mode from sysfs to override perf mode + * settings from user driver + * @dcvs_mode - dcvs mode from sysfs to turn on dcvs mode + * settings from user driver + * @devbw - bw device + */ +struct npu_pwrctrl { + int32_t pwr_vote_num; + + struct npu_pwrlevel pwrlevels[NPU_MAX_PWRLEVELS]; + uint32_t active_pwrlevel; + uint32_t default_pwrlevel; + uint32_t max_pwrlevel; + uint32_t min_pwrlevel; + uint32_t num_pwrlevels; + + struct device *devbw; + uint32_t bwmon_enabled; + uint32_t uc_pwrlevel; + uint32_t cdsprm_pwrlevel; + uint32_t fmax_pwrlevel; + uint32_t perf_mode_override; + uint32_t dcvs_mode; + uint32_t cur_dcvs_activity; +}; + +/** + * struct npu_thermalctrl - Thermal control settings for a NPU device + * @max_state - maximum thermal mitigation state + * @current_state - current thermal mitigation state + * @pwr_level -power level that thermal control requested + */ +struct npu_thermalctrl { + unsigned long max_state; + unsigned long current_state; + uint32_t pwr_level; +}; + +#define NPU_MAX_IRQ 3 + +struct npu_irq { + char *name; + int irq; + int irq_type; +}; + +struct npu_io_data { + size_t size; + void __iomem *base; +}; + +struct npu_fw_io_data { + phys_addr_t mem_phys; + phys_addr_t mem_reloc; + void *mem_region; + size_t mem_size; +}; + +struct npu_device { + struct mutex dev_lock; + + struct platform_device *pdev; + + dev_t dev_num; + struct cdev cdev; + struct class *class; + struct device *device; + + struct npu_io_data core_io; + struct npu_io_data tcm_io; + struct npu_io_data bwmon_io; + struct npu_io_data qfprom_io; + struct npu_fw_io_data fw_io; + + uint32_t core_clk_num; + struct npu_clk core_clks[NUM_MAX_CLK_NUM]; + + uint32_t regulator_num; + struct npu_regulator regulators[NPU_MAX_DT_NAME_LEN]; + + struct npu_irq irq[NPU_MAX_IRQ]; + + struct device *cb_device; + + struct npu_host_ctx host_ctx; + struct npu_smmu_ctx smmu_ctx; + struct npu_debugfs_ctx debugfs_ctx; + + struct npu_mbox mbox_aop; + + struct thermal_cooling_device *tcdev; + struct npu_pwrctrl pwrctrl; + struct npu_thermalctrl thermalctrl; + + struct llcc_slice_desc *sys_cache; + uint32_t execute_v2_flag; + bool cxlimit_registered; + struct icc_path *icc_npu_cdspmem; + struct icc_path *icc_cpu_imemcfg; + uint32_t hw_version; +}; + +struct npu_client { + struct npu_device *npu_dev; + struct mutex list_lock; + struct list_head mapped_buffer_list; +}; + +/* ------------------------------------------------------------------------- + * Function Prototypes + * ------------------------------------------------------------------------- + */ +int npu_debugfs_init(struct npu_device *npu_dev); +void npu_debugfs_deinit(struct npu_device *npu_dev); + +int npu_enable_core_power(struct npu_device *npu_dev); +void npu_disable_core_power(struct npu_device *npu_dev); +int npu_enable_post_pil_clocks(struct npu_device *npu_dev); +void npu_disable_post_pil_clocks(struct npu_device *npu_dev); + +irqreturn_t npu_intr_hdler(int irq, void *ptr); + +int npu_set_uc_power_level(struct npu_device *npu_dev, + uint32_t pwr_level); + +int fw_init(struct npu_device *npu_dev); +void fw_deinit(struct npu_device *npu_dev, bool ssr, bool fw_alive); +int npu_notify_cdsprm_cxlimit_activity(struct npu_device *npu_dev, bool enable); + +#endif /* _NPU_COMMON_H */ diff --git a/drivers/media/platform/msm/npu/npu_dbg.c b/drivers/media/platform/msm/npu/npu_dbg.c new file mode 100644 index 000000000000..6a600a497534 --- /dev/null +++ b/drivers/media/platform/msm/npu/npu_dbg.c @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2018-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2023-2024, Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +/* ------------------------------------------------------------------------- + * Includes + * ------------------------------------------------------------------------- + */ +#include "npu_common.h" +#include "npu_firmware.h" +#include "npu_hw.h" +#include "npu_hw_access.h" +#include "npu_mgr.h" + +/* ------------------------------------------------------------------------- + * Function Definitions - Debug + * ------------------------------------------------------------------------- + */ +void npu_dump_debug_timeout_stats(struct npu_device *npu_dev) +{ + uint32_t reg_val; + + reg_val = REGR(npu_dev, REG_FW_JOB_CNT_START); + pr_info("fw jobs execute started count = %d\n", reg_val); + reg_val = REGR(npu_dev, REG_FW_JOB_CNT_END); + pr_info("fw jobs execute finished count = %d\n", reg_val); + reg_val = REGR(npu_dev, REG_NPU_FW_DEBUG_DATA); + pr_info("fw jobs aco parser debug = %d\n", reg_val); +} diff --git a/drivers/media/platform/msm/npu/npu_debugfs.c b/drivers/media/platform/msm/npu/npu_debugfs.c new file mode 100644 index 000000000000..c7ac86e4fc6c --- /dev/null +++ b/drivers/media/platform/msm/npu/npu_debugfs.c @@ -0,0 +1,184 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2018-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2023-2024, Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +/* ------------------------------------------------------------------------- + * Includes + * ------------------------------------------------------------------------- + */ +#include + +#include "npu_hw.h" +#include "npu_hw_access.h" +#include "npu_common.h" + +/* ------------------------------------------------------------------------- + * Defines + * ------------------------------------------------------------------------- + */ +#define NPU_LOG_BUF_SIZE 4096 + +/* ------------------------------------------------------------------------- + * Function Prototypes + * ------------------------------------------------------------------------- + */ +static int npu_debug_open(struct inode *inode, struct file *file); +static int npu_debug_release(struct inode *inode, struct file *file); +static ssize_t npu_debug_ctrl_write(struct file *file, + const char __user *user_buf, size_t count, loff_t *ppos); + +/* ------------------------------------------------------------------------- + * Variables + * ------------------------------------------------------------------------- + */ +static struct npu_device *g_npu_dev; + +static const struct file_operations npu_ctrl_fops = { + .open = npu_debug_open, + .release = npu_debug_release, + .read = NULL, + .write = npu_debug_ctrl_write, +}; + +/* ------------------------------------------------------------------------- + * Function Implementations + * ------------------------------------------------------------------------- + */ +static int npu_debug_open(struct inode *inode, struct file *file) +{ + /* non-seekable */ + file->f_mode &= ~(FMODE_LSEEK | FMODE_PREAD | FMODE_PWRITE); + file->private_data = inode->i_private; + return 0; +} + +static int npu_debug_release(struct inode *inode, struct file *file) +{ + return 0; +} + + +/* ------------------------------------------------------------------------- + * Function Implementations - DebugFS Control + * ------------------------------------------------------------------------- + */ +static ssize_t npu_debug_ctrl_write(struct file *file, + const char __user *user_buf, size_t count, loff_t *ppos) +{ + char buf[24]; + struct npu_device *npu_dev = file->private_data; + struct npu_debugfs_ctx *debugfs; + int32_t rc = 0; + uint32_t val; + + pr_debug("npu_dev %pK %pK\n", npu_dev, g_npu_dev); + npu_dev = g_npu_dev; + debugfs = &npu_dev->debugfs_ctx; + + if (count >= sizeof(buf)) + return -EINVAL; + + if (copy_from_user(buf, user_buf, count)) + return -EFAULT; + + buf[count] = 0; /* end of string */ + + if (count >= 2) + buf[count-1] = 0;/* remove line feed */ + + if (strcmp(buf, "on") == 0) { + pr_info("triggering fw_init\n"); + if (fw_init(npu_dev) != 0) + pr_info("error in fw_init\n"); + } else if (strcmp(buf, "off") == 0) { + pr_info("triggering fw_deinit\n"); + fw_deinit(npu_dev, false, true); + } else if (strcmp(buf, "ssr") == 0) { + pr_info("trigger error irq\n"); + if (npu_enable_core_power(npu_dev)) + return -EPERM; + + REGW(npu_dev, NPU_MASTERn_ERROR_IRQ_SET(1), 2); + REGW(npu_dev, NPU_MASTERn_ERROR_IRQ_SET(0), 2); + npu_disable_core_power(npu_dev); + } else if (strcmp(buf, "ssr_wdt") == 0) { + pr_info("trigger wdt irq\n"); + npu_disable_post_pil_clocks(npu_dev); + } else if (strcmp(buf, "loopback") == 0) { + pr_debug("loopback test\n"); + rc = npu_host_loopback_test(npu_dev); + pr_debug("loopback test end: %d\n", rc); + } else { + rc = kstrtou32(buf, 10, &val); + if (rc) { + pr_err("Invalid input for power level settings\n"); + } else { + val = min(val, npu_dev->pwrctrl.max_pwrlevel); + npu_dev->pwrctrl.active_pwrlevel = val; + pr_info("setting power state to %d\n", val); + } + } + + return count; +} +/* ------------------------------------------------------------------------- + * Function Implementations - DebugFS + * ------------------------------------------------------------------------- + */ +int npu_debugfs_init(struct npu_device *npu_dev) +{ + struct npu_debugfs_ctx *debugfs = &npu_dev->debugfs_ctx; + struct npu_host_ctx *host_ctx = &npu_dev->host_ctx; + struct npu_pwrctrl *pwr = &npu_dev->pwrctrl; + + g_npu_dev = npu_dev; + + debugfs->root = debugfs_create_dir("npu", NULL); + if (IS_ERR_OR_NULL(debugfs->root)) { + pr_err("debugfs_create_dir for npu failed, error %ld\n", + PTR_ERR(debugfs->root)); + return -ENODEV; + } + + if (!debugfs_create_file("ctrl", 0644, debugfs->root, + npu_dev, &npu_ctrl_fops)) { + pr_err("debugfs_create_file ctrl fail\n"); + goto err; + } + + debugfs_create_bool("sys_cache_disable", 0644, + debugfs->root, &(host_ctx->sys_cache_disable)); + + debugfs_create_u32("fw_dbg_mode", 0644, + debugfs->root, &(host_ctx->fw_dbg_mode)); + + debugfs_create_u32("fw_state", 0444, + debugfs->root, &(host_ctx->fw_state)); + + debugfs_create_u32("pwr_level", 0444, + debugfs->root, &(pwr->active_pwrlevel)); + + debugfs_create_u32("exec_flags", 0644, + debugfs->root, &(host_ctx->exec_flags_override)); + + + return 0; + +err: + npu_debugfs_deinit(npu_dev); + return -ENODEV; +} + +void npu_debugfs_deinit(struct npu_device *npu_dev) +{ + struct npu_debugfs_ctx *debugfs = &npu_dev->debugfs_ctx; + + if (!IS_ERR_OR_NULL(debugfs->root)) { + debugfs_remove_recursive(debugfs->root); + debugfs->root = NULL; + } +} diff --git a/drivers/media/platform/msm/npu/npu_dev.c b/drivers/media/platform/msm/npu/npu_dev.c new file mode 100644 index 000000000000..da980013f921 --- /dev/null +++ b/drivers/media/platform/msm/npu/npu_dev.c @@ -0,0 +1,2428 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2017-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2023-2024, Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +/* ------------------------------------------------------------------------- + * Includes + * ------------------------------------------------------------------------- + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "npu_common.h" +#include "npu_hw.h" + +/* ------------------------------------------------------------------------- + * Defines + * ------------------------------------------------------------------------- + */ +#define CLASS_NAME "npu" +#define DRIVER_NAME "msm_npu" + +#define MBOX_OP_TIMEOUTMS 1000 + +#define ICC_NPU_CDSPMEM "icc-npu-cdspmem" +#define ICC_CPU_IMEMCFG "icc-cpu-imemcfg" +/* ------------------------------------------------------------------------- + * File Scope Prototypes + * ------------------------------------------------------------------------- + */ +static int npu_enable_regulators(struct npu_device *npu_dev); +static void npu_disable_regulators(struct npu_device *npu_dev); +static int npu_enable_clocks(struct npu_device *npu_dev, bool post_pil); +static void npu_disable_clocks(struct npu_device *npu_dev, bool post_pil); +static int npu_enable_core_clocks(struct npu_device *npu_dev); +static void npu_disable_core_clocks(struct npu_device *npu_dev); +static uint32_t npu_calc_power_level(struct npu_device *npu_dev); +static ssize_t caps_show(struct device *dev, + struct device_attribute *attr, char *buf); +static ssize_t pwr_show(struct device *dev, + struct device_attribute *attr, + char *buf); +static ssize_t pwr_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count); +static ssize_t perf_mode_override_show(struct device *dev, + struct device_attribute *attr, + char *buf); +static ssize_t perf_mode_override_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count); +static ssize_t dcvs_mode_show(struct device *dev, + struct device_attribute *attr, + char *buf); +static ssize_t dcvs_mode_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count); +static ssize_t fw_unload_delay_ms_show(struct device *dev, + struct device_attribute *attr, + char *buf); +static ssize_t fw_unload_delay_ms_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count); +static ssize_t fw_state_show(struct device *dev, + struct device_attribute *attr, + char *buf); +static ssize_t fw_state_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count); +static void npu_suspend_devbw(struct npu_device *npu_dev); +static void npu_resume_devbw(struct npu_device *npu_dev); +static bool npu_is_post_clock(const char *clk_name); +static bool npu_is_exclude_rate_clock(const char *clk_name); +static int npu_get_max_state(struct thermal_cooling_device *cdev, + unsigned long *state); +static int npu_get_cur_state(struct thermal_cooling_device *cdev, + unsigned long *state); +static int npu_set_cur_state(struct thermal_cooling_device *cdev, + unsigned long state); +static int npu_open(struct inode *inode, struct file *file); +static int npu_close(struct inode *inode, struct file *file); +static int npu_get_info(struct npu_client *client, unsigned long arg); +static int npu_map_buf(struct npu_client *client, unsigned long arg); +static int npu_unmap_buf(struct npu_client *client, + unsigned long arg); +static int npu_load_network(struct npu_client *client, + unsigned long arg); +static int npu_load_network_v2(struct npu_client *client, + unsigned long arg); +static int npu_unload_network(struct npu_client *client, + unsigned long arg); +static int npu_exec_network(struct npu_client *client, + unsigned long arg); +static int npu_exec_network_v2(struct npu_client *client, + unsigned long arg); +static int npu_set_fw_state(struct npu_client *client, uint32_t enable); +static int npu_set_property(struct npu_client *client, + unsigned long arg); +static int npu_get_property(struct npu_client *client, + unsigned long arg); +static long npu_ioctl(struct file *file, unsigned int cmd, + unsigned long arg); +static int npu_parse_dt_clock(struct npu_device *npu_dev); +static int npu_parse_dt_regulator(struct npu_device *npu_dev); +static int npu_of_parse_pwrlevels(struct npu_device *npu_dev, + struct device_node *node); +static int npu_pwrctrl_init(struct npu_device *npu_dev); +static int npu_probe(struct platform_device *pdev); +static int npu_remove(struct platform_device *pdev); +static int npu_suspend(struct platform_device *dev, pm_message_t state); +static int npu_resume(struct platform_device *dev); +static int __init npu_init(void); +static void __exit npu_exit(void); +static int npu_set_power_level(struct npu_device *npu_dev, bool notify_cxlimit); +static uint32_t npu_notify_cdsprm_cxlimit_corner(struct npu_device *npu_dev, + uint32_t pwr_lvl); +static void npu_icc_init(struct npu_device *npu_dev); +static void npu_icc_deinit(struct npu_device *npu_dev); +static void npu_disable_icc_bw(struct npu_device *npu_dev); +static void npu_enable_icc_bw(struct npu_device *npu_dev); +/* ------------------------------------------------------------------------- + * File Scope Variables + * ------------------------------------------------------------------------- + */ +static const char * const npu_post_clocks[] = { + "npu_cpc_clk", + "npu_cpc_timer_clk" +}; + +static const char * const npu_exclude_rate_clocks[] = { + "qdss_clk", + "at_clk", + "trig_clk", + "sleep_clk", + "conf_noc_ahb_clk", + "comp_noc_axi_clk", + "npu_core_cti_clk", + "npu_core_apb_clk", + "npu_core_atb_clk", + "npu_cpc_timer_clk", + "qtimer_core_clk", + "bwmon_clk", + "bto_core_clk" +}; + + +static struct npu_reg npu_saved_bw_registers[] = { + { BWMON2_SAMPLING_WINDOW, 0, false }, + { BWMON2_BYTE_COUNT_THRESHOLD_HIGH, 0, false }, + { BWMON2_BYTE_COUNT_THRESHOLD_MEDIUM, 0, false }, + { BWMON2_BYTE_COUNT_THRESHOLD_LOW, 0, false }, + { BWMON2_ZONE_ACTIONS, 0, false }, + { BWMON2_ZONE_COUNT_THRESHOLD, 0, false }, +}; + +static const struct npu_irq npu_irq_info[NPU_MAX_IRQ] = { + {"ipc_irq", 0, IRQF_TRIGGER_HIGH}, + {"error_irq", 0, IRQF_TRIGGER_RISING | IRQF_ONESHOT}, + {"wdg_bite_irq", 0, IRQF_TRIGGER_RISING | IRQF_ONESHOT}, +}; + +static struct npu_device *g_npu_dev; + +/* ------------------------------------------------------------------------- + * Entry Points for Probe + * ------------------------------------------------------------------------- + */ +/* Sys FS */ +static DEVICE_ATTR_RO(caps); +static DEVICE_ATTR_RW(pwr); +static DEVICE_ATTR_RW(perf_mode_override); +static DEVICE_ATTR_RW(dcvs_mode); +static DEVICE_ATTR_RW(fw_unload_delay_ms); +static DEVICE_ATTR_RW(fw_state); + +static struct attribute *npu_fs_attrs[] = { + &dev_attr_caps.attr, + &dev_attr_pwr.attr, + &dev_attr_perf_mode_override.attr, + &dev_attr_dcvs_mode.attr, + &dev_attr_fw_state.attr, + &dev_attr_fw_unload_delay_ms.attr, + NULL +}; + +static struct attribute_group npu_fs_attr_group = { + .attrs = npu_fs_attrs +}; + +static const struct of_device_id npu_dt_match[] = { + { .compatible = "qcom,msm-npu",}, + {} +}; + +static struct platform_driver npu_driver = { + .probe = npu_probe, + .remove = npu_remove, +#if defined(CONFIG_PM) + .suspend = npu_suspend, + .resume = npu_resume, +#endif + .driver = { + .name = "msm_npu", + .of_match_table = npu_dt_match, + .pm = NULL, + }, +}; + +static const struct file_operations npu_fops = { + .owner = THIS_MODULE, + .open = npu_open, + .release = npu_close, + .unlocked_ioctl = npu_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = npu_ioctl, +#endif +}; + +static const struct thermal_cooling_device_ops npu_cooling_ops = { + .get_max_state = npu_get_max_state, + .get_cur_state = npu_get_cur_state, + .set_cur_state = npu_set_cur_state, +}; + +/* ------------------------------------------------------------------------- + * SysFS - Capabilities + * ------------------------------------------------------------------------- + */ +static ssize_t caps_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + size_t ret = 0; + struct npu_device *npu_dev = dev_get_drvdata(dev); + + if (!npu_enable_core_power(npu_dev)) { + if (scnprintf(buf, PAGE_SIZE, "hw_version :0x%X", + REGR(npu_dev, NPU_HW_VERSION)) < 0) + ret = -EINVAL; + npu_disable_core_power(npu_dev); + } else + ret = -EPERM; + + return ret; +} + +/* ------------------------------------------------------------------------- + * SysFS - Power State + * ------------------------------------------------------------------------- + */ +static ssize_t pwr_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct npu_device *npu_dev = dev_get_drvdata(dev); + struct npu_pwrctrl *pwr = &npu_dev->pwrctrl; + + return scnprintf(buf, PAGE_SIZE, "%s\n", + (pwr->pwr_vote_num > 0) ? "on" : "off"); +} + +static ssize_t pwr_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct npu_device *npu_dev = dev_get_drvdata(dev); + bool pwr_on = false; + + if (strtobool(buf, &pwr_on) < 0) + return -EINVAL; + + if (pwr_on) { + if (npu_enable_core_power(npu_dev)) + return -EPERM; + } else { + npu_disable_core_power(npu_dev); + } + + return count; +} + +/* ------------------------------------------------------------------------- + * SysFS - perf_mode_override + * ------------------------------------------------------------------------- + */ +static ssize_t perf_mode_override_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct npu_device *npu_dev = dev_get_drvdata(dev); + struct npu_pwrctrl *pwr = &npu_dev->pwrctrl; + + return scnprintf(buf, PAGE_SIZE, "%d\n", pwr->perf_mode_override); +} + +static ssize_t perf_mode_override_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct npu_client client; + struct npu_device *npu_dev = dev_get_drvdata(dev); + uint32_t val; + int rc; + + rc = kstrtou32(buf, 10, &val); + if (rc) { + pr_err("Invalid input for perf mode setting\n"); + return -EINVAL; + } + + val = min(val, npu_dev->pwrctrl.num_pwrlevels); + npu_dev->pwrctrl.perf_mode_override = val; + pr_info("setting uc_pwrlevel_override to %d\n", val); + + client.npu_dev = npu_dev; + npu_host_set_perf_mode(&client, 0, val); + + return count; +} + +static ssize_t dcvs_mode_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct npu_device *npu_dev = dev_get_drvdata(dev); + struct npu_pwrctrl *pwr = &npu_dev->pwrctrl; + + return scnprintf(buf, PAGE_SIZE, "%d\n", pwr->dcvs_mode); +} + +static ssize_t dcvs_mode_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct npu_device *npu_dev = dev_get_drvdata(dev); + struct msm_npu_property prop; + uint32_t val; + int ret = 0; + + ret = kstrtou32(buf, 10, &val); + if (ret) { + pr_err("Invalid input for dcvs mode setting\n"); + return -EINVAL; + } + + val = min(val, (uint32_t)(npu_dev->pwrctrl.num_pwrlevels - 1)); + pr_debug("sysfs: setting dcvs_mode to %d\n", val); + + prop.prop_id = MSM_NPU_PROP_ID_DCVS_MODE; + prop.num_of_params = 1; + prop.network_hdl = 0; + prop.prop_param[0] = val; + + ret = npu_host_set_fw_property(npu_dev, &prop); + if (ret) { + pr_err("npu_host_set_fw_property failed %d\n", ret); + return ret; + } + + npu_dev->pwrctrl.dcvs_mode = val; + + return count; +} + +/* ------------------------------------------------------------------------- + * SysFS - Delayed FW unload + * ------------------------------------------------------------------------- + */ +static ssize_t fw_unload_delay_ms_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct npu_device *npu_dev = dev_get_drvdata(dev); + + return scnprintf(buf, PAGE_SIZE, "%d\n", + npu_dev->host_ctx.fw_unload_delay_ms); +} + +static ssize_t fw_unload_delay_ms_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct npu_device *npu_dev = dev_get_drvdata(dev); + uint32_t val; + int rc; + + rc = kstrtou32(buf, 10, &val); + if (rc) { + pr_err("Invalid input for fw unload delay setting\n"); + return -EINVAL; + } + + npu_dev->host_ctx.fw_unload_delay_ms = val; + pr_debug("setting fw_unload_delay_ms to %d\n", val); + + return count; +} + +/* ------------------------------------------------------------------------- + * SysFS - firmware state + * ------------------------------------------------------------------------- + */ +static ssize_t fw_state_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct npu_device *npu_dev = dev_get_drvdata(dev); + + return scnprintf(buf, PAGE_SIZE, "%s\n", + (npu_dev->host_ctx.fw_state == FW_ENABLED) ? + "on" : "off"); +} + +static ssize_t fw_state_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct npu_device *npu_dev = dev_get_drvdata(dev); + struct npu_client client; + bool enable = false; + int rc; + + if (strtobool(buf, &enable) < 0) + return -EINVAL; + + client.npu_dev = npu_dev; + rc = npu_set_fw_state(&client, enable ? 1 : 0); + + if (rc) { + pr_err("%s fw failed\n", enable ? "enable" : "disable"); + return rc; + } + + return count; +} + +/* ------------------------------------------------------------------------- + * Power Related + * ------------------------------------------------------------------------- + */ +static enum npu_power_level cdsprm_corner_to_npu_power_level( + enum cdsprm_npu_corner corner) +{ + enum npu_power_level pwr_lvl = NPU_PWRLEVEL_TURBO_L1; + + switch (corner) { + case CDSPRM_NPU_CLK_OFF: + pwr_lvl = NPU_PWRLEVEL_OFF; + break; + case CDSPRM_NPU_MIN_SVS: + pwr_lvl = NPU_PWRLEVEL_MINSVS; + break; + case CDSPRM_NPU_LOW_SVS: + pwr_lvl = NPU_PWRLEVEL_LOWSVS; + break; + case CDSPRM_NPU_SVS: + pwr_lvl = NPU_PWRLEVEL_SVS; + break; + case CDSPRM_NPU_SVS_L1: + pwr_lvl = NPU_PWRLEVEL_SVS_L1; + break; + case CDSPRM_NPU_NOM: + pwr_lvl = NPU_PWRLEVEL_NOM; + break; + case CDSPRM_NPU_NOM_L1: + pwr_lvl = NPU_PWRLEVEL_NOM_L1; + break; + case CDSPRM_NPU_TURBO: + pwr_lvl = NPU_PWRLEVEL_TURBO; + break; + case CDSPRM_NPU_TURBO_L1: + default: + pwr_lvl = NPU_PWRLEVEL_TURBO_L1; + break; + } + + return pwr_lvl; +} + +static enum cdsprm_npu_corner npu_power_level_to_cdsprm_corner( + enum npu_power_level pwr_lvl) +{ + enum cdsprm_npu_corner corner = CDSPRM_NPU_MIN_SVS; + + switch (pwr_lvl) { + case NPU_PWRLEVEL_OFF: + corner = CDSPRM_NPU_CLK_OFF; + break; + case NPU_PWRLEVEL_MINSVS: + corner = CDSPRM_NPU_MIN_SVS; + break; + case NPU_PWRLEVEL_LOWSVS: + corner = CDSPRM_NPU_LOW_SVS; + break; + case NPU_PWRLEVEL_SVS: + corner = CDSPRM_NPU_SVS; + break; + case NPU_PWRLEVEL_SVS_L1: + corner = CDSPRM_NPU_SVS_L1; + break; + case NPU_PWRLEVEL_NOM: + corner = CDSPRM_NPU_NOM; + break; + case NPU_PWRLEVEL_NOM_L1: + corner = CDSPRM_NPU_NOM_L1; + break; + case NPU_PWRLEVEL_TURBO: + corner = CDSPRM_NPU_TURBO; + break; + case NPU_PWRLEVEL_TURBO_L1: + default: + corner = CDSPRM_NPU_TURBO_L1; + break; + } + + return corner; +} + +static int npu_set_cdsprm_corner_limit(enum cdsprm_npu_corner corner) +{ + struct npu_pwrctrl *pwr; + enum npu_power_level pwr_lvl; + + if (!g_npu_dev) + return 0; + + pwr = &g_npu_dev->pwrctrl; + pwr_lvl = cdsprm_corner_to_npu_power_level(corner); + pwr->cdsprm_pwrlevel = pwr_lvl; + pr_debug("power level from cdsp %d\n", pwr_lvl); + + return npu_set_power_level(g_npu_dev, false); +} + +static const struct cdsprm_npu_limit_cbs cdsprm_npu_limit_cbs = { + .set_corner_limit = npu_set_cdsprm_corner_limit, +}; + +int npu_notify_cdsprm_cxlimit_activity(struct npu_device *npu_dev, bool enable) +{ + if (!npu_dev->cxlimit_registered) + return 0; + + pr_debug("notify cxlimit %s activity\n", enable ? "enable" : "disable"); + + return cdsprm_cxlimit_npu_activity_notify(enable ? 1 : 0); + return 0; +} + +static uint32_t npu_notify_cdsprm_cxlimit_corner( + struct npu_device *npu_dev, uint32_t pwr_lvl) +{ + uint32_t corner, pwr_lvl_to_set; + + if (!npu_dev->cxlimit_registered) + return pwr_lvl; + + corner = npu_power_level_to_cdsprm_corner(pwr_lvl); + corner = cdsprm_cxlimit_npu_corner_notify(corner); + pwr_lvl_to_set = cdsprm_corner_to_npu_power_level(corner); + pr_debug("Notify cdsprm %d:%d\n", pwr_lvl, + pwr_lvl_to_set); + + return pwr_lvl_to_set; +} + +static int npu_cdsprm_cxlimit_init(struct npu_device *npu_dev) +{ + bool enabled; + int ret = 0; + + enabled = of_property_read_bool(npu_dev->pdev->dev.of_node, + "qcom,npu-cxlimit-enable"); + pr_debug("qcom,npu-xclimit-enable is %s\n", enabled ? "true" : "false"); + + npu_dev->cxlimit_registered = false; + if (enabled) { + ret = cdsprm_cxlimit_npu_limit_register(&cdsprm_npu_limit_cbs); + ret = 0; + if (ret) { + pr_err("register cxlimit npu limit failed\n"); + } else { + pr_debug("register cxlimit npu limit succeeds\n"); + npu_dev->cxlimit_registered = true; + } + } + + return ret; +} + +static int npu_cdsprm_cxlimit_deinit(struct npu_device *npu_dev) +{ + int ret = 0; + + if (npu_dev->cxlimit_registered) { + ret = cdsprm_cxlimit_npu_limit_deregister(); + ret = 0; + if (ret) + pr_err("deregister cxlimit npu limit failed\n"); + npu_dev->cxlimit_registered = false; + } + + return ret; +} + +int npu_enable_core_power(struct npu_device *npu_dev) +{ + struct npu_pwrctrl *pwr = &npu_dev->pwrctrl; + int ret = 0; + + mutex_lock(&npu_dev->dev_lock); + if (!pwr->pwr_vote_num) { + npu_enable_icc_bw(npu_dev); + ret = npu_enable_regulators(npu_dev); + if (ret) { + npu_disable_icc_bw(npu_dev); + goto fail; + } + + + ret = npu_enable_core_clocks(npu_dev); + if (ret) { + npu_disable_regulators(npu_dev); + pwr->pwr_vote_num = 0; + npu_disable_icc_bw(npu_dev); + goto fail; + } + npu_resume_devbw(npu_dev); + } + pwr->pwr_vote_num++; +fail: + mutex_unlock(&npu_dev->dev_lock); + + return ret; +} + +void npu_disable_core_power(struct npu_device *npu_dev) +{ + struct npu_pwrctrl *pwr = &npu_dev->pwrctrl; + + mutex_lock(&npu_dev->dev_lock); + if (!pwr->pwr_vote_num) { + mutex_unlock(&npu_dev->dev_lock); + return; + } + + pwr->pwr_vote_num--; + if (!pwr->pwr_vote_num) { + npu_suspend_devbw(npu_dev); + npu_disable_core_clocks(npu_dev); + npu_disable_regulators(npu_dev); + npu_disable_icc_bw(npu_dev); + pwr->active_pwrlevel = pwr->default_pwrlevel; + pwr->uc_pwrlevel = pwr->max_pwrlevel; + pwr->cdsprm_pwrlevel = pwr->max_pwrlevel; + pwr->cur_dcvs_activity = pwr->num_pwrlevels; + pr_debug("setting back to power level=%d\n", + pwr->active_pwrlevel); + } + mutex_unlock(&npu_dev->dev_lock); +} + +static int npu_enable_core_clocks(struct npu_device *npu_dev) +{ + return npu_enable_clocks(npu_dev, false); +} + +static void npu_disable_core_clocks(struct npu_device *npu_dev) +{ + return npu_disable_clocks(npu_dev, false); +} + +int npu_enable_post_pil_clocks(struct npu_device *npu_dev) +{ + return npu_enable_clocks(npu_dev, true); +} + +void npu_disable_post_pil_clocks(struct npu_device *npu_dev) +{ + npu_disable_clocks(npu_dev, true); +} + +static uint32_t npu_power_level_from_index(struct npu_device *npu_dev, + uint32_t index) +{ + struct npu_pwrctrl *pwr = &npu_dev->pwrctrl; + + if (index >= pwr->num_pwrlevels) + index = pwr->num_pwrlevels - 1; + + return pwr->pwrlevels[index].pwr_level; +} + +static uint32_t npu_power_level_to_index(struct npu_device *npu_dev, + uint32_t pwr_lvl) +{ + struct npu_pwrctrl *pwr = &npu_dev->pwrctrl; + int i; + + for (i = 0; i < pwr->num_pwrlevels; i++) { + if (pwr->pwrlevels[i].pwr_level > pwr_lvl) + break; + } + + + return i == 0 ? 0 : i - 1; +} + +static uint32_t npu_calc_power_level(struct npu_device *npu_dev) +{ + uint32_t ret_level; + uint32_t therm_pwr_level = npu_dev->thermalctrl.pwr_level; + uint32_t active_pwr_level = npu_dev->pwrctrl.active_pwrlevel; + uint32_t uc_pwr_level = npu_dev->pwrctrl.uc_pwrlevel; + + /* + * pick the lowese power level between thermal power and usecase power + * settings + */ + ret_level = min(therm_pwr_level, uc_pwr_level); + pr_debug("%s therm=%d active=%d uc=%d set level=%d\n", + __func__, therm_pwr_level, active_pwr_level, uc_pwr_level, + ret_level); + + return ret_level; +} + +static int npu_set_power_level(struct npu_device *npu_dev, bool notify_cxlimit) +{ + struct npu_pwrctrl *pwr = &npu_dev->pwrctrl; + struct npu_pwrlevel *pwrlevel; + int i, ret = 0; + uint32_t pwr_level_to_set, pwr_level_to_cdsprm, pwr_level_idx; + + /* get power level to set */ + pwr_level_to_set = npu_calc_power_level(npu_dev); + pwr_level_to_cdsprm = pwr_level_to_set; + + if (!pwr->pwr_vote_num) { + pr_debug("power is not enabled during set request\n"); + pwr->active_pwrlevel = min(pwr_level_to_set, + npu_dev->pwrctrl.cdsprm_pwrlevel); + return 0; + } + + /* notify cxlimit to get allowed power level */ + if ((pwr_level_to_set > pwr->active_pwrlevel) && notify_cxlimit) + pwr_level_to_set = npu_notify_cdsprm_cxlimit_corner( + npu_dev, pwr_level_to_cdsprm); + + pwr_level_to_set = min(pwr_level_to_set, + npu_dev->pwrctrl.cdsprm_pwrlevel); + + /* if the same as current, dont do anything */ + if (pwr_level_to_set == pwr->active_pwrlevel) { + pr_debug("power level %d doesn't change\n", pwr_level_to_set); + return 0; + } + + pr_debug("setting power level to [%d]\n", pwr_level_to_set); + pwr_level_idx = npu_power_level_to_index(npu_dev, pwr_level_to_set); + pwrlevel = &npu_dev->pwrctrl.pwrlevels[pwr_level_idx]; + + for (i = 0; i < npu_dev->core_clk_num; i++) { + if (npu_is_exclude_rate_clock( + npu_dev->core_clks[i].clk_name)) + continue; + + if (npu_dev->host_ctx.fw_state == FW_DISABLED) { + if (npu_is_post_clock( + npu_dev->core_clks[i].clk_name)) + continue; + } + + pr_debug("requested rate of clock [%s] to [%ld]\n", + npu_dev->core_clks[i].clk_name, pwrlevel->clk_freq[i]); + + ret = clk_set_rate(npu_dev->core_clks[i].clk, + pwrlevel->clk_freq[i]); + if (ret) { + pr_debug("clk_set_rate %s to %ld failed with %d\n", + npu_dev->core_clks[i].clk_name, + pwrlevel->clk_freq[i], ret); + break; + } + } + + if ((pwr_level_to_cdsprm < pwr->active_pwrlevel) && notify_cxlimit) { + npu_notify_cdsprm_cxlimit_corner(npu_dev, + pwr_level_to_cdsprm); + pr_debug("Notify cdsprm(post) %d\n", pwr_level_to_cdsprm); + } + + pwr->active_pwrlevel = pwr_level_to_set; + return ret; +} + +int npu_set_uc_power_level(struct npu_device *npu_dev, + uint32_t perf_mode) +{ + struct npu_pwrctrl *pwr = &npu_dev->pwrctrl; + uint32_t uc_pwrlevel_to_set; + + uc_pwrlevel_to_set = npu_power_level_from_index(npu_dev, + perf_mode - 1); + + if (uc_pwrlevel_to_set > pwr->max_pwrlevel) + uc_pwrlevel_to_set = pwr->max_pwrlevel; + + pwr->uc_pwrlevel = uc_pwrlevel_to_set; + return npu_set_power_level(npu_dev, true); +} + +/* ------------------------------------------------------------------------- + * Bandwidth Related + * ------------------------------------------------------------------------- + */ +static void npu_save_bw_registers(struct npu_device *npu_dev) +{ + int i; + + if (!npu_dev->bwmon_io.base) + return; + + for (i = 0; i < ARRAY_SIZE(npu_saved_bw_registers); i++) { + npu_saved_bw_registers[i].val = npu_bwmon_reg_read(npu_dev, + npu_saved_bw_registers[i].off); + npu_saved_bw_registers[i].valid = true; + } +} + +static void npu_restore_bw_registers(struct npu_device *npu_dev) +{ + int i; + + if (!npu_dev->bwmon_io.base) + return; + + for (i = 0; i < ARRAY_SIZE(npu_saved_bw_registers); i++) { + if (npu_saved_bw_registers[i].valid) { + npu_bwmon_reg_write(npu_dev, + npu_saved_bw_registers[i].off, + npu_saved_bw_registers[i].val); + npu_saved_bw_registers[i].valid = false; + } + } +} + +static void npu_suspend_devbw(struct npu_device *npu_dev) +{ + struct npu_pwrctrl *pwr = &npu_dev->pwrctrl; + int ret; + + if (pwr->bwmon_enabled && pwr->devbw) { + pwr->bwmon_enabled = 0; + //TODO + //ret = devfreq_suspend_icc(pwr->devbw); + ret = 0; + if (ret) + pr_err("devfreq_suspend_devbw failed rc:%d\n", + ret); + npu_save_bw_registers(npu_dev); + } +} + +static void npu_resume_devbw(struct npu_device *npu_dev) +{ + struct npu_pwrctrl *pwr = &npu_dev->pwrctrl; + int ret; + + if (!pwr->bwmon_enabled && pwr->devbw) { + pwr->bwmon_enabled = 1; + npu_restore_bw_registers(npu_dev); + //TODO + //ret = devfreq_resume_icc(pwr->devbw); + ret = 0; + if (ret) + pr_err("devfreq_resume_devbw failed rc:%d\n", ret); + } +} + +/* ------------------------------------------------------------------------- + * Clocks Related + * ------------------------------------------------------------------------- + */ +static bool npu_is_post_clock(const char *clk_name) +{ + int ret = false; + int i; + + for (i = 0; i < ARRAY_SIZE(npu_post_clocks); i++) { + if (!strcmp(clk_name, npu_post_clocks[i])) { + ret = true; + break; + } + } + return ret; +} + +static bool npu_is_exclude_rate_clock(const char *clk_name) +{ + int ret = false; + int i; + + for (i = 0; i < ARRAY_SIZE(npu_exclude_rate_clocks); i++) { + if (!strcmp(clk_name, npu_exclude_rate_clocks[i])) { + ret = true; + break; + } + } + return ret; +} + +static int npu_enable_clocks(struct npu_device *npu_dev, bool post_pil) +{ + int i, rc = 0; + struct npu_clk *core_clks = npu_dev->core_clks; + struct npu_pwrctrl *pwr = &npu_dev->pwrctrl; + struct npu_pwrlevel *pwrlevel; + uint32_t pwrlevel_to_set, pwrlevel_idx; + + pwrlevel_to_set = pwr->active_pwrlevel; + if (!post_pil) { + pwrlevel_to_set = npu_notify_cdsprm_cxlimit_corner( + npu_dev, pwrlevel_to_set); + pr_debug("Notify cdsprm %d\n", pwrlevel_to_set); + pwr->active_pwrlevel = pwrlevel_to_set; + } + + pwrlevel_idx = npu_power_level_to_index(npu_dev, pwrlevel_to_set); + pwrlevel = &pwr->pwrlevels[pwrlevel_idx]; + for (i = 0; i < npu_dev->core_clk_num; i++) { + if (post_pil) { + if (!npu_is_post_clock(core_clks[i].clk_name)) + continue; + } else { + if (npu_is_post_clock(core_clks[i].clk_name)) + continue; + } + + pr_debug("enabling clock %s\n", core_clks[i].clk_name); + + rc = clk_prepare_enable(core_clks[i].clk); + if (rc) { + pr_err("%s enable failed\n", + core_clks[i].clk_name); + break; + } + + if (npu_is_exclude_rate_clock(core_clks[i].clk_name)) + continue; + + pr_debug("setting rate of clock %s to %ld\n", + core_clks[i].clk_name, pwrlevel->clk_freq[i]); + + rc = clk_set_rate(core_clks[i].clk, + pwrlevel->clk_freq[i]); + /* not fatal error, keep using previous clk rate */ + if (rc) { + pr_err("clk_set_rate %s to %ld failed\n", + core_clks[i].clk_name, + pwrlevel->clk_freq[i]); + rc = 0; + } + } + + if (rc) { + for (i--; i >= 0; i--) { + if (post_pil) { + if (!npu_is_post_clock(core_clks[i].clk_name)) + continue; + } else { + if (npu_is_post_clock(core_clks[i].clk_name)) + continue; + } + pr_debug("disabling clock %s\n", core_clks[i].clk_name); + clk_disable_unprepare(core_clks[i].clk); + } + } + + return rc; +} + +static void npu_disable_clocks(struct npu_device *npu_dev, bool post_pil) +{ + int i = 0; + struct npu_clk *core_clks = npu_dev->core_clks; + + if (!post_pil) { + npu_notify_cdsprm_cxlimit_corner(npu_dev, NPU_PWRLEVEL_OFF); + pr_debug("Notify cdsprm clock off\n"); + } + + for (i = npu_dev->core_clk_num - 1; i >= 0 ; i--) { + if (post_pil) { + if (!npu_is_post_clock(core_clks[i].clk_name)) + continue; + } else { + if (npu_is_post_clock(core_clks[i].clk_name)) + continue; + } + + pr_debug("disabling clock %s\n", core_clks[i].clk_name); + clk_disable_unprepare(core_clks[i].clk); + } +} + +/* ------------------------------------------------------------------------- + * Thermal Functions + * ------------------------------------------------------------------------- + */ +static int npu_get_max_state(struct thermal_cooling_device *cdev, + unsigned long *state) +{ + struct npu_device *npu_dev = cdev->devdata; + struct npu_thermalctrl *thermalctrl = &npu_dev->thermalctrl; + + pr_debug("enter %s thermal max state=%lu\n", __func__, + thermalctrl->max_state); + + *state = thermalctrl->max_state; + + return 0; +} + +static int npu_get_cur_state(struct thermal_cooling_device *cdev, + unsigned long *state) +{ + struct npu_device *npu_dev = cdev->devdata; + struct npu_thermalctrl *thermal = &npu_dev->thermalctrl; + + pr_debug("enter %s thermal current state=%lu\n", __func__, + thermal->current_state); + + *state = thermal->current_state; + + return 0; +} + +static int +npu_set_cur_state(struct thermal_cooling_device *cdev, unsigned long state) +{ + struct npu_device *npu_dev = cdev->devdata; + struct npu_thermalctrl *thermal = &npu_dev->thermalctrl; + + pr_debug("enter %s request state=%lu\n", __func__, state); + if (state > thermal->max_state) + return -EINVAL; + + thermal->current_state = state; + thermal->pwr_level = npu_power_level_from_index(npu_dev, + thermal->max_state - state); + + return npu_set_power_level(npu_dev, true); +} + +/* ------------------------------------------------------------------------- + * Regulator Related + * ------------------------------------------------------------------------- + */ +static int npu_enable_regulators(struct npu_device *npu_dev) +{ + int i = 0; + int rc = 0; + struct npu_host_ctx *host_ctx = &npu_dev->host_ctx; + struct npu_regulator *regulators = npu_dev->regulators; + + if (!host_ctx->power_vote_num) { + for (i = 0; i < npu_dev->regulator_num; i++) { + rc = regulator_enable(regulators[i].regulator); + if (rc < 0) { + pr_err("%s enable failed\n", + regulators[i].regulator_name); + break; + } + pr_debug("regulator %s enabled\n", + regulators[i].regulator_name); + } + } + host_ctx->power_vote_num++; + return rc; +} + +static void npu_disable_regulators(struct npu_device *npu_dev) +{ + int i = 0; + struct npu_host_ctx *host_ctx = &npu_dev->host_ctx; + struct npu_regulator *regulators = npu_dev->regulators; + + if (host_ctx->power_vote_num > 0) { + for (i = 0; i < npu_dev->regulator_num; i++) { + regulator_disable(regulators[i].regulator); + pr_debug("regulator %s disabled\n", + regulators[i].regulator_name); + } + host_ctx->power_vote_num--; + } +} + +/* ------------------------------------------------------------------------- + * Interrupt Related + * ------------------------------------------------------------------------- + */ +int npu_enable_irq(struct npu_device *npu_dev) +{ + int i; + + /* clear pending irq state */ + REGW(npu_dev, NPU_MASTERn_IPC_IRQ_OUT(0), 0x0); + REGW(npu_dev, NPU_MASTERn_ERROR_IRQ_CLEAR(0), NPU_ERROR_IRQ_MASK); + REGW(npu_dev, NPU_MASTERn_ERROR_IRQ_ENABLE(0), NPU_ERROR_IRQ_MASK); + REGW(npu_dev, NPU_MASTERn_ERROR_IRQ_OWNER(0), NPU_ERROR_IRQ_MASK); + REGW(npu_dev, NPU_MASTERn_WDOG_IRQ_OWNER(0), NPU_WDOG_IRQ_MASK); + + for (i = 0; i < NPU_MAX_IRQ; i++) { + if (npu_dev->irq[i].irq != 0) { + enable_irq(npu_dev->irq[i].irq); + pr_debug("enable irq %d\n", npu_dev->irq[i].irq); + } + } + + return 0; +} + +void npu_disable_irq(struct npu_device *npu_dev) +{ + int i; + + for (i = 0; i < NPU_MAX_IRQ; i++) { + if (npu_dev->irq[i].irq != 0) { + disable_irq(npu_dev->irq[i].irq); + pr_debug("disable irq %d\n", npu_dev->irq[i].irq); + } + } + + REGW(npu_dev, NPU_MASTERn_ERROR_IRQ_ENABLE(0), 0); + /* clear pending irq state */ + REGW(npu_dev, NPU_MASTERn_IPC_IRQ_OUT(0), 0x0); + REGW(npu_dev, NPU_MASTERn_ERROR_IRQ_CLEAR(0), NPU_ERROR_IRQ_MASK); +} + +/* ------------------------------------------------------------------------- + * System Cache + * ------------------------------------------------------------------------- + */ +int npu_enable_sys_cache(struct npu_device *npu_dev) +{ + int rc = 0; + uint32_t reg_val = 0; + + if (!npu_dev->host_ctx.sys_cache_disable) { + npu_dev->sys_cache = llcc_slice_getd(LLCC_NPU); + if (IS_ERR_OR_NULL(npu_dev->sys_cache)) { + pr_warn("unable to init sys cache\n"); + npu_dev->sys_cache = NULL; + npu_dev->host_ctx.sys_cache_disable = true; + return 0; + } + + /* set npu side regs - program SCID */ + reg_val = NPU_CACHE_ATTR_IDn___POR | SYS_CACHE_SCID; + + REGW(npu_dev, NPU_CACHE_ATTR_IDn(0), reg_val); + REGW(npu_dev, NPU_CACHE_ATTR_IDn(1), reg_val); + REGW(npu_dev, NPU_CACHE_ATTR_IDn(2), reg_val); + REGW(npu_dev, NPU_CACHE_ATTR_IDn(3), reg_val); + REGW(npu_dev, NPU_CACHE_ATTR_IDn(4), reg_val); + + pr_debug("prior to activate sys cache\n"); + rc = llcc_slice_activate(npu_dev->sys_cache); + if (rc) { + pr_warn("failed to activate sys cache\n"); + llcc_slice_putd(npu_dev->sys_cache); + npu_dev->sys_cache = NULL; + return 0; + } + + pr_debug("sys cache activated\n"); + } + + return rc; +} + +void npu_disable_sys_cache(struct npu_device *npu_dev) +{ + int rc = 0; + + if (!npu_dev->host_ctx.sys_cache_disable) { + if (npu_dev->sys_cache) { + rc = llcc_slice_deactivate(npu_dev->sys_cache); + if (rc) { + pr_err("failed to deactivate sys cache\n"); + return; + } + pr_debug("sys cache deactivated\n"); + llcc_slice_putd(npu_dev->sys_cache); + npu_dev->sys_cache = NULL; + } + } +} + +/* ------------------------------------------------------------------------- + * Open/Close + * ------------------------------------------------------------------------- + */ +static int npu_open(struct inode *inode, struct file *file) +{ + struct npu_device *npu_dev = container_of(inode->i_cdev, + struct npu_device, cdev); + struct npu_client *client; + + client = kmalloc(sizeof(*client), GFP_KERNEL); + if (!client) + return -ENOMEM; + + client->npu_dev = npu_dev; + mutex_init(&client->list_lock); + INIT_LIST_HEAD(&(client->mapped_buffer_list)); + file->private_data = client; + + return 0; +} + +static int npu_close(struct inode *inode, struct file *file) +{ + struct npu_client *client = file->private_data; + + npu_host_cleanup_networks(client); + mutex_destroy(&client->list_lock); + kfree(client); + return 0; +} + +/* ------------------------------------------------------------------------- + * IOCTL Implementations + * ------------------------------------------------------------------------- + */ +static int npu_get_info(struct npu_client *client, unsigned long arg) +{ + struct npu_device *npu_dev = client->npu_dev; + struct msm_npu_get_info_ioctl req; + void __user *argp = (void __user *)arg; + int ret = 0; + + ret = copy_from_user(&req, argp, sizeof(req)); + + if (ret) { + pr_err("fail to copy from user\n"); + return -EFAULT; + } + + ret = npu_host_get_info(npu_dev, &req); + + if (ret) { + pr_err("npu_host_get_info failed\n"); + return ret; + } + + ret = copy_to_user(argp, &req, sizeof(req)); + + if (ret) { + pr_err("fail to copy to user\n"); + return -EFAULT; + } + return 0; +} + +static int npu_map_buf(struct npu_client *client, unsigned long arg) +{ + struct msm_npu_map_buf_ioctl req; + void __user *argp = (void __user *)arg; + int ret = 0; + + ret = copy_from_user(&req, argp, sizeof(req)); + + if (ret) { + pr_err("fail to copy from user\n"); + return -EFAULT; + } + + ret = npu_host_map_buf(client, &req); + + if (ret) { + pr_err("npu_host_map_buf failed\n"); + return ret; + } + + ret = copy_to_user(argp, &req, sizeof(req)); + + if (ret) { + pr_err("fail to copy to user\n"); + return -EFAULT; + } + return 0; +} + +static int npu_unmap_buf(struct npu_client *client, unsigned long arg) +{ + struct msm_npu_unmap_buf_ioctl req; + void __user *argp = (void __user *)arg; + int ret = 0; + + ret = copy_from_user(&req, argp, sizeof(req)); + + if (ret) { + pr_err("fail to copy from user\n"); + return -EFAULT; + } + + ret = npu_host_unmap_buf(client, &req); + + if (ret) { + pr_err("npu_host_unmap_buf failed\n"); + return ret; + } + + ret = copy_to_user(argp, &req, sizeof(req)); + + if (ret) { + pr_err("fail to copy to user\n"); + return -EFAULT; + } + return 0; +} + +static int npu_load_network(struct npu_client *client, + unsigned long arg) +{ + struct msm_npu_load_network_ioctl req; + struct msm_npu_unload_network_ioctl unload_req; + void __user *argp = (void __user *)arg; + int ret = 0; + + ret = copy_from_user(&req, argp, sizeof(req)); + + if (ret) { + pr_err("fail to copy from user\n"); + return -EFAULT; + } + + pr_debug("network load with perf request %d\n", req.perf_mode); + + ret = npu_host_load_network(client, &req); + if (ret) { + pr_err("npu_host_load_network failed %d\n", ret); + return ret; + } + + ret = copy_to_user(argp, &req, sizeof(req)); + if (ret) { + pr_err("fail to copy to user\n"); + ret = -EFAULT; + unload_req.network_hdl = req.network_hdl; + npu_host_unload_network(client, &unload_req); + } + return ret; +} + +static int npu_load_network_v2(struct npu_client *client, + unsigned long arg) +{ + struct msm_npu_load_network_ioctl_v2 req; + struct msm_npu_unload_network_ioctl unload_req; + void __user *argp = (void __user *)arg; + struct msm_npu_patch_info_v2 *patch_info = NULL; + int ret; + + ret = copy_from_user(&req, argp, sizeof(req)); + if (ret) { + pr_err("fail to copy from user\n"); + return -EFAULT; + } + + if (req.patch_info_num > NPU_MAX_PATCH_NUM) { + pr_err("Invalid patch info num %d[max:%d]\n", + req.patch_info_num, NPU_MAX_PATCH_NUM); + return -EINVAL; + } + + if (req.patch_info_num) { + patch_info = kmalloc_array(req.patch_info_num, + sizeof(*patch_info), GFP_KERNEL); + if (!patch_info) + return -ENOMEM; + + ret = copy_from_user(patch_info, + (void __user *)req.patch_info, + req.patch_info_num * sizeof(*patch_info)); + if (ret) { + pr_err("fail to copy patch info\n"); + kfree(patch_info); + return -EFAULT; + } + } + + pr_debug("network load with perf request %d\n", req.perf_mode); + + ret = npu_host_load_network_v2(client, &req, patch_info); + + kfree(patch_info); + if (ret) { + pr_err("npu_host_load_network_v2 failed %d\n", ret); + return ret; + } + + ret = copy_to_user(argp, &req, sizeof(req)); + if (ret) { + pr_err("fail to copy to user\n"); + ret = -EFAULT; + unload_req.network_hdl = req.network_hdl; + npu_host_unload_network(client, &unload_req); + } + + return ret; +} + +static int npu_unload_network(struct npu_client *client, + unsigned long arg) +{ + struct msm_npu_unload_network_ioctl req; + void __user *argp = (void __user *)arg; + int ret = 0; + + ret = copy_from_user(&req, argp, sizeof(req)); + + if (ret) { + pr_err("fail to copy from user\n"); + return -EFAULT; + } + + ret = npu_host_unload_network(client, &req); + + if (ret) { + pr_err("npu_host_unload_network failed %d\n", ret); + return ret; + } + + ret = copy_to_user(argp, &req, sizeof(req)); + + if (ret) { + pr_err("fail to copy to user\n"); + return -EFAULT; + } + return 0; +} + +static int npu_exec_network(struct npu_client *client, + unsigned long arg) +{ + struct msm_npu_exec_network_ioctl req; + void __user *argp = (void __user *)arg; + int ret = 0; + + ret = copy_from_user(&req, argp, sizeof(req)); + + if (ret) { + pr_err("fail to copy from user\n"); + return -EFAULT; + } + + if ((req.input_layer_num > MSM_NPU_MAX_INPUT_LAYER_NUM) || + (req.output_layer_num > MSM_NPU_MAX_OUTPUT_LAYER_NUM)) { + pr_err("Invalid input/out layer num %d[max:%d] %d[max:%d]\n", + req.input_layer_num, MSM_NPU_MAX_INPUT_LAYER_NUM, + req.output_layer_num, MSM_NPU_MAX_OUTPUT_LAYER_NUM); + return -EINVAL; + } + + if (!req.patching_required) { + pr_err("Only support patched network\n"); + return -EINVAL; + } + + ret = npu_host_exec_network(client, &req); + + if (ret) { + pr_err("npu_host_exec_network failed %d\n", ret); + return ret; + } + + ret = copy_to_user(argp, &req, sizeof(req)); + + if (ret) { + pr_err("fail to copy to user\n"); + return -EFAULT; + } + return 0; +} + +static int npu_exec_network_v2(struct npu_client *client, + unsigned long arg) +{ + struct msm_npu_exec_network_ioctl_v2 req; + void __user *argp = (void __user *)arg; + struct msm_npu_patch_buf_info *patch_buf_info = NULL; + int ret; + + ret = copy_from_user(&req, argp, sizeof(req)); + if (ret) { + pr_err("fail to copy from user\n"); + return -EFAULT; + } + + if ((req.patch_buf_info_num > NPU_MAX_PATCH_NUM) || + (req.patch_buf_info_num == 0)) { + pr_err("Invalid patch buf info num %d[max:%d]\n", + req.patch_buf_info_num, NPU_MAX_PATCH_NUM); + return -EINVAL; + } + + if (req.stats_buf_size > NPU_MAX_STATS_BUF_SIZE) { + pr_err("Invalid stats buffer size %d max %d\n", + req.stats_buf_size, NPU_MAX_STATS_BUF_SIZE); + return -EINVAL; + } + + if (req.patch_buf_info_num) { + patch_buf_info = kmalloc_array(req.patch_buf_info_num, + sizeof(*patch_buf_info), GFP_KERNEL); + if (!patch_buf_info) + return -ENOMEM; + + ret = copy_from_user(patch_buf_info, + (void __user *)req.patch_buf_info, + req.patch_buf_info_num * sizeof(*patch_buf_info)); + if (ret) { + pr_err("fail to copy patch buf info\n"); + kfree(patch_buf_info); + return -EFAULT; + } + } + + ret = npu_host_exec_network_v2(client, &req, patch_buf_info); + + kfree(patch_buf_info); + if (ret) { + pr_err("npu_host_exec_network_v2 failed %d\n", ret); + return ret; + } + + ret = copy_to_user(argp, &req, sizeof(req)); + if (ret) { + pr_err("fail to copy to user\n"); + ret = -EFAULT; + } + + return ret; +} + +static int npu_set_fw_state(struct npu_client *client, uint32_t enable) +{ + struct npu_device *npu_dev = client->npu_dev; + struct npu_host_ctx *host_ctx = &npu_dev->host_ctx; + int rc = 0; + + if (host_ctx->network_num > 0) { + pr_err("Need to unload network first\n"); + mutex_unlock(&npu_dev->dev_lock); + return -EINVAL; + } + + if (enable) { + pr_debug("enable fw\n"); + rc = fw_init(npu_dev); + if (rc) { + pr_err("enable fw failed\n"); + } else { + host_ctx->npu_init_cnt++; + pr_debug("npu_init_cnt %d\n", + host_ctx->npu_init_cnt); + /* set npu to lowest power level */ + if (npu_set_uc_power_level(npu_dev, 1)) + pr_warn("Failed to set uc power level\n"); + } + } else if (host_ctx->npu_init_cnt > 0) { + pr_debug("disable fw\n"); + fw_deinit(npu_dev, false, true); + host_ctx->npu_init_cnt--; + pr_debug("npu_init_cnt %d\n", host_ctx->npu_init_cnt); + } else { + pr_err("can't disable fw %d\n", host_ctx->npu_init_cnt); + } + + return rc; +} + +static int npu_set_property(struct npu_client *client, + unsigned long arg) +{ + struct msm_npu_property prop; + void __user *argp = (void __user *)arg; + int ret = -EINVAL; + + ret = copy_from_user(&prop, argp, sizeof(prop)); + if (ret) { + pr_err("fail to copy from user\n"); + return -EFAULT; + } + + switch (prop.prop_id) { + case MSM_NPU_PROP_ID_FW_STATE: + ret = npu_set_fw_state(client, + (uint32_t)prop.prop_param[0]); + break; + case MSM_NPU_PROP_ID_PERF_MODE: + ret = npu_host_set_perf_mode(client, + (uint32_t)prop.network_hdl, + (uint32_t)prop.prop_param[0]); + break; + default: + ret = npu_host_set_fw_property(client->npu_dev, &prop); + if (ret) + pr_err("npu_host_set_fw_property failed\n"); + break; + } + + return ret; +} + +static int npu_get_property(struct npu_client *client, + unsigned long arg) +{ + struct msm_npu_property prop; + void __user *argp = (void __user *)arg; + int ret = -EINVAL; + struct npu_device *npu_dev = client->npu_dev; + struct npu_host_ctx *host_ctx = &npu_dev->host_ctx; + + ret = copy_from_user(&prop, argp, sizeof(prop)); + if (ret) { + pr_err("fail to copy from user\n"); + return -EFAULT; + } + + switch (prop.prop_id) { + case MSM_NPU_PROP_ID_FW_STATE: + prop.prop_param[0] = host_ctx->fw_state; + break; + case MSM_NPU_PROP_ID_PERF_MODE: + prop.prop_param[0] = npu_host_get_perf_mode(client, + (uint32_t)prop.network_hdl); + break; + case MSM_NPU_PROP_ID_PERF_MODE_MAX: + prop.prop_param[0] = npu_dev->pwrctrl.num_pwrlevels; + break; + case MSM_NPU_PROP_ID_DRV_VERSION: + prop.prop_param[0] = 0; + break; + case MSM_NPU_PROP_ID_HARDWARE_VERSION: + prop.prop_param[0] = npu_dev->hw_version; + break; + default: + ret = npu_host_get_fw_property(client->npu_dev, &prop); + if (ret) { + pr_err("npu_host_set_fw_property failed\n"); + return ret; + } + break; + } + + ret = copy_to_user(argp, &prop, sizeof(prop)); + if (ret) { + pr_err("fail to copy to user\n"); + return -EFAULT; + } + + return ret; +} + +static long npu_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + int ret = -ENOIOCTLCMD; + struct npu_client *client = file->private_data; + + switch (cmd) { + case MSM_NPU_GET_INFO: + ret = npu_get_info(client, arg); + break; + case MSM_NPU_MAP_BUF: + ret = npu_map_buf(client, arg); + break; + case MSM_NPU_UNMAP_BUF: + ret = npu_unmap_buf(client, arg); + break; + case MSM_NPU_LOAD_NETWORK: + ret = npu_load_network(client, arg); + break; + case MSM_NPU_LOAD_NETWORK_V2: + ret = npu_load_network_v2(client, arg); + break; + case MSM_NPU_UNLOAD_NETWORK: + ret = npu_unload_network(client, arg); + break; + case MSM_NPU_EXEC_NETWORK: + ret = npu_exec_network(client, arg); + break; + case MSM_NPU_EXEC_NETWORK_V2: + ret = npu_exec_network_v2(client, arg); + break; + case MSM_NPU_SET_PROP: + ret = npu_set_property(client, arg); + break; + case MSM_NPU_GET_PROP: + ret = npu_get_property(client, arg); + break; + default: + pr_err("unexpected IOCTL %x\n", cmd); + } + + return ret; +} + +/* ------------------------------------------------------------------------- + * Device Tree Parsing + * ------------------------------------------------------------------------- + */ +static int npu_parse_dt_clock(struct npu_device *npu_dev) +{ + int rc = 0; + uint32_t i; + const char *clock_name; + int num_clk; + struct npu_clk *core_clks = npu_dev->core_clks; + struct platform_device *pdev = npu_dev->pdev; + + num_clk = of_property_count_strings(pdev->dev.of_node, + "clock-names"); + if (num_clk <= 0) { + pr_err("clocks are not defined\n"); + rc = -EINVAL; + goto clk_err; + } else if (num_clk > NUM_MAX_CLK_NUM) { + pr_err("number of clocks %d exceeds limit\n", num_clk); + rc = -EINVAL; + goto clk_err; + } + + npu_dev->core_clk_num = num_clk; + for (i = 0; i < num_clk; i++) { + of_property_read_string_index(pdev->dev.of_node, "clock-names", + i, &clock_name); + strscpy(core_clks[i].clk_name, clock_name, + sizeof(core_clks[i].clk_name)); + core_clks[i].clk = devm_clk_get(&pdev->dev, clock_name); + if (IS_ERR(core_clks[i].clk)) { + pr_err("unable to get clk: %s\n", clock_name); + rc = -EINVAL; + break; + } + } + +clk_err: + return rc; +} + +static int npu_parse_dt_regulator(struct npu_device *npu_dev) +{ + int rc = 0; + uint32_t i; + const char *name; + int num; + struct npu_regulator *regulators = npu_dev->regulators; + struct platform_device *pdev = npu_dev->pdev; + + num = of_property_count_strings(pdev->dev.of_node, + "qcom,proxy-reg-names"); + if (num <= 0) { + rc = -EINVAL; + pr_err("regulator not defined\n"); + goto regulator_err; + } + if (num > NPU_MAX_REGULATOR_NUM) { + rc = -EINVAL; + pr_err("regulator number %d is over the limit %d\n", num, + NPU_MAX_REGULATOR_NUM); + num = NPU_MAX_REGULATOR_NUM; + } + + npu_dev->regulator_num = num; + for (i = 0; i < num; i++) { + of_property_read_string_index(pdev->dev.of_node, + "qcom,proxy-reg-names", i, &name); + strscpy(regulators[i].regulator_name, name, + sizeof(regulators[i].regulator_name)); + regulators[i].regulator = devm_regulator_get(&pdev->dev, name); + if (IS_ERR(regulators[i].regulator)) { + pr_err("unable to get regulator: %s\n", name); + rc = -EINVAL; + break; + } + } + +regulator_err: + return rc; +} + +static int npu_of_parse_pwrlevels(struct npu_device *npu_dev, + struct device_node *node) +{ + struct npu_pwrctrl *pwr = &npu_dev->pwrctrl; + struct device_node *child; + uint32_t init_level_index = 0, init_power_level; + uint32_t fmax, fmax_pwrlvl; + + pwr->num_pwrlevels = 0; + pwr->min_pwrlevel = NPU_PWRLEVEL_TURBO_L1; + pwr->max_pwrlevel = NPU_PWRLEVEL_MINSVS; + + for_each_available_child_of_node(node, child) { + uint32_t i = 0; + uint32_t index; + uint32_t pwr_level; + uint32_t clk_array_values[NUM_MAX_CLK_NUM]; + uint32_t clk_rate; + struct npu_pwrlevel *level; + + if (of_property_read_u32(child, "reg", &index)) { + pr_err("Can't find reg property\n"); + return -EINVAL; + } + + if (of_property_read_u32(child, "vreg", &pwr_level)) { + pr_err("Can't find vreg property\n"); + return -EINVAL; + } + + if (index >= NPU_MAX_PWRLEVELS) { + pr_err("pwrlevel index %d is out of range\n", + index); + continue; + } + + if (index >= pwr->num_pwrlevels) + pwr->num_pwrlevels = index + 1; + + if (of_property_read_u32_array(child, "clk-freq", + clk_array_values, npu_dev->core_clk_num)) { + pr_err("pwrlevel index %d read clk-freq failed %d\n", + index, npu_dev->core_clk_num); + return -EINVAL; + } + + level = &pwr->pwrlevels[index]; + level->pwr_level = pwr_level; + if (pwr->min_pwrlevel > pwr_level) + pwr->min_pwrlevel = pwr_level; + if (pwr->max_pwrlevel < pwr_level) + pwr->max_pwrlevel = pwr_level; + + for (i = 0; i < npu_dev->core_clk_num; i++) { + if (npu_is_exclude_rate_clock( + npu_dev->core_clks[i].clk_name)) + continue; + + clk_rate = clk_round_rate(npu_dev->core_clks[i].clk, + clk_array_values[i]); + pr_debug("clk %s rate [%u]:[%u]\n", + npu_dev->core_clks[i].clk_name, + clk_array_values[i], clk_rate); + level->clk_freq[i] = clk_rate; + } + } + + /* Read FMAX info if available */ + if (npu_dev->qfprom_io.base) { + fmax = (npu_qfprom_reg_read(npu_dev, + QFPROM_FMAX_REG_OFFSET) & QFPROM_FMAX_BITS_MASK) >> + QFPROM_FMAX_BITS_SHIFT; + pr_debug("fmax %x\n", fmax); + + switch (fmax) { + case 1: + case 2: + fmax_pwrlvl = NPU_PWRLEVEL_NOM; + break; + case 3: + fmax_pwrlvl = NPU_PWRLEVEL_SVS_L1; + break; + default: + fmax_pwrlvl = pwr->max_pwrlevel; + break; + } + + if (fmax_pwrlvl < pwr->max_pwrlevel) + pwr->max_pwrlevel = fmax_pwrlvl; + } + + of_property_read_u32(node, "initial-pwrlevel", &init_level_index); + pr_debug("initial-pwrlevel %d\n", init_level_index); + + if (init_level_index >= pwr->num_pwrlevels) + init_level_index = pwr->num_pwrlevels - 1; + + init_power_level = npu_power_level_from_index(npu_dev, + init_level_index); + if (init_power_level > pwr->max_pwrlevel) { + init_power_level = pwr->max_pwrlevel; + pr_debug("Adjust init power level to %d\n", init_power_level); + } + + pr_debug("init power level %d max %d min %d\n", init_power_level, + pwr->max_pwrlevel, pwr->min_pwrlevel); + pwr->active_pwrlevel = pwr->default_pwrlevel = init_power_level; + pwr->uc_pwrlevel = pwr->max_pwrlevel; + pwr->perf_mode_override = 0; + pwr->cdsprm_pwrlevel = pwr->max_pwrlevel; + pwr->cur_dcvs_activity = pwr->num_pwrlevels; + + return 0; +} + +static int npu_pwrctrl_init(struct npu_device *npu_dev) +{ + struct platform_device *pdev = npu_dev->pdev; + struct device_node *node; + int ret = 0; + struct platform_device *p2dev; + struct npu_pwrctrl *pwr = &npu_dev->pwrctrl; + + /* Power levels */ + node = of_find_node_by_name(pdev->dev.of_node, "qcom,npu-pwrlevels"); + + if (!node) { + pr_err("unable to find 'qcom,npu-pwrlevels'\n"); + return -EINVAL; + } + + ret = npu_of_parse_pwrlevels(npu_dev, node); + if (ret) + return ret; + + /* Parse Bandwidth */ + node = of_parse_phandle(pdev->dev.of_node, + "qcom,npubw-dev", 0); + + if (node) { + /* Set to 1 initially - we assume bwmon is on */ + pwr->bwmon_enabled = 1; + p2dev = of_find_device_by_node(node); + if (p2dev) { + pwr->devbw = &p2dev->dev; + } else { + pr_err("parser power level failed\n"); + ret = -EINVAL; + return ret; + } + } else { + pr_warn("bwdev is not defined in dts\n"); + pwr->devbw = NULL; + } + + return ret; +} + +static int npu_thermalctrl_init(struct npu_device *npu_dev) +{ + struct npu_pwrctrl *pwr = &npu_dev->pwrctrl; + struct npu_thermalctrl *thermalctrl = &npu_dev->thermalctrl; + + thermalctrl->max_state = pwr->num_pwrlevels - 1; + thermalctrl->current_state = 0; + return 0; +} + +static int npu_irq_init(struct npu_device *npu_dev) +{ + unsigned long irq_type; + int ret = 0, i; + + memcpy(npu_dev->irq, npu_irq_info, sizeof(npu_irq_info)); + for (i = 0; i < NPU_MAX_IRQ; i++) { + irq_type = npu_irq_info[i].irq_type; + npu_dev->irq[i].irq = platform_get_irq_byname( + npu_dev->pdev, npu_dev->irq[i].name); + if (npu_dev->irq[i].irq < 0) { + pr_err("get_irq for %s failed\n\n", + npu_dev->irq[i].name); + ret = -EINVAL; + break; + } + + pr_debug("irq %s: %d\n", npu_dev->irq[i].name, + npu_dev->irq[i].irq); + irq_set_status_flags(npu_dev->irq[i].irq, + IRQ_NOAUTOEN); + ret = devm_request_irq(&npu_dev->pdev->dev, + npu_dev->irq[i].irq, npu_intr_hdler, + irq_type, npu_dev->irq[i].name, + npu_dev); + if (ret) { + pr_err("devm_request_irq(%s:%d) failed\n", + npu_dev->irq[i].name, + npu_dev->irq[i].irq); + break; + } + } + + return ret; +} + +static int npu_mbox_init(struct npu_device *npu_dev) +{ + struct platform_device *pdev = npu_dev->pdev; + struct npu_mbox *mbox_aop = &npu_dev->mbox_aop; + + if (of_find_property(pdev->dev.of_node, "mboxes", NULL)) { + mbox_aop->client.dev = &pdev->dev; + mbox_aop->client.tx_block = true; + mbox_aop->client.tx_tout = MBOX_OP_TIMEOUTMS; + mbox_aop->client.knows_txdone = false; + + mbox_aop->chan = mbox_request_channel(&mbox_aop->client, 0); + if (IS_ERR(mbox_aop->chan)) { + pr_warn("aop mailbox is not available\n"); + mbox_aop->chan = NULL; + } + } + return 0; +} + +static void npu_mbox_deinit(struct npu_device *npu_dev) +{ + if (npu_dev->mbox_aop.chan) { + mbox_free_channel(npu_dev->mbox_aop.chan); + npu_dev->mbox_aop.chan = NULL; + } +} + +static int npu_hw_info_init(struct npu_device *npu_dev) +{ + int rc = 0; + + rc = npu_enable_core_power(npu_dev); + if (rc) { + pr_err("Failed to enable power\n"); + return rc; + } + + npu_dev->hw_version = REGR(npu_dev, NPU_HW_VERSION); + pr_debug("NPU_HW_VERSION 0x%x\n", npu_dev->hw_version); + npu_disable_core_power(npu_dev); + + return rc; +} + +static void npu_icc_init(struct npu_device *npu_dev) +{ + struct platform_device *pdev = npu_dev->pdev; + + npu_dev->icc_npu_cdspmem = of_icc_get(&pdev->dev, ICC_NPU_CDSPMEM); + if (IS_ERR_OR_NULL(npu_dev->icc_npu_cdspmem)) { + dev_err(&pdev->dev, "(%ld): failed getting %s path\n", + PTR_ERR(npu_dev->icc_npu_cdspmem), ICC_NPU_CDSPMEM); + npu_dev->icc_npu_cdspmem = NULL; + } else { + pr_info("get interconnects between npu and cdsp_mem successfully\n"); + } + npu_dev->icc_cpu_imemcfg = of_icc_get(&pdev->dev, ICC_CPU_IMEMCFG); + if (IS_ERR_OR_NULL(npu_dev->icc_cpu_imemcfg)) { + dev_err(&pdev->dev, "(%ld): failed getting %s path\n", + PTR_ERR(npu_dev->icc_cpu_imemcfg), ICC_CPU_IMEMCFG); + npu_dev->icc_cpu_imemcfg = NULL; + } else { + pr_info("get interconnects between cpu and imemcfg successfully\n"); + } +} + +static void npu_icc_deinit(struct npu_device *npu_dev) +{ + if (npu_dev->icc_npu_cdspmem) { + icc_put(npu_dev->icc_npu_cdspmem); + npu_dev->icc_npu_cdspmem = NULL; + } + if (npu_dev->icc_cpu_imemcfg) { + icc_put(npu_dev->icc_cpu_imemcfg); + npu_dev->icc_cpu_imemcfg = NULL; + } +} + +static void npu_enable_icc_bw(struct npu_device *npu_dev) +{ + int ret = 0; + + if (npu_dev->icc_npu_cdspmem) { + ret = icc_set_bw(npu_dev->icc_npu_cdspmem, 100, 100); + if (ret) + pr_err("set interconnects npu-cdspmem bw failed, ret: (%d)\n", ret); + } else { + pr_err("icc_path icc_npu_cdspmem is nullptr\n"); + } + + if (npu_dev->icc_cpu_imemcfg) { + ret = icc_set_bw(npu_dev->icc_cpu_imemcfg, 100, 100); + if (ret) + pr_err("set interconnects cpu-imemcfg bw failed, ret: (%d)\n", ret); + } else { + pr_err("icc_path icc_cpu_imemcfg is nullptr\n"); + } +} + +static void npu_disable_icc_bw(struct npu_device *npu_dev) +{ + icc_set_bw(npu_dev->icc_npu_cdspmem, 0, 0); + icc_set_bw(npu_dev->icc_cpu_imemcfg, 0, 0); +} +static int npu_alloc_memory_region(struct npu_device *npu_dev) +{ + struct device *dev = &npu_dev->pdev->dev; + struct device_node *node; + struct resource r; + int ret; + + node = of_parse_phandle(dev->of_node, "memory-region", 0); + if (!node) { + pr_err("no memory-region specified\n"); + return -EINVAL; + } + + ret = of_address_to_resource(node, 0, &r); + of_node_put(node); + if (ret) + return ret; + + npu_dev->fw_io.mem_phys = npu_dev->fw_io.mem_reloc = r.start; + npu_dev->fw_io.mem_size = resource_size(&r); + npu_dev->fw_io.mem_region = devm_ioremap_wc(dev, npu_dev->fw_io.mem_phys, + npu_dev->fw_io.mem_size); + if (!npu_dev->fw_io.mem_region) { + pr_err("unable to map memory region: %pa+%zx\n", + &r.start, npu_dev->fw_io.mem_size); + return -EBUSY; + } + + return 0; +} + +/* ------------------------------------------------------------------------- + * Probe/Remove + * ------------------------------------------------------------------------- + */ +static int npu_probe(struct platform_device *pdev) +{ + int rc = 0; + struct resource *res = NULL; + struct npu_device *npu_dev = NULL; + struct thermal_cooling_device *tcdev = NULL; + + if (!qcom_scm_is_available()) { + pr_err("qcom scm is not available, npu probe defer\n"); + return -EPROBE_DEFER; + } + npu_dev = devm_kzalloc(&pdev->dev, + sizeof(struct npu_device), GFP_KERNEL); + if (!npu_dev) + return -EFAULT; + npu_dev->pdev = pdev; + mutex_init(&npu_dev->dev_lock); + + platform_set_drvdata(pdev, npu_dev); + res = platform_get_resource_byname(pdev, + IORESOURCE_MEM, "core"); + if (!res) { + pr_err("unable to get core resource\n"); + rc = -ENODEV; + goto error_get_dev_num; + } + npu_dev->core_io.size = resource_size(res); + npu_dev->core_io.base = devm_ioremap(&pdev->dev, res->start, + npu_dev->core_io.size); + if (unlikely(!npu_dev->core_io.base)) { + pr_err("unable to map core\n"); + rc = -ENOMEM; + goto error_get_dev_num; + } + pr_debug("core phy address=0x%llx virt=%pK\n", + res->start, npu_dev->core_io.base); + + res = platform_get_resource_byname(pdev, + IORESOURCE_MEM, "tcm"); + if (!res) { + pr_err("unable to get tcm resource\n"); + rc = -ENODEV; + goto error_get_dev_num; + } + npu_dev->tcm_io.size = resource_size(res); + npu_dev->tcm_io.base = devm_ioremap(&pdev->dev, res->start, + npu_dev->tcm_io.size); + if (unlikely(!npu_dev->tcm_io.base)) { + pr_err("unable to map tcm\n"); + rc = -ENOMEM; + goto error_get_dev_num; + } + pr_debug("core phy address=0x%llx virt=%pK\n", + res->start, npu_dev->tcm_io.base); + + res = platform_get_resource_byname(pdev, + IORESOURCE_MEM, "bwmon"); + if (!res) { + pr_err("unable to get bwmon resource\n"); + rc = -ENODEV; + goto error_get_dev_num; + } + npu_dev->bwmon_io.size = resource_size(res); + npu_dev->bwmon_io.base = devm_ioremap(&pdev->dev, res->start, + npu_dev->bwmon_io.size); + if (unlikely(!npu_dev->bwmon_io.base)) { + pr_err("unable to map bwmon\n"); + rc = -ENOMEM; + goto error_get_dev_num; + } + pr_debug("bwmon phy address=0x%llx virt=%pK\n", + res->start, npu_dev->bwmon_io.base); + + res = platform_get_resource_byname(pdev, + IORESOURCE_MEM, "qfprom_physical"); + if (!res) { + pr_info("unable to get qfprom_physical resource\n"); + } else { + npu_dev->qfprom_io.size = resource_size(res); + npu_dev->qfprom_io.base = devm_ioremap(&pdev->dev, res->start, + npu_dev->qfprom_io.size); + if (unlikely(!npu_dev->qfprom_io.base)) { + pr_err("unable to map qfprom_physical\n"); + rc = -ENOMEM; + goto error_get_dev_num; + } + pr_debug("qfprom_physical phy address=0x%llx virt=%pK\n", + res->start, npu_dev->qfprom_io.base); + } + if (npu_alloc_memory_region(npu_dev)) { + rc = -ENOMEM; + goto error_get_dev_num; + } + rc = npu_parse_dt_regulator(npu_dev); + if (rc) + goto error_get_dev_num; + + rc = npu_parse_dt_clock(npu_dev); + if (rc) + goto error_get_dev_num; + + //init interconnets path if exists in devicetree file + npu_icc_init(npu_dev); + + rc = npu_hw_info_init(npu_dev); + if (rc) + goto error_get_dev_num; + + rc = npu_pwrctrl_init(npu_dev); + if (rc) + goto error_get_dev_num; + + rc = npu_thermalctrl_init(npu_dev); + if (rc) + goto error_get_dev_num; + + rc = npu_irq_init(npu_dev); + if (rc) + goto error_get_dev_num; + + rc = npu_mbox_init(npu_dev); + if (rc) + goto error_get_dev_num; + + /* character device might be optional */ + rc = alloc_chrdev_region(&npu_dev->dev_num, 0, 1, DRIVER_NAME); + if (rc < 0) { + pr_err("alloc_chrdev_region failed: %d\n", rc); + goto error_get_dev_num; + } + + npu_dev->class = class_create(THIS_MODULE, CLASS_NAME); + if (IS_ERR(npu_dev->class)) { + rc = PTR_ERR(npu_dev->class); + pr_err("class_create failed: %d\n", rc); + goto error_class_create; + } + + npu_dev->device = device_create(npu_dev->class, NULL, + npu_dev->dev_num, NULL, DRIVER_NAME); + if (IS_ERR(npu_dev->device)) { + rc = PTR_ERR(npu_dev->device); + pr_err("device_create failed: %d\n", rc); + goto error_class_device_create; + } + + cdev_init(&npu_dev->cdev, &npu_fops); + rc = cdev_add(&npu_dev->cdev, + MKDEV(MAJOR(npu_dev->dev_num), 0), 1); + if (rc < 0) { + pr_err("cdev_add failed %d\n", rc); + goto error_cdev_add; + } + dev_set_drvdata(npu_dev->device, npu_dev); + pr_debug("drvdata %pK %pK\n", dev_get_drvdata(&pdev->dev), + dev_get_drvdata(npu_dev->device)); + rc = sysfs_create_group(&npu_dev->device->kobj, &npu_fs_attr_group); + if (rc) { + pr_err("unable to register npu sysfs nodes\n"); + goto error_res_init; + } + + if (IS_ENABLED(CONFIG_THERMAL)) { + tcdev = thermal_of_cooling_device_register(pdev->dev.of_node, + "npu", npu_dev, + &npu_cooling_ops); + if (IS_ERR(tcdev)) { + dev_err(&pdev->dev, + "npu: failed to register npu as cooling device\n"); + rc = PTR_ERR(tcdev); + goto error_driver_init; + } + npu_dev->tcdev = tcdev; + } + + rc = npu_cdsprm_cxlimit_init(npu_dev); + if (rc) + goto error_driver_init; +#ifdef CONFIG_DEBUG_FS + npu_debugfs_init(npu_dev); +#endif + rc = npu_host_init(npu_dev); + if (rc) { + pr_err("unable to init host\n"); + goto error_driver_init; + } + + g_npu_dev = npu_dev; + + return rc; +error_driver_init: + npu_cdsprm_cxlimit_deinit(npu_dev); + if (npu_dev->tcdev) + thermal_cooling_device_unregister(npu_dev->tcdev); + sysfs_remove_group(&npu_dev->device->kobj, &npu_fs_attr_group); +error_res_init: + cdev_del(&npu_dev->cdev); +error_cdev_add: + device_destroy(npu_dev->class, npu_dev->dev_num); +error_class_device_create: + class_destroy(npu_dev->class); +error_class_create: + unregister_chrdev_region(npu_dev->dev_num, 1); + npu_mbox_deinit(npu_dev); +error_get_dev_num: + npu_icc_deinit(npu_dev); + return rc; +} + +static int npu_remove(struct platform_device *pdev) +{ + struct npu_device *npu_dev; + + npu_dev = platform_get_drvdata(pdev); + npu_host_deinit(npu_dev); + npu_debugfs_deinit(npu_dev); + npu_cdsprm_cxlimit_deinit(npu_dev); + if (npu_dev->tcdev) + thermal_cooling_device_unregister(npu_dev->tcdev); + sysfs_remove_group(&npu_dev->device->kobj, &npu_fs_attr_group); + cdev_del(&npu_dev->cdev); + device_destroy(npu_dev->class, npu_dev->dev_num); + class_destroy(npu_dev->class); + unregister_chrdev_region(npu_dev->dev_num, 1); + platform_set_drvdata(pdev, NULL); + npu_mbox_deinit(npu_dev); + npu_icc_deinit(npu_dev); + g_npu_dev = NULL; + + return 0; +} + +/* ------------------------------------------------------------------------- + * Suspend/Resume + * ------------------------------------------------------------------------- + */ +#if defined(CONFIG_PM) +static int npu_suspend(struct platform_device *dev, pm_message_t state) +{ + return 0; +} + +static int npu_resume(struct platform_device *dev) +{ + return 0; +} +#endif + +/* ------------------------------------------------------------------------- + * Module Entry Points + * ------------------------------------------------------------------------- + */ +static int __init npu_init(void) +{ + int rc; + + rc = platform_driver_register(&npu_driver); + if (rc) + pr_err("register failed %d\n", rc); + return rc; +} + +static void __exit npu_exit(void) +{ + platform_driver_unregister(&npu_driver); +} + +module_init(npu_init); +module_exit(npu_exit); + +MODULE_DEVICE_TABLE(of, npu_dt_match); +MODULE_DESCRIPTION("MSM NPU driver"); +MODULE_LICENSE("GPL"); +MODULE_INFO(intree, "Y"); diff --git a/drivers/media/platform/msm/npu/npu_firmware.h b/drivers/media/platform/msm/npu/npu_firmware.h new file mode 100644 index 000000000000..33ef141282f7 --- /dev/null +++ b/drivers/media/platform/msm/npu/npu_firmware.h @@ -0,0 +1,176 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2018-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2023-2024, Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#ifndef _NPU_FIRMWARE_H +#define _NPU_FIRMWARE_H + +/* ------------------------------------------------------------------------- + * Includes + * ------------------------------------------------------------------------- + */ +#include + +/* ------------------------------------------------------------------------- + * Defines + * ------------------------------------------------------------------------- + */ +/* NPU Firmware Control/Status Register, written by FW and read HOST */ +#define REG_NPU_FW_CTRL_STATUS NPU_GPR0 +/* written by HOST and read by FW for control */ +#define REG_NPU_HOST_CTRL_STATUS NPU_GPR1 +/* Data value for control */ +#define REG_NPU_HOST_CTRL_VALUE NPU_GPR2 +/* Simulates an interrupt for FW->HOST, used for pre-silicon */ +#define REG_FW_TO_HOST_EVENT NPU_GPR3 +/* Read/Written by both host and dsp for sync between driver and dsp */ +#define REG_HOST_DSP_CTRL_STATUS NPU_GPR4 +/* Data value for debug */ +#define REG_NPU_FW_DEBUG_DATA NPU_GPR13 + +/* Started job count */ +#define REG_FW_JOB_CNT_START NPU_GPR14 +/* Finished job count */ +#define REG_FW_JOB_CNT_END NPU_GPR15 + +/* NPU FW Control/Status Register */ +/* bit fields definitions in CTRL STATUS REG */ +#define FW_CTRL_STATUS_IPC_READY_BIT 0 +#define FW_CTRL_STATUS_LOG_READY_BIT 1 +#define FW_CTRL_STATUS_EXECUTE_THREAD_READY_BIT 2 +#define FW_CTRL_STATUS_MAIN_THREAD_READY_BIT 3 +#define FW_CTRL_STATUS_LOADED_ACO_BIT 4 +#define FW_CTRL_STATUS_EXECUTING_ACO_BIT 5 +#define FW_CTRL_STATUS_SHUTDOWN_DONE_BIT 12 +#define FW_CTRL_STATUS_STACK_CORRUPT_BIT 13 + +/* 32 bit values of the bit fields above */ +#define FW_CTRL_STATUS_IPC_READY_VAL (1 << FW_CTRL_STATUS_IPC_READY_BIT) +#define FW_CTRL_STATUS_LOG_READY_VAL (1 << FW_CTRL_STATUS_LOG_READY_BIT) +#define FW_CTRL_STATUS_EXECUTE_THREAD_READY_VAL \ + (1 << FW_CTRL_STATUS_EXECUTE_THREAD_READY_BIT) +#define FW_CTRL_STATUS_MAIN_THREAD_READY_VAL \ + (1 << FW_CTRL_STATUS_MAIN_THREAD_READY_BIT) +#define FW_CTRL_STATUS_LOADED_ACO_VAL \ + (1 << FW_CTRL_STATUS_LOADED_ACO_BIT) +#define FW_CTRL_STATUS_EXECUTING_ACO_VAL \ + (1 << FW_CTRL_STATUS_EXECUTING_ACO_BIT) +#define FW_CTRL_STATUS_SHUTDOWN_DONE_VAL \ + (1 << FW_CTRL_STATUS_SHUTDOWN_DONE_BIT) +#define FW_CTRL_STATUS_STACK_CORRUPT_VAL \ + (1 << FW_CTRL_STATUS_STACK_CORRUPT_BIT) + +/* NPU HOST Control/Status Register */ +/* bit fields definitions in CTRL STATUS REG */ +/* Host has programmed IPC address into the REG_NPU_HOST_CTRL_VALUE register */ +#define HOST_CTRL_STATUS_IPC_ADDRESS_READY_BIT 0 +/* Host has enabled logging during boot */ +#define HOST_CTRL_STATUS_BOOT_ENABLE_LOGGING_BIT 1 +/* Host has enabled the clk gating of CAL during boot */ +#define HOST_CTRL_STATUS_BOOT_ENABLE_CLK_GATE_BIT 2 +/* Host requests to pause fw during boot up */ +#define HOST_CTRL_STATUS_FW_PAUSE 3 +/* Host requests to disable watchdog */ +#define HOST_CTRL_STATUS_DISABLE_WDOG_BIT 4 + +/* 32 bit values of the bit fields above */ +#define HOST_CTRL_STATUS_IPC_ADDRESS_READY_VAL \ + (1 << HOST_CTRL_STATUS_IPC_ADDRESS_READY_BIT) +#define HOST_CTRL_STATUS_BOOT_ENABLE_LOGGING_VAL \ + (1 << HOST_CTRL_STATUS_BOOT_ENABLE_LOGGING_BIT) +#define HOST_CTRL_STATUS_BOOT_ENABLE_CLK_GATE_VAL \ + (1 << HOST_CTRL_STATUS_BOOT_ENABLE_CLK_GATE_BIT) +#define HOST_CTRL_STATUS_FW_PAUSE_VAL \ + (1 << HOST_CTRL_STATUS_FW_PAUSE) +#define HOST_CTRL_STATUS_DISABLE_WDOG_VAL \ + (1 << HOST_CTRL_STATUS_DISABLE_WDOG_BIT) + + +/* NPU HOST DSP Control/Status Register */ +/* notification of power up */ +/* following bits are set by host and read by dsp */ +#define HOST_DSP_CTRL_STATUS_PWR_UP_BIT 0 +/* notification of power dwn */ +#define HOST_DSP_CTRL_STATUS_PWR_DWN_BIT 1 +/* following bits are set by dsp and read by host */ +/* notification of power up acknowlegement*/ +#define HOST_DSP_CTRL_STATUS_PWR_UP_ACK_BIT 4 +/* notification of power down acknowlegement*/ +#define HOST_DSP_CTRL_STATUS_PWR_DWN_ACK_BIT 5 + + +/* 32 bit values of the bit fields above */ +#define HOST_DSP_CTRL_STATUS_PWR_UP_VAL \ + (1 << HOST_DSP_CTRL_STATUS_PWR_UP_BIT) +#define HOST_DSP_CTRL_STATUS_PWR_DWN_VAL \ + (1 << HOST_DSP_CTRL_STATUS_PWR_DWN_BIT) +#define HOST_DSP_CTRL_STATUS_PWR_UP_ACK_VAL \ + (1 << HOST_DSP_CTRL_STATUS_PWR_UP_ACK_BIT) +#define HOST_DSP_CTRL_STATUS_PWR_DWN_ACK_VAL \ + (1 << HOST_DSP_CTRL_STATUS_PWR_DWN_ACK_BIT) + +/* Queue table header definition */ +struct hfi_queue_tbl_header { + uint32_t qtbl_version; /* queue table version number */ + uint32_t qtbl_size; /* total tables+queues size in bytes */ + uint32_t qtbl_qhdr0_offset; /* offset of the 1st queue header entry */ + uint32_t qtbl_qhdr_size; /* queue header size */ + uint32_t qtbl_num_q; /* total number of queues */ + uint32_t qtbl_num_active_q; /* number of active queues */ +}; + +/* Queue header definition */ +struct hfi_queue_header { + uint32_t qhdr_status; /* 0 == inactive, 1 == active */ + /* 4 byte-aligned start offset from start of q table */ + uint32_t qhdr_start_offset; + /* queue type */ + uint32_t qhdr_type; + /* in bytes, value of 0 means packets are variable size.*/ + uint32_t qhdr_q_size; + /* size of the Queue packet entries, in bytes, 0 means variable size */ + uint32_t qhdr_pkt_size; + + uint32_t qhdr_pkt_drop_cnt; + /* receiver watermark in # of queue packets */ + uint32_t qhdr_rx_wm; + /* transmitter watermark in # of queue packets */ + uint32_t qhdr_tx_wm; + /* + * set to request an interrupt from transmitter + * if qhdr_tx_wm is reached + */ + uint32_t qhdr_rx_req; + /* + * set to request an interrupt from receiver + * if qhdr_rx_wm is reached + */ + uint32_t qhdr_tx_req; + uint32_t qhdr_rx_irq_status; /* Not used */ + uint32_t qhdr_tx_irq_status; /* Not used */ + uint32_t qhdr_read_idx; /* read index in bytes */ + uint32_t qhdr_write_idx; /* write index in bytes */ +}; + +/* in bytes */ +#define HFI_QUEUE_TABLE_HEADER_SIZE (sizeof(struct hfi_queue_tbl_header)) +#define HFI_QUEUE_HEADER_SIZE (sizeof(struct hfi_queue_header)) +#define HFI_QUEUE_TABLE_SIZE (HFI_QUEUE_TABLE_HEADER_SIZE + \ + (NPU_HFI_NUMBER_OF_QS * HFI_QUEUE_HEADER_SIZE)) + +/* Queue Indexes */ +#define IPC_QUEUE_CMD_HIGH_PRIORITY 0 /* High priority Queue APPS->M0 */ +#define IPC_QUEUE_APPS_EXEC 1 /* APPS Execute Queue APPS->M0 */ +#define IPC_QUEUE_DSP_EXEC 2 /* DSP Execute Queue DSP->M0 */ +#define IPC_QUEUE_APPS_RSP 3 /* APPS Message Queue M0->APPS */ +#define IPC_QUEUE_DSP_RSP 4 /* DSP Message Queue DSP->APPS */ +#define IPC_QUEUE_LOG 5 /* Log Message Queue M0->APPS */ + +#define NPU_HFI_NUMBER_OF_QS 6 +#define NPU_HFI_NUMBER_OF_ACTIVE_QS 6 + +#define NPU_HFI_QUEUES_PER_CHANNEL 2 + +#endif /* _NPU_FIRMWARE_H */ diff --git a/drivers/media/platform/msm/npu/npu_host_ipc.c b/drivers/media/platform/msm/npu/npu_host_ipc.c new file mode 100644 index 000000000000..63cb62a40f53 --- /dev/null +++ b/drivers/media/platform/msm/npu/npu_host_ipc.c @@ -0,0 +1,438 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2018-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2023-2024, Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +/* ------------------------------------------------------------------------- + * Includes + * ------------------------------------------------------------------------- + */ +#include "npu_hw_access.h" +#include "npu_mgr.h" +#include "npu_firmware.h" +#include "npu_hw.h" +#include "npu_host_ipc.h" + +/* ------------------------------------------------------------------------- + * Defines + * ------------------------------------------------------------------------- + */ +/* HFI IPC interface */ +#define TX_HDR_TYPE 0x01000000 +#define RX_HDR_TYPE 0x00010000 +#define HFI_QTBL_STATUS_ENABLED 0x00000001 + +#define QUEUE_TBL_VERSION 0x87654321 + +/* ------------------------------------------------------------------------- + * Data Structures + * ------------------------------------------------------------------------- + */ +struct npu_queue_tuple { + uint32_t size; + uint32_t hdr; + uint32_t start_offset; +}; + +static struct npu_queue_tuple npu_q_setup[6] = { + { 1024, IPC_QUEUE_CMD_HIGH_PRIORITY | TX_HDR_TYPE | RX_HDR_TYPE, 0}, + { 4096, IPC_QUEUE_APPS_EXEC | TX_HDR_TYPE | RX_HDR_TYPE, 0}, + { 4096, IPC_QUEUE_DSP_EXEC | TX_HDR_TYPE | RX_HDR_TYPE, 0}, + { 4096, IPC_QUEUE_APPS_RSP | TX_HDR_TYPE | RX_HDR_TYPE, 0}, + { 4096, IPC_QUEUE_DSP_RSP | TX_HDR_TYPE | RX_HDR_TYPE, 0}, + { 1024, IPC_QUEUE_LOG | TX_HDR_TYPE | RX_HDR_TYPE, 0}, +}; + +/* ------------------------------------------------------------------------- + * File Scope Function Prototypes + * ------------------------------------------------------------------------- + */ +static int npu_host_ipc_init_hfi(struct npu_device *npu_dev); +static int npu_host_ipc_send_cmd_hfi(struct npu_device *npu_dev, + uint32_t q_idx, void *cmd_ptr); +static int npu_host_ipc_read_msg_hfi(struct npu_device *npu_dev, + uint32_t q_idx, uint32_t *msg_ptr); +static int ipc_queue_read(struct npu_device *npu_dev, uint32_t target_que, + uint8_t *packet, uint8_t *is_tx_req_set); +static int ipc_queue_write(struct npu_device *npu_dev, uint32_t target_que, + uint8_t *packet, uint8_t *is_rx_req_set); + +/* ------------------------------------------------------------------------- + * Function Definitions + * ------------------------------------------------------------------------- + */ +static int npu_host_ipc_init_hfi(struct npu_device *npu_dev) +{ + int status = 0; + struct hfi_queue_tbl_header *q_tbl_hdr = NULL; + struct hfi_queue_header *q_hdr_arr = NULL; + struct hfi_queue_header *q_hdr = NULL; + void *q_tbl_addr = NULL; + uint32_t reg_val = 0; + uint32_t q_idx = 0; + uint32_t q_tbl_size = sizeof(struct hfi_queue_tbl_header) + + (NPU_HFI_NUMBER_OF_QS * sizeof(struct hfi_queue_header)); + uint32_t q_size = 0; + uint32_t cur_start_offset = 0; + + reg_val = REGR(npu_dev, REG_NPU_FW_CTRL_STATUS); + + /* + * If the firmware is already running and we're just attaching, + * we do not need to do this + */ + if ((reg_val & FW_CTRL_STATUS_LOG_READY_VAL) != 0) + return status; + + /* check for valid interface queue table start address */ + q_tbl_addr = kzalloc(q_tbl_size, GFP_KERNEL); + if (q_tbl_addr == NULL) + return -ENOMEM; + + /* retrieve interface queue table start address */ + q_tbl_hdr = q_tbl_addr; + q_hdr_arr = (struct hfi_queue_header *)((uint8_t *)q_tbl_addr + + sizeof(struct hfi_queue_tbl_header)); + + /* initialize the interface queue table header */ + q_tbl_hdr->qtbl_version = QUEUE_TBL_VERSION; + q_tbl_hdr->qtbl_size = q_tbl_size; + q_tbl_hdr->qtbl_qhdr0_offset = sizeof(struct hfi_queue_tbl_header); + q_tbl_hdr->qtbl_qhdr_size = sizeof(struct hfi_queue_header); + q_tbl_hdr->qtbl_num_q = NPU_HFI_NUMBER_OF_QS; + q_tbl_hdr->qtbl_num_active_q = NPU_HFI_NUMBER_OF_ACTIVE_QS; + + cur_start_offset = q_tbl_size; + + for (q_idx = IPC_QUEUE_CMD_HIGH_PRIORITY; + q_idx <= IPC_QUEUE_LOG; q_idx++) { + q_hdr = &q_hdr_arr[q_idx]; + /* queue is active */ + q_hdr->qhdr_status = 0x01; + q_hdr->qhdr_start_offset = cur_start_offset; + npu_q_setup[q_idx].start_offset = cur_start_offset; + q_size = npu_q_setup[q_idx].size; + q_hdr->qhdr_type = npu_q_setup[q_idx].hdr; + /* in bytes */ + q_hdr->qhdr_q_size = q_size; + /* variable size packets */ + q_hdr->qhdr_pkt_size = 0; + q_hdr->qhdr_pkt_drop_cnt = 0; + q_hdr->qhdr_rx_wm = 0x1; + q_hdr->qhdr_tx_wm = 0x1; + /* since queue is initially empty */ + q_hdr->qhdr_rx_req = 0x1; + q_hdr->qhdr_tx_req = 0x0; + /* not used */ + q_hdr->qhdr_rx_irq_status = 0; + /* not used */ + q_hdr->qhdr_tx_irq_status = 0; + q_hdr->qhdr_read_idx = 0; + q_hdr->qhdr_write_idx = 0; + cur_start_offset += q_size; + } + + MEMW(npu_dev, IPC_ADDR, (uint8_t *)q_tbl_hdr, q_tbl_size); + kfree(q_tbl_addr); + /* Write in the NPU's address for where IPC starts */ + REGW(npu_dev, (uint32_t)REG_NPU_HOST_CTRL_VALUE, + (uint32_t)IPC_MEM_OFFSET_FROM_SSTCM); + /* Set value bit */ + reg_val = REGR(npu_dev, (uint32_t)REG_NPU_HOST_CTRL_STATUS); + REGW(npu_dev, (uint32_t)REG_NPU_HOST_CTRL_STATUS, reg_val | + HOST_CTRL_STATUS_IPC_ADDRESS_READY_VAL); + return status; +} + +static int npu_host_ipc_send_cmd_hfi(struct npu_device *npu_dev, + uint32_t q_idx, void *cmd_ptr) +{ + int status = 0; + uint8_t is_rx_req_set = 0; + uint32_t retry_cnt = 5; + + status = ipc_queue_write(npu_dev, q_idx, (uint8_t *)cmd_ptr, + &is_rx_req_set); + + if (status == -ENOSPC) { + do { + msleep(20); + status = ipc_queue_write(npu_dev, q_idx, + (uint8_t *)cmd_ptr, &is_rx_req_set); + } while ((status == -ENOSPC) && (--retry_cnt > 0)); + } + + if (status == 0) { + if (is_rx_req_set == 1) + status = INTERRUPT_RAISE_NPU(npu_dev); + } + + if (status == 0) + pr_debug("Cmd Msg put on Command Queue - SUCCESSS\n"); + else + pr_err("Cmd Msg put on Command Queue - FAILURE\n"); + + return status; +} + +static int npu_host_ipc_read_msg_hfi(struct npu_device *npu_dev, + uint32_t q_idx, uint32_t *msg_ptr) +{ + int status = 0; + uint8_t is_tx_req_set; + + status = ipc_queue_read(npu_dev, q_idx, (uint8_t *)msg_ptr, + &is_tx_req_set); + + if (status == 0) { + /* raise interrupt if qhdr_tx_req is set */ + if (is_tx_req_set == 1) + status = INTERRUPT_RAISE_NPU(npu_dev); + } + + return status; +} + +static int ipc_queue_read(struct npu_device *npu_dev, + uint32_t target_que, uint8_t *packet, + uint8_t *is_tx_req_set) +{ + int status = 0; + struct hfi_queue_header queue; + uint32_t packet_size, new_read_idx; + size_t read_ptr; + size_t offset = 0; + + offset = (size_t)IPC_ADDR + sizeof(struct hfi_queue_tbl_header) + + target_que * sizeof(struct hfi_queue_header); + + if ((packet == NULL) || (is_tx_req_set == NULL)) + return -EINVAL; + + /* Read the queue */ + MEMR(npu_dev, (void *)((size_t)offset), (uint8_t *)&queue, + HFI_QUEUE_HEADER_SIZE); + + if (queue.qhdr_type != npu_q_setup[target_que].hdr || + queue.qhdr_q_size != npu_q_setup[target_que].size || + queue.qhdr_read_idx >= queue.qhdr_q_size || + queue.qhdr_write_idx >= queue.qhdr_q_size || + queue.qhdr_start_offset != + npu_q_setup[target_que].start_offset) { + pr_err("Invalid Queue header\n"); + status = -EIO; + goto exit; + } + + /* check if queue is empty */ + if (queue.qhdr_read_idx == queue.qhdr_write_idx) { + /* + * set qhdr_rx_req, to inform the sender that the Interrupt + * needs to be raised with the next packet queued + */ + queue.qhdr_rx_req = 1; + *is_tx_req_set = 0; + status = -EPERM; + goto exit; + } + + read_ptr = ((size_t)(size_t)IPC_ADDR + + queue.qhdr_start_offset + queue.qhdr_read_idx); + + /* Read packet size */ + MEMR(npu_dev, (void *)((size_t)read_ptr), packet, 4); + packet_size = *((uint32_t *)packet); + + pr_debug("target_que: %d, packet_size: %d\n", + target_que, + packet_size); + + if ((packet_size == 0) || + (packet_size > NPU_IPC_BUF_LENGTH)) { + pr_err("Invalid packet size %d\n", packet_size); + status = -EINVAL; + goto exit; + } + new_read_idx = queue.qhdr_read_idx + packet_size; + + if (new_read_idx < (queue.qhdr_q_size)) { + MEMR(npu_dev, (void *)((size_t)read_ptr), packet, packet_size); + } else { + new_read_idx -= (queue.qhdr_q_size); + + MEMR(npu_dev, (void *)((size_t)read_ptr), packet, + packet_size - new_read_idx); + + MEMR(npu_dev, (void *)((size_t)IPC_ADDR + + queue.qhdr_start_offset), + (void *)((size_t)packet + (packet_size-new_read_idx)), + new_read_idx); + } + + queue.qhdr_read_idx = new_read_idx; + + if (queue.qhdr_read_idx == queue.qhdr_write_idx) + /* + * receiver wants an interrupt from transmitter + * (when next item queued) because queue is empty + */ + queue.qhdr_rx_req = 1; + else + /* clear qhdr_rx_req since the queue is not empty */ + queue.qhdr_rx_req = 0; + + if (queue.qhdr_tx_req == 1) + /* transmitter requested an interrupt */ + *is_tx_req_set = 1; + else + *is_tx_req_set = 0; +exit: + /* Update RX interrupt request -- queue.qhdr_rx_req */ + MEMW(npu_dev, (void *)((size_t)offset + + (uint32_t)((size_t)&(queue.qhdr_rx_req) - + (size_t)&queue)), (uint8_t *)&queue.qhdr_rx_req, + sizeof(queue.qhdr_rx_req)); + /* Update Read pointer -- queue.qhdr_read_idx */ + MEMW(npu_dev, (void *)((size_t)offset + (uint32_t)( + (size_t)&(queue.qhdr_read_idx) - (size_t)&queue)), + (uint8_t *)&queue.qhdr_read_idx, sizeof(queue.qhdr_read_idx)); + + return status; +} + +static int ipc_queue_write(struct npu_device *npu_dev, + uint32_t target_que, uint8_t *packet, + uint8_t *is_rx_req_set) +{ + int status = 0; + struct hfi_queue_header queue; + uint32_t packet_size, new_write_idx; + uint32_t empty_space; + void *write_ptr; + uint32_t read_idx; + + size_t offset = (size_t)IPC_ADDR + + sizeof(struct hfi_queue_tbl_header) + + target_que * sizeof(struct hfi_queue_header); + + if ((packet == NULL) || (is_rx_req_set == NULL)) + return -EINVAL; + + MEMR(npu_dev, (void *)((size_t)offset), (uint8_t *)&queue, + HFI_QUEUE_HEADER_SIZE); + + if (queue.qhdr_type != npu_q_setup[target_que].hdr || + queue.qhdr_q_size != npu_q_setup[target_que].size || + queue.qhdr_read_idx >= queue.qhdr_q_size || + queue.qhdr_write_idx >= queue.qhdr_q_size || + queue.qhdr_start_offset != + npu_q_setup[target_que].start_offset) { + pr_err("Invalid Queue header\n"); + status = -EIO; + goto exit; + } + + packet_size = (*(uint32_t *)packet); + if (packet_size == 0) { + /* assign failed status and return */ + status = -EPERM; + goto exit; + } + + /* sample Read Idx */ + read_idx = queue.qhdr_read_idx; + + /* Calculate Empty Space(UWord32) in the Queue */ + empty_space = (queue.qhdr_write_idx >= read_idx) ? + ((queue.qhdr_q_size) - (queue.qhdr_write_idx - read_idx)) : + (read_idx - queue.qhdr_write_idx); + + if (empty_space <= packet_size) { + /* + * If Queue is FULL/ no space for message + * set qhdr_tx_req. + */ + queue.qhdr_tx_req = 1; + + /* + * Queue is FULL, force raise an interrupt to Receiver + */ + *is_rx_req_set = 1; + + status = -ENOSPC; + goto exit; + } + + /* + * clear qhdr_tx_req so that receiver does not raise an interrupt + * on reading packets from Queue, since there is space to write + * the next packet + */ + queue.qhdr_tx_req = 0; + + new_write_idx = (queue.qhdr_write_idx + packet_size); + + write_ptr = (void *)(size_t)((size_t)IPC_ADDR + + queue.qhdr_start_offset + queue.qhdr_write_idx); + + if (new_write_idx < queue.qhdr_q_size) { + MEMW(npu_dev, (void *)((size_t)write_ptr), (uint8_t *)packet, + packet_size); + } else { + /* wraparound case */ + new_write_idx -= (queue.qhdr_q_size); + + MEMW(npu_dev, (void *)((size_t)write_ptr), (uint8_t *)packet, + packet_size - new_write_idx); + + MEMW(npu_dev, (void *)((size_t)((size_t)IPC_ADDR + + queue.qhdr_start_offset)), (uint8_t *)(packet + + (packet_size - new_write_idx)), new_write_idx); + } + + /* Update qhdr_write_idx */ + queue.qhdr_write_idx = new_write_idx; + + *is_rx_req_set = (queue.qhdr_rx_req == 1) ? 1 : 0; + + /* Update Write pointer -- queue.qhdr_write_idx */ +exit: + /* Update TX request -- queue.qhdr_tx_req */ + MEMW(npu_dev, (void *)((size_t)(offset + (uint32_t)( + (size_t)&(queue.qhdr_tx_req) - (size_t)&queue))), + &queue.qhdr_tx_req, sizeof(queue.qhdr_tx_req)); + MEMW(npu_dev, (void *)((size_t)(offset + (uint32_t)( + (size_t)&(queue.qhdr_write_idx) - (size_t)&queue))), + &queue.qhdr_write_idx, sizeof(queue.qhdr_write_idx)); + + return status; +} + +/* ------------------------------------------------------------------------- + * IPC Interface functions + * ------------------------------------------------------------------------- + */ +int npu_host_ipc_send_cmd(struct npu_device *npu_dev, uint32_t q_idx, + void *cmd_ptr) +{ + return npu_host_ipc_send_cmd_hfi(npu_dev, q_idx, cmd_ptr); +} + +int npu_host_ipc_read_msg(struct npu_device *npu_dev, uint32_t q_idx, + uint32_t *msg_ptr) +{ + return npu_host_ipc_read_msg_hfi(npu_dev, q_idx, msg_ptr); +} + +int npu_host_ipc_pre_init(struct npu_device *npu_dev) +{ + return npu_host_ipc_init_hfi(npu_dev); +} + +int npu_host_ipc_post_init(struct npu_device *npu_dev) +{ + return 0; +} diff --git a/drivers/media/platform/msm/npu/npu_host_ipc.h b/drivers/media/platform/msm/npu/npu_host_ipc.h new file mode 100644 index 000000000000..a78cd39edf29 --- /dev/null +++ b/drivers/media/platform/msm/npu/npu_host_ipc.h @@ -0,0 +1,464 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2018-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2023-2024, Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#ifndef NPU_HOST_IPC_H +#define NPU_HOST_IPC_H + +/* ------------------------------------------------------------------------- + * Defines + * ------------------------------------------------------------------------- + */ +/* Messages sent **to** NPU */ +/* IPC Message Commands -- uint32_t */ +/* IPC command start base */ +#define NPU_IPC_CMD_BASE 0x00000000 +/* ipc_cmd_load_pkt */ +#define NPU_IPC_CMD_LOAD 0x00000001 +/* ipc_cmd_unload_pkt */ +#define NPU_IPC_CMD_UNLOAD 0x00000002 +/* ipc_cmd_execute_pkt */ +#define NPU_IPC_CMD_EXECUTE 0x00000003 +/* ipc_cmd_set_logging_state */ +#define NPU_IPC_CMD_CONFIG_LOG 0x00000004 +#define NPU_IPC_CMD_CONFIG_PERFORMANCE 0x00000005 +#define NPU_IPC_CMD_CONFIG_DEBUG 0x00000006 +#define NPU_IPC_CMD_SHUTDOWN 0x00000007 +/* ipc_cmd_loopback_packet */ +#define NPU_IPC_CMD_LOOPBACK 0x00000008 +/* ipc_cmd_load_packet_v2_t */ +#define NPU_IPC_CMD_LOAD_V2 0x00000009 +/* ipc_cmd_execute_packet_v2 */ +#define NPU_IPC_CMD_EXECUTE_V2 0x0000000A +/* ipc_cmd_set_property_packet */ +#define NPU_IPC_CMD_SET_PROPERTY 0x0000000B +/* ipc_cmd_get_property_packet */ +#define NPU_IPC_CMD_GET_PROPERTY 0x0000000C + +/* Messages sent **from** NPU */ +/* IPC Message Response -- uint32_t */ +/* IPC response start base */ +#define NPU_IPC_MSG_BASE 0x00010000 +/* ipc_msg_load_pkt */ +#define NPU_IPC_MSG_LOAD_DONE 0x00010001 +/* ipc_msg_header_pkt */ +#define NPU_IPC_MSG_UNLOAD_DONE 0x00010002 +/* ipc_msg_header_pkt */ +#define NPU_IPC_MSG_EXECUTE_DONE 0x00010003 +/* ipc_msg_event_notify_pkt */ +#define NPU_IPC_MSG_EVENT_NOTIFY 0x00010004 +/* ipc_msg_loopback_pkt */ +#define NPU_IPC_MSG_LOOPBACK_DONE 0x00010005 +/* ipc_msg_execute_pkt_v2 */ +#define NPU_IPC_MSG_EXECUTE_V2_DONE 0x00010006 +/* ipc_msg_set_property_packet */ +#define NPU_IPC_MSG_SET_PROPERTY_DONE 0x00010007 +/* ipc_msg_get_property_packet */ +#define NPU_IPC_MSG_GET_PROPERTY_DONE 0x00010008 +/* ipc_msg_general_notify_pkt */ +#define NPU_IPC_MSG_GENERAL_NOTIFY 0x00010010 + +/* IPC Notify Message Type -- uint32_t */ +#define NPU_NOTIFY_DCVS_MODE 0x00002000 + +/* Logging message size */ +/* Number 32-bit elements for the maximum log message size */ +#define NPU_LOG_MSG_MAX_SIZE 4 + +/* Performance */ +/* Performance counters for current network layer */ +/* Amount of data read from all the DMA read channels */ +#define NPU_PERFORMANCE_DMA_DATA_READ 0x01 +/* Amount of data written from all the DMA write channels */ +#define NPU_PERFORMANCE_DMA_DATA_WRITTEN 0x02 +/* Number of blocks read by DMA channels */ +#define NPU_PERFORMANCE_DMA_NUM_BLOCKS_READ 0x03 +/* Number of blocks written by DMA channels */ +#define NPU_PERFORMANCE_DMA_NUM_BLOCKS_WRITTEN 0x04 +/* Number of instructions executed by CAL */ +#define NPU_PERFORMANCE_INSTRUCTIONS_CAL 0x05 +/* Number of instructions executed by CUB */ +#define NPU_PERFORMANCE_INSTRUCTIONS_CUB 0x06 +/* Timestamp of start of network load */ +#define NPU_PERFORMANCE_TIMESTAMP_LOAD_START 0x07 +/* Timestamp of end of network load */ +#define NPU_PERFORMANCE_TIMESTAMP_LOAD_END 0x08 +/* Timestamp of start of network execute */ +#define NPU_PERFORMANCE_TIMESTAMP_EXECUTE_START 0x09 +/* Timestamp of end of network execute */ +#define NPU_PERFORMANCE_TIMESTAMP_EXECUTE_END 0x10 +/* Timestamp of CAL start */ +#define NPU_PERFORMANCE_TIMESTAMP_CAL_START 0x11 +/* Timestamp of CAL end */ +#define NPU_PERFORMANCE_TIMESTAMP_CAL_END 0x12 +/* Timestamp of CUB start */ +#define NPU_PERFORMANCE_TIMESTAMP_CUB_START 0x13 +/* Timestamp of CUB end */ +#define NPU_PERFORMANCE_TIMESTAMP_CUB_END 0x14 + +/* Performance enable */ +/* Select which counters you want back per layer */ + +/* Shutdown */ +/* Immediate shutdown, discard any state, etc */ +#define NPU_SHUTDOWN_IMMEDIATE 0x01 +/* Shutdown after current execution (if any) is completed */ +#define NPU_SHUTDOWN_WAIT_CURRENT_EXECUTION 0x02 + +/* Debug stats */ +#define NUM_LAYER_STATS_PER_EXE_MSG_MAX 110 + +/* DCVS */ +#define NPU_DCVS_ACTIVITY_MAX_PERF 0x100 + +/* ------------------------------------------------------------------------- + * Data Structures + * ------------------------------------------------------------------------- + */ +/* Command Header - Header for all Messages **TO** NPU */ +/* + * command header packet definition for + * messages sent from host->NPU + */ +struct ipc_cmd_header_pkt { + uint32_t size; + uint32_t cmd_type; + uint32_t trans_id; + uint32_t flags; /* TDO what flags and why */ +}; + +/* Message Header - Header for all messages **FROM** NPU */ +/* + * message header packet definition for + * mesasges sent from NPU->host + */ +struct ipc_msg_header_pkt { + uint32_t size; + uint32_t msg_type; + uint32_t status; + uint32_t trans_id; + uint32_t flags; +}; + +/* Execute */ +/* + * FIRMWARE + * keep lastNetworkIDRan = uint32 + * keep wasLastNetworkChunky = BOOLEAN + */ +/* + * ACO Buffer definition + */ +struct npu_aco_buffer { + /* + * used to track if previous network is the same and already loaded, + * we can save a dma + */ + uint32_t network_id; + /* + * size of header + first chunk ACO buffer - + * this saves a dma by dmaing both header and first chunk + */ + uint32_t buf_size; + /* + * SMMU 32-bit mapped address that the DMA engine can read - + * uses lower 32 bits + */ + uint64_t address; +}; + +/* + * ACO Buffer V2 definition + */ +struct npu_aco_buffer_v2 { + /* + * used to track if previous network is the same and already loaded, + * we can save a dma + */ + uint32_t network_id; + /* + * size of header + first chunk ACO buffer - + * this saves a dma by dmaing both header and first chunk + */ + uint32_t buf_size; + /* + * SMMU 32-bit mapped address that the DMA engine can read - + * uses lower 32 bits + */ + uint32_t address; + /* + * number of layers in the network + */ + uint32_t num_layers; +}; + +/* + * ACO Patch Parameters + */ +struct npu_patch_tuple { + uint32_t value; + uint32_t chunk_id; + uint16_t instruction_size_in_bytes; + uint16_t variable_size_in_bits; + uint16_t shift_value_in_bits; + uint32_t loc_offset; +}; + +/* + * ACO Patch Tuple V2 + */ +struct npu_patch_tuple_v2 { + uint32_t value; + uint32_t chunk_id; + uint32_t instruction_size_in_bytes; + uint32_t variable_size_in_bits; + uint32_t shift_value_in_bits; + uint32_t loc_offset; +}; + +struct npu_patch_params { + uint32_t num_params; + struct npu_patch_tuple param[2]; +}; + +/* + * LOAD command packet definition + */ +struct ipc_cmd_load_pkt { + struct ipc_cmd_header_pkt header; + struct npu_aco_buffer buf_pkt; +}; + +/* + * LOAD command packet V2 definition + */ +struct ipc_cmd_load_pkt_v2 { + struct ipc_cmd_header_pkt header; + struct npu_aco_buffer_v2 buf_pkt; + uint32_t num_patch_params; + struct npu_patch_tuple_v2 patch_params[]; +}; + +/* + * UNLOAD command packet definition + */ +struct ipc_cmd_unload_pkt { + struct ipc_cmd_header_pkt header; + uint32_t network_hdl; +}; + +/* + * Execute packet definition + */ +struct ipc_cmd_execute_pkt { + struct ipc_cmd_header_pkt header; + struct npu_patch_params patch_params; + uint32_t network_hdl; +}; + +struct npu_patch_params_v2 { + uint32_t value; + uint32_t id; +}; + +/* + * Execute packet V2 definition + */ +struct ipc_cmd_execute_pkt_v2 { + struct ipc_cmd_header_pkt header; + uint32_t network_hdl; + uint32_t num_patch_params; + struct npu_patch_params_v2 patch_params[]; +}; + +/* + * Loopback packet definition + */ +struct ipc_cmd_loopback_pkt { + struct ipc_cmd_header_pkt header; + uint32_t loopbackParams; +}; + +/* + * Generic property definition + */ +struct ipc_cmd_prop_pkt { + struct ipc_cmd_header_pkt header; + uint32_t prop_id; + uint32_t num_params; + uint32_t network_hdl; + uint32_t prop_param[]; +}; + +/* + * Generic property response packet definition + */ +struct ipc_msg_prop_pkt { + struct ipc_msg_header_pkt header; + uint32_t prop_id; + uint32_t num_params; + uint32_t network_hdl; + uint32_t prop_param[]; +}; + +/* + * Generic notify message packet definition + */ +struct ipc_msg_general_notify_pkt { + struct ipc_msg_header_pkt header; + uint32_t notify_id; + uint32_t num_params; + uint32_t network_hdl; + uint32_t notify_param[]; +}; + + +/* + * LOAD response packet definition + */ +struct ipc_msg_load_pkt { + struct ipc_msg_header_pkt header; + uint32_t network_hdl; +}; + +/* + * UNLOAD response packet definition + */ +struct ipc_msg_unload_pkt { + struct ipc_msg_header_pkt header; + uint32_t network_hdl; +}; + +/* + * Layer Stats information returned back during EXECUTE_DONE response + */ +struct ipc_layer_stats { + /* + * hardware tick count per layer + */ + uint32_t tick_count; +}; + +struct ipc_execute_layer_stats { + /* + * total number of layers associated with the execution + */ + uint32_t total_num_layers; + /* + * pointer to each layer stats + */ + struct ipc_layer_stats + layer_stats_list[NUM_LAYER_STATS_PER_EXE_MSG_MAX]; +}; + +struct ipc_execute_stats { + /* + * total e2e IPC tick count during EXECUTE cmd + */ + uint32_t e2e_ipc_tick_count; + /* + * tick count on ACO loading + */ + uint32_t aco_load_tick_count; + /* + * tick count on ACO execution + */ + uint32_t aco_execution_tick_count; + /* + * individual layer stats + */ + struct ipc_execute_layer_stats exe_stats; +}; + +/* + * EXECUTE response packet definition + */ +struct ipc_msg_execute_pkt { + struct ipc_msg_header_pkt header; + struct ipc_execute_stats stats; + uint32_t network_hdl; +}; + +/* + * EXECUTE V2 response packet definition + */ +struct ipc_msg_execute_pkt_v2 { + struct ipc_msg_header_pkt header; + uint32_t network_hdl; + uint32_t stats_data[]; +}; + +/* + * LOOPBACK response packet definition + */ +struct ipc_msg_loopback_pkt { + struct ipc_msg_header_pkt header; + uint32_t loopbackParams; +}; + +/* Logging Related */ + +/* + * ipc_log_state_t - Logging state + */ +struct ipc_log_state { + uint32_t module_msk; + uint32_t level_msk; +}; + +struct ipc_cmd_log_state_pkt { + struct ipc_cmd_header_pkt header; + struct ipc_log_state log_state; +}; + +struct ipc_msg_log_state_pkt { + struct ipc_msg_header_pkt header; + struct ipc_log_state log_state; +}; + +/* + * Logging message + * This is a message from the NPU that contains the + * logging message. The values of part1-4 are not exposed + * the receiver has to refer to the logging implementation to + * intrepret what these mean and how to parse + */ +struct ipc_msg_log_pkt { + struct ipc_msg_header_pkt header; + uint32_t log_msg[NPU_LOG_MSG_MAX_SIZE]; +}; + +/* Performance Related */ + +/* + * Set counter mask of which counters we want + * This is a message from HOST->NPU Firmware + */ +struct ipc_cmd_set_performance_query { + struct ipc_cmd_header_pkt header; + uint32_t cnt_msk; +}; + +/* + * Set counter mask of which counters we want + * This is a message from HOST->NPU Firmware + */ +struct ipc_msg_performance_counters { + struct ipc_cmd_header_pkt header; + uint32_t layer_id; + uint32_t num_tulpes; + /* Array of tuples [HEADER,value] */ + uint32_t cnt_tulpes[]; +}; + +/* + * ipc_cmd_shutdown - Shutdown command + */ +struct ipc_cmd_shutdown_pkt { + struct ipc_cmd_header_pkt header; + uint32_t shutdown_flags; +}; + +#endif /* NPU_HOST_IPC_H */ diff --git a/drivers/media/platform/msm/npu/npu_hw.h b/drivers/media/platform/msm/npu/npu_hw.h new file mode 100644 index 000000000000..3ca79896b492 --- /dev/null +++ b/drivers/media/platform/msm/npu/npu_hw.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2018-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2023-2024, Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#ifndef NPU_HW_H +#define NPU_HW_H + +/* ------------------------------------------------------------------------- + * Defines + * ------------------------------------------------------------------------- + */ +#define NPU_HW_VERSION (0x00000000) +#define NPU_MASTERn_IPC_IRQ_OUT(n) (0x00001004+0x1000*(n)) +#define NPU_CACHE_ATTR_IDn___POR 0x00011100 +#define NPU_CACHE_ATTR_IDn(n) (0x00000800+0x4*(n)) +#define NPU_MASTERn_IPC_IRQ_IN_CTRL(n) (0x00001008+0x1000*(n)) +#define NPU_MASTER0_IPC_IRQ_IN_CTRL__IRQ_SOURCE_SELECT___S 4 +#define NPU_MASTERn_IPC_IRQ_OUT_CTRL(n) (0x00001004+0x1000*(n)) +#define NPU_MASTER0_IPC_IRQ_OUT_CTRL__IRQ_TYPE_PULSE 4 +#define NPU_GPR0 (0x00000100) +#define NPU_MASTERn_ERROR_IRQ_STATUS(n) (0x00001010+0x1000*(n)) +#define NPU_MASTERn_ERROR_IRQ_INCLUDE(n) (0x00001014+0x1000*(n)) +#define NPU_MASTERn_ERROR_IRQ_ENABLE(n) (0x00001018+0x1000*(n)) +#define NPU_MASTERn_ERROR_IRQ_CLEAR(n) (0x0000101C+0x1000*(n)) +#define NPU_MASTERn_ERROR_IRQ_SET(n) (0x00001020+0x1000*(n)) +#define NPU_MASTERn_ERROR_IRQ_OWNER(n) (0x00007000+4*(n)) +#define NPU_ERROR_IRQ_MASK 0x000000E3 +#define NPU_MASTERn_WDOG_IRQ_STATUS(n) (0x00001030+0x1000*(n)) +#define NPU_WDOG_BITE_IRQ_STATUS (1 << 1) +#define NPU_MASTERn_WDOG_IRQ_INCLUDE(n) (0x00001034+0x1000*(n)) +#define NPU_WDOG_BITE_IRQ_INCLUDE (1 << 1) +#define NPU_MASTERn_WDOG_IRQ_OWNER(n) (0x00007010+4*(n)) +#define NPU_WDOG_IRQ_MASK 0x00000002 + + +#define NPU_GPR1 (0x00000104) +#define NPU_GPR2 (0x00000108) +#define NPU_GPR3 (0x0000010C) +#define NPU_GPR4 (0x00000110) +#define NPU_GPR13 (0x00000134) +#define NPU_GPR14 (0x00000138) +#define NPU_GPR15 (0x0000013C) + +#define BWMON2_SAMPLING_WINDOW (0x000003A8) +#define BWMON2_BYTE_COUNT_THRESHOLD_HIGH (0x000003AC) +#define BWMON2_BYTE_COUNT_THRESHOLD_MEDIUM (0x000003B0) +#define BWMON2_BYTE_COUNT_THRESHOLD_LOW (0x000003B4) +#define BWMON2_ZONE_ACTIONS (0x000003B8) +#define BWMON2_ZONE_COUNT_THRESHOLD (0x000003BC) + +#endif /* NPU_HW_H */ diff --git a/drivers/media/platform/msm/npu/npu_hw_access.c b/drivers/media/platform/msm/npu/npu_hw_access.c new file mode 100644 index 000000000000..c851cfd01ed4 --- /dev/null +++ b/drivers/media/platform/msm/npu/npu_hw_access.c @@ -0,0 +1,485 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2018-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2023-2024, Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +/* ------------------------------------------------------------------------- + * Includes + * ------------------------------------------------------------------------- + */ +#include +#include +#include +#include +#include +#include +#include + +#include "npu_hw_access.h" +#include "npu_common.h" +#include "npu_hw.h" + +/* ------------------------------------------------------------------------- + * Functions - Register + * ------------------------------------------------------------------------- + */ +static uint32_t npu_reg_read(void __iomem *base, size_t size, uint32_t off) +{ + if (!base) { + pr_err("NULL base address\n"); + return 0; + } + + if ((off % 4) != 0) { + pr_err("offset %x is not aligned\n", off); + return 0; + } + + if (off >= size) { + pr_err("offset exceeds io region %x:%x\n", off, size); + return 0; + } + + return readl_relaxed(base + off); +} + +static void npu_reg_write(void __iomem *base, size_t size, uint32_t off, + uint32_t val) +{ + if (!base) { + pr_err("NULL base address\n"); + return; + } + + if ((off % 4) != 0) { + pr_err("offset %x is not aligned\n", off); + return; + } + + if (off >= size) { + pr_err("offset exceeds io region %x:%x\n", off, size); + return; + } + + writel_relaxed(val, base + off); + __iowmb(); +} + +uint32_t npu_core_reg_read(struct npu_device *npu_dev, uint32_t off) +{ + return npu_reg_read(npu_dev->core_io.base, npu_dev->core_io.size, off); +} + +void npu_core_reg_write(struct npu_device *npu_dev, uint32_t off, uint32_t val) +{ + npu_reg_write(npu_dev->core_io.base, npu_dev->core_io.size, + off, val); +} + +uint32_t npu_bwmon_reg_read(struct npu_device *npu_dev, uint32_t off) +{ + return npu_reg_read(npu_dev->bwmon_io.base, npu_dev->bwmon_io.size, + off); +} + +void npu_bwmon_reg_write(struct npu_device *npu_dev, uint32_t off, + uint32_t val) +{ + npu_reg_write(npu_dev->bwmon_io.base, npu_dev->bwmon_io.size, + off, val); +} + +uint32_t npu_qfprom_reg_read(struct npu_device *npu_dev, uint32_t off) +{ + return npu_reg_read(npu_dev->qfprom_io.base, + npu_dev->qfprom_io.size, off); +} + +/* ------------------------------------------------------------------------- + * Functions - Memory + * ------------------------------------------------------------------------- + */ +void npu_mem_write(struct npu_device *npu_dev, void *dst, void *src, + uint32_t size) +{ + size_t dst_off = (size_t)dst; + uint32_t *src_ptr32 = (uint32_t *)src; + uint8_t *src_ptr8 = NULL; + uint32_t i = 0; + uint32_t num = 0; + + if (dst_off >= npu_dev->tcm_io.size || + (npu_dev->tcm_io.size - dst_off) < size) { + pr_err("memory write exceeds io region %x:%x:%x\n", + dst_off, size, npu_dev->tcm_io.size); + return; + } + + num = size/4; + for (i = 0; i < num; i++) { + writel_relaxed(src_ptr32[i], npu_dev->tcm_io.base + dst_off); + dst_off += 4; + } + + if (size%4 != 0) { + src_ptr8 = (uint8_t *)((size_t)src + (num*4)); + num = size%4; + for (i = 0; i < num; i++) { + writeb_relaxed(src_ptr8[i], npu_dev->tcm_io.base + + dst_off); + dst_off += 1; + } + } + + __iowmb(); +} + +int32_t npu_mem_read(struct npu_device *npu_dev, void *src, void *dst, + uint32_t size) +{ + size_t src_off = (size_t)src; + uint32_t *out32 = (uint32_t *)dst; + uint8_t *out8 = NULL; + uint32_t i = 0; + uint32_t num = 0; + + if (src_off >= npu_dev->tcm_io.size || + (npu_dev->tcm_io.size - src_off) < size) { + pr_err("memory read exceeds io region %x:%x:%x\n", + src_off, size, npu_dev->tcm_io.size); + return 0; + } + + num = size/4; + for (i = 0; i < num; i++) { + out32[i] = readl_relaxed(npu_dev->tcm_io.base + src_off); + src_off += 4; + } + + if (size%4 != 0) { + out8 = (uint8_t *)((size_t)dst + (num*4)); + num = size%4; + for (i = 0; i < num; i++) { + out8[i] = readb_relaxed(npu_dev->tcm_io.base + src_off); + src_off += 1; + } + } + return 0; +} + +void *npu_ipc_addr(void) +{ + return (void *)(IPC_MEM_OFFSET_FROM_SSTCM); +} + +/* ------------------------------------------------------------------------- + * Functions - Interrupt + * ------------------------------------------------------------------------- + */ +void npu_interrupt_ack(struct npu_device *npu_dev, uint32_t intr_num) +{ + struct npu_host_ctx *host_ctx = &npu_dev->host_ctx; + uint32_t wdg_irq_sts = 0, error_irq_sts = 0; + + /* Clear irq state */ + REGW(npu_dev, NPU_MASTERn_IPC_IRQ_OUT(0), 0x0); + + wdg_irq_sts = REGR(npu_dev, NPU_MASTERn_WDOG_IRQ_STATUS(0)); + if (wdg_irq_sts != 0) { + pr_err("wdg irq %x\n", wdg_irq_sts); + host_ctx->wdg_irq_sts |= wdg_irq_sts; + host_ctx->fw_error = true; + } + + error_irq_sts = REGR(npu_dev, NPU_MASTERn_ERROR_IRQ_STATUS(0)); + error_irq_sts &= REGR(npu_dev, NPU_MASTERn_ERROR_IRQ_ENABLE(0)); + if (error_irq_sts != 0) { + REGW(npu_dev, NPU_MASTERn_ERROR_IRQ_CLEAR(0), error_irq_sts); + pr_err("error irq %x\n", error_irq_sts); + host_ctx->err_irq_sts |= error_irq_sts; + host_ctx->fw_error = true; + } +} + +int32_t npu_interrupt_raise_m0(struct npu_device *npu_dev) +{ + /* Bit 4 is setting IRQ_SOURCE_SELECT to local + * and we're triggering a pulse to NPU_MASTER0_IPC_IN_IRQ0 + */ + npu_core_reg_write(npu_dev, NPU_MASTERn_IPC_IRQ_IN_CTRL(0), 0x1 + << NPU_MASTER0_IPC_IRQ_IN_CTRL__IRQ_SOURCE_SELECT___S | 0x1); + + return 0; +} + +int32_t npu_interrupt_raise_dsp(struct npu_device *npu_dev) +{ + npu_core_reg_write(npu_dev, NPU_MASTERn_IPC_IRQ_OUT_CTRL(1), 0x8); + + return 0; +} + +/* ------------------------------------------------------------------------- + * Functions - ION Memory + * ------------------------------------------------------------------------- + */ +static struct npu_ion_buf *npu_alloc_npu_ion_buffer(struct npu_client + *client, int buf_hdl, uint32_t size) +{ + struct npu_ion_buf *ret_val = NULL, *tmp; + struct list_head *pos = NULL; + + mutex_lock(&client->list_lock); + list_for_each(pos, &(client->mapped_buffer_list)) { + tmp = list_entry(pos, struct npu_ion_buf, list); + if (tmp->fd == buf_hdl) { + ret_val = tmp; + break; + } + } + + if (ret_val) { + /* mapped already, treat as invalid request */ + pr_err("ion buf has been mapped\n"); + ret_val = NULL; + } else { + ret_val = kzalloc(sizeof(*ret_val), GFP_KERNEL); + if (ret_val) { + ret_val->fd = buf_hdl; + ret_val->size = size; + ret_val->iova = 0; + list_add(&(ret_val->list), + &(client->mapped_buffer_list)); + } + } + mutex_unlock(&client->list_lock); + + return ret_val; +} + +static struct npu_ion_buf *npu_get_npu_ion_buffer(struct npu_client + *client, int buf_hdl) +{ + struct list_head *pos = NULL; + struct npu_ion_buf *ret_val = NULL, *tmp; + + mutex_lock(&client->list_lock); + list_for_each(pos, &(client->mapped_buffer_list)) { + tmp = list_entry(pos, struct npu_ion_buf, list); + if (tmp->fd == buf_hdl) { + ret_val = tmp; + break; + } + } + mutex_unlock(&client->list_lock); + + return ret_val; +} + +static void npu_free_npu_ion_buffer(struct npu_client + *client, int buf_hdl) +{ + struct list_head *pos = NULL; + struct npu_ion_buf *npu_ion_buf = NULL; + + mutex_lock(&client->list_lock); + list_for_each(pos, &(client->mapped_buffer_list)) { + npu_ion_buf = list_entry(pos, struct npu_ion_buf, list); + if (npu_ion_buf->fd == buf_hdl) { + list_del(&npu_ion_buf->list); + kfree(npu_ion_buf); + break; + } + } + mutex_unlock(&client->list_lock); +} + +int npu_mem_map(struct npu_client *client, int buf_hdl, uint32_t size, + uint64_t *addr) +{ + MODULE_IMPORT_NS(DMA_BUF); + int ret = 0; + struct npu_device *npu_dev = client->npu_dev; + struct npu_ion_buf *ion_buf = NULL; + struct npu_smmu_ctx *smmu_ctx = &npu_dev->smmu_ctx; + + if (buf_hdl == 0) + return -EINVAL; + + ion_buf = npu_alloc_npu_ion_buffer(client, buf_hdl, size); + if (!ion_buf) { + pr_err("%s fail to alloc npu_ion_buffer\n", __func__); + ret = -ENOMEM; + return ret; + } + + smmu_ctx->attach_cnt++; + + ion_buf->dma_buf = dma_buf_get(ion_buf->fd); + if (IS_ERR_OR_NULL(ion_buf->dma_buf)) { + pr_err("dma_buf_get failed %d\n", ion_buf->fd); + ret = -ENOMEM; + ion_buf->dma_buf = NULL; + goto map_end; + } + + ion_buf->attachment = dma_buf_attach(ion_buf->dma_buf, + &(npu_dev->pdev->dev)); + if (IS_ERR(ion_buf->attachment)) { + ret = -ENOMEM; + ion_buf->attachment = NULL; + goto map_end; + } + + ion_buf->attachment->dma_map_attrs = DMA_ATTR_IOMMU_USE_UPSTREAM_HINT; + + ion_buf->table = dma_buf_map_attachment(ion_buf->attachment, + DMA_BIDIRECTIONAL); + if (IS_ERR(ion_buf->table)) { + pr_err("npu dma_buf_map_attachment failed\n"); + ret = -ENOMEM; + ion_buf->table = NULL; + goto map_end; + } + + ion_buf->iova = ion_buf->table->sgl->dma_address; + ion_buf->size = ion_buf->dma_buf->size; + *addr = ion_buf->iova; + pr_debug("mapped mem addr:0x%llx size:0x%x\n", ion_buf->iova, + ion_buf->size); +map_end: + if (ret) + npu_mem_unmap(client, buf_hdl, 0); + + return ret; +} + +void npu_mem_invalidate(struct npu_client *client, int buf_hdl) +{ + struct npu_device *npu_dev = client->npu_dev; + struct npu_ion_buf *ion_buf = npu_get_npu_ion_buffer(client, + buf_hdl); + + if (!ion_buf) + pr_err("%s can't find ion buf\n", __func__); + else + dma_sync_sg_for_cpu(&(npu_dev->pdev->dev), ion_buf->table->sgl, + ion_buf->table->nents, DMA_BIDIRECTIONAL); +} + +bool npu_mem_verify_addr(struct npu_client *client, uint64_t addr) +{ + struct npu_ion_buf *ion_buf = NULL; + struct list_head *pos = NULL; + bool valid = false; + + mutex_lock(&client->list_lock); + list_for_each(pos, &(client->mapped_buffer_list)) { + ion_buf = list_entry(pos, struct npu_ion_buf, list); + if (ion_buf->iova == addr) { + valid = true; + break; + } + } + mutex_unlock(&client->list_lock); + + return valid; +} + +void npu_mem_unmap(struct npu_client *client, int buf_hdl, uint64_t addr) +{ + MODULE_IMPORT_NS(DMA_BUF); + struct npu_device *npu_dev = client->npu_dev; + struct npu_ion_buf *ion_buf = NULL; + + /* clear entry and retrieve the corresponding buffer */ + ion_buf = npu_get_npu_ion_buffer(client, buf_hdl); + if (!ion_buf) { + pr_err("%s could not find buffer\n", __func__); + return; + } + + if (ion_buf->iova != addr) + pr_warn("unmap address %llu doesn't match %llu\n", addr, + ion_buf->iova); + + if (ion_buf->table) + dma_buf_unmap_attachment(ion_buf->attachment, ion_buf->table, + DMA_BIDIRECTIONAL); + if (ion_buf->dma_buf && ion_buf->attachment) + dma_buf_detach(ion_buf->dma_buf, ion_buf->attachment); + if (ion_buf->dma_buf) + dma_buf_put(ion_buf->dma_buf); + npu_dev->smmu_ctx.attach_cnt--; + + pr_debug("unmapped mem addr:0x%llx size:0x%x\n", ion_buf->iova, + ion_buf->size); + npu_free_npu_ion_buffer(client, buf_hdl); +} + +/* ------------------------------------------------------------------------- + * Functions - Features + * ------------------------------------------------------------------------- + */ +uint8_t npu_hw_clk_gating_enabled(void) +{ + return 1; +} + +uint8_t npu_hw_log_enabled(void) +{ + return 1; +} + +/* ------------------------------------------------------------------------- + * Functions - Subsystem/PIL + * ------------------------------------------------------------------------- + */ +#define NPU_PAS_ID (23) + +int npu_subsystem_get(struct npu_device *npu_dev, const char *fw_name) +{ + struct device *dev = npu_dev->device; + const struct firmware *firmware_p; + ssize_t fw_size; + /* load firmware */ + int ret = request_firmware(&firmware_p, fw_name, dev); + + if (ret < 0) { + pr_err("request_firmware %s failed: %d\n", fw_name, ret); + return ret; + } + fw_size = qcom_mdt_get_size(firmware_p); + if (fw_size < 0 || fw_size > npu_dev->fw_io.mem_size) { + pr_err("npu fw size invalid, %lld\n", fw_size); + return -EINVAL; + } + /* load the ELF segments to memory */ + ret = qcom_mdt_load(dev, firmware_p, fw_name, NPU_PAS_ID, + npu_dev->fw_io.mem_region, npu_dev->fw_io.mem_phys, + npu_dev->fw_io.mem_size, &npu_dev->fw_io.mem_reloc); + release_firmware(firmware_p); + if (ret) { + pr_err("qcom_mdt_load failure, %d\n", ret); + return ret; + } + ret = qcom_scm_pas_auth_and_reset(NPU_PAS_ID); + if (ret) { + pr_err("failed to authenticate image and release reset\n"); + return -2; + } + pr_debug("done pas auth\n"); + return 0; +} + +void npu_subsystem_put(struct npu_device *npu_dev) +{ + int ret = qcom_scm_pas_shutdown(NPU_PAS_ID); + + if (ret) + pr_err("failed to shutdown: %d\n", ret); + +} diff --git a/drivers/media/platform/msm/npu/npu_hw_access.h b/drivers/media/platform/msm/npu/npu_hw_access.h new file mode 100644 index 000000000000..5e5ac67ffc75 --- /dev/null +++ b/drivers/media/platform/msm/npu/npu_hw_access.h @@ -0,0 +1,87 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2018-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2023-2024, Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#ifndef _NPU_HW_ACCESS_H +#define _NPU_HW_ACCESS_H + +/* ------------------------------------------------------------------------- + * Includes + * ------------------------------------------------------------------------- + */ +#include "npu_common.h" + +/* ------------------------------------------------------------------------- + * Defines + * ------------------------------------------------------------------------- + */ +#define IPC_MEM_OFFSET_FROM_SSTCM 0x00010000 +#define SYS_CACHE_SCID 23 + +#define QFPROM_FMAX_REG_OFFSET 0x000001C8 +#define QFPROM_FMAX_BITS_MASK 0x0000000C +#define QFPROM_FMAX_BITS_SHIFT 2 + +#define REGW(npu_dev, off, val) npu_core_reg_write(npu_dev, off, val) +#define REGR(npu_dev, off) npu_core_reg_read(npu_dev, off) +#define MEMW(npu_dev, dst, src, size) npu_mem_write(npu_dev, (void *)(dst),\ + (void *)(src), size) +#define MEMR(npu_dev, src, dst, size) npu_mem_read(npu_dev, (void *)(src),\ + (void *)(dst), size) +#define IPC_ADDR npu_ipc_addr() +#define INTERRUPT_ACK(npu_dev, num) npu_interrupt_ack(npu_dev, num) +#define INTERRUPT_RAISE_NPU(npu_dev) npu_interrupt_raise_m0(npu_dev) +#define INTERRUPT_RAISE_DSP(npu_dev) npu_interrupt_raise_dsp(npu_dev) + +/* ------------------------------------------------------------------------- + * Data Structures + * ------------------------------------------------------------------------- + */ +struct npu_device; +struct npu_ion_buf_t; +struct npu_host_ctx; +struct npu_client; +typedef irqreturn_t (*intr_hdlr_fn)(int32_t irq, void *ptr); +typedef void (*wq_hdlr_fn) (struct work_struct *work); + +/* ------------------------------------------------------------------------- + * Function Prototypes + * ------------------------------------------------------------------------- + */ +uint32_t npu_core_reg_read(struct npu_device *npu_dev, uint32_t off); +void npu_core_reg_write(struct npu_device *npu_dev, uint32_t off, uint32_t val); +uint32_t npu_bwmon_reg_read(struct npu_device *npu_dev, uint32_t off); +void npu_bwmon_reg_write(struct npu_device *npu_dev, uint32_t off, + uint32_t val); +void npu_mem_write(struct npu_device *npu_dev, void *dst, void *src, + uint32_t size); +int32_t npu_mem_read(struct npu_device *npu_dev, void *src, void *dst, + uint32_t size); +uint32_t npu_qfprom_reg_read(struct npu_device *npu_dev, uint32_t off); + +int npu_mem_map(struct npu_client *client, int buf_hdl, uint32_t size, + uint64_t *addr); +void npu_mem_unmap(struct npu_client *client, int buf_hdl, uint64_t addr); +void npu_mem_invalidate(struct npu_client *client, int buf_hdl); +bool npu_mem_verify_addr(struct npu_client *client, uint64_t addr); + +void *npu_ipc_addr(void); +void npu_interrupt_ack(struct npu_device *npu_dev, uint32_t intr_num); +int32_t npu_interrupt_raise_m0(struct npu_device *npu_dev); +int32_t npu_interrupt_raise_dsp(struct npu_device *npu_dev); + +uint8_t npu_hw_clk_gating_enabled(void); +uint8_t npu_hw_log_enabled(void); + +int npu_enable_irq(struct npu_device *npu_dev); +void npu_disable_irq(struct npu_device *npu_dev); + +int npu_enable_sys_cache(struct npu_device *npu_dev); +void npu_disable_sys_cache(struct npu_device *npu_dev); + +int npu_subsystem_get(struct npu_device *npu_dev, const char *fw_name); +void npu_subsystem_put(struct npu_device *npu_dev); + +#endif /* _NPU_HW_ACCESS_H*/ diff --git a/drivers/media/platform/msm/npu/npu_mgr.c b/drivers/media/platform/msm/npu/npu_mgr.c new file mode 100644 index 000000000000..1aea3a971ffa --- /dev/null +++ b/drivers/media/platform/msm/npu/npu_mgr.c @@ -0,0 +1,2112 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2018-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2023-2024, Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +/* ------------------------------------------------------------------------- + * Includes + * ------------------------------------------------------------------------- + */ +#include "npu_hw_access.h" +#include "npu_mgr.h" +#include "npu_firmware.h" +#include "npu_hw.h" +#include "npu_host_ipc.h" +#include "npu_common.h" + +/* ------------------------------------------------------------------------- + * Defines + * ------------------------------------------------------------------------- + */ +#define NPU_FW_TIMEOUT_POLL_INTERVAL_MS 10 +#define NPU_FW_TIMEOUT_MS 1000 + +/* ------------------------------------------------------------------------- + * File Scope Function Prototypes + * ------------------------------------------------------------------------- + */ +static void host_irq_wq(struct work_struct *work); +static void fw_deinit_wq(struct work_struct *work); +static void turn_off_fw_logging(struct npu_device *npu_dev); +static int wait_for_status_ready(struct npu_device *npu_dev, + uint32_t status_reg, uint32_t status_bits, bool poll); +static struct npu_network *alloc_network(struct npu_host_ctx *ctx, + struct npu_client *client); +static struct npu_network *get_network_by_hdl(struct npu_host_ctx *ctx, + struct npu_client *client, uint32_t hdl); +static struct npu_network *get_network_by_id(struct npu_host_ctx *ctx, + struct npu_client *client, int64_t id); +static void free_network(struct npu_host_ctx *ctx, struct npu_client *client, + int64_t id); +static int network_get(struct npu_network *network); +static int network_put(struct npu_network *network); +static void app_msg_proc(struct npu_host_ctx *host_ctx, uint32_t *msg); +static void host_session_msg_hdlr(struct npu_device *npu_dev); +static int host_error_hdlr(struct npu_device *npu_dev, bool force); +static int npu_send_network_cmd(struct npu_device *npu_dev, + struct npu_network *network, void *cmd_ptr); +static int npu_send_misc_cmd(struct npu_device *npu_dev, uint32_t q_idx, + void *cmd_ptr); +static int npu_notify_dsp(struct npu_device *npu_dev, bool pwr_up); +static int npu_notify_aop(struct npu_device *npu_dev, bool on); +static int update_dcvs_activity(struct npu_device *npu_dev, uint32_t activity); +static void npu_destroy_wq(struct npu_host_ctx *host_ctx); +static struct workqueue_struct *npu_create_wq(struct npu_host_ctx *host_ctx, + const char *name); + +/* ------------------------------------------------------------------------- + * Function Definitions - Init / Deinit + * ------------------------------------------------------------------------- + */ +int fw_init(struct npu_device *npu_dev) +{ + uint32_t reg_val; + struct npu_host_ctx *host_ctx = &npu_dev->host_ctx; + int ret = 0, retry_cnt = 3; + bool need_retry; + + mutex_lock(&host_ctx->lock); + if (host_ctx->fw_state == FW_ENABLED) { + host_ctx->fw_ref_cnt++; + pr_debug("fw_ref_cnt %d\n", host_ctx->fw_ref_cnt); + mutex_unlock(&host_ctx->lock); + return 0; + } + +retry: + need_retry = false; + npu_notify_aop(npu_dev, true); + + if (npu_enable_core_power(npu_dev)) { + ret = -EPERM; + goto enable_pw_fail; + } + + if (npu_enable_sys_cache(npu_dev)) { + ret = -EPERM; + goto enable_sys_cache_fail; + } + /* Boot the NPU subsystem */ + if (npu_subsystem_get(npu_dev, "npu.mdt")) { + pr_err("pil load npu fw failed\n"); + ret = -ENODEV; + goto subsystem_get_fail; + } + + /* Clear control/status registers */ + REGW(npu_dev, REG_NPU_FW_CTRL_STATUS, 0x0); + REGW(npu_dev, REG_NPU_HOST_CTRL_VALUE, 0x0); + REGW(npu_dev, REG_FW_TO_HOST_EVENT, 0x0); + pr_debug("fw_dbg_mode %x\n", host_ctx->fw_dbg_mode); + reg_val = 0; + if (host_ctx->fw_dbg_mode & FW_DBG_MODE_PAUSE) + reg_val |= HOST_CTRL_STATUS_FW_PAUSE_VAL; + + if (host_ctx->fw_dbg_mode & FW_DBG_DISABLE_WDOG) + reg_val |= HOST_CTRL_STATUS_DISABLE_WDOG_VAL; + + REGW(npu_dev, REG_NPU_HOST_CTRL_STATUS, reg_val); + /* Read back to flush all registers for fw to read */ + REGR(npu_dev, REG_NPU_HOST_CTRL_STATUS); + + /* Post PIL clocks */ + if (npu_enable_post_pil_clocks(npu_dev)) { + ret = -EPERM; + goto enable_post_clk_fail; + } + + /* + * Set logging state and clock gating state + * during FW bootup initialization + */ + reg_val = REGR(npu_dev, REG_NPU_HOST_CTRL_STATUS); + + /* Enable clock gating only if the HW access platform allows it */ + if (npu_hw_clk_gating_enabled()) + reg_val |= HOST_CTRL_STATUS_BOOT_ENABLE_CLK_GATE_VAL; + if (host_ctx->fw_dbg_mode & FW_DBG_ENABLE_LOGGING) { + //Enable logging + reg_val |= HOST_CTRL_STATUS_BOOT_ENABLE_LOGGING_VAL; + } + REGW(npu_dev, REG_NPU_HOST_CTRL_STATUS, reg_val); + + /* Initialize the host side IPC */ + ret = npu_host_ipc_pre_init(npu_dev); + if (ret) { + pr_err("npu_host_ipc_pre_init failed %d\n", ret); + goto enable_post_clk_fail; + } + + /* Keep reading ctrl status until NPU is ready */ + pr_debug("waiting for status ready from fw\n"); + + if (wait_for_status_ready(npu_dev, REG_NPU_FW_CTRL_STATUS, + FW_CTRL_STATUS_MAIN_THREAD_READY_VAL, true)) { + ret = -EPERM; + need_retry = true; + goto wait_fw_ready_fail; + } + + npu_host_ipc_post_init(npu_dev); + + if (npu_enable_irq(npu_dev)) { + ret = -EPERM; + goto wait_fw_ready_fail; + } + + npu_notify_dsp(npu_dev, true); + host_ctx->fw_state = FW_ENABLED; + host_ctx->fw_error = false; + host_ctx->fw_ref_cnt++; + reinit_completion(&host_ctx->fw_deinit_done); + + mutex_unlock(&host_ctx->lock); + pr_debug("firmware init complete\n"); + pr_debug("fw_ref_cnt %d\n", host_ctx->fw_ref_cnt); + + /* Set logging state */ + if (!npu_hw_log_enabled()) { + pr_debug("fw logging disabled\n"); + turn_off_fw_logging(npu_dev); + } + + return ret; + +wait_fw_ready_fail: + npu_disable_post_pil_clocks(npu_dev); +enable_post_clk_fail: + npu_subsystem_put(npu_dev); +subsystem_get_fail: + npu_disable_sys_cache(npu_dev); +enable_sys_cache_fail: + npu_disable_core_power(npu_dev); +enable_pw_fail: + npu_notify_aop(npu_dev, false); + host_ctx->fw_state = FW_DISABLED; + if (need_retry && (retry_cnt > 0)) { + retry_cnt--; + pr_warn("retry fw init %d\n", retry_cnt); + goto retry; + } + mutex_unlock(&host_ctx->lock); + return ret; +} + +void fw_deinit(struct npu_device *npu_dev, bool ssr, bool fw_alive) +{ + struct npu_host_ctx *host_ctx = &npu_dev->host_ctx; + struct ipc_cmd_shutdown_pkt cmd_shutdown_pkt; + struct npu_network *network = NULL; + int ret = 0, i; + + mutex_lock(&host_ctx->lock); + if (!ssr && (host_ctx->fw_ref_cnt > 0)) + host_ctx->fw_ref_cnt--; + + pr_debug("fw_ref_cnt %d\n", host_ctx->fw_ref_cnt); + + if (host_ctx->fw_state != FW_ENABLED) { + pr_err("fw is not enabled\n"); + mutex_unlock(&host_ctx->lock); + return; + } + + if ((host_ctx->fw_ref_cnt > 0) && !ssr) { + mutex_unlock(&host_ctx->lock); + return; + } + + npu_disable_irq(npu_dev); + + if (fw_alive) { + /* Command header */ + cmd_shutdown_pkt.header.cmd_type = NPU_IPC_CMD_SHUTDOWN; + cmd_shutdown_pkt.header.size = + sizeof(struct ipc_cmd_shutdown_pkt); + cmd_shutdown_pkt.header.trans_id = + atomic_add_return(1, &host_ctx->ipc_trans_id); + cmd_shutdown_pkt.header.flags = 0xF; + ret = npu_host_ipc_send_cmd(npu_dev, + IPC_QUEUE_CMD_HIGH_PRIORITY, &cmd_shutdown_pkt); + + pr_debug("NPU_IPC_CMD_SHUTDOWN sent status: %d\n", ret); + + if (ret) { + pr_err("npu_host_ipc_send_cmd failed\n"); + } else { + /* Keep reading ctrl status until NPU shuts down */ + pr_debug("waiting for shutdown status from fw\n"); + if (wait_for_status_ready(npu_dev, + REG_NPU_FW_CTRL_STATUS, + FW_CTRL_STATUS_SHUTDOWN_DONE_VAL, true)) { + pr_err("wait for fw shutdown timedout\n"); + ret = -ETIMEDOUT; + } + } + } + + npu_disable_post_pil_clocks(npu_dev); + npu_disable_sys_cache(npu_dev); + npu_subsystem_put(npu_dev); + host_ctx->fw_state = FW_DISABLED; + + /* + * if fw is still alive, notify dsp before power off + * otherwise delay 500 ms to make sure dsp has finished + * its own ssr handling. + */ + if (fw_alive) + npu_notify_dsp(npu_dev, false); + else + msleep(500); + + npu_disable_core_power(npu_dev); + + if (ssr) { + /* mark all existing network to error state */ + for (i = 0; i < MAX_LOADED_NETWORK; i++) { + network = &host_ctx->networks[i]; + if (network->is_valid) + network->fw_error = true; + } + } + + complete(&host_ctx->fw_deinit_done); + mutex_unlock(&host_ctx->lock); + pr_debug("firmware deinit complete\n"); + npu_notify_aop(npu_dev, false); +} + +int npu_host_init(struct npu_device *npu_dev) +{ + struct npu_host_ctx *host_ctx = &npu_dev->host_ctx; + + memset(host_ctx, 0, sizeof(*host_ctx)); + init_completion(&host_ctx->misc_done); + init_completion(&host_ctx->fw_deinit_done); + mutex_init(&host_ctx->lock); + atomic_set(&host_ctx->ipc_trans_id, 1); + host_ctx->npu_dev = npu_dev; + + host_ctx->wq = npu_create_wq(host_ctx, "npu_wq"); + if (!host_ctx->wq) + return -EPERM; + + host_ctx->prop_buf = kzalloc(sizeof(struct msm_npu_property), + GFP_KERNEL); + if (!host_ctx->prop_buf) + return -ENOMEM; + + host_ctx->misc_pending = false; + + return 0; +} + +void npu_host_deinit(struct npu_device *npu_dev) +{ + struct npu_host_ctx *host_ctx = &npu_dev->host_ctx; + + kfree(host_ctx->prop_buf); + npu_destroy_wq(host_ctx); + mutex_destroy(&host_ctx->lock); +} + +/* ------------------------------------------------------------------------- + * Function Definitions - Interrupt Handler + * ------------------------------------------------------------------------- + */ +irqreturn_t npu_intr_hdler(int irq, void *ptr) +{ + /* Check the interrupt we received */ + /* Currently this is the IPC interrupt */ + struct npu_device *npu_dev = (struct npu_device *)ptr; + struct npu_host_ctx *host_ctx = &npu_dev->host_ctx; + + INTERRUPT_ACK(npu_dev, irq); + + /* Check that the event thread currently is running */ + if (host_ctx->wq) + queue_work(host_ctx->wq, &host_ctx->irq_work); + + return IRQ_HANDLED; +} + +/* ------------------------------------------------------------------------- + * Function Definitions - Control + * ------------------------------------------------------------------------- + */ +static int host_error_hdlr(struct npu_device *npu_dev, bool force) +{ + struct npu_host_ctx *host_ctx = &npu_dev->host_ctx; + struct npu_network *network = NULL; + int i; + + if ((host_ctx->wdg_irq_sts == 0) && (host_ctx->err_irq_sts == 0) + && !force) + return 0; + + if (host_ctx->wdg_irq_sts) + pr_info("watchdog irq triggered\n"); + + fw_deinit(npu_dev, true, force); + host_ctx->wdg_irq_sts = 0; + host_ctx->err_irq_sts = 0; + + /* flush all pending npu cmds */ + mutex_lock(&host_ctx->lock); + for (i = 0; i < MAX_LOADED_NETWORK; i++) { + network = &host_ctx->networks[i]; + if (network->is_valid && network->cmd_pending && + network->fw_error) { + network->cmd_pending = false; + pr_debug("complete network %llx\n", + network->id); + complete(&network->cmd_done); + } + } + host_ctx->misc_pending = false; + complete_all(&host_ctx->misc_done); + mutex_unlock(&host_ctx->lock); + + return 1; +} + +static void host_irq_wq(struct work_struct *work) +{ + struct npu_host_ctx *host_ctx; + struct npu_device *npu_dev; + + host_ctx = container_of(work, struct npu_host_ctx, irq_work); + npu_dev = container_of(host_ctx, struct npu_device, host_ctx); + + if (host_error_hdlr(npu_dev, false)) + return; + + host_session_msg_hdlr(npu_dev); +} + +static void fw_deinit_wq(struct work_struct *work) +{ + struct npu_host_ctx *host_ctx; + struct npu_device *npu_dev; + + pr_debug("%s: deinit fw\n", __func__); + host_ctx = container_of(work, struct npu_host_ctx, fw_deinit_work.work); + npu_dev = container_of(host_ctx, struct npu_device, host_ctx); + + if (atomic_read(&host_ctx->fw_deinit_work_cnt) == 0) + return; + + do { + fw_deinit(npu_dev, false, true); + } while (!atomic_dec_and_test(&host_ctx->fw_deinit_work_cnt)); +} + +static void npu_destroy_wq(struct npu_host_ctx *host_ctx) +{ + flush_delayed_work(&host_ctx->fw_deinit_work); + destroy_workqueue(host_ctx->wq); +} + +static struct workqueue_struct *npu_create_wq(struct npu_host_ctx *host_ctx, + const char *name) +{ + struct workqueue_struct *wq = + alloc_workqueue(name, WQ_HIGHPRI | WQ_UNBOUND, 0); + + INIT_WORK(&host_ctx->irq_work, host_irq_wq); + INIT_DELAYED_WORK(&host_ctx->fw_deinit_work, fw_deinit_wq); + + return wq; +} + +static void turn_off_fw_logging(struct npu_device *npu_dev) +{ + struct ipc_cmd_log_state_pkt log_packet; + struct npu_host_ctx *host_ctx = &npu_dev->host_ctx; + int ret = 0; + + log_packet.header.cmd_type = NPU_IPC_CMD_CONFIG_LOG; + log_packet.header.size = sizeof(struct ipc_cmd_log_state_pkt); + log_packet.header.trans_id = + atomic_add_return(1, &host_ctx->ipc_trans_id); + log_packet.header.flags = 0xF; + log_packet.log_state.module_msk = 0; + log_packet.log_state.level_msk = 0; + ret = npu_send_misc_cmd(npu_dev, IPC_QUEUE_CMD_HIGH_PRIORITY, + &log_packet); + + pr_debug("NPU_IPC_CMD_CONFIG_LOG sent status: %d\n", ret); + + if (ret) + pr_err("npu_host_ipc_send_cmd failed\n"); +} + +static int wait_for_status_ready(struct npu_device *npu_dev, + uint32_t status_reg, uint32_t status_bits, bool poll) +{ + uint32_t ctrl_sts = 0; + uint32_t wait_cnt = 0, max_wait_ms; + struct npu_host_ctx *host_ctx = &npu_dev->host_ctx; + + max_wait_ms = (host_ctx->fw_dbg_mode & FW_DBG_MODE_INC_TIMEOUT) ? + NW_DEBUG_TIMEOUT_MS : NPU_FW_TIMEOUT_MS; + if (poll) + wait_cnt = max_wait_ms * 10; + else + wait_cnt = max_wait_ms / NPU_FW_TIMEOUT_POLL_INTERVAL_MS; + + /* keep reading status register until bits are set */ + do { + ctrl_sts = REGR(npu_dev, status_reg); + if ((ctrl_sts & status_bits) == status_bits) { + pr_debug("status %x[reg %x] ready received\n", + status_bits, status_reg); + break; + } + + if (!wait_cnt) { + pr_err("timeout wait for status %x[%x] in reg %x\n", + status_bits, ctrl_sts, status_reg); + return -ETIMEDOUT; + } + + if (poll) + udelay(100); + else + msleep(NPU_FW_TIMEOUT_POLL_INTERVAL_MS); + + wait_cnt--; + } while (1); + + return 0; + +} + +static int npu_notify_dsp(struct npu_device *npu_dev, bool pwr_up) +{ + uint32_t ack_val, notify_val; + int ret = 0; + + if (pwr_up) { + notify_val = HOST_DSP_CTRL_STATUS_PWR_UP_VAL; + ack_val = HOST_DSP_CTRL_STATUS_PWR_UP_ACK_VAL; + } else { + notify_val = HOST_DSP_CTRL_STATUS_PWR_DWN_VAL; + ack_val = HOST_DSP_CTRL_STATUS_PWR_DWN_ACK_VAL; + } + + REGW(npu_dev, REG_HOST_DSP_CTRL_STATUS, + notify_val); + /* Read back to flush register for dsp to read */ + REGR(npu_dev, REG_HOST_DSP_CTRL_STATUS); + + INTERRUPT_RAISE_DSP(npu_dev); + + ret = wait_for_status_ready(npu_dev, REG_HOST_DSP_CTRL_STATUS, + ack_val, true); + if (ret) + pr_warn("No response from dsp\n"); + + return ret; +} + +#define MAX_LEN 128 + +static int npu_notify_aop(struct npu_device *npu_dev, bool on) +{ + char buf[MAX_LEN]; + struct qmp_pkt pkt; + int buf_size, rc = 0; + + if (!npu_dev->mbox_aop.chan) { + pr_warn("aop mailbox channel is not available\n"); + return 0; + } + + buf_size = scnprintf(buf, MAX_LEN, "{class: bcm, res: npu_on, val: %d}", + on ? 1 : 0); + if (buf_size < 0) { + pr_err("prepare qmp notify buf failed\n"); + return -EINVAL; + } + + pr_debug("send msg %s to aop\n", buf); + memset(&pkt, 0, sizeof(pkt)); + pkt.size = (buf_size + 3) & ~0x3; + pkt.data = buf; + + rc = mbox_send_message(npu_dev->mbox_aop.chan, &pkt); + if (rc < 0) + pr_err("qmp message send failed, ret=%d\n", rc); + + return rc; +} + +/* ------------------------------------------------------------------------- + * Function Definitions - Network Management + * ------------------------------------------------------------------------- + */ +static int network_put(struct npu_network *network) +{ + if (!network) + return 0; + + return atomic_dec_return(&network->ref_cnt); +} + +static int network_get(struct npu_network *network) +{ + if (!network) + return 0; + + return atomic_inc_return(&network->ref_cnt); +} + +static struct npu_network *alloc_network(struct npu_host_ctx *ctx, + struct npu_client *client) +{ + int32_t i; + struct npu_network *network = ctx->networks; + + WARN_ON(!mutex_is_locked(&ctx->lock)); + + for (i = 0; i < MAX_LOADED_NETWORK; i++) { + if (network->id == 0) + break; + + network++; + } + + if (i == MAX_LOADED_NETWORK) { + pr_err("No free network\n"); + return NULL; + } + + memset(network, 0, sizeof(struct npu_network)); + network->id = i + 1; + init_completion(&network->cmd_done); + network->is_valid = true; + network->client = client; + network->stats_buf = kzalloc(NPU_MAX_STATS_BUF_SIZE, + GFP_KERNEL); + if (!network->stats_buf) { + memset(network, 0, sizeof(struct npu_network)); + return NULL; + } + + ctx->network_num++; + pr_debug("%s:Active network num %d\n", __func__, ctx->network_num); + + return network; +} + +static struct npu_network *get_network_by_hdl(struct npu_host_ctx *ctx, + struct npu_client *client, uint32_t hdl) +{ + int32_t i; + struct npu_network *network = ctx->networks; + + WARN_ON(!mutex_is_locked(&ctx->lock)); + + for (i = 0; i < MAX_LOADED_NETWORK; i++) { + if (network->network_hdl == hdl) + break; + + network++; + } + + if ((i == MAX_LOADED_NETWORK) || !network->is_valid) { + pr_err("network hdl invalid %d\n", hdl); + return NULL; + } + + if (client && (client != network->client)) { + pr_err("network %lld doesn't belong to this client\n", + network->id); + return NULL; + } + + network_get(network); + return network; +} + +static struct npu_network *get_network_by_id(struct npu_host_ctx *ctx, + struct npu_client *client, int64_t id) +{ + struct npu_network *network = NULL; + + WARN_ON(!mutex_is_locked(&ctx->lock)); + + if (id < 1 || id > MAX_LOADED_NETWORK || + !ctx->networks[id - 1].is_valid) { + pr_err("Invalid network id %d\n", (int32_t)id); + return NULL; + } + + network = &ctx->networks[id - 1]; + if (client && (client != network->client)) { + pr_err("network %lld doesn't belong to this client\n", id); + return NULL; + } + + network_get(network); + return network; +} + +static void free_network(struct npu_host_ctx *ctx, struct npu_client *client, + int64_t id) +{ + struct npu_network *network = NULL; + + WARN_ON(!mutex_is_locked(&ctx->lock)); + + network = get_network_by_id(ctx, client, id); + if (network) { + network_put(network); + if (atomic_read(&network->ref_cnt) == 0) { + kfree(network->stats_buf); + memset(network, 0, sizeof(struct npu_network)); + ctx->network_num--; + pr_debug("%s:Active network num %d\n", __func__, + ctx->network_num); + } else { + pr_warn("network %lld:%d is in use\n", network->id, + atomic_read(&network->ref_cnt)); + } + } +} + +/* ------------------------------------------------------------------------- + * Function Definitions - IPC + * ------------------------------------------------------------------------- + */ + +static void app_msg_proc(struct npu_host_ctx *host_ctx, uint32_t *msg) +{ + uint32_t msg_id; + struct npu_network *network = NULL; + struct npu_device *npu_dev = host_ctx->npu_dev; + + msg_id = msg[1]; + switch (msg_id) { + case NPU_IPC_MSG_EXECUTE_DONE: + { + struct ipc_msg_execute_pkt *exe_rsp_pkt = + (struct ipc_msg_execute_pkt *)msg; + + pr_debug("NPU_IPC_MSG_EXECUTE_DONE status: %d\n", + exe_rsp_pkt->header.status); + pr_debug("trans_id : %d\n", exe_rsp_pkt->header.trans_id); + pr_debug("e2e_IPC_time: %d (in tick count)\n", + exe_rsp_pkt->stats.e2e_ipc_tick_count); + pr_debug("aco_load_time: %d (in tick count)\n", + exe_rsp_pkt->stats.aco_load_tick_count); + pr_debug("aco_execute_time: %d (in tick count)\n", + exe_rsp_pkt->stats.aco_execution_tick_count); + pr_debug("total_num_layers: %d\n", + exe_rsp_pkt->stats.exe_stats.total_num_layers); + + network = get_network_by_hdl(host_ctx, NULL, + exe_rsp_pkt->network_hdl); + if (!network) { + pr_err("can't find network %x\n", + exe_rsp_pkt->network_hdl); + break; + } + + if (network->trans_id != exe_rsp_pkt->header.trans_id) { + pr_err("execute_pkt trans_id is not match %d:%d\n", + network->trans_id, + exe_rsp_pkt->header.trans_id); + network_put(network); + break; + } + + network->cmd_pending = false; + network->cmd_ret_status = exe_rsp_pkt->header.status; + + complete(&network->cmd_done); + network_put(network); + + break; + } + case NPU_IPC_MSG_EXECUTE_V2_DONE: + { + struct ipc_msg_execute_pkt_v2 *exe_rsp_pkt = + (struct ipc_msg_execute_pkt_v2 *)msg; + uint32_t stats_size = 0; + + pr_debug("NPU_IPC_MSG_EXECUTE_V2_DONE status: %d\n", + exe_rsp_pkt->header.status); + pr_debug("trans_id : %d\n", exe_rsp_pkt->header.trans_id); + + network = get_network_by_hdl(host_ctx, NULL, + exe_rsp_pkt->network_hdl); + if (!network) { + pr_err("can't find network %x\n", + exe_rsp_pkt->network_hdl); + break; + } + + if (network->trans_id != exe_rsp_pkt->header.trans_id) { + pr_err("execute_pkt_v2 trans_id is not match %d:%d\n", + network->trans_id, + exe_rsp_pkt->header.trans_id); + network_put(network); + break; + } + + pr_debug("network id : %llu\n", network->id); + if (exe_rsp_pkt->header.size < sizeof(*exe_rsp_pkt)) { + pr_err("invalid packet header size, header.size: %d\n", + exe_rsp_pkt->header.size); + network_put(network); + break; + } + stats_size = exe_rsp_pkt->header.size - sizeof(*exe_rsp_pkt); + pr_debug("stats_size %d:%d\n", exe_rsp_pkt->header.size, + stats_size); + stats_size = stats_size < network->stats_buf_size ? + stats_size : network->stats_buf_size; + if (stats_size) + memcpy(network->stats_buf, exe_rsp_pkt->stats_data, + stats_size); + + network->stats_buf_size = stats_size; + network->cmd_pending = false; + network->cmd_ret_status = exe_rsp_pkt->header.status; + complete(&network->cmd_done); + network_put(network); + break; + } + case NPU_IPC_MSG_LOAD_DONE: + { + uint32_t network_id = 0; + struct ipc_msg_load_pkt *load_rsp_pkt = + (struct ipc_msg_load_pkt *)msg; + + pr_debug("NPU_IPC_MSG_LOAD_DONE status: %d, trans_id: %d\n", + load_rsp_pkt->header.status, + load_rsp_pkt->header.trans_id); + + /* + * The upper 8 bits in flags is the current active + * network count in fw + */ + pr_debug("Current active network count in FW is %d\n", + load_rsp_pkt->header.flags >> 24); + + /* + * the upper 16 bits in returned network_hdl is + * the network ID + */ + pr_debug("network_hdl: %x\n", load_rsp_pkt->network_hdl); + network_id = load_rsp_pkt->network_hdl >> 16; + network = get_network_by_id(host_ctx, NULL, network_id); + if (!network) { + pr_err("can't find network %d\n", network_id); + break; + } + + if (network->trans_id != load_rsp_pkt->header.trans_id) { + pr_err("load_rsp_pkt trans_id is not match %d:%d\n", + network->trans_id, + load_rsp_pkt->header.trans_id); + network_put(network); + break; + } + + network->network_hdl = load_rsp_pkt->network_hdl; + network->cmd_pending = false; + network->cmd_ret_status = load_rsp_pkt->header.status; + + complete(&network->cmd_done); + network_put(network); + break; + } + case NPU_IPC_MSG_UNLOAD_DONE: + { + struct ipc_msg_unload_pkt *unload_rsp_pkt = + (struct ipc_msg_unload_pkt *)msg; + + pr_debug("NPU_IPC_MSG_UNLOAD_DONE status: %d, trans_id: %d\n", + unload_rsp_pkt->header.status, + unload_rsp_pkt->header.trans_id); + + /* + * The upper 8 bits in flags is the current active + * network count in fw + */ + pr_debug("Current active network count in FW is %d\n", + unload_rsp_pkt->header.flags >> 24); + + network = get_network_by_hdl(host_ctx, NULL, + unload_rsp_pkt->network_hdl); + if (!network) { + pr_err("can't find network %x\n", + unload_rsp_pkt->network_hdl); + break; + } + + if (network->trans_id != unload_rsp_pkt->header.trans_id) { + pr_err("unload_rsp_pkt trans_id is not match %d:%d\n", + network->trans_id, + unload_rsp_pkt->header.trans_id); + network_put(network); + break; + } + + network->cmd_pending = false; + network->cmd_ret_status = unload_rsp_pkt->header.status; + + complete(&network->cmd_done); + network_put(network); + break; + } + case NPU_IPC_MSG_LOOPBACK_DONE: + { + struct ipc_msg_loopback_pkt *lb_rsp_pkt = + (struct ipc_msg_loopback_pkt *)msg; + + pr_debug("NPU_IPC_MSG_LOOPBACK_DONE loopbackParams: 0x%x\n", + lb_rsp_pkt->loopbackParams); + host_ctx->misc_pending = false; + + complete_all(&host_ctx->misc_done); + break; + } + case NPU_IPC_MSG_SET_PROPERTY_DONE: + { + struct ipc_msg_prop_pkt *prop_rsp_pkt = + (struct ipc_msg_prop_pkt *)msg; + uint32_t *param = (uint32_t *)((uint8_t *)prop_rsp_pkt + + sizeof(struct ipc_msg_prop_pkt)); + pr_debug("NPU_IPC_MSG_SET_PROPERTY_DONE %d:0x%x:%d\n", + prop_rsp_pkt->network_hdl, + prop_rsp_pkt->prop_id, + param[0]); + + host_ctx->cmd_ret_status = prop_rsp_pkt->header.status; + host_ctx->misc_pending = false; + + complete_all(&host_ctx->misc_done); + break; + } + case NPU_IPC_MSG_GET_PROPERTY_DONE: + { + struct ipc_msg_prop_pkt *prop_rsp_pkt = + (struct ipc_msg_prop_pkt *)msg; + uint32_t prop_size = 0; + uint32_t *prop_data = (uint32_t *)((uint8_t *)prop_rsp_pkt + + sizeof(struct ipc_msg_header_pkt)); + + pr_debug("NPU_IPC_MSG_GET_PROPERTY_DONE %d:0x%x:%d:%d\n", + prop_rsp_pkt->network_hdl, + prop_rsp_pkt->prop_id, + prop_rsp_pkt->num_params, + prop_rsp_pkt->prop_param[0]); + + if (prop_rsp_pkt->header.size < + sizeof(struct ipc_msg_header_pkt)) { + pr_err("Invalid rsp pkt size %d\n", + prop_rsp_pkt->header.size); + break; + } + + host_ctx->cmd_ret_status = prop_rsp_pkt->header.status; + + if (prop_rsp_pkt->num_params > 0) { + /* Copy prop data to kernel buffer */ + prop_size = prop_rsp_pkt->header.size - + sizeof(struct ipc_msg_header_pkt); + memcpy(host_ctx->prop_buf, prop_data, prop_size); + } + host_ctx->misc_pending = false; + + complete_all(&host_ctx->misc_done); + break; + } + case NPU_IPC_MSG_GENERAL_NOTIFY: + { + struct ipc_msg_general_notify_pkt *notify_msg_pkt = + (struct ipc_msg_general_notify_pkt *)msg; + + pr_debug("NPU_IPC_MSG_GENERAL_NOTIFY %d:0x%x:%d\n", + notify_msg_pkt->network_hdl, + notify_msg_pkt->notify_id, + notify_msg_pkt->notify_param[0]); + + switch (notify_msg_pkt->notify_id) { + case NPU_NOTIFY_DCVS_MODE: + pr_debug("NPU_IPC_MSG_GENERAL_NOTIFY DCVS_MODE %d\n", + notify_msg_pkt->notify_param[0]); + update_dcvs_activity(npu_dev, + notify_msg_pkt->notify_param[0]); + break; + default: + pr_err("Nothing to do\n"); + break; + } + break; + } + default: + pr_err("Not supported apps response received %d\n", + msg_id); + break; + } +} + +static void host_session_msg_hdlr(struct npu_device *npu_dev) +{ + uint32_t *msg; + struct npu_host_ctx *host_ctx = &npu_dev->host_ctx; + + msg = kzalloc(sizeof(uint32_t) * NPU_IPC_BUF_LENGTH, GFP_KERNEL); + if (!msg) + return; + + mutex_lock(&host_ctx->lock); + if (host_ctx->fw_state == FW_DISABLED) { + pr_warn("handle npu session msg when FW is disabled\n"); + goto skip_read_msg; + } + + while (npu_host_ipc_read_msg(npu_dev, IPC_QUEUE_APPS_RSP, msg) == 0) { + pr_debug("received from msg queue\n"); + app_msg_proc(host_ctx, msg); + } + +skip_read_msg: + mutex_unlock(&host_ctx->lock); + kfree(msg); +} + + +/* ------------------------------------------------------------------------- + * Function Definitions - Functionality + * ------------------------------------------------------------------------- + */ +int32_t npu_host_get_info(struct npu_device *npu_dev, + struct msm_npu_get_info_ioctl *get_info_ioctl) +{ + get_info_ioctl->firmware_version = FIRMWARE_VERSION; + get_info_ioctl->flags = npu_dev->pwrctrl.num_pwrlevels; + return 0; +} + +int32_t npu_host_map_buf(struct npu_client *client, + struct msm_npu_map_buf_ioctl *map_ioctl) +{ + struct npu_device *npu_dev = client->npu_dev; + struct npu_host_ctx *host_ctx = &npu_dev->host_ctx; + int ret; + + mutex_lock(&host_ctx->lock); + ret = npu_mem_map(client, map_ioctl->buf_ion_hdl, map_ioctl->size, + &map_ioctl->npu_phys_addr); + mutex_unlock(&host_ctx->lock); + + return ret; +} + +int32_t npu_host_unmap_buf(struct npu_client *client, + struct msm_npu_unmap_buf_ioctl *unmap_ioctl) +{ + struct npu_device *npu_dev = client->npu_dev; + struct npu_host_ctx *host_ctx = &npu_dev->host_ctx; + + /* + * Once SSR occurs, all buffers only can be unmapped until + * fw is disabled + */ + if (host_ctx->fw_error && (host_ctx->fw_state == FW_ENABLED) && + !wait_for_completion_timeout( + &host_ctx->fw_deinit_done, NW_CMD_TIMEOUT)) + pr_warn("npu: wait for fw_deinit_done time out\n"); + + mutex_lock(&host_ctx->lock); + npu_mem_unmap(client, unmap_ioctl->buf_ion_hdl, + unmap_ioctl->npu_phys_addr); + mutex_unlock(&host_ctx->lock); + return 0; +} + +static int npu_send_network_cmd(struct npu_device *npu_dev, + struct npu_network *network, void *cmd_ptr) +{ + struct npu_host_ctx *host_ctx = &npu_dev->host_ctx; + int ret = 0; + + if (network->fw_error || host_ctx->fw_error || + (host_ctx->fw_state == FW_DISABLED)) { + pr_err("fw is in error state or disabled, can't send network cmd\n"); + ret = -EIO; + } else if (network->cmd_pending) { + pr_err("Another cmd is pending\n"); + ret = -EBUSY; + } else { + pr_debug("Send cmd %d network id %lld\n", + ((struct ipc_cmd_header_pkt *)cmd_ptr)->cmd_type, + network->id); + network->cmd_ret_status = 0; + network->cmd_pending = true; + network->trans_id = ((struct ipc_cmd_header_pkt *)cmd_ptr)->trans_id; + ret = npu_host_ipc_send_cmd(npu_dev, + IPC_QUEUE_APPS_EXEC, cmd_ptr); + if (ret) + network->cmd_pending = false; + } + + return ret; +} + +static int npu_send_misc_cmd(struct npu_device *npu_dev, uint32_t q_idx, + void *cmd_ptr) +{ + struct npu_host_ctx *host_ctx = &npu_dev->host_ctx; + int ret = 0; + + mutex_lock(&host_ctx->lock); + if (host_ctx->fw_error || (host_ctx->fw_state == FW_DISABLED)) { + pr_err("fw is in error state or disabled, can't send misc cmd\n"); + ret = -EIO; + } else if (host_ctx->misc_pending) { + pr_err("Another misc cmd is pending\n"); + ret = -EBUSY; + } else { + pr_debug("Send cmd %d\n", + ((struct ipc_cmd_header_pkt *)cmd_ptr)->cmd_type); + host_ctx->cmd_ret_status = 0; + reinit_completion(&host_ctx->misc_done); + host_ctx->misc_pending = true; + ret = npu_host_ipc_send_cmd(npu_dev, q_idx, cmd_ptr); + if (ret) + host_ctx->misc_pending = false; + } + mutex_unlock(&host_ctx->lock); + + return ret; +} + +static void host_copy_patch_data(struct npu_patch_tuple *param, uint32_t value, + struct msm_npu_layer *layer_info) +{ + param->value = value; + param->chunk_id = layer_info->patch_info.chunk_id; + param->loc_offset = layer_info->patch_info.loc_offset; + param->instruction_size_in_bytes = + layer_info->patch_info.instruction_size_in_bytes; + param->shift_value_in_bits = + layer_info->patch_info.shift_value_in_bits; + param->variable_size_in_bits = + layer_info->patch_info.variable_size_in_bits; + + pr_debug("copy_patch_data: %x %d %x %x %x %x\n", + param->value, + param->chunk_id, + param->loc_offset, + param->instruction_size_in_bytes, + param->shift_value_in_bits, + param->variable_size_in_bits); +} + +static void host_copy_patch_data_v2(struct npu_patch_tuple_v2 *param, + struct msm_npu_patch_info_v2 *patch_info) +{ + param->value = patch_info->value; + param->chunk_id = patch_info->chunk_id; + param->loc_offset = patch_info->loc_offset; + param->instruction_size_in_bytes = + patch_info->instruction_size_in_bytes; + param->shift_value_in_bits = patch_info->shift_value_in_bits; + param->variable_size_in_bits = patch_info->variable_size_in_bits; + pr_debug("copy_patch_data_v2: %x %d %x %x %x %x\n", + param->value, + param->chunk_id, + param->loc_offset, + param->instruction_size_in_bytes, + param->shift_value_in_bits, + param->variable_size_in_bits); +} + +static uint32_t find_networks_perf_mode(struct npu_host_ctx *host_ctx) +{ + struct npu_network *network; + uint32_t max_perf_mode = 0; + int i = 0; + + network = host_ctx->networks; + + if (!host_ctx->network_num) { + /* if no network exists, set to the lowest level */ + max_perf_mode = 1; + } else { + /* find the max level among all the networks */ + for (i = 0; i < MAX_LOADED_NETWORK; i++) { + if ((network->id != 0) && + (network->cur_perf_mode != 0) && + (network->cur_perf_mode > max_perf_mode)) + max_perf_mode = network->cur_perf_mode; + network++; + } + } + pr_debug("max perf mode for networks: %d\n", max_perf_mode); + + return max_perf_mode; +} + +static int set_perf_mode(struct npu_device *npu_dev) +{ + int ret = 0; + uint32_t networks_perf_mode; + struct npu_host_ctx *host_ctx = &npu_dev->host_ctx; + + networks_perf_mode = find_networks_perf_mode(host_ctx); + + if (npu_dev->pwrctrl.perf_mode_override) + networks_perf_mode = npu_dev->pwrctrl.perf_mode_override; + + if (npu_dev->pwrctrl.cur_dcvs_activity != NPU_DCVS_ACTIVITY_MAX_PERF) + networks_perf_mode = min_t(uint32_t, networks_perf_mode, + npu_dev->pwrctrl.cur_dcvs_activity); + + ret = npu_set_uc_power_level(npu_dev, networks_perf_mode); + if (ret) + pr_err("network load failed due to power level set\n"); + + return ret; +} + +static int update_dcvs_activity(struct npu_device *npu_dev, uint32_t activity) +{ + npu_dev->pwrctrl.cur_dcvs_activity = activity; + pr_debug("update dcvs activity to %d\n", activity); + + return set_perf_mode(npu_dev); +} + +int32_t npu_host_set_fw_property(struct npu_device *npu_dev, + struct msm_npu_property *property) +{ + int ret = 0, i; + uint32_t prop_param, prop_id; + struct ipc_cmd_prop_pkt *prop_packet = NULL; + struct npu_host_ctx *host_ctx = &npu_dev->host_ctx; + uint32_t num_of_params, pkt_size; + + prop_id = property->prop_id; + num_of_params = min_t(uint32_t, property->num_of_params, + (uint32_t)PROP_PARAM_MAX_SIZE); + pkt_size = sizeof(*prop_packet) + num_of_params * sizeof(uint32_t); + prop_packet = kzalloc(pkt_size, GFP_KERNEL); + + if (!prop_packet) + return -ENOMEM; + + switch (prop_id) { + case MSM_NPU_PROP_ID_DCVS_MODE: + prop_param = min_t(uint32_t, property->prop_param[0], + (uint32_t)(npu_dev->pwrctrl.num_pwrlevels - 1)); + property->prop_param[0] = prop_param; + pr_debug("setting dcvs_mode to %d\n", prop_param); + + if (property->network_hdl == 0) { + npu_dev->pwrctrl.dcvs_mode = prop_param; + pr_debug("Set global dcvs mode %d\n", prop_param); + } + break; + default: + pr_err("unsupported property received %d\n", property->prop_id); + goto set_prop_exit; + } + + ret = fw_init(npu_dev); + if (ret) { + pr_err("fw_init fail\n"); + goto set_prop_exit; + } + + prop_packet->header.cmd_type = NPU_IPC_CMD_SET_PROPERTY; + prop_packet->header.size = pkt_size; + prop_packet->header.trans_id = + atomic_add_return(1, &host_ctx->ipc_trans_id); + prop_packet->header.flags = 0; + + prop_packet->prop_id = prop_id; + prop_packet->num_params = num_of_params; + prop_packet->network_hdl = property->network_hdl; + for (i = 0; i < num_of_params; i++) + prop_packet->prop_param[i] = property->prop_param[i]; + + ret = npu_send_misc_cmd(npu_dev, IPC_QUEUE_APPS_EXEC, + prop_packet); + + pr_debug("NPU_IPC_CMD_SET_PROPERTY sent status: %d\n", ret); + + if (ret) { + pr_err("NPU_IPC_CMD_SET_PROPERTY failed\n"); + goto deinit_fw; + } + + ret = wait_for_completion_interruptible_timeout( + &host_ctx->misc_done, + (host_ctx->fw_dbg_mode & FW_DBG_MODE_INC_TIMEOUT) ? + NW_DEBUG_TIMEOUT : NW_CMD_TIMEOUT); + + if (!ret) { + pr_err_ratelimited("npu: NPU_IPC_CMD_SET_PROPERTY time out\n"); + ret = -ETIMEDOUT; + goto deinit_fw; + } else if (ret < 0) { + pr_err("Wait for set_property done interrupted by signal\n"); + goto deinit_fw; + } + + ret = host_ctx->cmd_ret_status; + if (ret) + pr_err("set fw property failed %d\n", ret); + +deinit_fw: + fw_deinit(npu_dev, false, true); +set_prop_exit: + kfree(prop_packet); + return ret; +} + +int32_t npu_host_get_fw_property(struct npu_device *npu_dev, + struct msm_npu_property *property) +{ + int ret = 0, i; + struct ipc_cmd_prop_pkt *prop_packet = NULL; + struct npu_host_ctx *host_ctx = &npu_dev->host_ctx; + struct msm_npu_property *prop_from_fw; + uint32_t num_of_params, pkt_size; + + num_of_params = min_t(uint32_t, property->num_of_params, + (uint32_t)PROP_PARAM_MAX_SIZE); + pkt_size = sizeof(*prop_packet) + num_of_params * sizeof(uint32_t); + prop_packet = kzalloc(pkt_size, GFP_KERNEL); + + if (!prop_packet) + return -ENOMEM; + + ret = fw_init(npu_dev); + if (ret) { + pr_err("fw_init fail\n"); + goto get_prop_exit; + } + + prop_packet->header.cmd_type = NPU_IPC_CMD_GET_PROPERTY; + prop_packet->header.size = pkt_size; + prop_packet->header.trans_id = + atomic_add_return(1, &host_ctx->ipc_trans_id); + prop_packet->header.flags = 0; + + prop_packet->prop_id = property->prop_id; + prop_packet->num_params = num_of_params; + prop_packet->network_hdl = property->network_hdl; + for (i = 0; i < num_of_params; i++) + prop_packet->prop_param[i] = property->prop_param[i]; + + ret = npu_send_misc_cmd(npu_dev, IPC_QUEUE_APPS_EXEC, + prop_packet); + pr_debug("NPU_IPC_CMD_GET_PROPERTY sent status: %d\n", ret); + + if (ret) { + pr_err("NPU_IPC_CMD_GET_PROPERTY failed\n"); + goto deinit_fw; + } + + ret = wait_for_completion_interruptible_timeout( + &host_ctx->misc_done, + (host_ctx->fw_dbg_mode & FW_DBG_MODE_INC_TIMEOUT) ? + NW_DEBUG_TIMEOUT : NW_CMD_TIMEOUT); + + if (!ret) { + pr_err_ratelimited("npu: NPU_IPC_CMD_GET_PROPERTY time out\n"); + ret = -ETIMEDOUT; + goto deinit_fw; + } else if (ret < 0) { + pr_err("Wait for get_property done interrupted by signal\n"); + goto deinit_fw; + } + + ret = host_ctx->cmd_ret_status; + if (!ret) { + /* Return prop data retrieved from fw to user */ + prop_from_fw = (struct msm_npu_property *)(host_ctx->prop_buf); + if (property->prop_id == prop_from_fw->prop_id && + property->network_hdl == prop_from_fw->network_hdl) { + property->num_of_params = num_of_params; + for (i = 0; i < num_of_params; i++) + property->prop_param[i] = + prop_from_fw->prop_param[i]; + } + } else { + pr_err("get fw property failed %d\n", ret); + } + +deinit_fw: + fw_deinit(npu_dev, false, true); +get_prop_exit: + kfree(prop_packet); + return ret; +} + +int32_t npu_host_load_network(struct npu_client *client, + struct msm_npu_load_network_ioctl *load_ioctl) +{ + int ret = 0; + struct npu_device *npu_dev = client->npu_dev; + struct npu_pwrctrl *pwr = &npu_dev->pwrctrl; + struct npu_network *network; + struct ipc_cmd_load_pkt load_packet; + struct npu_host_ctx *host_ctx = &npu_dev->host_ctx; + + ret = fw_init(npu_dev); + if (ret) + return ret; + + mutex_lock(&host_ctx->lock); + network = alloc_network(host_ctx, client); + if (!network) { + ret = -ENOMEM; + goto err_deinit_fw; + } + + network_get(network); + network->buf_hdl = load_ioctl->buf_ion_hdl; + network->size = load_ioctl->buf_size; + network->phy_add = load_ioctl->buf_phys_addr; + network->first_block_size = load_ioctl->first_block_size; + network->priority = load_ioctl->priority; + network->cur_perf_mode = network->init_perf_mode = + (load_ioctl->perf_mode == PERF_MODE_DEFAULT) ? + pwr->num_pwrlevels : load_ioctl->perf_mode; + + /* verify mapped physical address */ + if (!npu_mem_verify_addr(client, network->phy_add)) { + ret = -EINVAL; + goto error_free_network; + } + + load_packet.header.cmd_type = NPU_IPC_CMD_LOAD; + load_packet.header.size = sizeof(struct ipc_cmd_load_pkt); + load_packet.header.trans_id = + atomic_add_return(1, &host_ctx->ipc_trans_id); + load_packet.header.flags = 0; + + /* ACO Buffer. Use the npu mapped aco address */ + load_packet.buf_pkt.address = (uint64_t)network->phy_add; + load_packet.buf_pkt.buf_size = network->first_block_size; + load_packet.buf_pkt.network_id = network->id; + + set_perf_mode(npu_dev); + /* NPU_IPC_CMD_LOAD will go onto IPC_QUEUE_APPS_EXEC */ + reinit_completion(&network->cmd_done); + ret = npu_send_network_cmd(npu_dev, network, &load_packet); + if (ret) { + pr_err("NPU_IPC_CMD_LOAD sent failed: %d\n", ret); + goto error_free_network; + } + + mutex_unlock(&host_ctx->lock); + + ret = wait_for_completion_timeout( + &network->cmd_done, + (host_ctx->fw_dbg_mode & FW_DBG_MODE_INC_TIMEOUT) ? + NW_DEBUG_TIMEOUT : NW_CMD_TIMEOUT); + + mutex_lock(&host_ctx->lock); + if (!ret) { + pr_err_ratelimited("NPU_IPC_CMD_LOAD time out\n"); + ret = -ETIMEDOUT; + goto error_free_network; + } + + if (network->fw_error) { + ret = -EIO; + pr_err("fw is in error state during load network\n"); + goto error_free_network; + } + + ret = network->cmd_ret_status; + if (ret) + goto error_free_network; + + load_ioctl->network_hdl = network->network_hdl; + network->is_active = true; + network_put(network); + + mutex_unlock(&host_ctx->lock); + + return ret; + +error_free_network: + network_put(network); + free_network(host_ctx, client, network->id); +err_deinit_fw: + mutex_unlock(&host_ctx->lock); + fw_deinit(npu_dev, false, true); + return ret; +} + +int32_t npu_host_load_network_v2(struct npu_client *client, + struct msm_npu_load_network_ioctl_v2 *load_ioctl, + struct msm_npu_patch_info_v2 *patch_info) +{ + int ret = 0, i; + struct npu_device *npu_dev = client->npu_dev; + struct npu_pwrctrl *pwr = &npu_dev->pwrctrl; + struct npu_network *network; + struct ipc_cmd_load_pkt_v2 *load_packet = NULL; + struct npu_host_ctx *host_ctx = &npu_dev->host_ctx; + uint32_t num_patch_params, pkt_size; + + ret = fw_init(npu_dev); + if (ret) + return ret; + + mutex_lock(&host_ctx->lock); + network = alloc_network(host_ctx, client); + if (!network) { + ret = -ENOMEM; + goto err_deinit_fw; + } + + network_get(network); + num_patch_params = load_ioctl->patch_info_num; + pkt_size = sizeof(*load_packet) + + num_patch_params * sizeof(struct npu_patch_tuple_v2); + load_packet = kzalloc(pkt_size, GFP_KERNEL); + + if (!load_packet) { + ret = -ENOMEM; + goto error_free_network; + } + + for (i = 0; i < num_patch_params; i++) + host_copy_patch_data_v2(&load_packet->patch_params[i], + &patch_info[i]); + + network->buf_hdl = load_ioctl->buf_ion_hdl; + network->size = load_ioctl->buf_size; + network->phy_add = load_ioctl->buf_phys_addr; + network->first_block_size = load_ioctl->first_block_size; + network->priority = load_ioctl->priority; + network->cur_perf_mode = network->init_perf_mode = + (load_ioctl->perf_mode == PERF_MODE_DEFAULT) ? + pwr->num_pwrlevels : load_ioctl->perf_mode; + network->num_layers = load_ioctl->num_layers; + + /* verify mapped physical address */ + if (!npu_mem_verify_addr(client, network->phy_add)) { + pr_err("Invalid network address %llx\n", network->phy_add); + ret = -EINVAL; + goto error_free_network; + } + + load_packet->header.cmd_type = NPU_IPC_CMD_LOAD_V2; + load_packet->header.size = pkt_size; + load_packet->header.trans_id = + atomic_add_return(1, &host_ctx->ipc_trans_id); + load_packet->header.flags = 0; + + /* ACO Buffer. Use the npu mapped aco address */ + load_packet->buf_pkt.address = (uint32_t)network->phy_add; + load_packet->buf_pkt.buf_size = network->first_block_size; + load_packet->buf_pkt.network_id = network->id; + load_packet->buf_pkt.num_layers = network->num_layers; + load_packet->num_patch_params = num_patch_params; + + set_perf_mode(npu_dev); + /* NPU_IPC_CMD_LOAD_V2 will go onto IPC_QUEUE_APPS_EXEC */ + reinit_completion(&network->cmd_done); + ret = npu_send_network_cmd(npu_dev, network, load_packet); + if (ret) { + pr_debug("NPU_IPC_CMD_LOAD_V2 sent failed: %d\n", ret); + goto error_free_network; + } + + mutex_unlock(&host_ctx->lock); + + ret = wait_for_completion_timeout( + &network->cmd_done, + (host_ctx->fw_dbg_mode & FW_DBG_MODE_INC_TIMEOUT) ? + NW_DEBUG_TIMEOUT : NW_CMD_TIMEOUT); + + mutex_lock(&host_ctx->lock); + + if (!ret) { + pr_err_ratelimited("npu: NPU_IPC_CMD_LOAD_V2 time out\n"); + ret = -ETIMEDOUT; + goto error_free_network; + } + + if (network->fw_error) { + ret = -EIO; + pr_err("fw is in error state during load_v2 network\n"); + goto error_free_network; + } + + ret = network->cmd_ret_status; + if (ret) + goto error_free_network; + + load_ioctl->network_hdl = network->network_hdl; + network->is_active = true; + kfree(load_packet); + network_put(network); + + mutex_unlock(&host_ctx->lock); + + return ret; + +error_free_network: + kfree(load_packet); + network_put(network); + free_network(host_ctx, client, network->id); +err_deinit_fw: + mutex_unlock(&host_ctx->lock); + fw_deinit(npu_dev, false, true); + return ret; +} + +int32_t npu_host_unload_network(struct npu_client *client, + struct msm_npu_unload_network_ioctl *unload) +{ + int ret = 0, retry_cnt = 1; + struct npu_device *npu_dev = client->npu_dev; + struct ipc_cmd_unload_pkt unload_packet; + struct npu_network *network; + struct npu_host_ctx *host_ctx = &npu_dev->host_ctx; + + /* get the corresponding network for ipc trans id purpose */ + mutex_lock(&host_ctx->lock); + network = get_network_by_hdl(host_ctx, client, + unload->network_hdl); + if (!network) { + mutex_unlock(&host_ctx->lock); + return -EINVAL; + } + + if (!network->is_active) { + pr_err("network is not active\n"); + network_put(network); + mutex_unlock(&host_ctx->lock); + return -EINVAL; + } + + if (network->fw_error) { + pr_err("fw in error state, skip unload network in fw\n"); + goto free_network; + } + + pr_debug("Unload network %lld\n", network->id); + /* prepare IPC packet for UNLOAD */ + unload_packet.header.cmd_type = NPU_IPC_CMD_UNLOAD; + unload_packet.header.size = sizeof(struct ipc_cmd_unload_pkt); + unload_packet.header.trans_id = + atomic_add_return(1, &host_ctx->ipc_trans_id); + unload_packet.header.flags = 0; + unload_packet.network_hdl = (uint32_t)network->network_hdl; + +retry: + /* NPU_IPC_CMD_UNLOAD will go onto IPC_QUEUE_APPS_EXEC */ + reinit_completion(&network->cmd_done); + ret = npu_send_network_cmd(npu_dev, network, &unload_packet); + + if (ret) { + pr_err("NPU_IPC_CMD_UNLOAD sent failed: %d\n", ret); + /* + * If another command is running on this network, + * retry after 500ms. + */ + if ((ret == -EBUSY) && (retry_cnt > 0)) { + pr_err("Network is running, retry later\n"); + mutex_unlock(&host_ctx->lock); + retry_cnt--; + msleep(500); + mutex_lock(&host_ctx->lock); + goto retry; + } + goto free_network; + } + + mutex_unlock(&host_ctx->lock); + + ret = wait_for_completion_timeout( + &network->cmd_done, + (host_ctx->fw_dbg_mode & FW_DBG_MODE_INC_TIMEOUT) ? + NW_DEBUG_TIMEOUT : NW_CMD_TIMEOUT); + + mutex_lock(&host_ctx->lock); + + if (!ret) { + pr_err_ratelimited("npu: NPU_IPC_CMD_UNLOAD time out\n"); + network->cmd_pending = false; + ret = -ETIMEDOUT; + goto free_network; + } + + if (network->fw_error) { + ret = -EIO; + pr_err("fw is in error state during unload network\n"); + } else { + ret = network->cmd_ret_status; + pr_debug("unload network status %d\n", ret); + } + +free_network: + /* + * free the network on the kernel if the corresponding ACO + * handle is unloaded on the firmware side + */ + network_put(network); + free_network(host_ctx, client, network->id); + + /* recalculate uc_power_level after unload network */ + if (npu_dev->pwrctrl.cur_dcvs_activity) + set_perf_mode(npu_dev); + + mutex_unlock(&host_ctx->lock); + if (host_ctx->fw_unload_delay_ms) { + flush_delayed_work(&host_ctx->fw_deinit_work); + atomic_inc(&host_ctx->fw_deinit_work_cnt); + queue_delayed_work(host_ctx->wq, &host_ctx->fw_deinit_work, + msecs_to_jiffies(host_ctx->fw_unload_delay_ms)); + } else { + fw_deinit(npu_dev, false, true); + } + return ret; +} + +int32_t npu_host_exec_network(struct npu_client *client, + struct msm_npu_exec_network_ioctl *exec_ioctl) +{ + struct npu_device *npu_dev = client->npu_dev; + struct ipc_cmd_execute_pkt exec_packet; + /* npu mapped addr */ + uint64_t input_off, output_off; + int32_t ret; + struct npu_network *network; + struct npu_host_ctx *host_ctx = &npu_dev->host_ctx; + + mutex_lock(&host_ctx->lock); + network = get_network_by_hdl(host_ctx, client, + exec_ioctl->network_hdl); + + if (!network) { + mutex_unlock(&host_ctx->lock); + return -EINVAL; + } + + if (atomic_inc_return(&host_ctx->network_execute_cnt) == 1) + npu_notify_cdsprm_cxlimit_activity(npu_dev, true); + + if (!network->is_active) { + pr_err("network is not active\n"); + ret = -EINVAL; + goto exec_done; + } + + if (network->fw_error) { + pr_err("fw is in error state\n"); + ret = -EIO; + goto exec_done; + } + + pr_debug("execute network %lld\n", network->id); + memset(&exec_packet, 0, sizeof(exec_packet)); + if (exec_ioctl->patching_required) { + if ((exec_ioctl->input_layer_num != 1) || + (exec_ioctl->output_layer_num != 1)) { + pr_err("Invalid input/output layer num\n"); + ret = -EINVAL; + goto exec_done; + } + + input_off = exec_ioctl->input_layers[0].buf_phys_addr; + output_off = exec_ioctl->output_layers[0].buf_phys_addr; + /* verify mapped physical address */ + if (!npu_mem_verify_addr(client, input_off) || + !npu_mem_verify_addr(client, output_off)) { + pr_err("Invalid patch buf address\n"); + ret = -EINVAL; + goto exec_done; + } + + exec_packet.patch_params.num_params = 2; + host_copy_patch_data(&exec_packet.patch_params.param[0], + (uint32_t)input_off, &exec_ioctl->input_layers[0]); + host_copy_patch_data(&exec_packet.patch_params.param[1], + (uint32_t)output_off, &exec_ioctl->output_layers[0]); + } else { + exec_packet.patch_params.num_params = 0; + } + + exec_packet.header.cmd_type = NPU_IPC_CMD_EXECUTE; + exec_packet.header.size = sizeof(struct ipc_cmd_execute_pkt); + exec_packet.header.trans_id = + atomic_add_return(1, &host_ctx->ipc_trans_id); + exec_packet.header.flags = 0xF; + exec_packet.network_hdl = network->network_hdl; + + /* Send it on the high priority queue */ + reinit_completion(&network->cmd_done); + ret = npu_send_network_cmd(npu_dev, network, &exec_packet); + + if (ret) { + pr_err("NPU_IPC_CMD_EXECUTE sent failed: %d\n", ret); + goto exec_done; + } + + mutex_unlock(&host_ctx->lock); + + ret = wait_for_completion_timeout( + &network->cmd_done, + (host_ctx->fw_dbg_mode & FW_DBG_MODE_INC_TIMEOUT) ? + NW_DEBUG_TIMEOUT : NW_CMD_TIMEOUT); + + mutex_lock(&host_ctx->lock); + if (!ret) { + pr_err_ratelimited("npu: NPU_IPC_CMD_EXECUTE time out\n"); + /* dump debug stats */ + npu_dump_debug_timeout_stats(npu_dev); + network->cmd_pending = false; + ret = -ETIMEDOUT; + goto exec_done; + } + + if (network->fw_error) { + ret = -EIO; + pr_err("fw is in error state during execute network\n"); + } else { + ret = network->cmd_ret_status; + pr_debug("execution status %d\n", ret); + } + +exec_done: + network_put(network); + mutex_unlock(&host_ctx->lock); + + /* + * treat network execution timed our or interrupted by signal + * as error in order to force npu fw to stop execution + */ + if ((ret == -ETIMEDOUT) || (ret == -ERESTARTSYS)) { + pr_err("Error handling after execution failure\n"); + host_error_hdlr(npu_dev, true); + } + + if (atomic_dec_return(&host_ctx->network_execute_cnt) == 0) + npu_notify_cdsprm_cxlimit_activity(npu_dev, false); + + return ret; +} + +int32_t npu_host_exec_network_v2(struct npu_client *client, + struct msm_npu_exec_network_ioctl_v2 *exec_ioctl, + struct msm_npu_patch_buf_info *patch_buf_info) +{ + struct npu_device *npu_dev = client->npu_dev; + struct ipc_cmd_execute_pkt_v2 *exec_packet; + int32_t ret; + struct npu_network *network; + struct npu_host_ctx *host_ctx = &npu_dev->host_ctx; + uint32_t num_patch_params, pkt_size; + int i; + + mutex_lock(&host_ctx->lock); + network = get_network_by_hdl(host_ctx, client, + exec_ioctl->network_hdl); + + if (!network) { + mutex_unlock(&host_ctx->lock); + return -EINVAL; + } + + if (atomic_inc_return(&host_ctx->network_execute_cnt) == 1) + npu_notify_cdsprm_cxlimit_activity(npu_dev, true); + + if (!network->is_active) { + pr_err("network is not active\n"); + ret = -EINVAL; + goto exec_v2_done; + } + + if (network->fw_error) { + pr_err("fw is in error state\n"); + ret = -EIO; + goto exec_v2_done; + } + + pr_debug("execute_v2 network %lld\n", network->id); + num_patch_params = exec_ioctl->patch_buf_info_num; + pkt_size = num_patch_params * sizeof(struct npu_patch_params_v2) + + sizeof(*exec_packet); + exec_packet = kzalloc(pkt_size, GFP_KERNEL); + + if (!exec_packet) { + ret = -ENOMEM; + goto exec_v2_done; + } + + for (i = 0; i < num_patch_params; i++) { + exec_packet->patch_params[i].id = patch_buf_info[i].buf_id; + pr_debug("%d: patch_id: %x\n", i, + exec_packet->patch_params[i].id); + exec_packet->patch_params[i].value = + patch_buf_info[i].buf_phys_addr; + pr_debug("%d: patch value: %x\n", i, + exec_packet->patch_params[i].value); + + /* verify mapped physical address */ + if (!npu_mem_verify_addr(client, + patch_buf_info[i].buf_phys_addr)) { + pr_err("Invalid patch value\n"); + ret = -EINVAL; + goto free_exec_packet; + } + } + + exec_packet->header.cmd_type = NPU_IPC_CMD_EXECUTE_V2; + exec_packet->header.size = pkt_size; + exec_packet->header.trans_id = + atomic_add_return(1, &host_ctx->ipc_trans_id); + exec_packet->header.flags = host_ctx->exec_flags_override > 0 ? + host_ctx->exec_flags_override : exec_ioctl->flags; + exec_packet->network_hdl = network->network_hdl; + exec_packet->num_patch_params = num_patch_params; + + network->stats_buf_u = (void __user *)exec_ioctl->stats_buf_addr; + network->stats_buf_size = exec_ioctl->stats_buf_size; + + pr_debug("Execute_v2 flags %x stats_buf_size %d\n", + exec_packet->header.flags, exec_ioctl->stats_buf_size); + + /* Send it on the high priority queue */ + reinit_completion(&network->cmd_done); + ret = npu_send_network_cmd(npu_dev, network, exec_packet); + + if (ret) { + pr_err("NPU_IPC_CMD_EXECUTE_V2 sent failed: %d\n", ret); + goto free_exec_packet; + } + + mutex_unlock(&host_ctx->lock); + + ret = wait_for_completion_timeout( + &network->cmd_done, + (host_ctx->fw_dbg_mode & FW_DBG_MODE_INC_TIMEOUT) ? + NW_DEBUG_TIMEOUT : NW_CMD_TIMEOUT); + + mutex_lock(&host_ctx->lock); + if (!ret) { + pr_err_ratelimited("npu: NPU_IPC_CMD_EXECUTE_V2 time out\n"); + /* dump debug stats */ + npu_dump_debug_timeout_stats(npu_dev); + network->cmd_pending = false; + ret = -ETIMEDOUT; + goto free_exec_packet; + } + + if (network->fw_error) { + ret = -EIO; + pr_err("fw is in error state during execute_v2 network\n"); + goto free_exec_packet; + } + + ret = network->cmd_ret_status; + if (!ret) { + exec_ioctl->stats_buf_size = network->stats_buf_size; + if (copy_to_user( + (void __user *)exec_ioctl->stats_buf_addr, + network->stats_buf, + exec_ioctl->stats_buf_size)) { + pr_err("copy stats to user failed\n"); + exec_ioctl->stats_buf_size = 0; + } + } else { + pr_err("execution failed %d\n", ret); + } + +free_exec_packet: + kfree(exec_packet); +exec_v2_done: + network_put(network); + mutex_unlock(&host_ctx->lock); + + /* + * treat network execution timed our or interrupted by signal + * as error in order to force npu fw to stop execution + */ + if ((ret == -ETIMEDOUT) || (ret == -ERESTARTSYS)) { + pr_err("Error handling after execution failure\n"); + host_error_hdlr(npu_dev, true); + } + + if (atomic_dec_return(&host_ctx->network_execute_cnt) == 0) + npu_notify_cdsprm_cxlimit_activity(npu_dev, false); + + return ret; +} + +int32_t npu_host_loopback_test(struct npu_device *npu_dev) +{ + struct ipc_cmd_loopback_pkt loopback_packet; + struct npu_host_ctx *host_ctx = &npu_dev->host_ctx; + int32_t ret; + + ret = fw_init(npu_dev); + if (ret) + return ret; + + loopback_packet.header.cmd_type = NPU_IPC_CMD_LOOPBACK; + loopback_packet.header.size = sizeof(struct ipc_cmd_loopback_pkt); + loopback_packet.header.trans_id = + atomic_add_return(1, &host_ctx->ipc_trans_id); + loopback_packet.header.flags = 0; + loopback_packet.loopbackParams = 15; + + ret = npu_send_misc_cmd(npu_dev, IPC_QUEUE_APPS_EXEC, &loopback_packet); + + if (ret) { + pr_err("NPU_IPC_CMD_LOOPBACK sent failed: %d\n", ret); + goto loopback_exit; + } + + ret = wait_for_completion_interruptible_timeout( + &host_ctx->misc_done, + (host_ctx->fw_dbg_mode & FW_DBG_MODE_INC_TIMEOUT) ? + NW_DEBUG_TIMEOUT : NW_CMD_TIMEOUT); + + if (!ret) { + pr_err_ratelimited("npu: NPU_IPC_CMD_LOOPBACK time out\n"); + ret = -ETIMEDOUT; + } else if (ret < 0) { + pr_err("Wait for loopback done interrupted by signal\n"); + } + +loopback_exit: + fw_deinit(npu_dev, false, true); + + return ret; +} + +void npu_host_cleanup_networks(struct npu_client *client) +{ + int i; + struct npu_device *npu_dev = client->npu_dev; + struct npu_host_ctx *host_ctx = &npu_dev->host_ctx; + struct msm_npu_unload_network_ioctl unload_req; + struct msm_npu_unmap_buf_ioctl unmap_req; + struct npu_network *network; + struct npu_ion_buf *ion_buf; + + for (i = 0; i < MAX_LOADED_NETWORK; i++) { + network = &host_ctx->networks[i]; + if (network->client == client) { + pr_warn("network %d is not unloaded before close\n", + network->network_hdl); + unload_req.network_hdl = network->network_hdl; + npu_host_unload_network(client, &unload_req); + } + } + + /* unmap all remaining buffers */ + while (!list_empty(&client->mapped_buffer_list)) { + ion_buf = list_first_entry(&client->mapped_buffer_list, + struct npu_ion_buf, list); + pr_warn("unmap buffer %x:%llx\n", ion_buf->fd, ion_buf->iova); + unmap_req.buf_ion_hdl = ion_buf->fd; + unmap_req.npu_phys_addr = ion_buf->iova; + npu_host_unmap_buf(client, &unmap_req); + } +} + +/* + * set network or global perf_mode + * if network_hdl is 0, set global perf_mode_override + * otherwise set network perf_mode: if perf_mode is 0, + * change network perf_mode to initial perf_mode from + * load_network + */ +int32_t npu_host_set_perf_mode(struct npu_client *client, uint32_t network_hdl, + uint32_t perf_mode) +{ + int ret = 0; + struct npu_device *npu_dev = client->npu_dev; + struct npu_host_ctx *host_ctx = &npu_dev->host_ctx; + struct npu_network *network = NULL; + + mutex_lock(&host_ctx->lock); + + if (network_hdl == 0) { + pr_debug("change perf_mode_override to %d\n", perf_mode); + npu_dev->pwrctrl.perf_mode_override = perf_mode; + } else { + network = get_network_by_hdl(host_ctx, client, network_hdl); + if (!network) { + pr_err("invalid network handle %x\n", network_hdl); + mutex_unlock(&host_ctx->lock); + return -EINVAL; + } + + if (perf_mode == 0) { + network->cur_perf_mode = network->init_perf_mode; + pr_debug("change network %d perf_mode back to %d\n", + network_hdl, network->cur_perf_mode); + } else { + network->cur_perf_mode = perf_mode; + pr_debug("change network %d perf_mode to %d\n", + network_hdl, network->cur_perf_mode); + } + } + + ret = set_perf_mode(npu_dev); + if (ret) + pr_err("set_perf_mode failed\n"); + + if (network) + network_put(network); + mutex_unlock(&host_ctx->lock); + + return ret; +} + +/* + * get the currently set network or global perf_mode + * if network_hdl is 0, get global perf_mode_override + * otherwise get network perf_mode + */ +int32_t npu_host_get_perf_mode(struct npu_client *client, uint32_t network_hdl) +{ + int param_val = 0; + struct npu_device *npu_dev = client->npu_dev; + struct npu_host_ctx *host_ctx = &npu_dev->host_ctx; + struct npu_network *network = NULL; + + mutex_lock(&host_ctx->lock); + + if (network_hdl == 0) { + param_val = npu_dev->pwrctrl.perf_mode_override; + } else { + network = get_network_by_hdl(host_ctx, client, network_hdl); + if (!network) { + pr_err("invalid network handle %x\n", network_hdl); + mutex_unlock(&host_ctx->lock); + return -EINVAL; + } + param_val = network->cur_perf_mode; + network_put(network); + } + + mutex_unlock(&host_ctx->lock); + + return param_val; +} diff --git a/drivers/media/platform/msm/npu/npu_mgr.h b/drivers/media/platform/msm/npu/npu_mgr.h new file mode 100644 index 000000000000..ea2e24e36de9 --- /dev/null +++ b/drivers/media/platform/msm/npu/npu_mgr.h @@ -0,0 +1,147 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2018-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2023-2024, Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#ifndef _NPU_MGR_H +#define _NPU_MGR_H + +/* ------------------------------------------------------------------------- + * Includes + * ------------------------------------------------------------------------- + */ +#include +#include "npu_hw_access.h" +#include "npu_common.h" + +/* ------------------------------------------------------------------------- + * Defines + * ------------------------------------------------------------------------- + */ +#define NW_CMD_TIMEOUT_MS (1000 * 60 * 5) /* set for 5 minutes */ +#define NW_CMD_TIMEOUT msecs_to_jiffies(NW_CMD_TIMEOUT_MS) +#define NW_DEBUG_TIMEOUT_MS (1000 * 60 * 30) /* set for 30 minutes */ +#define NW_DEBUG_TIMEOUT msecs_to_jiffies(NW_DEBUG_TIMEOUT_MS) +#define FIRMWARE_VERSION 0x00001000 +#define MAX_LOADED_NETWORK 32 +#define NPU_IPC_BUF_LENGTH 512 + +#define FW_DBG_MODE_PAUSE (1 << 0) +#define FW_DBG_MODE_INC_TIMEOUT (1 << 1) +#define FW_DBG_DISABLE_WDOG (1 << 2) +#define FW_DBG_ENABLE_LOGGING (1 << 3) +/* ------------------------------------------------------------------------- + * Data Structures + * ------------------------------------------------------------------------- + */ +struct npu_network { + uint64_t id; + int buf_hdl; + uint64_t phy_add; + uint32_t size; + uint32_t first_block_size; + uint32_t network_hdl; + uint32_t priority; + uint32_t cur_perf_mode; + uint32_t init_perf_mode; + uint32_t num_layers; + void *stats_buf; + void __user *stats_buf_u; + uint32_t stats_buf_size; + uint32_t trans_id; + atomic_t ref_cnt; + bool is_valid; + bool is_active; + bool fw_error; + bool cmd_pending; + bool cmd_async; + int cmd_ret_status; + struct completion cmd_done; + struct npu_client *client; +}; + +enum fw_state { + FW_DISABLED = 0, + FW_ENABLED = 1, +}; + +struct npu_host_ctx { + struct mutex lock; + void *subsystem_handle; + struct npu_device *npu_dev; + enum fw_state fw_state; + int32_t fw_ref_cnt; + int32_t npu_init_cnt; + int32_t power_vote_num; + struct work_struct irq_work; + struct delayed_work fw_deinit_work; + atomic_t fw_deinit_work_cnt; + struct workqueue_struct *wq; + struct completion misc_done; + struct completion fw_deinit_done; + bool misc_pending; + void *prop_buf; + int32_t network_num; + struct npu_network networks[MAX_LOADED_NETWORK]; + bool sys_cache_disable; + uint32_t fw_dbg_mode; + uint32_t exec_flags_override; + uint32_t fw_unload_delay_ms; + atomic_t ipc_trans_id; + atomic_t network_execute_cnt; + int cmd_ret_status; + + uint32_t err_irq_sts; + uint32_t wdg_irq_sts; + bool fw_error; +}; + +struct npu_device; + +/* ------------------------------------------------------------------------- + * Function Prototypes + * ------------------------------------------------------------------------- + */ +int npu_host_init(struct npu_device *npu_dev); +void npu_host_deinit(struct npu_device *npu_dev); + +/* Host Driver IPC Interface */ +int npu_host_ipc_pre_init(struct npu_device *npu_dev); +int npu_host_ipc_post_init(struct npu_device *npu_dev); +void npu_host_ipc_deinit(struct npu_device *npu_dev); +int npu_host_ipc_send_cmd(struct npu_device *npu_dev, uint32_t queueIndex, + void *pCmd); +int npu_host_ipc_read_msg(struct npu_device *npu_dev, uint32_t queueIndex, + uint32_t *pMsg); + +int32_t npu_host_get_info(struct npu_device *npu_dev, + struct msm_npu_get_info_ioctl *get_info_ioctl); +int32_t npu_host_map_buf(struct npu_client *client, + struct msm_npu_map_buf_ioctl *map_ioctl); +int32_t npu_host_unmap_buf(struct npu_client *client, + struct msm_npu_unmap_buf_ioctl *unmap_ioctl); +int32_t npu_host_load_network(struct npu_client *client, + struct msm_npu_load_network_ioctl *load_ioctl); +int32_t npu_host_load_network_v2(struct npu_client *client, + struct msm_npu_load_network_ioctl_v2 *load_ioctl, + struct msm_npu_patch_info_v2 *patch_info); +int32_t npu_host_unload_network(struct npu_client *client, + struct msm_npu_unload_network_ioctl *unload); +int32_t npu_host_exec_network(struct npu_client *client, + struct msm_npu_exec_network_ioctl *exec_ioctl); +int32_t npu_host_exec_network_v2(struct npu_client *client, + struct msm_npu_exec_network_ioctl_v2 *exec_ioctl, + struct msm_npu_patch_buf_info *patch_buf_info); +int32_t npu_host_loopback_test(struct npu_device *npu_dev); +int32_t npu_host_set_fw_property(struct npu_device *npu_dev, + struct msm_npu_property *property); +int32_t npu_host_get_fw_property(struct npu_device *npu_dev, + struct msm_npu_property *property); +void npu_host_cleanup_networks(struct npu_client *client); +int32_t npu_host_set_perf_mode(struct npu_client *client, uint32_t network_hdl, + uint32_t perf_mode); +int32_t npu_host_get_perf_mode(struct npu_client *client, uint32_t network_hdl); +void npu_dump_debug_timeout_stats(struct npu_device *npu_dev); + +#endif /* _NPU_MGR_H */ diff --git a/include/soc/qcom/subsystem_restart.h b/include/soc/qcom/subsystem_restart.h new file mode 100644 index 000000000000..55722daea411 --- /dev/null +++ b/include/soc/qcom/subsystem_restart.h @@ -0,0 +1,279 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2014-2019 2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022, 2024 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#ifndef __SUBSYS_RESTART_H +#define __SUBSYS_RESTART_H + +#include +#include +#include + +struct subsys_device; +extern struct bus_type subsys_bus_type; + +enum { + RESET_SOC = 0, + RESET_SUBSYS_COUPLED, + RESET_LEVEL_MAX +}; + +enum crash_status { + CRASH_STATUS_NO_CRASH = 0, + CRASH_STATUS_ERR_FATAL, + CRASH_STATUS_WDOG_BITE, +}; + +struct device; +struct module; + +enum ssr_comm { + SUBSYS_TO_SUBSYS_SYSMON, + SUBSYS_TO_HLOS, + HLOS_TO_SUBSYS_SYSMON_SHUTDOWN, + HLOS_TO_SUBSYS_SYSMON_DSENTER, + NUM_SSR_COMMS, +}; + +/** + * struct subsys_notif_timeout - timeout data used by notification timeout hdlr + * @comm_type: Specifies if the type of communication being tracked is + * through sysmon between two subsystems, subsystem notifier call chain, or + * sysmon shutdown. + * @dest_name: subsystem to which sysmon notification is being sent to + * @source_name: subsystem which generated event that notification is being sent + * for + * @timer: timer for scheduling timeout + */ +struct subsys_notif_timeout { + enum ssr_comm comm_type; + const char *dest_name; + const char *source_name; + struct timer_list timer; +}; + +/** + * struct subsys_desc - subsystem descriptor + * @name: name of subsystem + * @fw_name: firmware name + * @pon_depends_on: subsystem this subsystem wants to power-on first. If the + * dependednt subsystem is already powered-on, the framework won't try to power + * it back up again. + * @poff_depends_on: subsystem this subsystem wants to power-off first. If the + * dependednt subsystem is already powered-off, the framework won't try to power + * it off again. + * @dev: parent device + * @owner: module the descriptor belongs to + * @shutdown: Stop a subsystem + * @powerup_notify: Notify about start of a subsystem + * @powerup: Start a subsystem + * @crash_shutdown: Shutdown a subsystem when the system crashes (can't sleep) + * @ramdump: Collect a ramdump of the subsystem + * @free_memory: Free the memory associated with this subsystem + * @no_auth: Set if subsystem does not rely on PIL to authenticate and bring + * it out of reset + * @ssctl_instance_id: Instance id used to connect with SSCTL service + * @sysmon_pid: pdev id that sysmon is probed with for the subsystem + * @sysmon_shutdown_ret: Return value for the call to sysmon_send_shutdown + * @system_debug: If "set", triggers a device restart when the + * subsystem's wdog bite handler is invoked. + * @ignore_ssr_failure: SSR failures are usually fatal and results in panic. If + * set will ignore failure. + * @edge: GLINK logical name of the subsystem + */ +struct subsys_desc { + const char *name; + char fw_name[256]; + const char *pon_depends_on; + const char *poff_depends_on; + struct device *dev; + struct module *owner; + + int (*shutdown)(const struct subsys_desc *desc, bool force_stop); + int (*enter_ds)(const struct subsys_desc *desc); + int (*powerup_notify)(const struct subsys_desc *desc); + int (*powerup)(const struct subsys_desc *desc); + void (*crash_shutdown)(const struct subsys_desc *desc); + int (*ramdump)(int need_dumps, const struct subsys_desc *desc); + void (*free_memory)(const struct subsys_desc *desc); + struct completion shutdown_ack; + struct completion dsentry_ack; + int err_fatal_gpio; + int force_stop_bit; + int ramdump_disable_irq; + int shutdown_ack_irq; + int dsentry_ack_irq; + int ramdump_disable; + bool no_auth; + bool pil_mss_memsetup; + int ssctl_instance_id; + u32 sysmon_pid; + int sysmon_shutdown_ret; + int sysmon_dsentry_ret; + bool system_debug; + bool ignore_ssr_failure; + const char *edge; + struct qcom_smem_state *state; +#ifdef CONFIG_SETUP_SSR_NOTIF_TIMEOUTS + struct subsys_notif_timeout timeout_data; +#endif /* CONFIG_SETUP_SSR_NOTIF_TIMEOUTS */ +}; + +/** + * struct notif_data - additional notif information + * @crashed: indicates if subsystem has crashed due to wdog bite or err fatal + * @enable_ramdump: ramdumps disabled if set to 0 + * @enable_mini_ramdumps: enable flag for minimized critical-memory-only + * ramdumps + * @no_auth: set if subsystem does not use PIL to bring it out of reset + * @pdev: subsystem platform device pointer + */ +struct notif_data { + enum crash_status crashed; + int enable_ramdump; + int enable_mini_ramdumps; + bool no_auth; + struct platform_device *pdev; +}; + +#if IS_ENABLED(CONFIG_MSM_SUBSYSTEM_RESTART) + +extern int subsys_get_restart_level(struct subsys_device *dev); +extern int subsystem_restart_dev(struct subsys_device *dev); +extern int subsystem_restart(const char *name); +extern int subsystem_crashed(const char *name); +extern int subsystem_start_notify(const char *name); +extern int subsystem_stop_notify(const char *subsystem); +extern int subsystem_ds_entry(const char *subsystem); +extern int subsystem_ds_exit(const char *name); +extern int subsystem_s2d_entry(const char *subsystem); +extern int subsystem_s2d_exit(const char *name); + +extern void *subsystem_get(const char *name); +extern void *subsystem_get_with_fwname(const char *name, const char *fw_name); +extern int subsystem_set_fwname(const char *name, const char *fw_name); +extern void subsystem_put(void *subsystem); + +extern struct subsys_device *subsys_register(struct subsys_desc *desc); +extern void subsys_unregister(struct subsys_device *dev); + +extern void subsys_set_crash_status(struct subsys_device *dev, + enum crash_status crashed); +extern enum crash_status subsys_get_crash_status(struct subsys_device *dev); +void notify_proxy_vote(struct device *device); +void notify_proxy_unvote(struct device *device); +void notify_before_auth_and_reset(struct device *device); +static inline void complete_shutdown_ack(struct subsys_desc *desc) +{ + complete(&desc->shutdown_ack); +} +static inline void complete_dsentry_ack(struct subsys_desc *desc) +{ + complete(&desc->dsentry_ack); +} +struct subsys_device *find_subsys_device(const char *str); +#else + +static inline int subsys_get_restart_level(struct subsys_device *dev) +{ + return 0; +} + +static inline int subsystem_restart_dev(struct subsys_device *dev) +{ + return 0; +} + +static inline int subsystem_restart(const char *name) +{ + return 0; +} + +static inline int subsystem_crashed(const char *name) +{ + return 0; +} + +extern int subsystem_start_notify(const char *name) +{ + return 0; +} + +extern int subsystem_stop_notify(const char *subsystem) +{ + return 0; +} + +/** + * static int subsystem_ds_entry(const char *subsystem) + * { + * return 0; + * } + * + * static int subsystem_ds_exit(const char *name) + * { + * return 0; + * } + * + * static int subsystem_s2d_exit(const char *name) + * { + * return 0; + * } + * + * static int subsystem_s2d_entry(const char *name) + * { + * return 0; + * } + */ + +static inline void *subsystem_get(const char *name) +{ + return NULL; +} + +static inline void *subsystem_get_with_fwname(const char *name, + const char *fw_name) { + return NULL; +} + +static inline int subsystem_set_fwname(const char *name, + const char *fw_name) { + return 0; +} + +static inline void subsystem_put(void *subsystem) { } + +static inline +struct subsys_device *subsys_register(struct subsys_desc *desc) +{ + return NULL; +} + +static inline void subsys_unregister(struct subsys_device *dev) { } + +static inline void subsys_set_crash_status(struct subsys_device *dev, + enum crash_status crashed) { } +static inline +enum crash_status subsys_get_crash_status(struct subsys_device *dev) +{ + return false; +} +static inline void notify_proxy_vote(struct device *device) { } +static inline void notify_proxy_unvote(struct device *device) { } +static inline void notify_before_auth_and_reset(struct device *device) { } +#endif +/* CONFIG_MSM_SUBSYSTEM_RESTART */ + +/* Helper wrappers */ +static inline void wakeup_source_trash(struct wakeup_source *ws) +{ + if (!ws) + return; + + wakeup_source_remove(ws); + __pm_relax(ws); +} + +#endif