android/android_kernel_samsung_sm8650-modules
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
lineage-22.1
android_kernel_samsung_sm86.../qcom/opensource/graphics-kernel/adreno_a6xx_gmu.c
David Wronek 880d405719 Add 'qcom/opensource/graphics-kernel/' from commit 'b4fdc4c04295ac59109ae19d64747522740c3f14' 
git-subtree-dir: qcom/opensource/graphics-kernel
git-subtree-mainline: 992813d9c1
git-subtree-split: b4fdc4c042
Change-Id:
repo: https://git.codelinaro.org/clo/la/platform/vendor/qcom/opensource/graphics-kernel
tag: GRAPHICS.LA.14.0.r1-07700-lanai.0
2024-10-06 16:44:56 +02:00

3864 lines
101 KiB
C
Raw Permalink Blame History

// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) 2018-2021, The Linux Foundation. All rights reserved.
* Copyright (c) 2022-2024, Qualcomm Innovation Center, Inc. All rights reserved.
*/
#include <dt-bindings/regulator/qcom,rpmh-regulator-levels.h>
#include <linux/clk.h>
#include <linux/component.h>
#include <linux/delay.h>
#include <linux/dma-map-ops.h>
#include <linux/firmware.h>
#include <linux/interconnect.h>
#include <linux/io.h>
#include <linux/kobject.h>
#include <linux/of_platform.h>
#include <linux/qcom-iommu-util.h>
#include <linux/regulator/consumer.h>
#include <linux/slab.h>
#include <linux/sysfs.h>
#include <linux/mailbox/qmp.h>
#include <soc/qcom/cmd-db.h>
#include "adreno.h"
#include "adreno_a6xx.h"
#include "adreno_trace.h"
#include "kgsl_bus.h"
#include "kgsl_device.h"
#include "kgsl_trace.h"
#include "kgsl_util.h"
#define ARC_VOTE_GET_PRI(_v) ((_v) & 0xFF)
#define ARC_VOTE_GET_SEC(_v) (((_v) >> 8) & 0xFF)
#define ARC_VOTE_GET_VLVL(_v) (((_v) >> 16) & 0xFFFF)
#define ARC_VOTE_SET(pri, sec, vlvl) \
((((vlvl) & 0xFFFF) << 16) | (((sec) & 0xFF) << 8) | ((pri) & 0xFF))
static struct gmu_vma_entry a6xx_gmu_vma_legacy[] = {
[GMU_ITCM] = {
.start = 0x00000,
.size = SZ_16K
},
[GMU_ICACHE] = {
.start = 0x04000,
.size = (SZ_256K - SZ_16K),
.next_va = 0x4000
},
[GMU_DTCM] = {
.start = 0x40000,
.size = SZ_16K
},
[GMU_DCACHE] = {
.start = 0x44000,
.size = (SZ_256K - SZ_16K),
.next_va = 0x44000
},
[GMU_NONCACHED_KERNEL] = {
.start = 0x60000000,
.size = SZ_512M,
.next_va = 0x60000000
},
};
static struct gmu_vma_entry a6xx_gmu_vma[] = {
[GMU_ITCM] = {
.start = 0x00000000,
.size = SZ_16K
},
[GMU_CACHE] = {
.start = SZ_16K,
.size = (SZ_16M - SZ_16K),
.next_va = SZ_16K
},
[GMU_DTCM] = {
.start = SZ_256M + SZ_16K,
.size = SZ_16K
},
[GMU_DCACHE] = {
.start = 0x0,
.size = 0x0
},
[GMU_NONCACHED_KERNEL] = {
.start = 0x60000000,
.size = SZ_512M,
.next_va = 0x60000000
},
};
static void _regwrite(void __iomem *regbase, u32 offsetwords, u32 value)
{
void __iomem *reg;
reg = regbase + (offsetwords << 2);
__raw_writel(value, reg);
}
static void _regrmw(void __iomem *regbase, u32 offsetwords, u32 mask, u32 or)
{
void __iomem *reg;
u32 val;
reg = regbase + (offsetwords << 2);
val = __raw_readl(reg);
/* Make sure the read posted and all pending writes are done */
mb();
__raw_writel((val & ~mask) | or, reg);
}
static ssize_t log_stream_enable_store(struct kobject *kobj,
struct kobj_attribute *attr, const char *buf, size_t count)
{
struct a6xx_gmu_device *gmu = container_of(kobj, struct a6xx_gmu_device, log_kobj);
bool val;
int ret;
ret = kstrtobool(buf, &val);
if (ret)
return ret;
gmu->log_stream_enable = val;
return count;
}
static ssize_t log_stream_enable_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
{
struct a6xx_gmu_device *gmu = container_of(kobj, struct a6xx_gmu_device, log_kobj);
return scnprintf(buf, PAGE_SIZE, "%d\n", gmu->log_stream_enable);
}
static ssize_t log_group_mask_store(struct kobject *kobj,
struct kobj_attribute *attr, const char *buf, size_t count)
{
struct a6xx_gmu_device *gmu = container_of(kobj, struct a6xx_gmu_device, log_kobj);
u32 val;
int ret;
ret = kstrtou32(buf, 0, &val);
if (ret)
return ret;
gmu->log_group_mask = val;
return count;
}
static ssize_t log_group_mask_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
{
struct a6xx_gmu_device *gmu = container_of(kobj, struct a6xx_gmu_device, log_kobj);
return scnprintf(buf, PAGE_SIZE, "%x\n", gmu->log_group_mask);
}
static struct kobj_attribute log_stream_enable_attr =
__ATTR(log_stream_enable, 0644, log_stream_enable_show, log_stream_enable_store);
static struct kobj_attribute log_group_mask_attr =
__ATTR(log_group_mask, 0644, log_group_mask_show, log_group_mask_store);
static struct attribute *log_attrs[] = {
&log_stream_enable_attr.attr,
&log_group_mask_attr.attr,
NULL,
};
ATTRIBUTE_GROUPS(log);
static struct kobj_type log_kobj_type = {
.sysfs_ops = &kobj_sysfs_ops,
.default_groups = log_groups,
};
static ssize_t stats_enable_store(struct kobject *kobj,
struct kobj_attribute *attr, const char *buf, size_t count)
{
struct a6xx_gmu_device *gmu = container_of(kobj, struct a6xx_gmu_device, stats_kobj);
bool val;
int ret;
ret = kstrtobool(buf, &val);
if (ret)
return ret;
gmu->stats_enable = val;
return count;
}
static ssize_t stats_enable_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
{
struct a6xx_gmu_device *gmu = container_of(kobj, struct a6xx_gmu_device, stats_kobj);
return scnprintf(buf, PAGE_SIZE, "%d\n", gmu->stats_enable);
}
static ssize_t stats_mask_store(struct kobject *kobj,
struct kobj_attribute *attr, const char *buf, size_t count)
{
struct a6xx_gmu_device *gmu = container_of(kobj, struct a6xx_gmu_device, stats_kobj);
u32 val;
int ret;
ret = kstrtou32(buf, 0, &val);
if (ret)
return ret;
gmu->stats_mask = val;
return count;
}
static ssize_t stats_mask_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
{
struct a6xx_gmu_device *gmu = container_of(kobj, struct a6xx_gmu_device, stats_kobj);
return scnprintf(buf, PAGE_SIZE, "%x\n", gmu->stats_mask);
}
static ssize_t stats_interval_store(struct kobject *kobj,
struct kobj_attribute *attr, const char *buf, size_t count)
{
struct a6xx_gmu_device *gmu = container_of(kobj, struct a6xx_gmu_device, stats_kobj);
u32 val;
int ret;
ret = kstrtou32(buf, 0, &val);
if (ret)
return ret;
gmu->stats_interval = val;
return count;
}
static ssize_t stats_interval_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
{
struct a6xx_gmu_device *gmu = container_of(kobj, struct a6xx_gmu_device, stats_kobj);
return scnprintf(buf, PAGE_SIZE, "%x\n", gmu->stats_interval);
}
static struct kobj_attribute stats_enable_attr =
__ATTR(stats_enable, 0644, stats_enable_show, stats_enable_store);
static struct kobj_attribute stats_mask_attr =
__ATTR(stats_mask, 0644, stats_mask_show, stats_mask_store);
static struct kobj_attribute stats_interval_attr =
__ATTR(stats_interval, 0644, stats_interval_show, stats_interval_store);
static struct attribute *stats_attrs[] = {
&stats_enable_attr.attr,
&stats_mask_attr.attr,
&stats_interval_attr.attr,
NULL,
};
ATTRIBUTE_GROUPS(stats);
static struct kobj_type stats_kobj_type = {
.sysfs_ops = &kobj_sysfs_ops,
.default_groups = stats_groups,
};
static int timed_poll_check_rscc(struct kgsl_device *device,
unsigned int offset, unsigned int expected_ret,
unsigned int timeout, unsigned int mask)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
u32 value;
if (!adreno_is_a650_family(adreno_dev))
return gmu_core_timed_poll_check(device,
offset + RSCC_OFFSET_LEGACY,
expected_ret, timeout, mask);
return readl_poll_timeout(gmu->rscc_virt + (offset << 2), value,
(value & mask) == expected_ret, 100, timeout * 1000);
}
struct a6xx_gmu_device *to_a6xx_gmu(struct adreno_device *adreno_dev)
{
struct a6xx_device *a6xx_dev = container_of(adreno_dev,
struct a6xx_device, adreno_dev);
return &a6xx_dev->gmu;
}
struct adreno_device *a6xx_gmu_to_adreno(struct a6xx_gmu_device *gmu)
{
struct a6xx_device *a6xx_dev =
container_of(gmu, struct a6xx_device, gmu);
return &a6xx_dev->adreno_dev;
}
#define RSC_CMD_OFFSET 2
#define PDC_CMD_OFFSET 4
#define PDC_ENABLE_REG_VALUE 0x80000001
void a6xx_load_rsc_ucode(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
void __iomem *rscc;
if (adreno_is_a650_family(adreno_dev))
rscc = gmu->rscc_virt;
else
rscc = kgsl_regmap_virt(&device->regmap, RSCC_OFFSET_LEGACY);
/* Disable SDE clock gating */
_regwrite(rscc, A6XX_GPU_RSCC_RSC_STATUS0_DRV0, BIT(24));
/* Setup RSC PDC handshake for sleep and wakeup */
_regwrite(rscc, A6XX_RSCC_PDC_SLAVE_ID_DRV0, 1);
_regwrite(rscc, A6XX_RSCC_HIDDEN_TCS_CMD0_DATA, 0);
_regwrite(rscc, A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR, 0);
_regwrite(rscc, A6XX_RSCC_HIDDEN_TCS_CMD0_DATA + RSC_CMD_OFFSET, 0);
_regwrite(rscc, A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR + RSC_CMD_OFFSET, 0);
_regwrite(rscc, A6XX_RSCC_HIDDEN_TCS_CMD0_DATA + RSC_CMD_OFFSET * 2,
0x80000000);
_regwrite(rscc, A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR + RSC_CMD_OFFSET * 2,
0);
_regwrite(rscc, A6XX_RSCC_OVERRIDE_START_ADDR, 0);
_regwrite(rscc, A6XX_RSCC_PDC_SEQ_START_ADDR, 0x4520);
_regwrite(rscc, A6XX_RSCC_PDC_MATCH_VALUE_LO, 0x4510);
_regwrite(rscc, A6XX_RSCC_PDC_MATCH_VALUE_HI, 0x4514);
/* Load RSC sequencer uCode for sleep and wakeup */
if (adreno_is_a650_family(adreno_dev)) {
_regwrite(rscc, A6XX_RSCC_SEQ_MEM_0_DRV0, 0xEAAAE5A0);
_regwrite(rscc, A6XX_RSCC_SEQ_MEM_0_DRV0 + 1, 0xE1A1EBAB);
_regwrite(rscc, A6XX_RSCC_SEQ_MEM_0_DRV0 + 2, 0xA2E0A581);
_regwrite(rscc, A6XX_RSCC_SEQ_MEM_0_DRV0 + 3, 0xECAC82E2);
_regwrite(rscc, A6XX_RSCC_SEQ_MEM_0_DRV0 + 4, 0x0020EDAD);
} else {
_regwrite(rscc, A6XX_RSCC_SEQ_MEM_0_DRV0, 0xA7A506A0);
_regwrite(rscc, A6XX_RSCC_SEQ_MEM_0_DRV0 + 1, 0xA1E6A6E7);
_regwrite(rscc, A6XX_RSCC_SEQ_MEM_0_DRV0 + 2, 0xA2E081E1);
_regwrite(rscc, A6XX_RSCC_SEQ_MEM_0_DRV0 + 3, 0xE9A982E2);
_regwrite(rscc, A6XX_RSCC_SEQ_MEM_0_DRV0 + 4, 0x0020E8A8);
}
}
int a6xx_load_pdc_ucode(struct adreno_device *adreno_dev)
{
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
struct resource *res_pdc, *res_cfg, *res_seq;
unsigned int cfg_offset, seq_offset;
void __iomem *cfg = NULL, *seq = NULL;
const struct adreno_a6xx_core *a6xx_core = to_a6xx_core(adreno_dev);
u32 vrm_resource_addr = cmd_db_read_addr("vrm.soc");
u32 xo_resource_addr = cmd_db_read_addr("xo.lvl");
u32 cx_res_addr = cmd_db_read_addr("cx.lvl");
u32 mx_res_addr = cmd_db_read_addr("mx.lvl");
if (!xo_resource_addr) {
dev_err(&gmu->pdev->dev,
"Failed to get 'xo.lvl' addr from cmd_db\n");
return -ENOENT;
}
if (!cx_res_addr) {
dev_err(&gmu->pdev->dev,
"Failed to get 'cx.lvl' addr from cmd_db\n");
return -ENOENT;
}
if (!mx_res_addr) {
dev_err(&gmu->pdev->dev,
"Failed to get 'mx.lvl' addr from cmd_db\n");
return -ENOENT;
}
/*
* Older A6x platforms specified PDC registers in the DT using a
* single base pointer that encompassed the entire PDC range. Current
* targets specify the individual GPU-owned PDC register blocks
* (sequence and config).
*
* This code handles both possibilities and generates individual
* pointers to the GPU PDC blocks, either as offsets from the single
* base, or as directly specified ranges.
*
* PDC programming has moved to AOP for newer A6x platforms.
* However registers to enable GPU PDC and set the sequence start
* address still need to be programmed.
*/
/* Offsets from the base PDC (if no PDC subsections in the DTSI) */
if (adreno_is_a640v2(adreno_dev)) {
cfg_offset = 0x90000;
seq_offset = 0x290000;
} else {
cfg_offset = 0x80000;
seq_offset = 0x280000;
}
/* Get pointers to each of the possible PDC resources */
res_pdc = platform_get_resource_byname(gmu->pdev, IORESOURCE_MEM,
"kgsl_gmu_pdc_reg");
res_cfg = platform_get_resource_byname(gmu->pdev, IORESOURCE_MEM,
"kgsl_gmu_pdc_cfg");
/*
* Map the starting address for pdc_cfg programming. If the pdc_cfg
* resource is not available use an offset from the base PDC resource.
*/
if (gmu->pdc_cfg_base == NULL) {
if (res_cfg)
gmu->pdc_cfg_base = devm_ioremap(&gmu->pdev->dev,
res_cfg->start, resource_size(res_cfg));
else if (res_pdc)
gmu->pdc_cfg_base = devm_ioremap(&gmu->pdev->dev,
res_pdc->start + cfg_offset, 0x10000);
if (!gmu->pdc_cfg_base) {
dev_err(&gmu->pdev->dev, "Failed to map PDC CFG\n");
return -ENODEV;
}
}
cfg = gmu->pdc_cfg_base;
/* PDC is programmed in AOP for newer platforms */
if (a6xx_core->pdc_in_aop)
goto done;
/*
* Map the starting address for pdc_seq programming. If the pdc_seq
* resource is not available use an offset from the base PDC resource.
*/
if (gmu->pdc_seq_base == NULL) {
res_seq = platform_get_resource_byname(gmu->pdev, IORESOURCE_MEM,
"kgsl_gmu_pdc_seq");
if (res_seq)
gmu->pdc_seq_base = devm_ioremap(&gmu->pdev->dev,
res_seq->start, resource_size(res_seq));
else if (res_pdc)
gmu->pdc_seq_base = devm_ioremap(&gmu->pdev->dev,
res_pdc->start + seq_offset, 0x10000);
if (!gmu->pdc_seq_base) {
dev_err(&gmu->pdev->dev, "Failed to map PDC SEQ\n");
return -ENODEV;
}
}
seq = gmu->pdc_seq_base;
/* Load PDC sequencer uCode for power up and power down sequence */
_regwrite(seq, PDC_GPU_SEQ_MEM_0, 0xFEBEA1E1);
_regwrite(seq, PDC_GPU_SEQ_MEM_0 + 1, 0xA5A4A3A2);
_regwrite(seq, PDC_GPU_SEQ_MEM_0 + 2, 0x8382A6E0);
_regwrite(seq, PDC_GPU_SEQ_MEM_0 + 3, 0xBCE3E284);
_regwrite(seq, PDC_GPU_SEQ_MEM_0 + 4, 0x002081FC);
/* Set TCS commands used by PDC sequence for low power modes */
_regwrite(cfg, PDC_GPU_TCS1_CMD_ENABLE_BANK, 7);
_regwrite(cfg, PDC_GPU_TCS1_CMD_WAIT_FOR_CMPL_BANK, 0);
_regwrite(cfg, PDC_GPU_TCS1_CONTROL, 0);
_regwrite(cfg, PDC_GPU_TCS1_CMD0_MSGID, 0x10108);
_regwrite(cfg, PDC_GPU_TCS1_CMD0_ADDR, mx_res_addr);
_regwrite(cfg, PDC_GPU_TCS1_CMD0_DATA, 1);
_regwrite(cfg, PDC_GPU_TCS1_CMD0_MSGID + PDC_CMD_OFFSET, 0x10108);
_regwrite(cfg, PDC_GPU_TCS1_CMD0_ADDR + PDC_CMD_OFFSET, cx_res_addr);
_regwrite(cfg, PDC_GPU_TCS1_CMD0_DATA + PDC_CMD_OFFSET, 0x0);
_regwrite(cfg, PDC_GPU_TCS1_CMD0_MSGID + PDC_CMD_OFFSET * 2, 0x10108);
_regwrite(cfg, PDC_GPU_TCS1_CMD0_ADDR + PDC_CMD_OFFSET * 2,
xo_resource_addr);
_regwrite(cfg, PDC_GPU_TCS1_CMD0_DATA + PDC_CMD_OFFSET * 2, 0x0);
if (vrm_resource_addr && adreno_is_a620(adreno_dev)) {
_regwrite(cfg, PDC_GPU_TCS1_CMD0_MSGID + PDC_CMD_OFFSET * 3,
0x10108);
_regwrite(cfg, PDC_GPU_TCS1_CMD0_ADDR + PDC_CMD_OFFSET * 3,
vrm_resource_addr + 0x4);
_regwrite(cfg, PDC_GPU_TCS1_CMD0_DATA + PDC_CMD_OFFSET * 3,
0x0);
}
_regwrite(cfg, PDC_GPU_TCS3_CMD_ENABLE_BANK, 7);
_regwrite(cfg, PDC_GPU_TCS3_CMD_WAIT_FOR_CMPL_BANK, 0);
_regwrite(cfg, PDC_GPU_TCS3_CONTROL, 0);
_regwrite(cfg, PDC_GPU_TCS3_CMD0_MSGID, 0x10108);
_regwrite(cfg, PDC_GPU_TCS3_CMD0_ADDR, mx_res_addr);
_regwrite(cfg, PDC_GPU_TCS3_CMD0_DATA, 2);
_regwrite(cfg, PDC_GPU_TCS3_CMD0_MSGID + PDC_CMD_OFFSET, 0x10108);
_regwrite(cfg, PDC_GPU_TCS3_CMD0_ADDR + PDC_CMD_OFFSET, cx_res_addr);
if (adreno_is_a618(adreno_dev) || adreno_is_a619(adreno_dev) ||
adreno_is_a650_family(adreno_dev))
_regwrite(cfg, PDC_GPU_TCS3_CMD0_DATA + PDC_CMD_OFFSET, 0x2);
else
_regwrite(cfg, PDC_GPU_TCS3_CMD0_DATA + PDC_CMD_OFFSET, 0x3);
_regwrite(cfg, PDC_GPU_TCS3_CMD0_MSGID + PDC_CMD_OFFSET * 2, 0x10108);
_regwrite(cfg, PDC_GPU_TCS3_CMD0_ADDR + PDC_CMD_OFFSET * 2,
xo_resource_addr);
_regwrite(cfg, PDC_GPU_TCS3_CMD0_DATA + PDC_CMD_OFFSET * 2, 0x3);
if (vrm_resource_addr && adreno_is_a620(adreno_dev)) {
_regwrite(cfg, PDC_GPU_TCS3_CMD0_MSGID + PDC_CMD_OFFSET * 3,
0x10108);
_regwrite(cfg, PDC_GPU_TCS3_CMD0_ADDR + PDC_CMD_OFFSET * 3,
vrm_resource_addr + 0x4);
_regwrite(cfg, PDC_GPU_TCS3_CMD0_DATA + PDC_CMD_OFFSET * 3,
0x1);
}
done:
/* Setup GPU PDC */
_regwrite(cfg, PDC_GPU_SEQ_START_ADDR, 0);
_regwrite(cfg, PDC_GPU_ENABLE_PDC, PDC_ENABLE_REG_VALUE);
/* ensure no writes happen before the uCode is fully written */
wmb();
return 0;
}
/* GMU timeouts */
#define GMU_IDLE_TIMEOUT 100 /* ms */
#define GMU_START_TIMEOUT 100 /* ms */
#define GPU_START_TIMEOUT 100 /* ms */
#define GPU_RESET_TIMEOUT 1 /* ms */
#define GPU_RESET_TIMEOUT_US 10 /* us */
/*
* The lowest 16 bits of this value are the number of XO clock cycles
* for main hysteresis. This is the first hysteresis. Here we set it
* to 0x1680 cycles, or 300 us. The highest 16 bits of this value are
* the number of XO clock cycles for short hysteresis. This happens
* after main hysteresis. Here we set it to 0xA cycles, or 0.5 us.
*/
#define A6X_GMU_LONG_IFPC_HYST FIELD_PREP(GENMASK(15, 0), 0x1680)
#define A6X_GMU_SHORT_IFPC_HYST FIELD_PREP(GENMASK(31, 16), 0xA)
/* Minimum IFPC timer (200usec) allowed to override default value */
#define A6X_GMU_LONG_IFPC_HYST_FLOOR FIELD_PREP(GENMASK(15, 0), 0x0F00)
/*
* a6xx_gmu_power_config() - Configure and enable GMU's low power mode
* setting based on ADRENO feature flags.
* @adreno_dev: Pointer to adreno device
*/
static void a6xx_gmu_power_config(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
/* Configure registers for idle setting. The setting is cumulative */
/* Disable GMU WB/RB buffer and caches at boot */
gmu_core_regwrite(device, A6XX_GMU_SYS_BUS_CONFIG, 0x1);
gmu_core_regwrite(device, A6XX_GMU_ICACHE_CONFIG, 0x1);
gmu_core_regwrite(device, A6XX_GMU_DCACHE_CONFIG, 0x1);
gmu_core_regwrite(device,
A6XX_GMU_PWR_COL_INTER_FRAME_CTRL, 0x9C40400);
if (gmu->idle_level == GPU_HW_IFPC) {
gmu_core_regwrite(device, A6XX_GMU_PWR_COL_INTER_FRAME_HYST,
A6X_GMU_SHORT_IFPC_HYST | adreno_dev->ifpc_hyst);
gmu_core_regrmw(device, A6XX_GMU_PWR_COL_INTER_FRAME_CTRL,
IFPC_ENABLE_MASK, IFPC_ENABLE_MASK);
gmu_core_regwrite(device, A6XX_GMU_PWR_COL_SPTPRAC_HYST,
A6X_GMU_SHORT_IFPC_HYST | adreno_dev->ifpc_hyst);
gmu_core_regrmw(device, A6XX_GMU_PWR_COL_INTER_FRAME_CTRL,
SPTP_ENABLE_MASK, SPTP_ENABLE_MASK);
}
/* Enable RPMh GPU client */
gmu_core_regrmw(device, A6XX_GMU_RPMH_CTRL, RPMH_ENABLE_MASK,
RPMH_ENABLE_MASK);
}
static void gmu_ao_sync_event(struct adreno_device *adreno_dev)
{
unsigned long flags;
u64 ticks;
local_irq_save(flags);
/* Read GMU always on register */
ticks = a6xx_read_alwayson(adreno_dev);
/* Trace the GMU time to create a mapping to ftrace time */
trace_gmu_ao_sync(ticks);
local_irq_restore(flags);
}
void a6xx_gmu_disable_gdsc(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct kgsl_pwrctrl *pwr = &device->pwrctrl;
if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_CX_GDSC))
regulator_set_mode(pwr->cx_gdsc, REGULATOR_MODE_IDLE);
kgsl_pwrctrl_disable_cx_gdsc(device);
if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_CX_GDSC))
regulator_set_mode(pwr->cx_gdsc, REGULATOR_MODE_NORMAL);
}
int a6xx_gmu_device_start(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
u32 val = 0x00000100;
u32 mask = 0x000001FF;
gmu_core_reset_trace_header(&gmu->trace);
gmu_ao_sync_event(adreno_dev);
/* Check for 0xBABEFACE on legacy targets */
if (gmu->ver.core <= 0x20010004) {
val = 0xBABEFACE;
mask = 0xFFFFFFFF;
}
/* Bring GMU out of reset */
gmu_core_regwrite(device, A6XX_GMU_CM3_SYSRESET, 0);
/* Make sure the write is posted before moving ahead */
wmb();
if (gmu_core_timed_poll_check(device,
A6XX_GMU_CM3_FW_INIT_RESULT,
val, GMU_START_TIMEOUT, mask)) {
dev_err(&gmu->pdev->dev, "GMU doesn't boot\n");
gmu_core_fault_snapshot(device);
return -ETIMEDOUT;
}
return 0;
}
/*
* a6xx_gmu_hfi_start() - Write registers and start HFI.
* @device: Pointer to KGSL device
*/
int a6xx_gmu_hfi_start(struct adreno_device *adreno_dev)
{
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
gmu_core_regwrite(device, A6XX_GMU_HFI_CTRL_INIT, 1);
if (gmu_core_timed_poll_check(device,
A6XX_GMU_HFI_CTRL_STATUS,
BIT(0),
GMU_START_TIMEOUT,
BIT(0))) {
dev_err(&gmu->pdev->dev, "GMU HFI init failed\n");
gmu_core_fault_snapshot(device);
return -ETIMEDOUT;
}
return 0;
}
int a6xx_rscc_wakeup_sequence(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
struct device *dev = &gmu->pdev->dev;
int val;
/* Skip wakeup sequence if we didn't do the sleep sequence */
if (!test_bit(GMU_PRIV_RSCC_SLEEP_DONE, &gmu->flags))
return 0;
/* A660 has a replacement register */
if (adreno_is_a662(adreno_dev) || adreno_is_a621(adreno_dev))
gmu_core_regread(device, A662_GPU_CC_GX_DOMAIN_MISC3, &val);
else if (adreno_is_a660(ADRENO_DEVICE(device)) ||
adreno_is_a663(adreno_dev))
gmu_core_regread(device, A6XX_GPU_CC_GX_DOMAIN_MISC3, &val);
else
gmu_core_regread(device, A6XX_GPU_CC_GX_DOMAIN_MISC, &val);
if (!(val & 0x1))
dev_info_ratelimited(&gmu->pdev->dev,
"GMEM CLAMP IO not set while GFX rail off\n");
/* RSC wake sequence */
gmu_core_regwrite(device, A6XX_GMU_RSCC_CONTROL_REQ, BIT(1));
/* Write request before polling */
wmb();
if (gmu_core_timed_poll_check(device,
A6XX_GMU_RSCC_CONTROL_ACK,
BIT(1),
GPU_START_TIMEOUT,
BIT(1))) {
dev_err(dev, "Failed to do GPU RSC power on\n");
return -ETIMEDOUT;
}
if (timed_poll_check_rscc(device,
A6XX_RSCC_SEQ_BUSY_DRV0,
0,
GPU_START_TIMEOUT,
0xFFFFFFFF))
goto error_rsc;
gmu_core_regwrite(device, A6XX_GMU_RSCC_CONTROL_REQ, 0);
clear_bit(GMU_PRIV_RSCC_SLEEP_DONE, &gmu->flags);
return 0;
error_rsc:
dev_err(dev, "GPU RSC sequence stuck in waking up GPU\n");
return -ETIMEDOUT;
}
int a6xx_rscc_sleep_sequence(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
int ret;
if (!test_bit(GMU_PRIV_FIRST_BOOT_DONE, &gmu->flags))
return 0;
if (test_bit(GMU_PRIV_RSCC_SLEEP_DONE, &gmu->flags))
return 0;
gmu_core_regwrite(device, A6XX_GMU_CM3_SYSRESET, 1);
/* Make sure M3 is in reset before going on */
wmb();
gmu_core_regread(device, A6XX_GPU_GMU_CX_GMU_PWR_COL_CP_RESP,
&gmu->log_wptr_retention);
gmu_core_regwrite(device, A6XX_GMU_RSCC_CONTROL_REQ, 1);
/* Make sure the request completes before continuing */
wmb();
ret = timed_poll_check_rscc(device,
A6XX_GPU_RSCC_RSC_STATUS0_DRV0,
BIT(16),
GPU_START_TIMEOUT,
BIT(16));
if (ret) {
dev_err(&gmu->pdev->dev, "GPU RSC power off fail\n");
return -ETIMEDOUT;
}
gmu_core_regwrite(device, A6XX_GMU_RSCC_CONTROL_REQ, 0);
if (adreno_dev->lm_enabled)
gmu_core_regwrite(device, A6XX_GMU_AO_SPARE_CNTL, 0);
set_bit(GMU_PRIV_RSCC_SLEEP_DONE, &gmu->flags);
return 0;
}
static struct kgsl_memdesc *find_gmu_memdesc(struct a6xx_gmu_device *gmu,
u32 addr, u32 size)
{
int i;
for (i = 0; i < gmu->global_entries; i++) {
struct kgsl_memdesc *md = &gmu->gmu_globals[i];
if ((addr >= md->gmuaddr) &&
(((addr + size) <= (md->gmuaddr + md->size))))
return md;
}
return NULL;
}
static int find_vma_block(struct a6xx_gmu_device *gmu, u32 addr, u32 size)
{
int i;
for (i = 0; i < GMU_MEM_TYPE_MAX; i++) {
struct gmu_vma_entry *vma = &gmu->vma[i];
if ((addr >= vma->start) &&
((addr + size) <= (vma->start + vma->size)))
return i;
}
return -ENOENT;
}
#define MAX_GMUFW_SIZE 0x8000 /* in bytes */
static int _load_legacy_gmu_fw(struct kgsl_device *device,
struct a6xx_gmu_device *gmu)
{
const struct firmware *fw = gmu->fw_image;
if (fw->size > MAX_GMUFW_SIZE)
return -EINVAL;
gmu_core_blkwrite(device, A6XX_GMU_CM3_ITCM_START, fw->data,
fw->size);
/* Proceed only after the FW is written */
wmb();
return 0;
}
static void load_tcm(struct adreno_device *adreno_dev, const u8 *src,
u32 tcm_start, u32 base, const struct gmu_block_header *blk)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
u32 tcm_offset = tcm_start + ((blk->addr - base)/sizeof(u32));
void __iomem *addr = kgsl_regmap_virt(&device->regmap, tcm_offset);
memcpy_toio(addr, src, blk->size);
}
int a6xx_gmu_load_fw(struct adreno_device *adreno_dev)
{
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
const u8 *fw = (const u8 *)gmu->fw_image->data;
if (adreno_is_a630(adreno_dev) || adreno_is_a615_family(adreno_dev))
return _load_legacy_gmu_fw(KGSL_DEVICE(adreno_dev), gmu);
while (fw < gmu->fw_image->data + gmu->fw_image->size) {
const struct gmu_block_header *blk =
(const struct gmu_block_header *)fw;
int id;
fw += sizeof(*blk);
/* Don't deal with zero size blocks */
if (blk->size == 0)
continue;
id = find_vma_block(gmu, blk->addr, blk->size);
if (id < 0) {
dev_err(&gmu->pdev->dev,
"Unknown block in GMU FW addr:0x%x size:0x%x\n",
blk->addr, blk->size);
return -EINVAL;
}
if (id == GMU_ITCM) {
load_tcm(adreno_dev, fw,
A6XX_GMU_CM3_ITCM_START,
gmu->vma[GMU_ITCM].start, blk);
} else if (id == GMU_DTCM) {
load_tcm(adreno_dev, fw,
A6XX_GMU_CM3_DTCM_START,
gmu->vma[GMU_DTCM].start, blk);
} else {
struct kgsl_memdesc *md =
find_gmu_memdesc(gmu, blk->addr, blk->size);
if (!md) {
dev_err(&gmu->pdev->dev,
"No backing memory for GMU FW block addr:0x%x size:0x%x\n",
blk->addr, blk->size);
return -EINVAL;
}
memcpy(md->hostptr + (blk->addr - md->gmuaddr), fw,
blk->size);
}
fw += blk->size;
}
/* Proceed only after the FW is written */
wmb();
return 0;
}
static const char *oob_to_str(enum oob_request req)
{
if (req == oob_gpu)
return "oob_gpu";
else if (req == oob_perfcntr)
return "oob_perfcntr";
else if (req == oob_boot_slumber)
return "oob_boot_slumber";
else if (req == oob_dcvs)
return "oob_dcvs";
return "unknown";
}
static void trigger_reset_recovery(struct adreno_device *adreno_dev,
enum oob_request req)
{
/*
* Trigger recovery for perfcounter oob only since only
* perfcounter oob can happen alongside an actively rendering gpu.
*/
if (req != oob_perfcntr)
return;
if (adreno_dev->dispatch_ops && adreno_dev->dispatch_ops->fault)
adreno_dev->dispatch_ops->fault(adreno_dev,
ADRENO_GMU_FAULT_SKIP_SNAPSHOT);
}
int a6xx_gmu_oob_set(struct kgsl_device *device,
enum oob_request req)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
int ret = 0;
int set, check;
if (req == oob_perfcntr && gmu->num_oob_perfcntr++)
return 0;
if (adreno_is_a630(adreno_dev) || adreno_is_a615_family(adreno_dev)) {
set = BIT(req + 16);
check = BIT(req + 24);
} else {
/*
* The legacy targets have special bits that aren't supported on
* newer implementations
*/
if (req >= oob_boot_slumber) {
dev_err(&gmu->pdev->dev,
"Unsupported OOB request %s\n",
oob_to_str(req));
return -EINVAL;
}
set = BIT(30 - req * 2);
check = BIT(31 - req);
}
gmu_core_regwrite(device, A6XX_GMU_HOST2GMU_INTR_SET, set);
if (gmu_core_timed_poll_check(device, A6XX_GMU_GMU2HOST_INTR_INFO,
check, GPU_START_TIMEOUT, check)) {
if (req == oob_perfcntr)
gmu->num_oob_perfcntr--;
gmu_core_fault_snapshot(device);
ret = -ETIMEDOUT;
WARN(1, "OOB request %s timed out\n", oob_to_str(req));
trigger_reset_recovery(adreno_dev, req);
}
gmu_core_regwrite(device, A6XX_GMU_GMU2HOST_INTR_CLR, check);
trace_kgsl_gmu_oob_set(set);
return ret;
}
void a6xx_gmu_oob_clear(struct kgsl_device *device,
enum oob_request req)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
int clear;
if (req == oob_perfcntr && --gmu->num_oob_perfcntr)
return;
if (adreno_is_a630(adreno_dev) || adreno_is_a615_family(adreno_dev)) {
clear = BIT(req + 24);
} else {
clear = BIT(31 - req * 2);
if (req >= oob_boot_slumber) {
dev_err(&gmu->pdev->dev, "Unsupported OOB clear %s\n",
oob_to_str(req));
return;
}
}
gmu_core_regwrite(device, A6XX_GMU_HOST2GMU_INTR_SET, clear);
trace_kgsl_gmu_oob_clear(clear);
}
void a6xx_gmu_irq_enable(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
struct a6xx_hfi *hfi = &gmu->hfi;
/* Clear pending IRQs and Unmask needed IRQs */
gmu_core_regwrite(device, A6XX_GMU_GMU2HOST_INTR_CLR, 0xffffffff);
gmu_core_regwrite(device, A6XX_GMU_AO_HOST_INTERRUPT_CLR, 0xffffffff);
gmu_core_regwrite(device, A6XX_GMU_GMU2HOST_INTR_MASK,
(unsigned int)~HFI_IRQ_MASK);
gmu_core_regwrite(device, A6XX_GMU_AO_HOST_INTERRUPT_MASK,
(unsigned int)~GMU_AO_INT_MASK);
/* Enable all IRQs on host */
enable_irq(hfi->irq);
enable_irq(gmu->irq);
}
void a6xx_gmu_irq_disable(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
struct a6xx_hfi *hfi = &gmu->hfi;
/* Disable all IRQs on host */
disable_irq(gmu->irq);
disable_irq(hfi->irq);
/* Mask all IRQs and clear pending IRQs */
gmu_core_regwrite(device, A6XX_GMU_GMU2HOST_INTR_MASK, 0xffffffff);
gmu_core_regwrite(device, A6XX_GMU_AO_HOST_INTERRUPT_MASK, 0xffffffff);
gmu_core_regwrite(device, A6XX_GMU_GMU2HOST_INTR_CLR, 0xffffffff);
gmu_core_regwrite(device, A6XX_GMU_AO_HOST_INTERRUPT_CLR, 0xffffffff);
}
static int a6xx_gmu_hfi_start_msg(struct adreno_device *adreno_dev)
{
struct hfi_start_cmd req;
/*
* This HFI was not supported in legacy firmware and this quirk
* serves as a better means to identify targets that depend on
* legacy firmware.
*/
if (!ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_HFI_USE_REG)) {
int ret;
ret = CMD_MSG_HDR(req, H2F_MSG_START);
if (ret)
return ret;
return a6xx_hfi_send_generic_req(adreno_dev, &req, sizeof(req));
}
return 0;
}
#define FREQ_VOTE(idx, ack) (((idx) & 0xFF) | (((ack) & 0xF) << 28))
#define BW_VOTE(idx) ((((idx) & 0xFFF) << 12) | ((idx) & 0xFFF))
#define CLKSET_OPTION_ATLEAST 3
/*
* a6xx_gmu_dcvs_nohfi() - request GMU to do DCVS without using HFI
* @device: Pointer to KGSL device
* @perf_idx: Index into GPU performance level table defined in
* HFI DCVS table message
* @bw_idx: Index into GPU b/w table defined in HFI b/w table message
*
*/
static int a6xx_gmu_dcvs_nohfi(struct kgsl_device *device,
unsigned int perf_idx, unsigned int bw_idx)
{
int ret;
gmu_core_regwrite(device, A6XX_GMU_DCVS_ACK_OPTION, DCVS_ACK_NONBLOCK);
gmu_core_regwrite(device, A6XX_GMU_DCVS_PERF_SETTING,
FREQ_VOTE(perf_idx, CLKSET_OPTION_ATLEAST));
gmu_core_regwrite(device, A6XX_GMU_DCVS_BW_SETTING, BW_VOTE(bw_idx));
ret = a6xx_gmu_oob_set(device, oob_dcvs);
if (ret == 0)
gmu_core_regread(device, A6XX_GMU_DCVS_RETURN, &ret);
a6xx_gmu_oob_clear(device, oob_dcvs);
return ret;
}
static u32 a6xx_rscc_tcsm_drv0_status_reglist[] = {
A6XX_RSCC_TCS0_DRV0_STATUS,
A6XX_RSCC_TCS1_DRV0_STATUS,
A6XX_RSCC_TCS2_DRV0_STATUS,
A6XX_RSCC_TCS3_DRV0_STATUS,
A6XX_RSCC_TCS4_DRV0_STATUS,
A6XX_RSCC_TCS5_DRV0_STATUS,
A6XX_RSCC_TCS6_DRV0_STATUS,
A6XX_RSCC_TCS7_DRV0_STATUS,
A6XX_RSCC_TCS8_DRV0_STATUS,
A6XX_RSCC_TCS9_DRV0_STATUS,
};
static int a6xx_complete_rpmh_votes(struct adreno_device *adreno_dev,
unsigned int timeout)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
/* Number of TCS commands are increased to 10 from A650 family onwards */
int count = adreno_is_a650_family(adreno_dev) ?
ARRAY_SIZE(a6xx_rscc_tcsm_drv0_status_reglist) : 4;
int i, ret = 0;
for (i = 0; i < count; i++)
ret |= timed_poll_check_rscc(device, a6xx_rscc_tcsm_drv0_status_reglist[i],
BIT(0), timeout, BIT(0));
if (ret)
dev_err(device->dev, "RPMH votes timedout: %d\n", ret);
return ret;
}
#define SPTPRAC_CTRL_TIMEOUT 10 /* ms */
/*
* a6xx_gmu_sptprac_enable() - Power on SPTPRAC
* @adreno_dev: Pointer to Adreno device
*/
int a6xx_gmu_sptprac_enable(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
/* Only certain targets have sptprac */
if (!adreno_is_a630(adreno_dev) && !adreno_is_a615_family(adreno_dev))
return 0;
if (test_bit(ADRENO_DEVICE_GPU_REGULATOR_ENABLED, &adreno_dev->priv))
return 0;
/* GMU enabled a630 and a615 targets */
gmu_core_regwrite(device, A6XX_GMU_GX_SPTPRAC_POWER_CONTROL,
SPTPRAC_POWERON_CTRL_MASK);
if (gmu_core_timed_poll_check(device,
A6XX_GMU_SPTPRAC_PWR_CLK_STATUS,
SPTPRAC_POWERON_STATUS_MASK,
SPTPRAC_CTRL_TIMEOUT,
SPTPRAC_POWERON_STATUS_MASK)) {
dev_err(&gmu->pdev->dev, "power on SPTPRAC fail\n");
gmu_core_fault_snapshot(device);
return -ETIMEDOUT;
}
set_bit(ADRENO_DEVICE_GPU_REGULATOR_ENABLED, &adreno_dev->priv);
return 0;
}
/*
* a6xx_gmu_sptprac_disable() - Power of SPTPRAC
* @adreno_dev: Pointer to Adreno device
*/
void a6xx_gmu_sptprac_disable(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
/* Only certain targets have sptprac */
if (!adreno_is_a630(adreno_dev) && !adreno_is_a615_family(adreno_dev))
return;
if (!test_and_clear_bit(ADRENO_DEVICE_GPU_REGULATOR_ENABLED,
&adreno_dev->priv))
return;
/* GMU enabled a630 and a615 targets */
/* Ensure that retention is on */
gmu_core_regrmw(device, A6XX_GPU_CC_GX_GDSCR, 0,
A6XX_RETAIN_FF_ENABLE_ENABLE_MASK);
gmu_core_regwrite(device, A6XX_GMU_GX_SPTPRAC_POWER_CONTROL,
SPTPRAC_POWEROFF_CTRL_MASK);
if (gmu_core_timed_poll_check(device,
A6XX_GMU_SPTPRAC_PWR_CLK_STATUS,
SPTPRAC_POWEROFF_STATUS_MASK,
SPTPRAC_CTRL_TIMEOUT,
SPTPRAC_POWEROFF_STATUS_MASK))
dev_err(&gmu->pdev->dev, "power off SPTPRAC fail\n");
}
#define SPTPRAC_POWER_OFF BIT(2)
#define SP_CLK_OFF BIT(4)
#define GX_GDSC_POWER_OFF BIT(6)
#define GX_CLK_OFF BIT(7)
#define is_on(val) (!(val & (GX_GDSC_POWER_OFF | GX_CLK_OFF)))
bool a6xx_gmu_gx_is_on(struct adreno_device *adreno_dev)
{
unsigned int val;
gmu_core_regread(KGSL_DEVICE(adreno_dev),
A6XX_GMU_SPTPRAC_PWR_CLK_STATUS, &val);
return is_on(val);
}
bool a619_holi_gx_is_on(struct adreno_device *adreno_dev)
{
unsigned int val;
gmu_core_regread(KGSL_DEVICE(adreno_dev),
A6XX_GMU_SPTPRAC_PWR_CLK_STATUS, &val);
return is_on(val);
}
/*
* a6xx_gmu_sptprac_is_on() - Check if SPTP is on using pwr status register
* @adreno_dev - Pointer to adreno_device
* This check should only be performed if the keepalive bit is set or it
* can be guaranteed that the power state of the GPU will remain unchanged
*/
bool a6xx_gmu_sptprac_is_on(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
unsigned int val;
if (!adreno_is_a630(adreno_dev) && !adreno_is_a615_family(adreno_dev))
return true;
if (adreno_is_a619_holi(adreno_dev))
kgsl_regread(device,
A6XX_GMU_SPTPRAC_PWR_CLK_STATUS, &val);
else
gmu_core_regread(device, A6XX_GMU_SPTPRAC_PWR_CLK_STATUS,
&val);
return !(val & (SPTPRAC_POWER_OFF | SP_CLK_OFF));
}
/*
* a6xx_gmu_gfx_rail_on() - request GMU to power GPU at given OPP.
* @device: Pointer to KGSL device
*
*/
static int a6xx_gmu_gfx_rail_on(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct kgsl_pwrctrl *pwr = &device->pwrctrl;
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
u32 perf_idx = gmu->hfi.dcvs_table.gpu_level_num -
pwr->default_pwrlevel - 1;
u32 default_opp = gmu->hfi.dcvs_table.gx_votes[perf_idx].vote;
gmu_core_regwrite(device, A6XX_GMU_BOOT_SLUMBER_OPTION,
OOB_BOOT_OPTION);
gmu_core_regwrite(device, A6XX_GMU_GX_VOTE_IDX,
ARC_VOTE_GET_PRI(default_opp));
gmu_core_regwrite(device, A6XX_GMU_MX_VOTE_IDX,
ARC_VOTE_GET_SEC(default_opp));
a6xx_rdpm_mx_freq_update(gmu,
gmu->hfi.dcvs_table.gx_votes[perf_idx].freq);
return a6xx_gmu_oob_set(device, oob_boot_slumber);
}
static bool idle_trandition_complete(unsigned int idle_level,
unsigned int gmu_power_reg,
unsigned int sptprac_clk_reg)
{
if (idle_level != gmu_power_reg)
return false;
if (idle_level == GPU_HW_IFPC && is_on(sptprac_clk_reg))
return false;
return true;
}
static const char *idle_level_name(int level)
{
if (level == GPU_HW_ACTIVE)
return "GPU_HW_ACTIVE";
else if (level == GPU_HW_IFPC)
return "GPU_HW_IFPC";
return "";
}
int a6xx_gmu_wait_for_lowest_idle(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
unsigned int reg, reg1, reg2, reg3, reg4, reg5, reg6, reg7, reg8;
unsigned long t;
uint64_t ts1, ts2, ts3;
ts1 = a6xx_read_alwayson(adreno_dev);
t = jiffies + msecs_to_jiffies(GMU_IDLE_TIMEOUT);
do {
gmu_core_regread(device,
A6XX_GPU_GMU_CX_GMU_RPMH_POWER_STATE, &reg);
gmu_core_regread(device,
A6XX_GMU_SPTPRAC_PWR_CLK_STATUS, &reg1);
if (idle_trandition_complete(gmu->idle_level, reg, reg1))
return 0;
/* Wait 100us to reduce unnecessary AHB bus traffic */
usleep_range(10, 100);
} while (!time_after(jiffies, t));
ts2 = a6xx_read_alwayson(adreno_dev);
/* Check one last time */
gmu_core_regread(device, A6XX_GPU_GMU_CX_GMU_RPMH_POWER_STATE, &reg);
gmu_core_regread(device, A6XX_GMU_SPTPRAC_PWR_CLK_STATUS, &reg1);
if (idle_trandition_complete(gmu->idle_level, reg, reg1))
return 0;
ts3 = a6xx_read_alwayson(adreno_dev);
/* Collect abort data to help with debugging */
gmu_core_regread(device, A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS, &reg2);
gmu_core_regread(device, A6XX_GMU_RBBM_INT_UNMASKED_STATUS, &reg3);
gmu_core_regread(device, A6XX_GMU_GMU_PWR_COL_KEEPALIVE, &reg4);
gmu_core_regread(device, A6XX_GMU_AO_SPARE_CNTL, &reg5);
dev_err(&gmu->pdev->dev,
"----------------------[ GMU error ]----------------------\n");
dev_err(&gmu->pdev->dev,
"Timeout waiting for lowest idle level %s\n",
idle_level_name(gmu->idle_level));
dev_err(&gmu->pdev->dev, "Start: %llx (absolute ticks)\n", ts1);
dev_err(&gmu->pdev->dev, "Poll: %llx (ticks relative to start)\n",
ts2-ts1);
dev_err(&gmu->pdev->dev, "Retry: %llx (ticks relative to poll)\n",
ts3-ts2);
dev_err(&gmu->pdev->dev,
"RPMH_POWER_STATE=%x SPTPRAC_PWR_CLK_STATUS=%x\n", reg, reg1);
dev_err(&gmu->pdev->dev, "CX_BUSY_STATUS=%x\n", reg2);
dev_err(&gmu->pdev->dev,
"RBBM_INT_UNMASKED_STATUS=%x PWR_COL_KEEPALIVE=%x\n",
reg3, reg4);
dev_err(&gmu->pdev->dev, "A6XX_GMU_AO_SPARE_CNTL=%x\n", reg5);
if (adreno_is_a660(adreno_dev)) {
u32 val;
gmu_core_regread(device, A6XX_GMU_PWR_COL_PREEMPT_KEEPALIVE, &val);
dev_err(&gmu->pdev->dev, "PWR_COL_PREEMPT_KEEPALIVE=%x\n", val);
}
/* Access GX registers only when GX is ON */
if (is_on(reg1)) {
kgsl_regread(device, A6XX_CP_STATUS_1, &reg6);
kgsl_regread(device, A6XX_CP_CP2GMU_STATUS, &reg7);
kgsl_regread(device, A6XX_CP_CONTEXT_SWITCH_CNTL, &reg8);
dev_err(&gmu->pdev->dev, "A6XX_CP_STATUS_1=%x\n", reg6);
dev_err(&gmu->pdev->dev,
"CP2GMU_STATUS=%x CONTEXT_SWITCH_CNTL=%x\n",
reg7, reg8);
}
WARN_ON(1);
gmu_core_fault_snapshot(device);
return -ETIMEDOUT;
}
/* Bitmask for GPU idle status check */
#define CXGXCPUBUSYIGNAHB BIT(30)
int a6xx_gmu_wait_for_idle(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
unsigned int status2;
uint64_t ts1;
ts1 = a6xx_read_alwayson(adreno_dev);
if (gmu_core_timed_poll_check(device, A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS,
0, GMU_START_TIMEOUT, CXGXCPUBUSYIGNAHB)) {
gmu_core_regread(device,
A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS2, &status2);
dev_err(&gmu->pdev->dev,
"GMU not idling: status2=0x%x %llx %llx\n",
status2, ts1,
a6xx_read_alwayson(ADRENO_DEVICE(device)));
gmu_core_fault_snapshot(device);
return -ETIMEDOUT;
}
return 0;
}
/* A6xx GMU FENCE RANGE MASK */
#define GMU_FENCE_RANGE_MASK ((0x1 << 31) | ((0xA << 2) << 18) | (0x8A0))
void a6xx_gmu_version_info(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
/* GMU version info is at a fixed offset in the DTCM */
gmu_core_regread(device, A6XX_GMU_CM3_DTCM_START + 0xFF8,
&gmu->ver.core);
gmu_core_regread(device, A6XX_GMU_CM3_DTCM_START + 0xFF9,
&gmu->ver.core_dev);
gmu_core_regread(device, A6XX_GMU_CM3_DTCM_START + 0xFFA,
&gmu->ver.pwr);
gmu_core_regread(device, A6XX_GMU_CM3_DTCM_START + 0xFFB,
&gmu->ver.pwr_dev);
gmu_core_regread(device, A6XX_GMU_CM3_DTCM_START + 0xFFC,
&gmu->ver.hfi);
}
int a6xx_gmu_itcm_shadow(struct adreno_device *adreno_dev)
{
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
u32 i, *dest;
if (gmu->itcm_shadow)
return 0;
gmu->itcm_shadow = vzalloc(gmu->vma[GMU_ITCM].size);
if (!gmu->itcm_shadow)
return -ENOMEM;
dest = (u32 *)gmu->itcm_shadow;
/* FIXME: use bulk read? */
for (i = 0; i < (gmu->vma[GMU_ITCM].size >> 2); i++)
gmu_core_regread(KGSL_DEVICE(adreno_dev),
A6XX_GMU_CM3_ITCM_START + i, dest++);
return 0;
}
static void a6xx_gmu_enable_throttle_counters(
struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
u32 val;
if (!(adreno_dev->lm_enabled || adreno_dev->bcl_enabled))
return;
if (adreno_dev->lm_enabled) {
/*
* For LM throttling -
* XOCLK1: countable: 0x10
* XOCLK2: countable: 0x16 for newer hardware / 0x15 for others
* XOCLK3: countable: 0xf for newer hardware / 0x19 for others
*
* POWER_CONTROL_SELECT_0 controls counters 0 - 3, each selector
* is 8 bits wide.
*/
if (adreno_is_a620(adreno_dev) || adreno_is_a650(adreno_dev))
val = (0x10 << 8) | (0x16 << 16) | (0x0f << 24);
else
val = (0x10 << 8) | (0x15 << 16) | (0x19 << 24);
} else {
/*
* When LM is not enabled, we can enable BCL throttling -
* XOCLK1: countable: 0x13 (25% throttle)
* XOCLK2: countable: 0x17 (58% throttle)
* XOCLK3: countable: 0x19 (75% throttle)
*
* POWER_CONTROL_SELECT_0 controls counters 0 - 3, each selector
* is 8 bits wide.
*/
val = (0x13 << 8) | (0x17 << 16) | (0x19 << 24);
}
/* Make sure not to write over XOCLK0 */
gmu_core_regrmw(device, A6XX_GMU_CX_GMU_POWER_COUNTER_SELECT_0,
0xffffff00, val);
gmu_core_regwrite(device, A6XX_GMU_AO_SPARE_CNTL, 1);
}
void a6xx_gmu_register_config(struct adreno_device *adreno_dev)
{
const struct adreno_a6xx_core *a6xx_core = to_a6xx_core(adreno_dev);
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
u32 gmu_log_info, chipid = 0;
/* Clear any previously set cm3 fault */
atomic_set(&gmu->cm3_fault, 0);
/* Vote veto for FAL10 feature if supported*/
if (a6xx_core->veto_fal10) {
gmu_core_regwrite(device,
A6XX_GPU_GMU_CX_GMU_CX_FALNEXT_INTF, 0x1);
gmu_core_regwrite(device, A6XX_GPU_GMU_CX_GMU_CX_FAL_INTF, 0x1);
}
/* Turn on TCM retention */
gmu_core_regwrite(device, A6XX_GMU_GENERAL_7, 1);
/* Clear init result to make sure we are getting fresh value */
gmu_core_regwrite(device, A6XX_GMU_CM3_FW_INIT_RESULT, 0);
gmu_core_regwrite(device, A6XX_GMU_CM3_BOOT_CONFIG, 0x2);
gmu_core_regwrite(device, A6XX_GMU_HFI_QTBL_ADDR,
gmu->hfi.hfi_mem->gmuaddr);
gmu_core_regwrite(device, A6XX_GMU_HFI_QTBL_INFO, 1);
/*
* For A6xx GMUAO interrupt line BIT[1] is combined for ipcc
* and doorbell. Enable dbdWakeupEn interrupt for GMU to receive
* IPC interrupt.
*/
if (ADRENO_FEATURE(adreno_dev, ADRENO_LSR))
gmu_core_regwrite(device, A6XX_GMU_AO_INTERRUPT_EN, BIT(1));
gmu_core_regwrite(device, A6XX_GMU_AHB_FENCE_RANGE_0,
GMU_FENCE_RANGE_MASK);
/*
* Make sure that CM3 state is at reset value. Snapshot is changing
* NMI bit and if we boot up GMU with NMI bit set GMU will boot
* straight in to NMI handler without executing __main code
*/
gmu_core_regwrite(device, A6XX_GMU_CM3_CFG, 0x4052);
/**
* We may have asserted gbif halt as part of reset sequence which may
* not get cleared if the gdsc was not reset. So clear it before
* attempting GMU boot.
*/
if (!adreno_is_a630(adreno_dev))
kgsl_regwrite(device, A6XX_GBIF_HALT, 0x0);
/* Set vrb address before starting GMU */
if (!IS_ERR_OR_NULL(gmu->vrb))
gmu_core_regwrite(device, A6XX_GMU_GENERAL_11, gmu->vrb->gmuaddr);
/* Set the log wptr index */
gmu_core_regwrite(device, A6XX_GPU_GMU_CX_GMU_PWR_COL_CP_RESP,
gmu->log_wptr_retention);
/* Pass chipid to GMU FW, must happen before starting GMU */
chipid = ADRENO_GMU_CHIPID(adreno_dev->chipid);
/*
* For A660 GPU variant, GMU firmware expects chipid as per below
* format to differentiate between A660 and A660 variant. In device
* tree, target version is specified as high nibble of patch to align
* with usermode driver expectation. Format the chipid according to
* firmware requirement.
*
* Bit 11-8: patch version
* Bit 15-12: minor version
* Bit 23-16: major version
* Bit 27-24: core version
* Bit 31-28: target version
*/
if (adreno_is_a660_shima(adreno_dev))
chipid |= ((ADRENO_CHIPID_PATCH(adreno_dev->chipid) >> 4) << 28);
gmu_core_regwrite(device, A6XX_GMU_HFI_SFR_ADDR, chipid);
/* Log size is encoded in (number of 4K units - 1) */
gmu_log_info = (gmu->gmu_log->gmuaddr & 0xFFFFF000) |
((GMU_LOG_SIZE/SZ_4K - 1) & 0xFF);
gmu_core_regwrite(device, A6XX_GPU_GMU_CX_GMU_PWR_COL_CP_MSG,
gmu_log_info);
/* Configure power control and bring the GMU out of reset */
a6xx_gmu_power_config(adreno_dev);
a6xx_gmu_enable_throttle_counters(adreno_dev);
}
struct kgsl_memdesc *reserve_gmu_kernel_block(struct a6xx_gmu_device *gmu,
u32 addr, u32 size, u32 vma_id, u32 align)
{
int ret;
struct kgsl_memdesc *md;
struct gmu_vma_entry *vma = &gmu->vma[vma_id];
struct kgsl_device *device = KGSL_DEVICE(a6xx_gmu_to_adreno(gmu));
u32 aligned_size = ALIGN(size, hfi_get_gmu_sz_alignment(align));
if (gmu->global_entries == ARRAY_SIZE(gmu->gmu_globals))
return ERR_PTR(-ENOMEM);
md = &gmu->gmu_globals[gmu->global_entries];
ret = kgsl_allocate_kernel(device, md, size, 0, KGSL_MEMDESC_SYSMEM);
if (ret) {
memset(md, 0x0, sizeof(*md));
return ERR_PTR(-ENOMEM);
}
if (!addr)
addr = ALIGN(vma->next_va, hfi_get_gmu_va_alignment(align));
ret = gmu_core_map_memdesc(gmu->domain, md, addr,
IOMMU_READ | IOMMU_WRITE | IOMMU_PRIV);
if (ret) {
dev_err(&gmu->pdev->dev,
"Unable to map GMU kernel block: addr:0x%08x size:0x%llx :%d\n",
addr, md->size, ret);
kgsl_sharedmem_free(md);
memset(md, 0, sizeof(*md));
return ERR_PTR(-ENOMEM);
}
md->gmuaddr = addr;
/* Take into account the size alignment when reserving the GMU VA */
vma->next_va = md->gmuaddr + aligned_size;
gmu->global_entries++;
return md;
}
struct kgsl_memdesc *reserve_gmu_kernel_block_fixed(struct a6xx_gmu_device *gmu,
u32 addr, u32 size, u32 vma_id, const char *resource, int attrs, u32 align)
{
int ret;
struct kgsl_memdesc *md;
struct gmu_vma_entry *vma = &gmu->vma[vma_id];
struct kgsl_device *device = KGSL_DEVICE(a6xx_gmu_to_adreno(gmu));
u32 aligned_size = ALIGN(size, hfi_get_gmu_sz_alignment(align));
if (gmu->global_entries == ARRAY_SIZE(gmu->gmu_globals))
return ERR_PTR(-ENOMEM);
md = &gmu->gmu_globals[gmu->global_entries];
ret = kgsl_memdesc_init_fixed(device, gmu->pdev, resource, md);
if (ret)
return ERR_PTR(ret);
if (!addr)
addr = ALIGN(vma->next_va, hfi_get_gmu_va_alignment(align));
if ((vma->next_va + aligned_size) > (vma->start + vma->size)) {
dev_err(&gmu->pdev->dev,
"GMU mapping too big. available: %d required: %d\n",
vma->next_va - vma->start, aligned_size);
md = ERR_PTR(-ENOMEM);
goto done;
}
ret = gmu_core_map_memdesc(gmu->domain, md, addr, attrs);
if (ret) {
dev_err(&gmu->pdev->dev,
"Unable to map GMU kernel block: addr:0x%08x size:0x%llx :%d\n",
addr, md->size, ret);
md = ERR_PTR(-ENOMEM);
goto done;
}
md->gmuaddr = addr;
/* Take into account the size alignment when reserving the GMU VA */
vma->next_va = md->gmuaddr + aligned_size;
gmu->global_entries++;
done:
sg_free_table(md->sgt);
kfree(md->sgt);
md->sgt = NULL;
return md;
}
static int reserve_entire_vma(struct a6xx_gmu_device *gmu, u32 vma_id)
{
struct kgsl_memdesc *md;
u32 start = gmu->vma[vma_id].start, size = gmu->vma[vma_id].size;
md = find_gmu_memdesc(gmu, start, size);
if (md)
return 0;
md = reserve_gmu_kernel_block(gmu, start, size, vma_id, 0);
return PTR_ERR_OR_ZERO(md);
}
static int a6xx_gmu_cache_finalize(struct adreno_device *adreno_dev)
{
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
struct kgsl_memdesc *md;
int ret;
/* Preallocations were made so no need to request all this memory */
if (gmu->preallocations)
return 0;
ret = reserve_entire_vma(gmu, GMU_ICACHE);
if (ret)
return ret;
if (!adreno_is_a650_family(adreno_dev)) {
ret = reserve_entire_vma(gmu, GMU_DCACHE);
if (ret)
return ret;
}
md = reserve_gmu_kernel_block(gmu, 0, SZ_4K, GMU_NONCACHED_KERNEL, 0);
if (IS_ERR(md))
return PTR_ERR(md);
gmu->preallocations = true;
return 0;
}
static int a6xx_gmu_process_prealloc(struct a6xx_gmu_device *gmu,
struct gmu_block_header *blk)
{
struct kgsl_memdesc *md;
int id = find_vma_block(gmu, blk->addr, blk->value);
if (id < 0) {
dev_err(&gmu->pdev->dev,
"Invalid prealloc block addr: 0x%x value:%d\n",
blk->addr, blk->value);
return id;
}
/* Nothing to do for TCM blocks or user uncached */
if (id == GMU_ITCM || id == GMU_DTCM || id == GMU_NONCACHED_USER)
return 0;
/* Check if the block is already allocated */
md = find_gmu_memdesc(gmu, blk->addr, blk->value);
if (md != NULL)
return 0;
md = reserve_gmu_kernel_block(gmu, blk->addr, blk->value, id, 0);
if (IS_ERR(md))
return PTR_ERR(md);
gmu->preallocations = true;
return 0;
}
int a6xx_gmu_parse_fw(struct adreno_device *adreno_dev)
{
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
const struct adreno_a6xx_core *a6xx_core = to_a6xx_core(adreno_dev);
struct gmu_block_header *blk;
int ret, offset = 0;
/* GMU fw already saved and verified so do nothing new */
if (!gmu->fw_image) {
if (a6xx_core->gmufw_name == NULL)
return -EINVAL;
ret = request_firmware(&gmu->fw_image, a6xx_core->gmufw_name,
&gmu->pdev->dev);
if (ret) {
dev_err(&gmu->pdev->dev, "request_firmware (%s) failed: %d\n",
a6xx_core->gmufw_name, ret);
return ret;
}
}
/*
* Zero payload fw blocks contain metadata and are
* guaranteed to precede fw load data. Parse the
* metadata blocks.
*/
while (offset < gmu->fw_image->size) {
blk = (struct gmu_block_header *)&gmu->fw_image->data[offset];
if (offset + sizeof(*blk) > gmu->fw_image->size) {
dev_err(&gmu->pdev->dev, "Invalid FW Block\n");
return -EINVAL;
}
/* Done with zero length blocks so return */
if (blk->size)
break;
offset += sizeof(*blk);
if (blk->type == GMU_BLK_TYPE_PREALLOC_REQ ||
blk->type == GMU_BLK_TYPE_PREALLOC_PERSIST_REQ) {
ret = a6xx_gmu_process_prealloc(gmu, blk);
if (ret)
return ret;
}
}
return 0;
}
int a6xx_gmu_memory_init(struct adreno_device *adreno_dev)
{
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
/* Allocates & maps GMU crash dump memory */
if (adreno_is_a630(adreno_dev) || adreno_is_a615_family(adreno_dev)) {
if (IS_ERR_OR_NULL(gmu->dump_mem))
gmu->dump_mem = reserve_gmu_kernel_block(gmu, 0, SZ_16K,
GMU_NONCACHED_KERNEL, 0);
if (IS_ERR(gmu->dump_mem))
return PTR_ERR(gmu->dump_mem);
}
/* GMU master log */
if (IS_ERR_OR_NULL(gmu->gmu_log))
gmu->gmu_log = reserve_gmu_kernel_block(gmu, 0, GMU_LOG_SIZE,
GMU_NONCACHED_KERNEL, 0);
return PTR_ERR_OR_ZERO(gmu->gmu_log);
}
static int a6xx_gmu_init(struct adreno_device *adreno_dev)
{
int ret;
ret = a6xx_gmu_parse_fw(adreno_dev);
if (ret)
return ret;
/* Request any other cache ranges that might be required */
ret = a6xx_gmu_cache_finalize(adreno_dev);
if (ret)
return ret;
ret = a6xx_gmu_memory_init(adreno_dev);
if (ret)
return ret;
return a6xx_hfi_init(adreno_dev);
}
#define A6XX_VBIF_XIN_HALT_CTRL1_ACKS (BIT(0) | BIT(1) | BIT(2) | BIT(3))
static void a6xx_gmu_pwrctrl_suspend(struct adreno_device *adreno_dev)
{
int ret = 0;
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct kgsl_pwrctrl *pwr = &device->pwrctrl;
/* If SPTP_RAC is on, turn off SPTP_RAC HS */
a6xx_gmu_sptprac_disable(adreno_dev);
/* Disconnect GPU from BUS is not needed if CX GDSC goes off later */
/*
* GEMNOC can enter power collapse state during GPU power down sequence.
* This could abort CX GDSC collapse. Assert Qactive to avoid this.
*/
if ((adreno_is_a662(adreno_dev) || adreno_is_a621(adreno_dev) ||
adreno_is_a635(adreno_dev)))
gmu_core_regwrite(device, A6XX_GPU_GMU_CX_GMU_CX_FALNEXT_INTF, 0x1);
/* Check no outstanding RPMh voting */
a6xx_complete_rpmh_votes(adreno_dev, GPU_RESET_TIMEOUT);
/* Clear the WRITEDROPPED fields and set fence to allow mode */
gmu_core_regwrite(device, A6XX_GMU_AHB_FENCE_STATUS_CLR, 0x7);
gmu_core_regwrite(device, A6XX_GMU_AO_AHB_FENCE_CTRL, 0);
/* Make sure above writes are committed before we proceed to recovery */
wmb();
gmu_core_regwrite(device, A6XX_GMU_CM3_SYSRESET, 1);
if (!adreno_is_a630(adreno_dev)) {
/* Halt GX traffic */
if (a6xx_gmu_gx_is_on(adreno_dev)) {
kgsl_regwrite(device, A6XX_RBBM_GBIF_HALT,
A6XX_GBIF_GX_HALT_MASK);
adreno_wait_for_halt_ack(device,
A6XX_RBBM_GBIF_HALT_ACK,
A6XX_GBIF_GX_HALT_MASK);
}
/* Halt CX traffic */
a6xx_halt_gbif(adreno_dev);
/* De-assert the halts */
kgsl_regwrite(device, A6XX_GBIF_HALT, 0x0);
}
if (a6xx_gmu_gx_is_on(adreno_dev))
kgsl_regwrite(device, A6XX_RBBM_SW_RESET_CMD, 0x1);
/* Make sure above writes are posted before turning off power resources */
wmb();
/* Allow the software reset to complete */
udelay(100);
/*
* This is based on the assumption that GMU is the only one controlling
* the GX HS. This code path is the only client voting for GX through
* the regulator interface.
*/
if (pwr->gx_gdsc) {
if (a6xx_gmu_gx_is_on(adreno_dev)) {
/* Switch gx gdsc control from GMU to CPU
* force non-zero reference count in clk driver
* so next disable call will turn
* off the GDSC
*/
ret = regulator_enable(pwr->gx_gdsc);
if (ret)
dev_err(&gmu->pdev->dev,
"suspend fail: gx enable %d\n", ret);
/*
* Toggle the loop_en bit, across disabling the gx gdsc,
* with a delay of 10 XO cycles before disabling gx
* gdsc. This is to prevent CPR measurements from
* failing.
*/
if (adreno_is_a660(adreno_dev)) {
gmu_core_regrmw(device, A6XX_GPU_CPR_FSM_CTL,
1, 0);
ndelay(520);
}
ret = regulator_disable(pwr->gx_gdsc);
if (ret)
dev_err(&gmu->pdev->dev,
"suspend fail: gx disable %d\n", ret);
if (adreno_is_a660(adreno_dev))
gmu_core_regrmw(device, A6XX_GPU_CPR_FSM_CTL,
1, 1);
if (a6xx_gmu_gx_is_on(adreno_dev))
dev_err(&gmu->pdev->dev,
"gx is stuck on\n");
}
}
}
/*
* a6xx_gmu_notify_slumber() - initiate request to GMU to prepare to slumber
* @device: Pointer to KGSL device
*/
static int a6xx_gmu_notify_slumber(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct kgsl_pwrctrl *pwr = &device->pwrctrl;
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
int bus_level = pwr->pwrlevels[pwr->default_pwrlevel].bus_freq;
int perf_idx = gmu->hfi.dcvs_table.gpu_level_num -
pwr->default_pwrlevel - 1;
int ret, state;
/* Disable the power counter so that the GMU is not busy */
gmu_core_regwrite(device, A6XX_GMU_CX_GMU_POWER_COUNTER_ENABLE, 0);
/* Turn off SPTPRAC if we own it */
if (gmu->idle_level == GPU_HW_ACTIVE)
a6xx_gmu_sptprac_disable(adreno_dev);
if (!ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_HFI_USE_REG)) {
struct hfi_prep_slumber_cmd req = {
.freq = perf_idx,
.bw = bus_level,
};
ret = CMD_MSG_HDR(req, H2F_MSG_PREPARE_SLUMBER);
if (!ret)
ret = a6xx_hfi_send_generic_req(adreno_dev, &req, sizeof(req));
goto out;
}
gmu_core_regwrite(device, A6XX_GMU_BOOT_SLUMBER_OPTION,
OOB_SLUMBER_OPTION);
gmu_core_regwrite(device, A6XX_GMU_GX_VOTE_IDX, perf_idx);
gmu_core_regwrite(device, A6XX_GMU_MX_VOTE_IDX, bus_level);
ret = a6xx_gmu_oob_set(device, oob_boot_slumber);
a6xx_gmu_oob_clear(device, oob_boot_slumber);
if (!ret) {
gmu_core_regread(device,
A6XX_GPU_GMU_CX_GMU_RPMH_POWER_STATE, &state);
if (state != GPU_HW_SLUMBER) {
dev_err(&gmu->pdev->dev,
"Failed to prepare for slumber: 0x%x\n",
state);
ret = -ETIMEDOUT;
}
}
out:
/* Make sure the fence is in ALLOW mode */
gmu_core_regwrite(device, A6XX_GMU_AO_AHB_FENCE_CTRL, 0);
/*
* GEMNOC can enter power collapse state during GPU power down sequence.
* This could abort CX GDSC collapse. Assert Qactive to avoid this.
*/
if ((adreno_is_a662(adreno_dev) || adreno_is_a621(adreno_dev) ||
adreno_is_a635(adreno_dev)))
gmu_core_regwrite(device, A6XX_GPU_GMU_CX_GMU_CX_FALNEXT_INTF, 0x1);
return ret;
}
void a6xx_gmu_suspend(struct adreno_device *adreno_dev)
{
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
a6xx_gmu_pwrctrl_suspend(adreno_dev);
clk_bulk_disable_unprepare(gmu->num_clks, gmu->clks);
a6xx_gmu_disable_gdsc(adreno_dev);
a6xx_rdpm_cx_freq_update(gmu, 0);
dev_err(&gmu->pdev->dev, "Suspended GMU\n");
kgsl_pwrctrl_set_state(device, KGSL_STATE_NONE);
}
static int a6xx_gmu_dcvs_set(struct adreno_device *adreno_dev,
int gpu_pwrlevel, int bus_level)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct kgsl_pwrctrl *pwr = &device->pwrctrl;
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
struct hfi_dcvstable_cmd *table = &gmu->hfi.dcvs_table;
struct hfi_gx_bw_perf_vote_cmd req = {
.ack_type = DCVS_ACK_BLOCK,
.freq = INVALID_DCVS_IDX,
.bw = INVALID_DCVS_IDX,
};
int ret = 0;
if (!test_bit(GMU_PRIV_HFI_STARTED, &gmu->flags))
return 0;
/* Do not set to XO and lower GPU clock vote from GMU */
if ((gpu_pwrlevel != INVALID_DCVS_IDX) &&
(gpu_pwrlevel >= table->gpu_level_num - 1))
return -EINVAL;
if (gpu_pwrlevel < table->gpu_level_num - 1)
req.freq = table->gpu_level_num - gpu_pwrlevel - 1;
if (bus_level < pwr->ddr_table_count && bus_level > 0)
req.bw = bus_level;
/* GMU will vote for slumber levels through the sleep sequence */
if ((req.freq == INVALID_DCVS_IDX) &&
(req.bw == INVALID_DCVS_IDX)) {
return 0;
}
ret = CMD_MSG_HDR(req, H2F_MSG_GX_BW_PERF_VOTE);
if (ret)
return ret;
if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_HFI_USE_REG))
ret = a6xx_gmu_dcvs_nohfi(device, req.freq, req.bw);
else
ret = a6xx_hfi_send_generic_req(adreno_dev, &req, sizeof(req));
if (ret) {
dev_err_ratelimited(&gmu->pdev->dev,
"Failed to set GPU perf idx %u, bw idx %u\n",
req.freq, req.bw);
/*
* If this was a dcvs request along side an active gpu, request
* dispatcher based reset and recovery.
*/
if (test_bit(GMU_PRIV_GPU_STARTED, &gmu->flags))
adreno_dispatcher_fault(adreno_dev, ADRENO_GMU_FAULT |
ADRENO_GMU_FAULT_SKIP_SNAPSHOT);
}
if (req.freq != INVALID_DCVS_IDX)
a6xx_rdpm_mx_freq_update(gmu,
gmu->hfi.dcvs_table.gx_votes[req.freq].freq);
return ret;
}
static int a6xx_gmu_clock_set(struct adreno_device *adreno_dev, u32 pwrlevel)
{
return a6xx_gmu_dcvs_set(adreno_dev, pwrlevel, INVALID_DCVS_IDX);
}
static int a6xx_gmu_ifpc_store(struct kgsl_device *device,
unsigned int val)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
unsigned int requested_idle_level;
if (!ADRENO_FEATURE(adreno_dev, ADRENO_IFPC))
return -EINVAL;
if (val)
requested_idle_level = GPU_HW_IFPC;
else
requested_idle_level = GPU_HW_ACTIVE;
if (gmu->idle_level == requested_idle_level)
return 0;
/* Power down the GPU before changing the idle level */
return adreno_power_cycle_u32(adreno_dev, &gmu->idle_level,
requested_idle_level);
}
static unsigned int a6xx_gmu_ifpc_isenabled(struct kgsl_device *device)
{
struct a6xx_gmu_device *gmu = to_a6xx_gmu(ADRENO_DEVICE(device));
return gmu->idle_level == GPU_HW_IFPC;
}
/* Send an NMI to the GMU */
void a6xx_gmu_send_nmi(struct kgsl_device *device, bool force)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
u32 val;
/*
* Do not send NMI if the SMMU is stalled because GMU will not be able
* to save cm3 state to DDR.
*/
if (a6xx_gmu_gx_is_on(adreno_dev) && adreno_smmu_is_stalled(adreno_dev)) {
dev_err(&gmu->pdev->dev,
"Skipping NMI because SMMU is stalled\n");
return;
}
if (force)
goto nmi;
/*
* We should not send NMI if there was a CM3 fault reported because we
* don't want to overwrite the critical CM3 state captured by gmu before
* it sent the CM3 fault interrupt. Also don't send NMI if GMU reset is
* already active. We could have hit a GMU assert and NMI might have
* already been triggered.
*/
/* make sure we're reading the latest cm3_fault */
smp_rmb();
if (atomic_read(&gmu->cm3_fault))
return;
gmu_core_regread(device, A6XX_GMU_CM3_FW_INIT_RESULT, &val);
if (val & 0xE00)
return;
nmi:
/* Mask so there's no interrupt caused by NMI */
gmu_core_regwrite(device, A6XX_GMU_GMU2HOST_INTR_MASK, 0xFFFFFFFF);
/* Make sure the interrupt is masked before causing it */
wmb();
if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_HFI_USE_REG))
gmu_core_regwrite(device,
A6XX_GMU_NMI_CONTROL_STATUS, 0);
/* This will cause the GMU to save it's internal state to ddr */
gmu_core_regread(device, A6XX_GMU_CM3_CFG, &val);
val |= BIT(9);
gmu_core_regwrite(device, A6XX_GMU_CM3_CFG, val);
/* Make sure the NMI is invoked before we proceed*/
wmb();
/* Wait for the NMI to be handled */
udelay(200);
}
static void a6xx_gmu_cooperative_reset(struct kgsl_device *device)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
unsigned int result;
gmu_core_regwrite(device, A6XX_GMU_CX_GMU_WDOG_CTRL, 0);
gmu_core_regwrite(device, A6XX_GMU_HOST2GMU_INTR_SET, BIT(17));
/*
* After triggering graceful death wait for snapshot ready
* indication from GMU.
*/
if (!gmu_core_timed_poll_check(device, A6XX_GMU_CM3_FW_INIT_RESULT,
0x800, 2, 0x800))
return;
gmu_core_regread(device, A6XX_GMU_CM3_FW_INIT_RESULT, &result);
dev_err(&gmu->pdev->dev,
"GMU cooperative reset timed out 0x%x\n", result);
/*
* If we dont get a snapshot ready from GMU, trigger NMI
* and if we still timeout then we just continue with reset.
*/
a6xx_gmu_send_nmi(device, true);
gmu_core_regread(device, A6XX_GMU_CM3_FW_INIT_RESULT, &result);
if ((result & 0x800) != 0x800)
dev_err(&gmu->pdev->dev,
"GMU cooperative reset NMI timed out 0x%x\n", result);
}
static int a6xx_gmu_wait_for_active_transition(
struct kgsl_device *device)
{
unsigned int reg;
struct a6xx_gmu_device *gmu = to_a6xx_gmu(ADRENO_DEVICE(device));
if (!gmu_core_isenabled(device))
return 0;
if (gmu_core_timed_poll_check(device, A6XX_GPU_GMU_CX_GMU_RPMH_POWER_STATE,
GPU_HW_ACTIVE, 100, GENMASK(3, 0))) {
gmu_core_regread(device, A6XX_GPU_GMU_CX_GMU_RPMH_POWER_STATE, &reg);
dev_err(&gmu->pdev->dev,
"GMU failed to move to ACTIVE state, Current state: 0x%x\n",
reg);
return -ETIMEDOUT;
}
return 0;
}
static bool a6xx_gmu_scales_bandwidth(struct kgsl_device *device)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
return (ADRENO_GPUREV(adreno_dev) >= ADRENO_REV_A640);
}
void a6xx_gmu_handle_watchdog(struct adreno_device *adreno_dev)
{
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
u32 mask;
/* Temporarily mask the watchdog interrupt to prevent a storm */
gmu_core_regread(device, A6XX_GMU_AO_HOST_INTERRUPT_MASK,
&mask);
gmu_core_regwrite(device, A6XX_GMU_AO_HOST_INTERRUPT_MASK,
(mask | GMU_INT_WDOG_BITE));
a6xx_gmu_send_nmi(device, false);
dev_err_ratelimited(&gmu->pdev->dev,
"GMU watchdog expired interrupt received\n");
}
static irqreturn_t a6xx_gmu_irq_handler(int irq, void *data)
{
struct kgsl_device *device = data;
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
const struct a6xx_gpudev *a6xx_gpudev =
to_a6xx_gpudev(ADRENO_GPU_DEVICE(adreno_dev));
unsigned int status = 0;
gmu_core_regread(device, A6XX_GMU_AO_HOST_INTERRUPT_STATUS, &status);
gmu_core_regwrite(device, A6XX_GMU_AO_HOST_INTERRUPT_CLR, status);
/* Ignore GMU_INT_RSCC_COMP and GMU_INT_DBD WAKEUP interrupts */
if (status & GMU_INT_WDOG_BITE)
a6xx_gpudev->handle_watchdog(adreno_dev);
if (status & GMU_INT_HOST_AHB_BUS_ERR)
dev_err_ratelimited(&gmu->pdev->dev,
"AHB bus error interrupt received\n");
if (status & GMU_INT_FENCE_ERR) {
unsigned int fence_status;
gmu_core_regread(device, A6XX_GMU_AHB_FENCE_STATUS,
&fence_status);
dev_err_ratelimited(&gmu->pdev->dev,
"FENCE error interrupt received %x\n", fence_status);
}
if (status & ~GMU_AO_INT_MASK)
dev_err_ratelimited(&gmu->pdev->dev,
"Unhandled GMU interrupts 0x%lx\n",
status & ~GMU_AO_INT_MASK);
return IRQ_HANDLED;
}
void a6xx_gmu_snapshot(struct adreno_device *adreno_dev,
struct kgsl_snapshot *snapshot)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
a6xx_gmu_device_snapshot(device, snapshot);
a6xx_snapshot(adreno_dev, snapshot);
gmu_core_regwrite(device, A6XX_GMU_GMU2HOST_INTR_CLR,
0xffffffff);
gmu_core_regwrite(device, A6XX_GMU_GMU2HOST_INTR_MASK,
HFI_IRQ_MASK);
}
void a6xx_gmu_aop_send_acd_state(struct a6xx_gmu_device *gmu, bool flag)
{
struct qmp_pkt msg;
char msg_buf[36];
u32 size;
int ret;
if (IS_ERR_OR_NULL(gmu->mailbox.channel))
return;
size = scnprintf(msg_buf, sizeof(msg_buf),
"{class: gpu, res: acd, val: %d}", flag);
/* mailbox controller expects 4-byte aligned buffer */
msg.size = ALIGN((size + 1), SZ_4);
msg.data = msg_buf;
ret = mbox_send_message(gmu->mailbox.channel, &msg);
if (ret < 0)
dev_err(&gmu->pdev->dev,
"AOP mbox send message failed: %d\n", ret);
}
int a6xx_gmu_enable_clks(struct adreno_device *adreno_dev, u32 level)
{
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
int ret;
a6xx_rdpm_cx_freq_update(gmu, gmu->freqs[level] / 1000);
ret = kgsl_clk_set_rate(gmu->clks, gmu->num_clks, "gmu_clk",
gmu->freqs[level]);
if (ret) {
dev_err(&gmu->pdev->dev, "GMU clock:%d set failed:%d\n",
gmu->freqs[level], ret);
return ret;
}
ret = kgsl_clk_set_rate(gmu->clks, gmu->num_clks, "hub_clk",
adreno_dev->gmu_hub_clk_freq);
if (ret && ret != -ENODEV) {
dev_err(&gmu->pdev->dev, "Unable to set the HUB clock\n");
return ret;
}
ret = clk_bulk_prepare_enable(gmu->num_clks, gmu->clks);
if (ret) {
dev_err(&gmu->pdev->dev, "Cannot enable GMU clocks\n");
return ret;
}
device->state = KGSL_STATE_AWARE;
return 0;
}
static void a6xx_gmu_force_first_boot(struct kgsl_device *device)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
u32 val = 0;
if (gmu->pdc_cfg_base) {
kgsl_pwrctrl_enable_cx_gdsc(device);
a6xx_gmu_enable_clks(adreno_dev, 0);
val = __raw_readl(gmu->pdc_cfg_base + (PDC_GPU_ENABLE_PDC << 2));
/* ensure this read operation is done before the next one */
rmb();
clk_bulk_disable_unprepare(gmu->num_clks, gmu->clks);
a6xx_gmu_disable_gdsc(adreno_dev);
a6xx_rdpm_cx_freq_update(gmu, 0);
}
if (val != PDC_ENABLE_REG_VALUE) {
clear_bit(GMU_PRIV_RSCC_SLEEP_DONE, &gmu->flags);
clear_bit(GMU_PRIV_PDC_RSC_LOADED, &gmu->flags);
}
}
static int a6xx_gmu_first_boot(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct kgsl_pwrctrl *pwr = &device->pwrctrl;
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
int level, ret;
kgsl_pwrctrl_request_state(device, KGSL_STATE_AWARE);
a6xx_gmu_aop_send_acd_state(gmu, adreno_dev->acd_enabled);
ret = kgsl_pwrctrl_enable_cx_gdsc(device);
if (ret)
return ret;
ret = a6xx_gmu_enable_clks(adreno_dev, 0);
if (ret)
goto gdsc_off;
ret = a6xx_gmu_load_fw(adreno_dev);
if (ret)
goto clks_gdsc_off;
ret = a6xx_gmu_itcm_shadow(adreno_dev);
if (ret)
goto clks_gdsc_off;
a6xx_gmu_register_config(adreno_dev);
a6xx_gmu_version_info(adreno_dev);
a6xx_gmu_irq_enable(adreno_dev);
/* Vote for minimal DDR BW for GMU to init */
level = pwr->pwrlevels[pwr->default_pwrlevel].bus_min;
icc_set_bw(pwr->icc_path, 0, kBps_to_icc(pwr->ddr_table[level]));
/* Clear any GPU faults that might have been left over */
adreno_clear_gpu_fault(adreno_dev);
ret = a6xx_gmu_device_start(adreno_dev);
if (ret)
goto err;
if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_HFI_USE_REG)) {
ret = a6xx_gmu_gfx_rail_on(adreno_dev);
if (ret) {
a6xx_gmu_oob_clear(device, oob_boot_slumber);
goto err;
}
}
if (gmu->idle_level == GPU_HW_ACTIVE) {
ret = a6xx_gmu_sptprac_enable(adreno_dev);
if (ret)
goto err;
}
if (!test_bit(GMU_PRIV_PDC_RSC_LOADED, &gmu->flags)) {
ret = a6xx_load_pdc_ucode(adreno_dev);
if (ret)
goto err;
a6xx_load_rsc_ucode(adreno_dev);
set_bit(GMU_PRIV_PDC_RSC_LOADED, &gmu->flags);
}
ret = a6xx_gmu_hfi_start(adreno_dev);
if (ret)
goto err;
ret = a6xx_hfi_start(adreno_dev);
if (ret)
goto err;
icc_set_bw(pwr->icc_path, 0, 0);
device->gmu_fault = false;
kgsl_pwrctrl_set_state(device, KGSL_STATE_AWARE);
return 0;
err:
a6xx_gmu_irq_disable(adreno_dev);
if (device->gmu_fault) {
a6xx_gmu_suspend(adreno_dev);
return ret;
}
clks_gdsc_off:
clk_bulk_disable_unprepare(gmu->num_clks, gmu->clks);
gdsc_off:
a6xx_gmu_disable_gdsc(adreno_dev);
a6xx_rdpm_cx_freq_update(gmu, 0);
return ret;
}
static int a6xx_gmu_boot(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
int ret = 0;
kgsl_pwrctrl_request_state(device, KGSL_STATE_AWARE);
ret = kgsl_pwrctrl_enable_cx_gdsc(device);
if (ret)
return ret;
ret = a6xx_gmu_enable_clks(adreno_dev, 0);
if (ret)
goto gdsc_off;
ret = a6xx_rscc_wakeup_sequence(adreno_dev);
if (ret)
goto clks_gdsc_off;
ret = a6xx_gmu_load_fw(adreno_dev);
if (ret)
goto clks_gdsc_off;
a6xx_gmu_register_config(adreno_dev);
a6xx_gmu_irq_enable(adreno_dev);
/* Clear any GPU faults that might have been left over */
adreno_clear_gpu_fault(adreno_dev);
ret = a6xx_gmu_device_start(adreno_dev);
if (ret)
goto err;
if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_HFI_USE_REG)) {
ret = a6xx_gmu_gfx_rail_on(adreno_dev);
if (ret) {
a6xx_gmu_oob_clear(device, oob_boot_slumber);
goto err;
}
}
if (gmu->idle_level == GPU_HW_ACTIVE) {
ret = a6xx_gmu_sptprac_enable(adreno_dev);
if (ret)
goto err;
}
ret = a6xx_gmu_hfi_start(adreno_dev);
if (ret)
goto err;
ret = a6xx_hfi_start(adreno_dev);
if (ret)
goto err;
device->gmu_fault = false;
kgsl_pwrctrl_set_state(device, KGSL_STATE_AWARE);
return 0;
err:
a6xx_gmu_irq_disable(adreno_dev);
if (device->gmu_fault) {
a6xx_gmu_suspend(adreno_dev);
return ret;
}
clks_gdsc_off:
clk_bulk_disable_unprepare(gmu->num_clks, gmu->clks);
gdsc_off:
a6xx_gmu_disable_gdsc(adreno_dev);
a6xx_rdpm_cx_freq_update(gmu, 0);
return ret;
}
static void set_acd(struct adreno_device *adreno_dev, void *priv)
{
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
adreno_dev->acd_enabled = *((bool *)priv);
a6xx_gmu_aop_send_acd_state(gmu, adreno_dev->acd_enabled);
}
static int a6xx_gmu_acd_set(struct kgsl_device *device, bool val)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
if (IS_ERR_OR_NULL(gmu->mailbox.channel))
return -EINVAL;
/* Don't do any unneeded work if ACD is already in the correct state */
if (adreno_dev->acd_enabled == val)
return 0;
/* Power cycle the GPU for changes to take effect */
return adreno_power_cycle(adreno_dev, set_acd, &val);
}
static void a6xx_send_tlb_hint(struct kgsl_device *device, bool val)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
if (!gmu->domain)
return;
#if (KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE)
qcom_skip_tlb_management(&gmu->pdev->dev, val);
#endif
if (!val)
iommu_flush_iotlb_all(gmu->domain);
}
static const struct gmu_dev_ops a6xx_gmudev = {
.oob_set = a6xx_gmu_oob_set,
.oob_clear = a6xx_gmu_oob_clear,
.ifpc_store = a6xx_gmu_ifpc_store,
.ifpc_isenabled = a6xx_gmu_ifpc_isenabled,
.cooperative_reset = a6xx_gmu_cooperative_reset,
.wait_for_active_transition = a6xx_gmu_wait_for_active_transition,
.scales_bandwidth = a6xx_gmu_scales_bandwidth,
.acd_set = a6xx_gmu_acd_set,
.force_first_boot = a6xx_gmu_force_first_boot,
.send_nmi = a6xx_gmu_send_nmi,
.send_tlb_hint = a6xx_send_tlb_hint,
};
static int a6xx_gmu_bus_set(struct adreno_device *adreno_dev, int buslevel,
u32 ab)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct kgsl_pwrctrl *pwr = &device->pwrctrl;
int ret = 0;
kgsl_icc_set_tag(pwr, buslevel);
if (buslevel != pwr->cur_buslevel) {
ret = a6xx_gmu_dcvs_set(adreno_dev, INVALID_DCVS_IDX, buslevel);
if (ret)
return ret;
pwr->cur_buslevel = buslevel;
}
if (ab != pwr->cur_ab) {
icc_set_bw(pwr->icc_path, MBps_to_icc(ab), 0);
pwr->cur_ab = ab;
}
trace_kgsl_buslevel(device, pwr->active_pwrlevel, pwr->cur_buslevel, pwr->cur_ab);
return ret;
}
static void a6xx_free_gmu_globals(struct a6xx_gmu_device *gmu)
{
int i;
for (i = 0; i < gmu->global_entries && i < ARRAY_SIZE(gmu->gmu_globals); i++) {
struct kgsl_memdesc *md = &gmu->gmu_globals[i];
if (!md->gmuaddr)
continue;
iommu_unmap(gmu->domain, md->gmuaddr, md->size);
if (md->priv & KGSL_MEMDESC_SYSMEM)
kgsl_sharedmem_free(md);
memset(md, 0, sizeof(*md));
}
if (gmu->domain) {
iommu_detach_device(gmu->domain, &gmu->pdev->dev);
iommu_domain_free(gmu->domain);
gmu->domain = NULL;
}
gmu->global_entries = 0;
}
static int a6xx_gmu_aop_mailbox_init(struct adreno_device *adreno_dev,
struct a6xx_gmu_device *gmu)
{
struct kgsl_mailbox *mailbox = &gmu->mailbox;
mailbox->client.dev = &gmu->pdev->dev;
mailbox->client.tx_block = true;
mailbox->client.tx_tout = 1000;
mailbox->client.knows_txdone = false;
mailbox->channel = mbox_request_channel(&mailbox->client, 0);
if (IS_ERR(mailbox->channel))
return PTR_ERR(mailbox->channel);
adreno_dev->acd_enabled = true;
return 0;
}
static void a6xx_gmu_acd_probe(struct kgsl_device *device,
struct a6xx_gmu_device *gmu, struct device_node *node)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
struct kgsl_pwrctrl *pwr = &device->pwrctrl;
struct kgsl_pwrlevel *pwrlevel =
&pwr->pwrlevels[pwr->num_pwrlevels - 1];
struct hfi_acd_table_cmd *cmd = &gmu->hfi.acd_table;
int ret, i, cmd_idx = 0;
if (!ADRENO_FEATURE(adreno_dev, ADRENO_ACD))
return;
cmd->hdr = CREATE_MSG_HDR(H2F_MSG_ACD_TBL, HFI_MSG_CMD);
cmd->version = 1;
cmd->stride = 1;
cmd->enable_by_level = 0;
/*
* Iterate through each gpu power level and generate a mask for GMU
* firmware for ACD enabled levels and store the corresponding control
* register configurations to the acd_table structure.
*/
for (i = 0; i < pwr->num_pwrlevels; i++) {
if (pwrlevel->acd_level) {
cmd->enable_by_level |= (1 << (i + 1));
cmd->data[cmd_idx++] = pwrlevel->acd_level;
}
pwrlevel--;
}
if (!cmd->enable_by_level)
return;
cmd->num_levels = cmd_idx;
ret = a6xx_gmu_aop_mailbox_init(adreno_dev, gmu);
if (ret)
dev_err(&gmu->pdev->dev,
"AOP mailbox init failed: %d\n", ret);
}
static int a6xx_gmu_reg_probe(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
int ret;
ret = kgsl_regmap_add_region(&device->regmap, gmu->pdev,
"kgsl_gmu_reg", NULL, NULL);
if (ret)
dev_err(&gmu->pdev->dev, "Unable to map the GMU registers\n");
return ret;
}
static int a6xx_gmu_clk_probe(struct adreno_device *adreno_dev)
{
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
int ret, i;
int tbl_size;
int num_freqs;
int offset;
ret = devm_clk_bulk_get_all(&gmu->pdev->dev, &gmu->clks);
if (ret < 0)
return ret;
/*
* Voting for apb_pclk will enable power and clocks required for
* QDSS path to function. However, if QCOM_KGSL_QDSS_STM is not enabled,
* QDSS is essentially unusable. Hence, if QDSS cannot be used,
* don't vote for this clock.
*/
if (!IS_ENABLED(CONFIG_QCOM_KGSL_QDSS_STM)) {
for (i = 0; i < ret; i++) {
if (!strcmp(gmu->clks[i].id, "apb_pclk")) {
gmu->clks[i].clk = NULL;
break;
}
}
}
gmu->num_clks = ret;
/* Read the optional list of GMU frequencies */
if (of_get_property(gmu->pdev->dev.of_node,
"qcom,gmu-freq-table", &tbl_size) == NULL)
goto default_gmu_freq;
num_freqs = (tbl_size / sizeof(u32)) / 2;
if (num_freqs != ARRAY_SIZE(gmu->freqs))
goto default_gmu_freq;
for (i = 0; i < num_freqs; i++) {
offset = i * 2;
ret = of_property_read_u32_index(gmu->pdev->dev.of_node,
"qcom,gmu-freq-table", offset, &gmu->freqs[i]);
if (ret)
goto default_gmu_freq;
ret = of_property_read_u32_index(gmu->pdev->dev.of_node,
"qcom,gmu-freq-table", offset + 1, &gmu->vlvls[i]);
if (ret)
goto default_gmu_freq;
}
return 0;
default_gmu_freq:
/* The GMU frequency table is missing or invalid. Go with a default */
gmu->freqs[0] = GMU_FREQ_MIN;
gmu->vlvls[0] = RPMH_REGULATOR_LEVEL_MIN_SVS;
gmu->freqs[1] = GMU_FREQ_MAX;
gmu->vlvls[1] = RPMH_REGULATOR_LEVEL_SVS;
if (adreno_is_a660(adreno_dev))
gmu->vlvls[0] = RPMH_REGULATOR_LEVEL_LOW_SVS;
return 0;
}
static void a6xx_gmu_rdpm_probe(struct a6xx_gmu_device *gmu,
struct kgsl_device *device)
{
struct resource *res;
res = platform_get_resource_byname(device->pdev, IORESOURCE_MEM,
"rdpm_cx");
if (res)
gmu->rdpm_cx_virt = devm_ioremap(&device->pdev->dev,
res->start, resource_size(res));
res = platform_get_resource_byname(device->pdev, IORESOURCE_MEM,
"rdpm_mx");
if (res)
gmu->rdpm_mx_virt = devm_ioremap(&device->pdev->dev,
res->start, resource_size(res));
}
void a6xx_gmu_remove(struct kgsl_device *device)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
if (!IS_ERR_OR_NULL(gmu->mailbox.channel))
mbox_free_channel(gmu->mailbox.channel);
adreno_dev->acd_enabled = false;
if (gmu->fw_image)
release_firmware(gmu->fw_image);
a6xx_free_gmu_globals(gmu);
vfree(gmu->itcm_shadow);
kobject_put(&gmu->log_kobj);
kobject_put(&gmu->stats_kobj);
}
static int a6xx_gmu_iommu_fault_handler(struct iommu_domain *domain,
struct device *dev, unsigned long addr, int flags, void *token)
{
char *fault_type = "unknown";
if (flags & IOMMU_FAULT_TRANSLATION)
fault_type = "translation";
else if (flags & IOMMU_FAULT_PERMISSION)
fault_type = "permission";
else if (flags & IOMMU_FAULT_EXTERNAL)
fault_type = "external";
else if (flags & IOMMU_FAULT_TRANSACTION_STALLED)
fault_type = "transaction stalled";
dev_err(dev, "GMU fault addr = %lX, context=kernel (%s %s fault)\n",
addr,
(flags & IOMMU_FAULT_WRITE) ? "write" : "read",
fault_type);
return 0;
}
static int a6xx_gmu_iommu_init(struct a6xx_gmu_device *gmu)
{
int ret;
gmu->domain = iommu_domain_alloc(&platform_bus_type);
if (gmu->domain == NULL) {
dev_err(&gmu->pdev->dev, "Unable to allocate GMU IOMMU domain\n");
return -ENODEV;
}
/*
* Disable stall on fault for the GMU context bank.
* This sets SCTLR.CFCFG = 0.
* Also note that, the smmu driver sets SCTLR.HUPCF = 0 by default.
*/
qcom_iommu_set_fault_model(gmu->domain, QCOM_IOMMU_FAULT_MODEL_NO_STALL);
ret = iommu_attach_device(gmu->domain, &gmu->pdev->dev);
if (!ret) {
iommu_set_fault_handler(gmu->domain,
a6xx_gmu_iommu_fault_handler, gmu);
return 0;
}
dev_err(&gmu->pdev->dev,
"Unable to attach GMU IOMMU domain: %d\n", ret);
iommu_domain_free(gmu->domain);
gmu->domain = NULL;
return ret;
}
int a6xx_gmu_probe(struct kgsl_device *device,
struct platform_device *pdev)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
struct device *dev = &pdev->dev;
struct resource *res;
int ret;
gmu->pdev = pdev;
dma_set_coherent_mask(&gmu->pdev->dev, DMA_BIT_MASK(64));
gmu->pdev->dev.dma_mask = &gmu->pdev->dev.coherent_dma_mask;
set_dma_ops(&gmu->pdev->dev, NULL);
res = platform_get_resource_byname(device->pdev, IORESOURCE_MEM,
"rscc");
if (res) {
gmu->rscc_virt = devm_ioremap(&device->pdev->dev, res->start,
resource_size(res));
if (gmu->rscc_virt == NULL) {
dev_err(&gmu->pdev->dev, "rscc ioremap failed\n");
return -ENOMEM;
}
}
/* Setup any rdpm register ranges */
a6xx_gmu_rdpm_probe(gmu, device);
/* Set up GMU regulators */
ret = kgsl_pwrctrl_probe_regulators(device, pdev);
if (ret)
return ret;
ret = a6xx_gmu_clk_probe(adreno_dev);
if (ret < 0)
return ret;
/* Set up GMU IOMMU and shared memory with GMU */
ret = a6xx_gmu_iommu_init(gmu);
if (ret)
goto error;
if (adreno_is_a650_family(adreno_dev))
gmu->vma = a6xx_gmu_vma;
else
gmu->vma = a6xx_gmu_vma_legacy;
/* Map and reserve GMU CSRs registers */
ret = a6xx_gmu_reg_probe(adreno_dev);
if (ret)
goto error;
/* Populates RPMh configurations */
ret = a6xx_build_rpmh_tables(adreno_dev);
if (ret)
goto error;
/* Set up GMU idle state */
if (ADRENO_FEATURE(adreno_dev, ADRENO_IFPC)) {
gmu->idle_level = GPU_HW_IFPC;
adreno_dev->ifpc_hyst = A6X_GMU_LONG_IFPC_HYST;
adreno_dev->ifpc_hyst_floor = A6X_GMU_LONG_IFPC_HYST_FLOOR;
} else {
gmu->idle_level = GPU_HW_ACTIVE;
}
a6xx_gmu_acd_probe(device, gmu, pdev->dev.of_node);
set_bit(GMU_ENABLED, &device->gmu_core.flags);
/* Initialize to zero to detect trace packet loss */
gmu->trace.seq_num = 0;
device->gmu_core.dev_ops = &a6xx_gmudev;
/* Set default GMU attributes */
gmu->log_stream_enable = false;
gmu->log_group_mask = 0x3;
/* Disabled by default */
gmu->stats_enable = false;
/* Set default to CM3 busy cycles countable */
gmu->stats_mask = BIT(A6XX_GMU_CM3_BUSY_CYCLES);
/* Interval is in 50 us units. Set default sampling frequency to 4x50 us */
gmu->stats_interval = HFI_FEATURE_GMU_STATS_INTERVAL;
/* GMU sysfs nodes setup */
(void) kobject_init_and_add(&gmu->log_kobj, &log_kobj_type, &dev->kobj, "log");
(void) kobject_init_and_add(&gmu->stats_kobj, &stats_kobj_type, &dev->kobj, "stats");
of_property_read_u32(gmu->pdev->dev.of_node, "qcom,gmu-perf-ddr-bw",
&gmu->perf_ddr_bw);
gmu->irq = kgsl_request_irq(gmu->pdev, "kgsl_gmu_irq",
a6xx_gmu_irq_handler, device);
if (gmu->irq >= 0)
return 0;
ret = gmu->irq;
error:
a6xx_gmu_remove(device);
return ret;
}
static void a6xx_gmu_active_count_put(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
if (WARN_ON(!mutex_is_locked(&device->mutex)))
return;
if (WARN(atomic_read(&device->active_cnt) == 0,
"Unbalanced get/put calls to KGSL active count\n"))
return;
if (atomic_dec_and_test(&device->active_cnt)) {
kgsl_pwrscale_update_stats(device);
kgsl_pwrscale_update(device);
kgsl_start_idle_timer(device);
}
trace_kgsl_active_count(device,
(unsigned long) __builtin_return_address(0));
wake_up(&device->active_cnt_wq);
}
int a6xx_halt_gbif(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
int ret;
/* Halt new client requests */
kgsl_regwrite(device, A6XX_GBIF_HALT, A6XX_GBIF_CLIENT_HALT_MASK);
ret = adreno_wait_for_halt_ack(device,
A6XX_GBIF_HALT_ACK, A6XX_GBIF_CLIENT_HALT_MASK);
/* Halt all AXI requests */
kgsl_regwrite(device, A6XX_GBIF_HALT, A6XX_GBIF_ARB_HALT_MASK);
ret = adreno_wait_for_halt_ack(device,
A6XX_GBIF_HALT_ACK, A6XX_GBIF_ARB_HALT_MASK);
return ret;
}
#define RPMH_VOTE_TIMEOUT 2 /* ms */
static int a6xx_gmu_power_off(struct adreno_device *adreno_dev)
{
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
int ret = 0;
if (device->gmu_fault)
goto error;
/* Wait for the lowest idle level we requested */
ret = a6xx_gmu_wait_for_lowest_idle(adreno_dev);
if (ret)
goto error;
ret = a6xx_complete_rpmh_votes(adreno_dev, RPMH_VOTE_TIMEOUT);
if (ret)
goto error;
ret = a6xx_gmu_notify_slumber(adreno_dev);
if (ret)
goto error;
ret = a6xx_gmu_wait_for_idle(adreno_dev);
if (ret)
goto error;
ret = a6xx_rscc_sleep_sequence(adreno_dev);
a6xx_rdpm_mx_freq_update(gmu, 0);
/* Now that we are done with GMU and GPU, Clear the GBIF */
if (!adreno_is_a630(adreno_dev)) {
ret = a6xx_halt_gbif(adreno_dev);
/* De-assert the halts */
kgsl_regwrite(device, A6XX_GBIF_HALT, 0x0);
}
a6xx_gmu_irq_disable(adreno_dev);
a6xx_hfi_stop(adreno_dev);
clk_bulk_disable_unprepare(gmu->num_clks, gmu->clks);
a6xx_gmu_disable_gdsc(adreno_dev);
a6xx_rdpm_cx_freq_update(gmu, 0);
kgsl_pwrctrl_set_state(device, KGSL_STATE_NONE);
return ret;
error:
a6xx_gmu_irq_disable(adreno_dev);
a6xx_hfi_stop(adreno_dev);
a6xx_gmu_suspend(adreno_dev);
return ret;
}
void a6xx_enable_gpu_irq(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
kgsl_pwrctrl_irq(device, true);
adreno_irqctrl(adreno_dev, 1);
}
void a6xx_disable_gpu_irq(struct adreno_device *adreno_dev)
{
kgsl_pwrctrl_irq(KGSL_DEVICE(adreno_dev), false);
if (a6xx_gmu_gx_is_on(adreno_dev))
adreno_irqctrl(adreno_dev, 0);
}
static void a6xx_fusa_init(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
void __iomem *fusa_virt = NULL;
struct resource *res;
if (!adreno_is_a663(adreno_dev))
return;
res = platform_get_resource_byname(device->pdev,
IORESOURCE_MEM, "fusa");
if (res)
fusa_virt = ioremap(res->start, resource_size(res));
if (!fusa_virt) {
dev_err(device->dev, "Failed to map fusa\n");
return;
}
/* Disable fusa mode in boot stage */
_regrmw(fusa_virt, A6XX_GPU_FUSA_REG_ECC_CTRL - A6XX_GPU_FUSA_REG_BASE,
A6XX_GPU_FUSA_DISABLE_MASK, A6XX_GPU_FUSA_DISABLE_BITS);
_regrmw(fusa_virt, A6XX_GPU_FUSA_REG_CSR_PRIY - A6XX_GPU_FUSA_REG_BASE,
A6XX_GPU_FUSA_DISABLE_MASK, A6XX_GPU_FUSA_DISABLE_BITS);
iounmap(fusa_virt);
}
static int a6xx_gpu_boot(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
int ret;
adreno_set_active_ctxs_null(adreno_dev);
ret = kgsl_mmu_start(device);
if (ret)
goto err;
ret = a6xx_gmu_oob_set(device, oob_gpu);
if (ret)
goto oob_clear;
ret = a6xx_gmu_hfi_start_msg(adreno_dev);
if (ret)
goto oob_clear;
/* Clear the busy_data stats - we're starting over from scratch */
memset(&adreno_dev->busy_data, 0, sizeof(adreno_dev->busy_data));
/* Restore performance counter registers with saved values */
adreno_perfcounter_restore(adreno_dev);
a6xx_start(adreno_dev);
/* Re-initialize the coresight registers if applicable */
adreno_coresight_start(adreno_dev);
adreno_perfcounter_start(adreno_dev);
/* Clear FSR here in case it is set from a previous pagefault */
kgsl_mmu_clear_fsr(&device->mmu);
a6xx_enable_gpu_irq(adreno_dev);
ret = a6xx_rb_start(adreno_dev);
if (ret) {
a6xx_disable_gpu_irq(adreno_dev);
goto oob_clear;
}
/*
* At this point it is safe to assume that we recovered. Setting
* this field allows us to take a new snapshot for the next failure
* if we are prioritizing the first unrecoverable snapshot.
*/
if (device->snapshot)
device->snapshot->recovered = true;
/* Start the dispatcher */
adreno_dispatcher_start(device);
device->reset_counter++;
a6xx_gmu_oob_clear(device, oob_gpu);
if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_HFI_USE_REG))
gmu_core_dev_oob_clear(device, oob_boot_slumber);
return 0;
oob_clear:
a6xx_gmu_oob_clear(device, oob_gpu);
if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_HFI_USE_REG))
gmu_core_dev_oob_clear(device, oob_boot_slumber);
err:
a6xx_gmu_power_off(adreno_dev);
return ret;
}
static void gmu_idle_timer(struct timer_list *t)
{
struct kgsl_device *device = container_of(t, struct kgsl_device,
idle_timer);
kgsl_schedule_work(&device->idle_check_ws);
}
static int a6xx_boot(struct adreno_device *adreno_dev)
{
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
int ret;
if (WARN_ON(test_bit(GMU_PRIV_GPU_STARTED, &gmu->flags)))
return 0;
kgsl_pwrctrl_request_state(device, KGSL_STATE_ACTIVE);
if (IS_ENABLED(CONFIG_QCOM_KGSL_HIBERNATION) &&
!test_bit(GMU_PRIV_PDC_RSC_LOADED, &gmu->flags))
ret = a6xx_gmu_first_boot(adreno_dev);
else
ret = a6xx_gmu_boot(adreno_dev);
if (ret)
return ret;
ret = a6xx_gpu_boot(adreno_dev);
if (ret)
return ret;
kgsl_start_idle_timer(device);
kgsl_pwrscale_wake(device);
set_bit(GMU_PRIV_GPU_STARTED, &gmu->flags);
device->pwrctrl.last_stat_updated = ktime_get();
kgsl_pwrctrl_set_state(device, KGSL_STATE_ACTIVE);
return ret;
}
static int a6xx_first_boot(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
int ret;
if (test_bit(GMU_PRIV_FIRST_BOOT_DONE, &gmu->flags)) {
if (!test_bit(GMU_PRIV_GPU_STARTED, &gmu->flags))
return a6xx_boot(adreno_dev);
return 0;
}
KGSL_BOOT_MARKER("ADRENO Init");
ret = a6xx_ringbuffer_init(adreno_dev);
if (ret)
return ret;
ret = a6xx_microcode_read(adreno_dev);
if (ret)
return ret;
ret = a6xx_init(adreno_dev);
if (ret)
return ret;
ret = a6xx_gmu_init(adreno_dev);
if (ret)
return ret;
kgsl_pwrctrl_request_state(device, KGSL_STATE_ACTIVE);
ret = a6xx_gmu_first_boot(adreno_dev);
if (ret)
return ret;
a6xx_fusa_init(adreno_dev);
ret = a6xx_gpu_boot(adreno_dev);
if (ret)
return ret;
adreno_get_bus_counters(adreno_dev);
adreno_dev->cooperative_reset = ADRENO_FEATURE(adreno_dev,
ADRENO_COOP_RESET);
adreno_create_profile_buffer(adreno_dev);
set_bit(GMU_PRIV_FIRST_BOOT_DONE, &gmu->flags);
set_bit(GMU_PRIV_GPU_STARTED, &gmu->flags);
/*
* BCL needs respective Central Broadcast register to
* be programed from TZ. This programing happens only
* when zap shader firmware load is successful. Zap firmware
* load can fail in boot up path hence enable BCL only after we
* successfully complete first boot to ensure that Central
* Broadcast register was programed before enabling BCL.
*/
if (ADRENO_FEATURE(adreno_dev, ADRENO_BCL))
adreno_dev->bcl_enabled = true;
/*
* There is a possible deadlock scenario during kgsl firmware reading
* (request_firmware) and devfreq update calls. During first boot, kgsl
* device mutex is held and then request_firmware is called for reading
* firmware. request_firmware internally takes dev_pm_qos_mtx lock.
* Whereas in case of devfreq update calls triggered by thermal/bcl or
* devfreq sysfs, it first takes the same dev_pm_qos_mtx lock and then
* tries to take kgsl device mutex as part of get_dev_status/target
* calls. This results in deadlock when both thread are unable to acquire
* the mutex held by other thread. Enable devfreq updates now as we are
* done reading all firmware files.
*/
device->pwrscale.devfreq_enabled = true;
device->pwrctrl.last_stat_updated = ktime_get();
kgsl_pwrctrl_set_state(device, KGSL_STATE_ACTIVE);
KGSL_BOOT_MARKER("ADRENO Ready");
return 0;
}
static int a630_vbif_halt(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
int ret;
kgsl_regwrite(device, A6XX_VBIF_XIN_HALT_CTRL0,
A6XX_VBIF_XIN_HALT_CTRL0_MASK);
ret = adreno_wait_for_halt_ack(device,
A6XX_VBIF_XIN_HALT_CTRL1,
A6XX_VBIF_XIN_HALT_CTRL0_MASK);
kgsl_regwrite(device, A6XX_VBIF_XIN_HALT_CTRL0, 0);
return ret;
}
static int a6xx_power_off(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
int ret;
WARN_ON(!test_bit(GMU_PRIV_GPU_STARTED, &gmu->flags));
adreno_suspend_context(device);
/*
* adreno_suspend_context() unlocks the device mutex, which
* could allow a concurrent thread to attempt SLUMBER sequence.
* Hence, check the flags again before proceeding with SLUMBER.
*/
if (!test_bit(GMU_PRIV_GPU_STARTED, &gmu->flags))
return 0;
kgsl_pwrctrl_request_state(device, KGSL_STATE_SLUMBER);
ret = a6xx_gmu_oob_set(device, oob_gpu);
if (ret) {
a6xx_gmu_oob_clear(device, oob_gpu);
goto no_gx_power;
}
if (a6xx_irq_pending(adreno_dev)) {
a6xx_gmu_oob_clear(device, oob_gpu);
return -EBUSY;
}
kgsl_pwrscale_update_stats(device);
/* Save active coresight registers if applicable */
adreno_coresight_stop(adreno_dev);
/* Save physical performance counter values before GPU power down*/
adreno_perfcounter_save(adreno_dev);
/*
* Clear GX halt on non-gbif targets. For targets with GBIF,
* GX halt is handled by the GMU FW.
*/
if (adreno_is_a630(adreno_dev))
a630_vbif_halt(adreno_dev);
adreno_irqctrl(adreno_dev, 0);
a6xx_gmu_oob_clear(device, oob_gpu);
no_gx_power:
kgsl_pwrctrl_irq(device, false);
a6xx_gmu_power_off(adreno_dev);
adreno_set_active_ctxs_null(adreno_dev);
adreno_dispatcher_stop(adreno_dev);
adreno_ringbuffer_stop(adreno_dev);
adreno_llcc_slice_deactivate(adreno_dev);
clear_bit(GMU_PRIV_GPU_STARTED, &gmu->flags);
del_timer_sync(&device->idle_timer);
kgsl_pwrscale_sleep(device);
kgsl_pwrctrl_clear_l3_vote(device);
kgsl_pwrctrl_set_state(device, KGSL_STATE_SLUMBER);
return ret;
}
static void gmu_idle_check(struct work_struct *work)
{
struct kgsl_device *device = container_of(work,
struct kgsl_device, idle_check_ws);
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
int ret;
mutex_lock(&device->mutex);
if (test_bit(GMU_DISABLE_SLUMBER, &device->gmu_core.flags))
goto done;
if (atomic_read(&device->active_cnt) || time_is_after_jiffies(device->idle_jiffies)) {
kgsl_pwrscale_update(device);
kgsl_start_idle_timer(device);
goto done;
}
if (!test_bit(GMU_PRIV_GPU_STARTED, &gmu->flags))
goto done;
spin_lock(&device->submit_lock);
if (device->submit_now) {
spin_unlock(&device->submit_lock);
kgsl_pwrscale_update(device);
kgsl_start_idle_timer(device);
goto done;
}
device->skip_inline_submit = true;
spin_unlock(&device->submit_lock);
ret = a6xx_power_off(adreno_dev);
if (ret == -EBUSY) {
kgsl_pwrscale_update(device);
kgsl_start_idle_timer(device);
}
done:
mutex_unlock(&device->mutex);
}
static int a6xx_gmu_first_open(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
int ret;
/*
* Do the one time settings that need to happen when we
* attempt to boot the gpu the very first time
*/
ret = a6xx_first_boot(adreno_dev);
if (ret)
return ret;
/*
* A client that does a first_open but never closes the device
* may prevent us from going back to SLUMBER. So trigger the idle
* check by incrementing the active count and immediately releasing it.
*/
atomic_inc(&device->active_cnt);
a6xx_gmu_active_count_put(adreno_dev);
return 0;
}
static int a6xx_gmu_last_close(struct adreno_device *adreno_dev)
{
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
if (test_bit(GMU_PRIV_GPU_STARTED, &gmu->flags))
return a6xx_power_off(adreno_dev);
return 0;
}
static int a6xx_gmu_active_count_get(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
int ret = 0;
if (WARN_ON(!mutex_is_locked(&device->mutex)))
return -EINVAL;
if (test_bit(GMU_PRIV_PM_SUSPEND, &gmu->flags))
return -EINVAL;
if ((atomic_read(&device->active_cnt) == 0) &&
!test_bit(GMU_PRIV_GPU_STARTED, &gmu->flags))
ret = a6xx_boot(adreno_dev);
if (ret == 0)
atomic_inc(&device->active_cnt);
trace_kgsl_active_count(device,
(unsigned long) __builtin_return_address(0));
return ret;
}
static int a6xx_gmu_pm_suspend(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
int ret;
if (test_bit(GMU_PRIV_PM_SUSPEND, &gmu->flags))
return 0;
kgsl_pwrctrl_request_state(device, KGSL_STATE_SUSPEND);
/* Halt any new submissions */
reinit_completion(&device->halt_gate);
/* wait for active count so device can be put in slumber */
ret = kgsl_active_count_wait(device, 0, HZ);
if (ret) {
dev_err(device->dev,
"Timed out waiting for the active count\n");
goto err;
}
ret = adreno_idle(device);
if (ret)
goto err;
if (test_bit(GMU_PRIV_GPU_STARTED, &gmu->flags))
a6xx_power_off(adreno_dev);
set_bit(GMU_PRIV_PM_SUSPEND, &gmu->flags);
adreno_get_gpu_halt(adreno_dev);
kgsl_pwrctrl_set_state(device, KGSL_STATE_SUSPEND);
return 0;
err:
adreno_dispatcher_start(device);
return ret;
}
static void a6xx_gmu_pm_resume(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
if (WARN(!test_bit(GMU_PRIV_PM_SUSPEND, &gmu->flags),
"resume invoked without a suspend\n"))
return;
adreno_put_gpu_halt(adreno_dev);
adreno_dispatcher_start(device);
clear_bit(GMU_PRIV_PM_SUSPEND, &gmu->flags);
}
static void a6xx_gmu_touch_wakeup(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
int ret;
/*
* Do not wake up a suspended device or until the first boot sequence
* has been completed.
*/
if (test_bit(GMU_PRIV_PM_SUSPEND, &gmu->flags) ||
!test_bit(GMU_PRIV_FIRST_BOOT_DONE, &gmu->flags))
return;
if (test_bit(GMU_PRIV_GPU_STARTED, &gmu->flags))
goto done;
kgsl_pwrctrl_request_state(device, KGSL_STATE_ACTIVE);
ret = a6xx_gmu_boot(adreno_dev);
if (ret)
return;
ret = a6xx_gpu_boot(adreno_dev);
if (ret)
return;
kgsl_pwrscale_wake(device);
set_bit(GMU_PRIV_GPU_STARTED, &gmu->flags);
device->pwrctrl.last_stat_updated = ktime_get();
kgsl_pwrctrl_set_state(device, KGSL_STATE_ACTIVE);
done:
/*
* When waking up from a touch event we want to stay active long enough
* for the user to send a draw command. The default idle timer timeout
* is shorter than we want so go ahead and push the idle timer out
* further for this special case
*/
mod_timer(&device->idle_timer, jiffies +
msecs_to_jiffies(adreno_wake_timeout));
}
const struct adreno_power_ops a6xx_gmu_power_ops = {
.first_open = a6xx_gmu_first_open,
.last_close = a6xx_gmu_last_close,
.active_count_get = a6xx_gmu_active_count_get,
.active_count_put = a6xx_gmu_active_count_put,
.pm_suspend = a6xx_gmu_pm_suspend,
.pm_resume = a6xx_gmu_pm_resume,
.touch_wakeup = a6xx_gmu_touch_wakeup,
.gpu_clock_set = a6xx_gmu_clock_set,
.gpu_bus_set = a6xx_gmu_bus_set,
};
const struct adreno_power_ops a630_gmu_power_ops = {
.first_open = a6xx_gmu_first_open,
.last_close = a6xx_gmu_last_close,
.active_count_get = a6xx_gmu_active_count_get,
.active_count_put = a6xx_gmu_active_count_put,
.pm_suspend = a6xx_gmu_pm_suspend,
.pm_resume = a6xx_gmu_pm_resume,
.touch_wakeup = a6xx_gmu_touch_wakeup,
.gpu_clock_set = a6xx_gmu_clock_set,
};
int a6xx_gmu_device_probe(struct platform_device *pdev,
u32 chipid, const struct adreno_gpu_core *gpucore)
{
struct adreno_device *adreno_dev;
struct kgsl_device *device;
struct a6xx_device *a6xx_dev;
int ret;
a6xx_dev = devm_kzalloc(&pdev->dev, sizeof(*a6xx_dev),
GFP_KERNEL);
if (!a6xx_dev)
return -ENOMEM;
adreno_dev = &a6xx_dev->adreno_dev;
adreno_dev->irq_mask = A6XX_INT_MASK;
ret = a6xx_probe_common(pdev, adreno_dev, chipid, gpucore);
if (ret)
return ret;
ret = adreno_dispatcher_init(adreno_dev);
if (ret)
return ret;
device = KGSL_DEVICE(adreno_dev);
INIT_WORK(&device->idle_check_ws, gmu_idle_check);
timer_setup(&device->idle_timer, gmu_idle_timer, 0);
return 0;
}
int a6xx_gmu_reset(struct adreno_device *adreno_dev)
{
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
a6xx_disable_gpu_irq(adreno_dev);
a6xx_gmu_irq_disable(adreno_dev);
a6xx_hfi_stop(adreno_dev);
/* Hard reset the gmu and gpu */
a6xx_gmu_suspend(adreno_dev);
a6xx_reset_preempt_records(adreno_dev);
adreno_llcc_slice_deactivate(adreno_dev);
clear_bit(GMU_PRIV_GPU_STARTED, &gmu->flags);
/* Attempt to reboot the gmu and gpu */
return a6xx_boot(adreno_dev);
}
int a6xx_gmu_hfi_probe(struct adreno_device *adreno_dev)
{
struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev);
struct a6xx_hfi *hfi = &gmu->hfi;
hfi->irq = kgsl_request_irq(gmu->pdev, "kgsl_hfi_irq",
a6xx_hfi_irq_handler, KGSL_DEVICE(adreno_dev));
return hfi->irq < 0 ? hfi->irq : 0;
}
int a6xx_gmu_add_to_minidump(struct adreno_device *adreno_dev)
{
struct a6xx_device *a6xx_dev = container_of(adreno_dev,
struct a6xx_device, adreno_dev);
int ret;
ret = kgsl_add_va_to_minidump(adreno_dev->dev.dev, KGSL_A6XX_DEVICE,
(void *)(a6xx_dev), sizeof(struct a6xx_device));
if (ret)
return ret;
ret = kgsl_add_va_to_minidump(adreno_dev->dev.dev, KGSL_GMU_LOG_ENTRY,
a6xx_dev->gmu.gmu_log->hostptr, a6xx_dev->gmu.gmu_log->size);
if (ret)
return ret;
ret = kgsl_add_va_to_minidump(adreno_dev->dev.dev, KGSL_HFIMEM_ENTRY,
a6xx_dev->gmu.hfi.hfi_mem->hostptr, a6xx_dev->gmu.hfi.hfi_mem->size);
if (ret)
return ret;
if (adreno_is_a630(adreno_dev) || adreno_is_a615_family(adreno_dev))
ret = kgsl_add_va_to_minidump(adreno_dev->dev.dev, KGSL_GMU_DUMPMEM_ENTRY,
a6xx_dev->gmu.dump_mem->hostptr, a6xx_dev->gmu.dump_mem->size);
return ret;
}
static int a6xx_gmu_bind(struct device *dev, struct device *master, void *data)
{
struct kgsl_device *device = dev_get_drvdata(master);
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
const struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev);
const struct a6xx_gpudev *a6xx_gpudev = to_a6xx_gpudev(gpudev);
int ret;
ret = a6xx_gmu_probe(device, to_platform_device(dev));
if (ret)
return ret;
if (a6xx_gpudev->hfi_probe) {
ret = a6xx_gpudev->hfi_probe(adreno_dev);
if (ret) {
a6xx_gmu_remove(device);
return ret;
}
}
return 0;
}
static void a6xx_gmu_unbind(struct device *dev, struct device *master,
void *data)
{
struct kgsl_device *device = dev_get_drvdata(master);
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
const struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev);
const struct a6xx_gpudev *a6xx_gpudev = to_a6xx_gpudev(gpudev);
if (a6xx_gpudev->hfi_remove)
a6xx_gpudev->hfi_remove(adreno_dev);
a6xx_gmu_remove(device);
}
static const struct component_ops a6xx_gmu_component_ops = {
.bind = a6xx_gmu_bind,
.unbind = a6xx_gmu_unbind,
};
static int a6xx_gmu_probe_dev(struct platform_device *pdev)
{
return component_add(&pdev->dev, &a6xx_gmu_component_ops);
}
static int a6xx_gmu_remove_dev(struct platform_device *pdev)
{
component_del(&pdev->dev, &a6xx_gmu_component_ops);
return 0;
}
static const struct of_device_id a6xx_gmu_match_table[] = {
{ .compatible = "qcom,gpu-gmu" },
{ },
};
struct platform_driver a6xx_gmu_driver = {
.probe = a6xx_gmu_probe_dev,
.remove = a6xx_gmu_remove_dev,
.driver = {
.name = "adreno-a6xx-gmu",
.of_match_table = a6xx_gmu_match_table,
},
};
Reference in New Issue View Git Blame Copy Permalink