arm-smmu: Restore the downstream SMMU module
The arm_smmu module had to be reverted in commitfe6e5ac9fb("Revert "arm-smmu: Port our downstream SMMU changes from 5.15"") due to an LTS merge conflict in commitd82e9dd3f4("Merge keystone/android-mainline-keystone-qcom-release.5.18-rc2 (2069378) into msm-pineapple"). Restore the module to its downstream state accordingly. To account for commit9a630a4b41("iommu: Split struct iommu_ops"), we rework arm_smmu_ops. To account for the struct page->freelist member being removed in commit07f910f9b7("mm: Remove slab from struct page"), use the struct page->lru.next pointer to form a single linked list of pages to be tended to during our deferred TLB maintenance. Change-Id: I6773a2e70bb1fabe9eef9f143bcf5df66f99dfa7 Signed-off-by: Chris Goldsworthy <quic_cgoldswo@quicinc.com>
This commit is contained in:
Chris Goldsworthy
parent
1a66e8cf59
commit
2306fec4f0
@ -2,4 +2,4 @@
|
||||
obj-$(CONFIG_QCOM_IOMMU) += qcom_iommu.o
|
||||
obj-$(CONFIG_ARM_SMMU) += arm_smmu.o
|
||||
arm_smmu-objs += arm-smmu.o arm-smmu-impl.o arm-smmu-nvidia.o
|
||||
arm_smmu-$(CONFIG_ARM_SMMU_QCOM) += arm-smmu-qcom.o
|
||||
arm_smmu-$(CONFIG_ARM_SMMU_QCOM) += arm-smmu-qcom.o arm-smmu-qcom-pm.o
|
||||
|
||||
@ -215,6 +215,9 @@ struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu)
|
||||
of_device_is_compatible(np, "nvidia,tegra186-smmu"))
|
||||
return nvidia_smmu_impl_init(smmu);
|
||||
|
||||
if (of_device_is_compatible(smmu->dev->of_node, "qcom,qsmmu-v500"))
|
||||
return qsmmuv500_impl_init(smmu);
|
||||
|
||||
if (IS_ENABLED(CONFIG_ARM_SMMU_QCOM))
|
||||
smmu = qcom_smmu_impl_init(smmu);
|
||||
|
||||
|
||||
336
drivers/iommu/arm/arm-smmu/arm-smmu-qcom-pm.c
Normal file
336
drivers/iommu/arm/arm-smmu/arm-smmu-qcom-pm.c
Normal file
@ -0,0 +1,336 @@
|
||||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
/*
|
||||
* Copyright (c) 2021, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
#include <linux/clk.h>
|
||||
#include <linux/regulator/consumer.h>
|
||||
#include <linux/interconnect.h>
|
||||
#include <linux/of_platform.h>
|
||||
#include <linux/iopoll.h>
|
||||
#include "arm-smmu.h"
|
||||
|
||||
#define ARM_SMMU_ICC_AVG_BW 0
|
||||
#define ARM_SMMU_ICC_PEAK_BW_HIGH 1000
|
||||
#define ARM_SMMU_ICC_PEAK_BW_LOW 0
|
||||
#define ARM_SMMU_ICC_ACTIVE_ONLY_TAG 0x3
|
||||
|
||||
/*
|
||||
* Theoretically, our interconnect does not guarantee the order between
|
||||
* writes to different "register blocks" even with device memory type.
|
||||
* It does guarantee that the completion of a read to a particular
|
||||
* register block implies that previously issued writes to that
|
||||
* register block have completed, with device memory type.
|
||||
*
|
||||
* In particular, we need to ensure that writes to iommu registers
|
||||
* complete before we turn off the power.
|
||||
*/
|
||||
static void arm_smmu_arch_write_sync(struct arm_smmu_device *smmu)
|
||||
{
|
||||
u32 id;
|
||||
|
||||
if (!smmu)
|
||||
return;
|
||||
|
||||
/* Read to complete prior write transcations */
|
||||
id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID0);
|
||||
|
||||
/* Wait for read to complete before off */
|
||||
rmb();
|
||||
}
|
||||
|
||||
static int arm_smmu_prepare_clocks(struct arm_smmu_power_resources *pwr)
|
||||
{
|
||||
|
||||
int i, ret = 0;
|
||||
|
||||
for (i = 0; i < pwr->num_clocks; ++i) {
|
||||
ret = clk_prepare(pwr->clocks[i]);
|
||||
if (ret) {
|
||||
dev_err(pwr->dev, "Couldn't prepare clock #%d\n", i);
|
||||
while (i--)
|
||||
clk_unprepare(pwr->clocks[i]);
|
||||
break;
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void arm_smmu_unprepare_clocks(struct arm_smmu_power_resources *pwr)
|
||||
{
|
||||
int i;
|
||||
|
||||
for (i = pwr->num_clocks; i; --i)
|
||||
clk_unprepare(pwr->clocks[i - 1]);
|
||||
}
|
||||
|
||||
static int arm_smmu_enable_clocks(struct arm_smmu_power_resources *pwr)
|
||||
{
|
||||
int i, ret = 0;
|
||||
|
||||
for (i = 0; i < pwr->num_clocks; ++i) {
|
||||
ret = clk_enable(pwr->clocks[i]);
|
||||
if (ret) {
|
||||
dev_err(pwr->dev, "Couldn't enable clock #%d\n", i);
|
||||
while (i--)
|
||||
clk_disable(pwr->clocks[i]);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void arm_smmu_disable_clocks(struct arm_smmu_power_resources *pwr)
|
||||
{
|
||||
int i;
|
||||
|
||||
for (i = pwr->num_clocks; i; --i)
|
||||
clk_disable(pwr->clocks[i - 1]);
|
||||
}
|
||||
|
||||
static int arm_smmu_raise_interconnect_bw(struct arm_smmu_power_resources *pwr)
|
||||
{
|
||||
if (!pwr->icc_path)
|
||||
return 0;
|
||||
return icc_set_bw(pwr->icc_path, ARM_SMMU_ICC_AVG_BW,
|
||||
ARM_SMMU_ICC_PEAK_BW_HIGH);
|
||||
}
|
||||
|
||||
static void arm_smmu_lower_interconnect_bw(struct arm_smmu_power_resources *pwr)
|
||||
{
|
||||
if (!pwr->icc_path)
|
||||
return;
|
||||
WARN_ON(icc_set_bw(pwr->icc_path, ARM_SMMU_ICC_AVG_BW,
|
||||
ARM_SMMU_ICC_PEAK_BW_LOW));
|
||||
}
|
||||
|
||||
static int arm_smmu_enable_regulators(struct arm_smmu_power_resources *pwr)
|
||||
{
|
||||
struct regulator_bulk_data *consumers;
|
||||
int num_consumers, ret;
|
||||
int i;
|
||||
|
||||
num_consumers = pwr->num_gdscs;
|
||||
consumers = pwr->gdscs;
|
||||
for (i = 0; i < num_consumers; i++) {
|
||||
ret = regulator_enable(consumers[i].consumer);
|
||||
if (ret)
|
||||
goto out;
|
||||
}
|
||||
return 0;
|
||||
|
||||
out:
|
||||
i -= 1;
|
||||
for (; i >= 0; i--)
|
||||
regulator_disable(consumers[i].consumer);
|
||||
return ret;
|
||||
}
|
||||
|
||||
int arm_smmu_power_on(struct arm_smmu_power_resources *pwr)
|
||||
{
|
||||
int ret;
|
||||
|
||||
mutex_lock(&pwr->power_lock);
|
||||
if (pwr->power_count > 0) {
|
||||
pwr->power_count += 1;
|
||||
mutex_unlock(&pwr->power_lock);
|
||||
return 0;
|
||||
}
|
||||
|
||||
ret = arm_smmu_raise_interconnect_bw(pwr);
|
||||
if (ret)
|
||||
goto out_unlock;
|
||||
|
||||
ret = arm_smmu_enable_regulators(pwr);
|
||||
if (ret)
|
||||
goto out_disable_bus;
|
||||
|
||||
ret = arm_smmu_prepare_clocks(pwr);
|
||||
if (ret)
|
||||
goto out_disable_regulators;
|
||||
|
||||
ret = arm_smmu_enable_clocks(pwr);
|
||||
if (ret)
|
||||
goto out_unprepare_clocks;
|
||||
|
||||
if (pwr->resume) {
|
||||
ret = pwr->resume(pwr);
|
||||
if (ret)
|
||||
goto out_disable_clocks;
|
||||
}
|
||||
|
||||
pwr->power_count = 1;
|
||||
mutex_unlock(&pwr->power_lock);
|
||||
return 0;
|
||||
out_disable_clocks:
|
||||
arm_smmu_disable_clocks(pwr);
|
||||
out_unprepare_clocks:
|
||||
arm_smmu_unprepare_clocks(pwr);
|
||||
out_disable_regulators:
|
||||
regulator_bulk_disable(pwr->num_gdscs, pwr->gdscs);
|
||||
out_disable_bus:
|
||||
arm_smmu_lower_interconnect_bw(pwr);
|
||||
out_unlock:
|
||||
mutex_unlock(&pwr->power_lock);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/*
|
||||
* Needing to pass smmu to this api for arm_smmu_arch_write_sync is awkward.
|
||||
*/
|
||||
void arm_smmu_power_off(struct arm_smmu_device *smmu,
|
||||
struct arm_smmu_power_resources *pwr)
|
||||
{
|
||||
mutex_lock(&pwr->power_lock);
|
||||
if (pwr->power_count == 0) {
|
||||
WARN(1, "%s: Bad power count\n", dev_name(pwr->dev));
|
||||
mutex_unlock(&pwr->power_lock);
|
||||
return;
|
||||
|
||||
} else if (pwr->power_count > 1) {
|
||||
pwr->power_count--;
|
||||
mutex_unlock(&pwr->power_lock);
|
||||
return;
|
||||
}
|
||||
|
||||
if (pwr->suspend)
|
||||
pwr->suspend(pwr);
|
||||
|
||||
arm_smmu_arch_write_sync(smmu);
|
||||
arm_smmu_disable_clocks(pwr);
|
||||
arm_smmu_unprepare_clocks(pwr);
|
||||
regulator_bulk_disable(pwr->num_gdscs, pwr->gdscs);
|
||||
arm_smmu_lower_interconnect_bw(pwr);
|
||||
pwr->power_count = 0;
|
||||
mutex_unlock(&pwr->power_lock);
|
||||
}
|
||||
|
||||
static int arm_smmu_init_clocks(struct arm_smmu_power_resources *pwr)
|
||||
{
|
||||
const char *cname;
|
||||
struct property *prop;
|
||||
int i;
|
||||
struct device *dev = pwr->dev;
|
||||
|
||||
pwr->num_clocks =
|
||||
of_property_count_strings(dev->of_node, "clock-names");
|
||||
|
||||
if (pwr->num_clocks < 1) {
|
||||
pwr->num_clocks = 0;
|
||||
return 0;
|
||||
}
|
||||
|
||||
pwr->clocks = devm_kzalloc(
|
||||
dev, sizeof(*pwr->clocks) * pwr->num_clocks,
|
||||
GFP_KERNEL);
|
||||
|
||||
if (!pwr->clocks)
|
||||
return -ENOMEM;
|
||||
|
||||
i = 0;
|
||||
of_property_for_each_string(dev->of_node, "clock-names",
|
||||
prop, cname) {
|
||||
struct clk *c = devm_clk_get(dev, cname);
|
||||
|
||||
if (IS_ERR(c)) {
|
||||
dev_err(dev, "Couldn't get clock: %s\n",
|
||||
cname);
|
||||
return PTR_ERR(c);
|
||||
}
|
||||
|
||||
if (clk_get_rate(c) == 0) {
|
||||
long rate = clk_round_rate(c, 1000);
|
||||
|
||||
clk_set_rate(c, rate);
|
||||
}
|
||||
|
||||
pwr->clocks[i] = c;
|
||||
|
||||
++i;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int arm_smmu_init_regulators(struct arm_smmu_power_resources *pwr)
|
||||
{
|
||||
const char *cname;
|
||||
struct property *prop;
|
||||
int i;
|
||||
struct device *dev = pwr->dev;
|
||||
|
||||
pwr->num_gdscs =
|
||||
of_property_count_strings(dev->of_node, "qcom,regulator-names");
|
||||
|
||||
if (pwr->num_gdscs < 1) {
|
||||
pwr->num_gdscs = 0;
|
||||
return 0;
|
||||
}
|
||||
|
||||
pwr->gdscs = devm_kzalloc(
|
||||
dev, sizeof(*pwr->gdscs) * pwr->num_gdscs, GFP_KERNEL);
|
||||
|
||||
if (!pwr->gdscs)
|
||||
return -ENOMEM;
|
||||
|
||||
i = 0;
|
||||
of_property_for_each_string(dev->of_node, "qcom,regulator-names",
|
||||
prop, cname)
|
||||
pwr->gdscs[i++].supply = cname;
|
||||
|
||||
return devm_regulator_bulk_get(dev, pwr->num_gdscs, pwr->gdscs);
|
||||
}
|
||||
|
||||
static int arm_smmu_init_interconnect(struct arm_smmu_power_resources *pwr)
|
||||
{
|
||||
struct device *dev = pwr->dev;
|
||||
|
||||
/* We don't want the interconnect APIs to print an error message */
|
||||
if (!of_find_property(dev->of_node, "interconnects", NULL)) {
|
||||
dev_dbg(dev, "No interconnect info\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
pwr->icc_path = devm_of_icc_get(dev, NULL);
|
||||
if (IS_ERR_OR_NULL(pwr->icc_path)) {
|
||||
if (PTR_ERR(pwr->icc_path) != -EPROBE_DEFER)
|
||||
dev_err(dev, "Unable to read interconnect path from devicetree rc: %ld\n",
|
||||
PTR_ERR(pwr->icc_path));
|
||||
return pwr->icc_path ? PTR_ERR(pwr->icc_path) : -EINVAL;
|
||||
}
|
||||
|
||||
if (of_property_read_bool(dev->of_node, "qcom,active-only"))
|
||||
icc_set_tag(pwr->icc_path, ARM_SMMU_ICC_ACTIVE_ONLY_TAG);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Cleanup done by devm. Any non-devm resources must clean up themselves.
|
||||
*/
|
||||
struct arm_smmu_power_resources *arm_smmu_init_power_resources(
|
||||
struct device *dev)
|
||||
{
|
||||
struct arm_smmu_power_resources *pwr;
|
||||
int ret;
|
||||
|
||||
pwr = devm_kzalloc(dev, sizeof(*pwr), GFP_KERNEL);
|
||||
if (!pwr)
|
||||
return ERR_PTR(-ENOMEM);
|
||||
|
||||
pwr->dev = dev;
|
||||
mutex_init(&pwr->power_lock);
|
||||
|
||||
ret = arm_smmu_init_clocks(pwr);
|
||||
if (ret)
|
||||
return ERR_PTR(ret);
|
||||
|
||||
ret = arm_smmu_init_regulators(pwr);
|
||||
if (ret)
|
||||
return ERR_PTR(ret);
|
||||
|
||||
ret = arm_smmu_init_interconnect(pwr);
|
||||
if (ret)
|
||||
return ERR_PTR(ret);
|
||||
|
||||
return pwr;
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
224
drivers/iommu/arm/arm-smmu/arm-smmu-trace.h
Normal file
224
drivers/iommu/arm/arm-smmu/arm-smmu-trace.h
Normal file
@ -0,0 +1,224 @@
|
||||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
/*
|
||||
* Copyright (c) 2019, 2021 The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#undef TRACE_SYSTEM
|
||||
#define TRACE_SYSTEM arm_smmu
|
||||
|
||||
#if !defined(_TRACE_ARM_SMMU_H) || defined(TRACE_HEADER_MULTI_READ)
|
||||
#define _TRACE_ARM_SMMU_H
|
||||
|
||||
#include <linux/types.h>
|
||||
#include <linux/tracepoint.h>
|
||||
#include <linux/scatterlist.h>
|
||||
#include "arm-smmu.h"
|
||||
|
||||
struct device;
|
||||
|
||||
DECLARE_EVENT_CLASS(iommu_tlbi,
|
||||
|
||||
TP_PROTO(struct arm_smmu_domain *domain),
|
||||
|
||||
TP_ARGS(domain),
|
||||
|
||||
TP_STRUCT__entry(
|
||||
__string(group_name, dev_name(domain->dev))
|
||||
),
|
||||
|
||||
TP_fast_assign(
|
||||
__assign_str(group_name, dev_name(domain->dev));
|
||||
),
|
||||
|
||||
TP_printk("group=%s",
|
||||
__get_str(group_name)
|
||||
)
|
||||
);
|
||||
|
||||
DEFINE_EVENT(iommu_tlbi, tlbi_start,
|
||||
|
||||
TP_PROTO(struct arm_smmu_domain *domain),
|
||||
|
||||
TP_ARGS(domain)
|
||||
);
|
||||
|
||||
DEFINE_EVENT(iommu_tlbi, tlbi_end,
|
||||
|
||||
TP_PROTO(struct arm_smmu_domain *domain),
|
||||
|
||||
TP_ARGS(domain)
|
||||
);
|
||||
|
||||
DECLARE_EVENT_CLASS(iommu_pgtable,
|
||||
|
||||
TP_PROTO(struct arm_smmu_domain *domain, unsigned long iova,
|
||||
unsigned long long ipa, size_t granule),
|
||||
|
||||
TP_ARGS(domain, iova, ipa, granule),
|
||||
|
||||
TP_STRUCT__entry(
|
||||
__string(group_name, dev_name(domain->dev))
|
||||
__field(unsigned long, iova)
|
||||
__field(unsigned long long, ipa)
|
||||
__field(size_t, granule)
|
||||
),
|
||||
|
||||
TP_fast_assign(
|
||||
__assign_str(group_name, dev_name(domain->dev));
|
||||
__entry->iova = iova;
|
||||
__entry->ipa = ipa;
|
||||
__entry->granule = granule;
|
||||
),
|
||||
|
||||
TP_printk("group=%s table_base_iova=%lx table_ipa=%llx table_size=%zx",
|
||||
__get_str(group_name), __entry->iova,
|
||||
__entry->ipa, __entry->granule
|
||||
)
|
||||
);
|
||||
|
||||
DEFINE_EVENT(iommu_pgtable, iommu_pgtable_add,
|
||||
|
||||
TP_PROTO(struct arm_smmu_domain *domain, unsigned long iova,
|
||||
unsigned long long ipa, size_t granule),
|
||||
|
||||
TP_ARGS(domain, iova, ipa, granule)
|
||||
);
|
||||
|
||||
DEFINE_EVENT(iommu_pgtable, iommu_pgtable_remove,
|
||||
|
||||
TP_PROTO(struct arm_smmu_domain *domain, unsigned long iova,
|
||||
unsigned long long ipa, size_t granule),
|
||||
|
||||
TP_ARGS(domain, iova, ipa, granule)
|
||||
);
|
||||
|
||||
DECLARE_EVENT_CLASS(iommu_map_pages,
|
||||
|
||||
TP_PROTO(struct arm_smmu_domain *domain, unsigned long iova,
|
||||
size_t pgsize, size_t pgcount),
|
||||
|
||||
TP_ARGS(domain, iova, pgsize, pgcount),
|
||||
|
||||
TP_STRUCT__entry(
|
||||
__string(group_name, dev_name(domain->dev))
|
||||
__field(unsigned long, iova)
|
||||
__field(size_t, pgsize)
|
||||
__field(size_t, pgcount)
|
||||
),
|
||||
|
||||
TP_fast_assign(
|
||||
__assign_str(group_name, dev_name(domain->dev));
|
||||
__entry->iova = iova;
|
||||
__entry->pgsize = pgsize;
|
||||
__entry->pgcount = pgcount;
|
||||
),
|
||||
|
||||
TP_printk("group=%s iova=%lx size=%zx pgsize=%zx pgcount=%zx",
|
||||
__get_str(group_name), __entry->iova,
|
||||
__entry->pgsize * __entry->pgcount,
|
||||
__entry->pgsize, __entry->pgcount
|
||||
)
|
||||
);
|
||||
|
||||
DEFINE_EVENT(iommu_map_pages, map_pages,
|
||||
|
||||
TP_PROTO(struct arm_smmu_domain *domain, unsigned long iova,
|
||||
size_t pgsize, size_t pgcount),
|
||||
|
||||
TP_ARGS(domain, iova, pgsize, pgcount)
|
||||
);
|
||||
|
||||
DEFINE_EVENT(iommu_map_pages, unmap_pages,
|
||||
|
||||
TP_PROTO(struct arm_smmu_domain *domain, unsigned long iova,
|
||||
size_t pgsize, size_t pgcount),
|
||||
|
||||
TP_ARGS(domain, iova, pgsize, pgcount)
|
||||
);
|
||||
|
||||
/* Refer to samples/ftrace_events */
|
||||
#ifndef __TRACE_EVENT_ARM_SMMU_HELPER_FUNCTIONS
|
||||
#define __TRACE_EVENT_ARM_SMMU_HELPER_FUNCTIONS
|
||||
static inline unsigned long sum_scatterlist_length(struct scatterlist *sgl,
|
||||
unsigned int nents)
|
||||
{
|
||||
int i = 0;
|
||||
unsigned long sum = 0;
|
||||
|
||||
for (i = 0; i < nents; i++, sgl = sg_next(sgl))
|
||||
sum += sgl->length;
|
||||
|
||||
return sum;
|
||||
}
|
||||
#endif
|
||||
|
||||
TRACE_EVENT(map_sg,
|
||||
|
||||
TP_PROTO(struct arm_smmu_domain *domain, unsigned long iova,
|
||||
struct scatterlist *sgl, unsigned int nents),
|
||||
|
||||
TP_ARGS(domain, iova, sgl, nents),
|
||||
|
||||
TP_STRUCT__entry(
|
||||
__string(group_name, dev_name(domain->dev))
|
||||
__field(unsigned long, iova)
|
||||
__field(unsigned long, size)
|
||||
),
|
||||
|
||||
TP_fast_assign(
|
||||
__assign_str(group_name, dev_name(domain->dev));
|
||||
__entry->iova = iova;
|
||||
__entry->size = sum_scatterlist_length(sgl, nents);
|
||||
),
|
||||
|
||||
TP_printk("group=%s iova=%lx size=%lx",
|
||||
__get_str(group_name), __entry->iova,
|
||||
__entry->size
|
||||
)
|
||||
);
|
||||
|
||||
TRACE_EVENT(tlbsync_timeout,
|
||||
|
||||
TP_PROTO(struct device *dev),
|
||||
|
||||
TP_ARGS(dev),
|
||||
|
||||
TP_STRUCT__entry(
|
||||
__string(device, dev_name(dev))
|
||||
),
|
||||
|
||||
TP_fast_assign(
|
||||
__assign_str(device, dev_name(dev));
|
||||
),
|
||||
|
||||
TP_printk("smmu=%s",
|
||||
__get_str(device)
|
||||
)
|
||||
);
|
||||
|
||||
TRACE_EVENT(smmu_init,
|
||||
|
||||
TP_PROTO(u64 time),
|
||||
|
||||
TP_ARGS(time),
|
||||
|
||||
TP_STRUCT__entry(
|
||||
__field(u64, time)
|
||||
),
|
||||
|
||||
TP_fast_assign(
|
||||
__entry->time = time;
|
||||
),
|
||||
|
||||
TP_printk("ARM SMMU init latency: %lld us", __entry->time)
|
||||
);
|
||||
#endif /* _TRACE_ARM_SMMU_H */
|
||||
|
||||
#undef TRACE_INCLUDE_PATH
|
||||
#define TRACE_INCLUDE_PATH ../../drivers/iommu/arm/arm-smmu
|
||||
|
||||
#undef TRACE_INCLUDE_FILE
|
||||
#define TRACE_INCLUDE_FILE arm-smmu-trace
|
||||
|
||||
/* This part must be outside protection */
|
||||
#include <trace/define_trace.h>
|
||||
File diff suppressed because it is too large
Load Diff
@ -5,6 +5,8 @@
|
||||
* Copyright (C) 2013 ARM Limited
|
||||
*
|
||||
* Author: Will Deacon <will.deacon@arm.com>
|
||||
*
|
||||
* Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
|
||||
*/
|
||||
|
||||
#ifndef _ARM_SMMU_H
|
||||
@ -22,10 +24,15 @@
|
||||
#include <linux/mutex.h>
|
||||
#include <linux/spinlock.h>
|
||||
#include <linux/types.h>
|
||||
#include <linux/qcom-iommu-util.h>
|
||||
|
||||
#include "../../qcom-io-pgtable.h"
|
||||
|
||||
/* Configuration registers */
|
||||
#define ARM_SMMU_GR0_sCR0 0x0
|
||||
#define ARM_SMMU_sCR0_VMID16EN BIT(31)
|
||||
#define ARM_SMMU_sCR0_SHCFG GENMASK(23, 22)
|
||||
#define ARM_SMMU_sCR0_SHCFG_NSH 0x3
|
||||
#define ARM_SMMU_sCR0_BSU GENMASK(15, 14)
|
||||
#define ARM_SMMU_sCR0_FB BIT(13)
|
||||
#define ARM_SMMU_sCR0_PTM BIT(12)
|
||||
@ -117,6 +124,8 @@ enum arm_smmu_s2cr_type {
|
||||
S2CR_TYPE_FAULT,
|
||||
};
|
||||
#define ARM_SMMU_S2CR_EXIDVALID BIT(10)
|
||||
#define ARM_SMMU_S2CR_SHCFG GENMASK(9, 8)
|
||||
#define ARM_SMMU_S2CR_SHCFG_NSH 0x3
|
||||
#define ARM_SMMU_S2CR_CBNDX GENMASK(7, 0)
|
||||
|
||||
/* Context bank attribute registers */
|
||||
@ -136,12 +145,23 @@ enum arm_smmu_cbar_type {
|
||||
#define ARM_SMMU_CBAR_VMID GENMASK(7, 0)
|
||||
|
||||
#define ARM_SMMU_GR1_CBFRSYNRA(n) (0x400 + ((n) << 2))
|
||||
#define CBFRSYNRA_SID_MASK (0xffff)
|
||||
|
||||
#define ARM_SMMU_GR1_CBA2R(n) (0x800 + ((n) << 2))
|
||||
#define ARM_SMMU_CBA2R_VMID16 GENMASK(31, 16)
|
||||
#define ARM_SMMU_CBA2R_VA64 BIT(0)
|
||||
|
||||
#define ARM_SMMU_CB_SCTLR 0x0
|
||||
#define ARM_SMMU_SCTLR_WACFG GENMASK(27, 26)
|
||||
#define ARM_SMMU_SCTLR_WACFG_WA 0x2
|
||||
#define ARM_SMMU_SCTLR_RACFG GENMASK(25, 24)
|
||||
#define ARM_SMMU_SCTLR_RACFG_RA 0x2
|
||||
#define ARM_SMMU_SCTLR_SHCFG GENMASK(23, 22)
|
||||
#define ARM_SMMU_SCTLR_SHCFG_OSH 0x1
|
||||
#define ARM_SMMU_SCTLR_SHCFG_NSH 0x3
|
||||
#define ARM_SMMU_SCTLR_MTCFG BIT(20)
|
||||
#define ARM_SMMU_SCTLR_MEM_ATTR GENMASK(19, 16)
|
||||
#define ARM_SMMU_SCTLR_MEM_ATTR_OISH_WB_CACHE 0xf
|
||||
#define ARM_SMMU_SCTLR_S1_ASIDPNE BIT(12)
|
||||
#define ARM_SMMU_SCTLR_CFCFG BIT(7)
|
||||
#define ARM_SMMU_SCTLR_HUPCF BIT(8)
|
||||
@ -156,6 +176,7 @@ enum arm_smmu_cbar_type {
|
||||
|
||||
#define ARM_SMMU_CB_RESUME 0x8
|
||||
#define ARM_SMMU_RESUME_TERMINATE BIT(0)
|
||||
#define ARM_SMMU_RESUME_RESUME 0
|
||||
|
||||
#define ARM_SMMU_CB_TCR2 0x10
|
||||
#define ARM_SMMU_TCR2_SEP GENMASK(17, 15)
|
||||
@ -219,10 +240,19 @@ enum arm_smmu_cbar_type {
|
||||
ARM_SMMU_FSR_TF | \
|
||||
ARM_SMMU_FSR_IGN)
|
||||
|
||||
#define ARM_SMMU_CB_FSRRESTORE 0x5c
|
||||
#define ARM_SMMU_CB_FAR 0x60
|
||||
|
||||
#define ARM_SMMU_CB_FSYNR0 0x68
|
||||
#define ARM_SMMU_FSYNR0_WNR BIT(4)
|
||||
#define ARM_SMMU_FSYNR0_PNU BIT(5)
|
||||
#define ARM_SMMU_FSYNR0_IND BIT(6)
|
||||
#define ARM_SMMU_FSYNR0_NSATTR BIT(8)
|
||||
|
||||
#define ARM_SMMU_CB_FSYNR1 0x6c
|
||||
#define ARM_SMMU_FSYNR1_BID GENMASK(15, 13)
|
||||
#define ARM_SMMU_FSYNR1_PID GENMASK(12, 8)
|
||||
#define ARM_SMMU_FSYNR1_MID GENMASK(7, 0)
|
||||
|
||||
#define ARM_SMMU_CB_FSYNR1 0x6c
|
||||
|
||||
@ -235,6 +265,24 @@ enum arm_smmu_cbar_type {
|
||||
#define ARM_SMMU_CB_TLBSTATUS 0x7f4
|
||||
#define ARM_SMMU_CB_ATS1PR 0x800
|
||||
|
||||
/* Implementation Defined Register Space 5 registers*/
|
||||
/* Relative to IMPL_DEF5 page */
|
||||
#define ARM_SMMU_STATS_SYNC_INV_TBU_ACK 0x5dc
|
||||
#define TBU_SYNC_ACK GENMASK(31, 17)
|
||||
#define TBU_SYNC_REQ BIT(16)
|
||||
#define TBU_INV_ACK GENMASK(15, 1)
|
||||
#define TBU_INV_REQ BIT(0)
|
||||
#define APPS_SMMU_TBU_REG_ACCESS_REQ_NS 0x5f8
|
||||
#define APPS_SMMU_TBU_REG_ACCESS_ACK_NS 0x5fc
|
||||
|
||||
/* Relative to SMMU_BASE */
|
||||
#define ARM_SMMU_TBU_PWR_STATUS 0x2204
|
||||
|
||||
/* Relative SMMU_BASE */
|
||||
#define ARM_SMMU_MMU2QSS_AND_SAFE_WAIT_CNTR 0x2670
|
||||
#define TCU_SYNC_IN_PRGSS BIT(20)
|
||||
#define TCU_INV_IN_PRGSS BIT(16)
|
||||
|
||||
#define ARM_SMMU_CB_ATSR 0x8f0
|
||||
#define ARM_SMMU_ATSR_ACTIVE BIT(0)
|
||||
|
||||
@ -242,8 +290,9 @@ enum arm_smmu_cbar_type {
|
||||
/* Maximum number of context banks per SMMU */
|
||||
#define ARM_SMMU_MAX_CBS 128
|
||||
|
||||
#define TLB_LOOP_TIMEOUT 1000000 /* 1s! */
|
||||
#define TLB_LOOP_TIMEOUT 500000 /* 500ms */
|
||||
#define TLB_SPIN_COUNT 10
|
||||
#define TLB_LOOP_INC_MAX 1000 /*1ms*/
|
||||
|
||||
/* Shared driver definitions */
|
||||
enum arm_smmu_arch_version {
|
||||
@ -260,12 +309,42 @@ enum arm_smmu_implementation {
|
||||
QCOM_SMMUV500,
|
||||
};
|
||||
|
||||
struct arm_smmu_impl_def_reg {
|
||||
u32 offset;
|
||||
u32 value;
|
||||
};
|
||||
|
||||
/*
|
||||
* Describes resources required for on/off power operation.
|
||||
* Separate reference count is provided for atomic/nonatomic
|
||||
* operations.
|
||||
*/
|
||||
struct arm_smmu_power_resources {
|
||||
struct device *dev;
|
||||
|
||||
struct clk **clocks;
|
||||
int num_clocks;
|
||||
|
||||
struct regulator_bulk_data *gdscs;
|
||||
int num_gdscs;
|
||||
|
||||
struct icc_path *icc_path;
|
||||
|
||||
/* Protects power_count */
|
||||
struct mutex power_lock;
|
||||
int power_count;
|
||||
|
||||
int (*resume)(struct arm_smmu_power_resources *pwr);
|
||||
void (*suspend)(struct arm_smmu_power_resources *pwr);
|
||||
};
|
||||
|
||||
struct arm_smmu_s2cr {
|
||||
struct iommu_group *group;
|
||||
int count;
|
||||
enum arm_smmu_s2cr_type type;
|
||||
enum arm_smmu_s2cr_privcfg privcfg;
|
||||
u8 cbndx;
|
||||
bool pinned;
|
||||
};
|
||||
|
||||
struct arm_smmu_smr {
|
||||
@ -297,6 +376,12 @@ struct arm_smmu_device {
|
||||
#define ARM_SMMU_FEAT_EXIDS (1 << 12)
|
||||
u32 features;
|
||||
|
||||
#define ARM_SMMU_OPT_FATAL_ASF (1 << 0)
|
||||
#define ARM_SMMU_OPT_3LVL_TABLES (1 << 2)
|
||||
#define ARM_SMMU_OPT_NO_ASID_RETENTION (1 << 3)
|
||||
#define ARM_SMMU_OPT_DISABLE_ATOS (1 << 4)
|
||||
#define ARM_SMMU_OPT_CONTEXT_FAULT_RETRY (1 << 5)
|
||||
u32 options;
|
||||
enum arm_smmu_arch_version version;
|
||||
enum arm_smmu_implementation model;
|
||||
const struct arm_smmu_impl *impl;
|
||||
@ -328,6 +413,17 @@ struct arm_smmu_device {
|
||||
|
||||
/* IOMMU core code handle */
|
||||
struct iommu_device iommu;
|
||||
|
||||
/* Specific to QCOM */
|
||||
struct arm_smmu_impl_def_reg *impl_def_attach_registers;
|
||||
unsigned int num_impl_def_attach_registers;
|
||||
|
||||
struct arm_smmu_power_resources *pwr;
|
||||
|
||||
/* used for qsmmuv500 scm_io_readl */
|
||||
phys_addr_t phys_addr;
|
||||
|
||||
unsigned long sync_timed_out;
|
||||
};
|
||||
|
||||
enum arm_smmu_context_fmt {
|
||||
@ -344,6 +440,19 @@ struct arm_smmu_cfg {
|
||||
u16 asid;
|
||||
u16 vmid;
|
||||
};
|
||||
u32 procid;
|
||||
struct {
|
||||
u32 wacfg:2;
|
||||
u32 racfg:2;
|
||||
u32 shcfg:2;
|
||||
u32 mtcfg:1;
|
||||
u32 memattr:4;
|
||||
u32 hupcf:1;
|
||||
u32 cfcfg:1;
|
||||
u32 cfre:1;
|
||||
u32 m:1;
|
||||
} sctlr;
|
||||
|
||||
enum arm_smmu_cbar_type cbar;
|
||||
enum arm_smmu_context_fmt fmt;
|
||||
bool flush_walk_prefer_tlbiasid;
|
||||
@ -354,6 +463,7 @@ struct arm_smmu_cb {
|
||||
u64 ttbr[2];
|
||||
u32 tcr[2];
|
||||
u32 mair[2];
|
||||
u32 sctlr;
|
||||
struct arm_smmu_cfg *cfg;
|
||||
};
|
||||
|
||||
@ -364,16 +474,53 @@ enum arm_smmu_domain_stage {
|
||||
ARM_SMMU_DOMAIN_BYPASS,
|
||||
};
|
||||
|
||||
struct arm_smmu_fault_model {
|
||||
char non_fatal : 1;
|
||||
char no_cfre : 1;
|
||||
char no_stall : 1;
|
||||
char hupcf : 1;
|
||||
};
|
||||
|
||||
struct arm_smmu_mapping_cfg {
|
||||
char s1_bypass : 1;
|
||||
char atomic : 1;
|
||||
char fast : 1;
|
||||
};
|
||||
|
||||
struct arm_smmu_domain {
|
||||
struct arm_smmu_device *smmu;
|
||||
struct device *dev;
|
||||
struct io_pgtable_ops *pgtbl_ops;
|
||||
unsigned long pgtbl_quirks;
|
||||
bool force_coherent_walk;
|
||||
const struct iommu_flush_ops *flush_ops;
|
||||
struct arm_smmu_cfg cfg;
|
||||
enum arm_smmu_domain_stage stage;
|
||||
struct mutex init_mutex; /* Protects smmu pointer */
|
||||
spinlock_t cb_lock; /* Serialises ATS1* ops and TLB syncs */
|
||||
spinlock_t cb_lock; /* Serialises ATS1* ops */
|
||||
spinlock_t sync_lock; /* Serialises TLB syncs */
|
||||
struct arm_smmu_fault_model fault_model;
|
||||
struct arm_smmu_mapping_cfg mapping_cfg;
|
||||
bool delayed_s1_trans_enable;
|
||||
u32 secure_vmid;
|
||||
|
||||
/*
|
||||
* Track PMDs which require tlb invalidate prior to being
|
||||
* freed, or before their iovas can be reused by iommu_map().
|
||||
*/
|
||||
spinlock_t iotlb_gather_lock;
|
||||
struct list_head *freelist;
|
||||
bool deferred_sync;
|
||||
|
||||
struct iommu_debug_attachment *logger;
|
||||
struct iommu_domain domain;
|
||||
/* mapping_cfg.atomic indicates that runtime power management should be disabled. */
|
||||
bool rpm_always_on;
|
||||
|
||||
#ifdef CONFIG_ARM_SMMU_CONTEXT_FAULT_RETRY
|
||||
u64 prev_fault_address;
|
||||
u32 fault_retry_counter;
|
||||
#endif
|
||||
};
|
||||
|
||||
struct arm_smmu_master_cfg {
|
||||
@ -420,7 +567,28 @@ static inline u32 arm_smmu_lpae_vtcr(const struct io_pgtable_cfg *cfg)
|
||||
FIELD_PREP(ARM_SMMU_VTCR_T0SZ, cfg->arm_lpae_s2_cfg.vtcr.tsz);
|
||||
}
|
||||
|
||||
static inline u32 arm_smmu_lpae_sctlr(struct arm_smmu_cfg *cfg)
|
||||
{
|
||||
bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
|
||||
|
||||
return FIELD_PREP(ARM_SMMU_SCTLR_WACFG, cfg->sctlr.wacfg) |
|
||||
FIELD_PREP(ARM_SMMU_SCTLR_RACFG, cfg->sctlr.racfg) |
|
||||
FIELD_PREP(ARM_SMMU_SCTLR_SHCFG, cfg->sctlr.shcfg) |
|
||||
FIELD_PREP(ARM_SMMU_SCTLR_MTCFG, cfg->sctlr.mtcfg) |
|
||||
FIELD_PREP(ARM_SMMU_SCTLR_MEM_ATTR, cfg->sctlr.memattr) |
|
||||
FIELD_PREP(ARM_SMMU_SCTLR_S1_ASIDPNE, stage1) |
|
||||
FIELD_PREP(ARM_SMMU_SCTLR_HUPCF, cfg->sctlr.hupcf) |
|
||||
FIELD_PREP(ARM_SMMU_SCTLR_CFCFG, cfg->sctlr.cfcfg) |
|
||||
ARM_SMMU_SCTLR_CFIE |
|
||||
FIELD_PREP(ARM_SMMU_SCTLR_CFRE, cfg->sctlr.cfre) |
|
||||
FIELD_PREP(ARM_SMMU_SCTLR_E, IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) |
|
||||
ARM_SMMU_SCTLR_AFE |
|
||||
ARM_SMMU_SCTLR_TRE |
|
||||
FIELD_PREP(ARM_SMMU_SCTLR_M, cfg->sctlr.m);
|
||||
}
|
||||
|
||||
/* Implementation details, yay! */
|
||||
|
||||
struct arm_smmu_impl {
|
||||
u32 (*read_reg)(struct arm_smmu_device *smmu, int page, int offset);
|
||||
void (*write_reg)(struct arm_smmu_device *smmu, int page, int offset,
|
||||
@ -432,6 +600,13 @@ struct arm_smmu_impl {
|
||||
int (*reset)(struct arm_smmu_device *smmu);
|
||||
int (*init_context)(struct arm_smmu_domain *smmu_domain,
|
||||
struct io_pgtable_cfg *cfg, struct device *dev);
|
||||
void (*init_context_bank)(struct arm_smmu_domain *smmu_domain,
|
||||
struct device *dev);
|
||||
phys_addr_t (*iova_to_phys_hard)(struct arm_smmu_domain *smmu_domain,
|
||||
struct qcom_iommu_atos_txn *txn);
|
||||
void (*tlb_sync_timeout)(struct arm_smmu_device *smmu);
|
||||
void (*device_remove)(struct arm_smmu_device *smmu);
|
||||
int (*device_group)(struct device *dev, struct iommu_group *group);
|
||||
void (*tlb_sync)(struct arm_smmu_device *smmu, int page, int sync,
|
||||
int status);
|
||||
int (*def_domain_type)(struct device *dev);
|
||||
@ -503,6 +678,13 @@ static inline void arm_smmu_writeq(struct arm_smmu_device *smmu, int page,
|
||||
|
||||
#define ARM_SMMU_GR0 0
|
||||
#define ARM_SMMU_GR1 1
|
||||
|
||||
/*
|
||||
* Implementation defined space starts after SMMU GR space, so IMPL_DEF page n
|
||||
* is page n + 2 in the SMMU register space.
|
||||
*/
|
||||
#define ARM_SMMU_IMPL_DEF5 7
|
||||
|
||||
#define ARM_SMMU_CB(s, n) ((s)->numpage + (n))
|
||||
|
||||
#define arm_smmu_gr0_read(s, o) \
|
||||
@ -527,8 +709,21 @@ static inline void arm_smmu_writeq(struct arm_smmu_device *smmu, int page,
|
||||
struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu);
|
||||
struct arm_smmu_device *nvidia_smmu_impl_init(struct arm_smmu_device *smmu);
|
||||
struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu);
|
||||
struct arm_smmu_device *qsmmuv500_impl_init(struct arm_smmu_device *smmu);
|
||||
struct arm_smmu_device *qcom_adreno_smmu_impl_init(struct arm_smmu_device *smmu);
|
||||
|
||||
void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx);
|
||||
int arm_mmu500_reset(struct arm_smmu_device *smmu);
|
||||
|
||||
int arm_smmu_power_on(struct arm_smmu_power_resources *pwr);
|
||||
void arm_smmu_power_off(struct arm_smmu_device *smmu,
|
||||
struct arm_smmu_power_resources *pwr);
|
||||
struct arm_smmu_power_resources *arm_smmu_init_power_resources(
|
||||
struct device *dev);
|
||||
|
||||
extern struct platform_driver qsmmuv500_tbu_driver;
|
||||
|
||||
/* Misc. constants */
|
||||
#define ARM_MMU500_ACR_CACHE_LOCK (1 << 26)
|
||||
|
||||
#endif /* _ARM_SMMU_H */
|
||||
|
||||
Loading…
Reference in New Issue
Block a user