android/android_kernel_samsung_sm8650-modules
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
lineage-22.1
android_kernel_samsung_sm86.../qcom/opensource/graphics-kernel/adreno_hwsched.c
David Wronek 880d405719 Add 'qcom/opensource/graphics-kernel/' from commit 'b4fdc4c04295ac59109ae19d64747522740c3f14' 
git-subtree-dir: qcom/opensource/graphics-kernel
git-subtree-mainline: 992813d9c1
git-subtree-split: b4fdc4c042
Change-Id:
repo: https://git.codelinaro.org/clo/la/platform/vendor/qcom/opensource/graphics-kernel
tag: GRAPHICS.LA.14.0.r1-07700-lanai.0
2024-10-06 16:44:56 +02:00

2511 lines
68 KiB
C
Raw Permalink Blame History

// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
* Copyright (c) 2022-2024, Qualcomm Innovation Center, Inc. All rights reserved.
*/
#include <linux/dma-fence-array.h>
#include <soc/qcom/msm_performance.h>
#include "adreno.h"
#include "adreno_hfi.h"
#include "adreno_snapshot.h"
#include "adreno_sysfs.h"
#include "adreno_trace.h"
#include "kgsl_timeline.h"
#include <linux/msm_kgsl.h>
/*
* Number of commands that can be queued in a context before it sleeps
*
* Our code that "puts back" a command from the context is much cleaner
* if we are sure that there will always be enough room in the ringbuffer
* so restrict the size of the context queue to ADRENO_CONTEXT_DRAWQUEUE_SIZE - 1
*/
static u32 _context_drawqueue_size = ADRENO_CONTEXT_DRAWQUEUE_SIZE - 1;
/* Number of milliseconds to wait for the context queue to clear */
static unsigned int _context_queue_wait = 10000;
/*
* GFT throttle parameters. If GFT recovered more than
* X times in Y ms invalidate the context and do not attempt recovery.
* X -> _fault_throttle_burst
* Y -> _fault_throttle_time
*/
static unsigned int _fault_throttle_time = 2000;
static unsigned int _fault_throttle_burst = 3;
/* Use a kmem cache to speed up allocations for dispatcher jobs */
static struct kmem_cache *jobs_cache;
/* Use a kmem cache to speed up allocations for inflight command objects */
static struct kmem_cache *obj_cache;
inline bool adreno_hwsched_context_queue_enabled(struct adreno_device *adreno_dev)
{
return test_bit(ADRENO_HWSCHED_CONTEXT_QUEUE, &adreno_dev->hwsched.flags);
}
static bool is_cmdobj(struct kgsl_drawobj *drawobj)
{
return (drawobj->type & CMDOBJ_TYPE);
}
static bool _check_context_queue(struct adreno_context *drawctxt, u32 count)
{
bool ret;
spin_lock(&drawctxt->lock);
/*
* Wake up if there is room in the context or if the whole thing got
* invalidated while we were asleep
*/
if (kgsl_context_invalid(&drawctxt->base))
ret = false;
else
ret = ((drawctxt->queued + count) < _context_drawqueue_size) ? 1 : 0;
spin_unlock(&drawctxt->lock);
return ret;
}
static void _pop_drawobj(struct adreno_context *drawctxt)
{
drawctxt->drawqueue_head = DRAWQUEUE_NEXT(drawctxt->drawqueue_head,
ADRENO_CONTEXT_DRAWQUEUE_SIZE);
drawctxt->queued--;
}
static int _retire_syncobj(struct adreno_device *adreno_dev,
struct kgsl_drawobj_sync *syncobj, struct adreno_context *drawctxt)
{
struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
if (!kgsl_drawobj_events_pending(syncobj)) {
_pop_drawobj(drawctxt);
kgsl_drawobj_destroy(DRAWOBJ(syncobj));
return 0;
}
/*
* If hardware fences are enabled, and this SYNCOBJ is backed by hardware fences,
* send it to the GMU
*/
if (test_bit(ADRENO_HWSCHED_HW_FENCE, &hwsched->flags) &&
((syncobj->flags & KGSL_SYNCOBJ_HW)))
return 1;
/*
* If we got here, there are pending events for sync object.
* Start the canary timer if it hasnt been started already.
*/
if (!syncobj->timeout_jiffies) {
syncobj->timeout_jiffies = jiffies + msecs_to_jiffies(5000);
mod_timer(&syncobj->timer, syncobj->timeout_jiffies);
}
return -EAGAIN;
}
static bool _marker_expired(struct kgsl_drawobj_cmd *markerobj)
{
struct kgsl_drawobj *drawobj = DRAWOBJ(markerobj);
return (drawobj->flags & KGSL_DRAWOBJ_MARKER) &&
kgsl_check_timestamp(drawobj->device, drawobj->context,
markerobj->marker_timestamp);
}
/* Only retire the timestamp. The drawobj will be destroyed later */
static void _retire_timestamp_only(struct kgsl_drawobj *drawobj)
{
struct kgsl_context *context = drawobj->context;
struct kgsl_device *device = context->device;
/*
* Write the start and end timestamp to the memstore to keep the
* accounting sane
*/
kgsl_sharedmem_writel(device->memstore,
KGSL_MEMSTORE_OFFSET(context->id, soptimestamp),
drawobj->timestamp);
kgsl_sharedmem_writel(device->memstore,
KGSL_MEMSTORE_OFFSET(context->id, eoptimestamp),
drawobj->timestamp);
msm_perf_events_update(MSM_PERF_GFX, MSM_PERF_RETIRED,
pid_nr(context->proc_priv->pid),
context->id, drawobj->timestamp,
!!(drawobj->flags & KGSL_DRAWOBJ_END_OF_FRAME));
if (drawobj->flags & KGSL_DRAWOBJ_END_OF_FRAME) {
atomic64_inc(&drawobj->context->proc_priv->frame_count);
atomic_inc(&drawobj->context->proc_priv->period->frames);
}
/* Retire pending GPU events for the object */
kgsl_process_event_group(device, &context->events);
}
static void _retire_timestamp(struct kgsl_drawobj *drawobj)
{
_retire_timestamp_only(drawobj);
kgsl_drawobj_destroy(drawobj);
}
static int _retire_markerobj(struct adreno_device *adreno_dev, struct kgsl_drawobj_cmd *cmdobj,
struct adreno_context *drawctxt)
{
struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
if (_marker_expired(cmdobj)) {
set_bit(CMDOBJ_MARKER_EXPIRED, &cmdobj->priv);
/*
* There may be pending hardware fences that need to be signaled upon retiring
* this MARKER object. Hence, send it to the target specific layers to trigger
* the hardware fences.
*/
if (test_bit(ADRENO_HWSCHED_HW_FENCE, &hwsched->flags)) {
_retire_timestamp_only(DRAWOBJ(cmdobj));
return 1;
}
_pop_drawobj(drawctxt);
_retire_timestamp(DRAWOBJ(cmdobj));
return 0;
}
/*
* If the marker isn't expired but the SKIP bit
* is set then there are real commands following
* this one in the queue. This means that we
* need to dispatch the command so that we can
* keep the timestamp accounting correct. If
* skip isn't set then we block this queue
* until the dependent timestamp expires
*/
return test_bit(CMDOBJ_SKIP, &cmdobj->priv) ? 1 : -EAGAIN;
}
static int _retire_timelineobj(struct kgsl_drawobj *drawobj,
struct adreno_context *drawctxt)
{
_pop_drawobj(drawctxt);
kgsl_drawobj_destroy(drawobj);
return 0;
}
static int drawqueue_retire_bindobj(struct kgsl_drawobj *drawobj,
struct adreno_context *drawctxt)
{
struct kgsl_drawobj_bind *bindobj = BINDOBJ(drawobj);
if (test_bit(KGSL_BINDOBJ_STATE_DONE, &bindobj->state)) {
_pop_drawobj(drawctxt);
_retire_timestamp(drawobj);
return 0;
}
if (!test_and_set_bit(KGSL_BINDOBJ_STATE_START, &bindobj->state)) {
/*
* Take a reference to the drawobj and the context because both
* get referenced in the bind callback
*/
_kgsl_context_get(&drawctxt->base);
kref_get(&drawobj->refcount);
kgsl_sharedmem_bind_ranges(bindobj->bind);
}
return -EAGAIN;
}
/*
* Retires all expired marker and sync objs from the context
* queue and returns one of the below
* a) next drawobj that needs to be sent to ringbuffer
* b) -EAGAIN for syncobj with syncpoints pending.
* c) -EAGAIN for markerobj whose marker timestamp has not expired yet.
* c) NULL for no commands remaining in drawqueue.
*/
static struct kgsl_drawobj *_process_drawqueue_get_next_drawobj(
struct adreno_device *adreno_dev, struct adreno_context *drawctxt)
{
struct kgsl_drawobj *drawobj;
unsigned int i = drawctxt->drawqueue_head;
struct kgsl_drawobj_cmd *cmdobj;
struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
int ret = 0;
if (drawctxt->drawqueue_head == drawctxt->drawqueue_tail)
return NULL;
for (i = drawctxt->drawqueue_head; i != drawctxt->drawqueue_tail;
i = DRAWQUEUE_NEXT(i, ADRENO_CONTEXT_DRAWQUEUE_SIZE)) {
drawobj = drawctxt->drawqueue[i];
if (!drawobj)
return NULL;
switch (drawobj->type) {
case CMDOBJ_TYPE:
cmdobj = CMDOBJ(drawobj);
/* We only support one big IB inflight */
if ((cmdobj->numibs > HWSCHED_MAX_DISPATCH_NUMIBS) &&
hwsched->big_cmdobj)
return ERR_PTR(-ENOSPC);
return drawobj;
case SYNCOBJ_TYPE:
ret = _retire_syncobj(adreno_dev, SYNCOBJ(drawobj), drawctxt);
if (ret == 1)
return drawobj;
break;
case MARKEROBJ_TYPE:
ret = _retire_markerobj(adreno_dev, CMDOBJ(drawobj), drawctxt);
/* Special case where marker needs to be sent to GPU */
if (ret == 1)
return drawobj;
break;
case BINDOBJ_TYPE:
ret = drawqueue_retire_bindobj(drawobj, drawctxt);
break;
case TIMELINEOBJ_TYPE:
ret = _retire_timelineobj(drawobj, drawctxt);
break;
default:
ret = -EINVAL;
break;
}
if (ret)
return ERR_PTR(ret);
}
return NULL;
}
/**
* hwsched_dispatcher_requeue_drawobj() - Put a draw objet back on the context
* queue
* @drawctxt: Pointer to the adreno draw context
* @drawobj: Pointer to the KGSL draw object to requeue
*
* Failure to submit a drawobj to the ringbuffer isn't the fault of the drawobj
* being submitted so if a failure happens, push it back on the head of the
* context queue to be reconsidered again unless the context got detached.
*/
static inline int hwsched_dispatcher_requeue_drawobj(
struct adreno_context *drawctxt,
struct kgsl_drawobj *drawobj)
{
unsigned int prev;
spin_lock(&drawctxt->lock);
if (kgsl_context_is_bad(&drawctxt->base)) {
spin_unlock(&drawctxt->lock);
/* get rid of this drawobj since the context is bad */
kgsl_drawobj_destroy(drawobj);
return -ENOENT;
}
prev = drawctxt->drawqueue_head == 0 ?
(ADRENO_CONTEXT_DRAWQUEUE_SIZE - 1) :
(drawctxt->drawqueue_head - 1);
/*
* The maximum queue size always needs to be one less then the size of
* the ringbuffer queue so there is "room" to put the drawobj back in
*/
WARN_ON(prev == drawctxt->drawqueue_tail);
drawctxt->drawqueue[prev] = drawobj;
drawctxt->queued++;
/* Reset the command queue head to reflect the newly requeued change */
drawctxt->drawqueue_head = prev;
if (is_cmdobj(drawobj)) {
struct kgsl_drawobj_cmd *cmdobj = CMDOBJ(drawobj);
cmdobj->requeue_cnt++;
}
spin_unlock(&drawctxt->lock);
return 0;
}
/**
* hwsched_queue_context() - Queue a context in the dispatcher list of jobs
* @adreno_dev: Pointer to the adreno device structure
* @drawctxt: Pointer to the adreno draw context
*
* Add a context to the dispatcher list of jobs.
*/
static int hwsched_queue_context(struct adreno_device *adreno_dev,
struct adreno_context *drawctxt)
{
struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
struct adreno_dispatch_job *job;
/* Refuse to queue a detached context */
if (kgsl_context_detached(&drawctxt->base))
return 0;
if (!_kgsl_context_get(&drawctxt->base))
return 0;
job = kmem_cache_alloc(jobs_cache, GFP_ATOMIC);
if (!job) {
kgsl_context_put(&drawctxt->base);
return -ENOMEM;
}
job->drawctxt = drawctxt;
trace_dispatch_queue_context(drawctxt);
llist_add(&job->node, &hwsched->jobs[drawctxt->base.priority]);
return 0;
}
void adreno_hwsched_flush(struct adreno_device *adreno_dev)
{
struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
kthread_flush_worker(hwsched->worker);
}
/**
* is_marker_skip() - Check if the draw object is a MARKEROBJ_TYPE and CMDOBJ_SKIP bit is set
*/
static bool is_marker_skip(struct kgsl_drawobj *drawobj)
{
struct kgsl_drawobj_cmd *cmdobj = NULL;
if (drawobj->type != MARKEROBJ_TYPE)
return false;
cmdobj = CMDOBJ(drawobj);
if (test_bit(CMDOBJ_SKIP, &cmdobj->priv))
return true;
return false;
}
static bool _abort_submission(struct adreno_device *adreno_dev)
{
struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
/* We only need a single barrier before reading all the atomic variables below */
smp_rmb();
if (atomic_read(&adreno_dev->halt) || atomic_read(&hwsched->fault))
return true;
return false;
}
/**
* sendcmd() - Send a drawobj to the GPU hardware
* @dispatcher: Pointer to the adreno dispatcher struct
* @drawobj: Pointer to the KGSL drawobj being sent
*
* Send a KGSL drawobj to the GPU hardware
*/
static int hwsched_sendcmd(struct adreno_device *adreno_dev,
struct kgsl_drawobj *drawobj)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
struct kgsl_context *context = drawobj->context;
int ret;
struct cmd_list_obj *obj;
obj = kmem_cache_alloc(obj_cache, GFP_KERNEL);
if (!obj)
return -ENOMEM;
mutex_lock(&device->mutex);
if (_abort_submission(adreno_dev)) {
mutex_unlock(&device->mutex);
kmem_cache_free(obj_cache, obj);
return -EBUSY;
}
if (kgsl_context_detached(context)) {
mutex_unlock(&device->mutex);
kmem_cache_free(obj_cache, obj);
return -ENOENT;
}
hwsched->inflight++;
if (hwsched->inflight == 1 &&
!test_bit(ADRENO_HWSCHED_POWER, &hwsched->flags)) {
ret = adreno_active_count_get(adreno_dev);
if (ret) {
hwsched->inflight--;
mutex_unlock(&device->mutex);
kmem_cache_free(obj_cache, obj);
return ret;
}
set_bit(ADRENO_HWSCHED_POWER, &hwsched->flags);
}
ret = hwsched->hwsched_ops->submit_drawobj(adreno_dev, drawobj);
if (ret) {
/*
* If the first submission failed, then put back the active
* count to relinquish active vote
*/
if (hwsched->inflight == 1) {
adreno_active_count_put(adreno_dev);
clear_bit(ADRENO_HWSCHED_POWER, &hwsched->flags);
}
hwsched->inflight--;
kmem_cache_free(obj_cache, obj);
mutex_unlock(&device->mutex);
return ret;
}
if ((hwsched->inflight == 1) &&
!test_and_set_bit(ADRENO_HWSCHED_ACTIVE, &hwsched->flags))
reinit_completion(&hwsched->idle_gate);
if (is_cmdobj(drawobj)) {
struct kgsl_drawobj_cmd *cmdobj = CMDOBJ(drawobj);
/* If this MARKER object is already retired, we can destroy it here */
if ((test_bit(CMDOBJ_MARKER_EXPIRED, &cmdobj->priv))) {
kmem_cache_free(obj_cache, obj);
kgsl_drawobj_destroy(drawobj);
goto done;
}
if (cmdobj->numibs > HWSCHED_MAX_DISPATCH_NUMIBS) {
hwsched->big_cmdobj = cmdobj;
kref_get(&drawobj->refcount);
}
}
obj->drawobj = drawobj;
list_add_tail(&obj->node, &hwsched->cmd_list);
done:
mutex_unlock(&device->mutex);
return 0;
}
/**
* hwsched_sendcmds() - Send commands from a context to the GPU
* @adreno_dev: Pointer to the adreno device struct
* @drawctxt: Pointer to the adreno context to dispatch commands from
*
* Dequeue and send a burst of commands from the specified context to the GPU
* Returns postive if the context needs to be put back on the pending queue
* 0 if the context is empty or detached and negative on error
*/
static int hwsched_sendcmds(struct adreno_device *adreno_dev,
struct adreno_context *drawctxt)
{
int count = 0;
int ret = 0;
while (1) {
struct kgsl_drawobj *drawobj;
struct kgsl_drawobj_cmd *cmdobj = NULL;
struct kgsl_context *context;
spin_lock(&drawctxt->lock);
drawobj = _process_drawqueue_get_next_drawobj(adreno_dev,
drawctxt);
/*
* adreno_context_get_drawobj returns -EAGAIN if the current
* drawobj has pending sync points so no more to do here.
* When the sync points are satisfied then the context will get
* reqeueued
*/
if (IS_ERR_OR_NULL(drawobj)) {
if (IS_ERR(drawobj))
ret = PTR_ERR(drawobj);
spin_unlock(&drawctxt->lock);
break;
}
_pop_drawobj(drawctxt);
spin_unlock(&drawctxt->lock);
if (is_cmdobj(drawobj) || is_marker_skip(drawobj)) {
cmdobj = CMDOBJ(drawobj);
context = drawobj->context;
trace_adreno_cmdbatch_ready(context->id,
context->priority, drawobj->timestamp,
cmdobj->requeue_cnt);
}
ret = hwsched_sendcmd(adreno_dev, drawobj);
/*
* On error from hwsched_sendcmd() try to requeue the cmdobj
* unless we got back -ENOENT which means that the context has
* been detached and there will be no more deliveries from here
*/
if (ret != 0) {
/* Destroy the cmdobj on -ENOENT */
if (ret == -ENOENT)
kgsl_drawobj_destroy(drawobj);
else {
/*
* If we couldn't put it on dispatch queue
* then return it to the context queue
*/
int r = hwsched_dispatcher_requeue_drawobj(
drawctxt, drawobj);
if (r)
ret = r;
}
break;
}
if (cmdobj)
drawctxt->submitted_timestamp = drawobj->timestamp;
count++;
}
/*
* Wake up any snoozing threads if we have consumed any real commands
* or marker commands and we have room in the context queue.
*/
if (_check_context_queue(drawctxt, 0))
wake_up_all(&drawctxt->wq);
if (!ret)
ret = count;
/* Return error or the number of commands queued */
return ret;
}
static void hwsched_handle_jobs_list(struct adreno_device *adreno_dev,
int id, unsigned long *map, struct llist_node *list)
{
struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
struct adreno_dispatch_job *job, *next;
if (!list)
return;
/* Reverse the list so we deal with oldest submitted contexts first */
list = llist_reverse_order(list);
llist_for_each_entry_safe(job, next, list, node) {
int ret;
if (kgsl_context_is_bad(&job->drawctxt->base)) {
kgsl_context_put(&job->drawctxt->base);
kmem_cache_free(jobs_cache, job);
continue;
}
/*
* Due to the nature of the lockless queue the same context
* might have multiple jobs on the list. We allow this so we
* don't have to query the list on the producer side but on the
* consumer side we only want each context to be considered
* once. Use a bitmap to remember which contexts we've already
* seen and quietly discard duplicate jobs
*/
if (test_and_set_bit(job->drawctxt->base.id, map)) {
kgsl_context_put(&job->drawctxt->base);
kmem_cache_free(jobs_cache, job);
continue;
}
ret = hwsched_sendcmds(adreno_dev, job->drawctxt);
/*
* If the context had nothing queued or the context has been
* destroyed then drop the job
*/
if (!ret || ret == -ENOENT) {
kgsl_context_put(&job->drawctxt->base);
kmem_cache_free(jobs_cache, job);
continue;
}
/*
* If the dispatch queue is full then requeue the job to be
* considered first next time. Otherwise the context
* either successfully submmitted to the GPU or another error
* happened and it should go back on the regular queue
*/
if (ret == -ENOSPC)
llist_add(&job->node, &hwsched->requeue[id]);
else
llist_add(&job->node, &hwsched->jobs[id]);
}
}
static void hwsched_handle_jobs(struct adreno_device *adreno_dev, int id)
{
struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
unsigned long map[BITS_TO_LONGS(KGSL_MEMSTORE_MAX)];
struct llist_node *requeue, *jobs;
memset(map, 0, sizeof(map));
requeue = llist_del_all(&hwsched->requeue[id]);
jobs = llist_del_all(&hwsched->jobs[id]);
hwsched_handle_jobs_list(adreno_dev, id, map, requeue);
hwsched_handle_jobs_list(adreno_dev, id, map, jobs);
}
/**
* hwsched_issuecmds() - Issue commmands from pending contexts
* @adreno_dev: Pointer to the adreno device struct
*
* Issue as many commands as possible (up to inflight) from the pending contexts
* This function assumes the dispatcher mutex has been locked.
*/
static void hwsched_issuecmds(struct adreno_device *adreno_dev)
{
struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
int i;
for (i = 0; i < ARRAY_SIZE(hwsched->jobs); i++)
hwsched_handle_jobs(adreno_dev, i);
}
void adreno_hwsched_trigger(struct adreno_device *adreno_dev)
{
struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
kthread_queue_work(hwsched->worker, &hwsched->work);
}
static inline void _decrement_submit_now(struct kgsl_device *device)
{
spin_lock(&device->submit_lock);
device->submit_now--;
spin_unlock(&device->submit_lock);
}
u32 adreno_hwsched_gpu_fault(struct adreno_device *adreno_dev)
{
/* make sure we're reading the latest value */
smp_rmb();
return atomic_read(&adreno_dev->hwsched.fault);
}
/**
* adreno_hwsched_issuecmds() - Issue commmands from pending contexts
* @adreno_dev: Pointer to the adreno device struct
*
* Lock the dispatcher and call hwsched_issuecmds
*/
static void adreno_hwsched_issuecmds(struct adreno_device *adreno_dev)
{
struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
spin_lock(&device->submit_lock);
/* If GPU state is not ACTIVE, schedule the work for later */
if (device->skip_inline_submit) {
spin_unlock(&device->submit_lock);
goto done;
}
device->submit_now++;
spin_unlock(&device->submit_lock);
/* If the dispatcher is busy then schedule the work for later */
if (!mutex_trylock(&hwsched->mutex)) {
_decrement_submit_now(device);
goto done;
}
if (!adreno_hwsched_gpu_fault(adreno_dev))
hwsched_issuecmds(adreno_dev);
if (hwsched->inflight > 0) {
mutex_lock(&device->mutex);
kgsl_pwrscale_update(device);
kgsl_start_idle_timer(device);
mutex_unlock(&device->mutex);
}
mutex_unlock(&hwsched->mutex);
_decrement_submit_now(device);
return;
done:
adreno_hwsched_trigger(adreno_dev);
}
/**
* get_timestamp() - Return the next timestamp for the context
* @drawctxt - Pointer to an adreno draw context struct
* @drawobj - Pointer to a drawobj
* @timestamp - Pointer to a timestamp value possibly passed from the user
* @user_ts - user generated timestamp
*
* Assign a timestamp based on the settings of the draw context and the command
* batch.
*/
static int get_timestamp(struct adreno_context *drawctxt,
struct kgsl_drawobj *drawobj, unsigned int *timestamp,
unsigned int user_ts)
{
if (drawctxt->base.flags & KGSL_CONTEXT_USER_GENERATED_TS) {
/*
* User specified timestamps need to be greater than the last
* issued timestamp in the context
*/
if (timestamp_cmp(drawctxt->timestamp, user_ts) >= 0)
return -ERANGE;
drawctxt->timestamp = user_ts;
} else
drawctxt->timestamp++;
*timestamp = drawctxt->timestamp;
drawobj->timestamp = *timestamp;
return 0;
}
static inline int _wait_for_room_in_context_queue(
struct adreno_context *drawctxt, u32 count)
{
int ret = 0;
/*
* There is always a possibility that dispatcher may end up pushing
* the last popped draw object back to the context drawqueue. Hence,
* we can only queue up to _context_drawqueue_size - 1 here to make
* sure we never let drawqueue->queued exceed _context_drawqueue_size.
*/
if ((drawctxt->queued + count) > (_context_drawqueue_size - 1)) {
trace_adreno_drawctxt_sleep(drawctxt);
spin_unlock(&drawctxt->lock);
ret = wait_event_interruptible_timeout(drawctxt->wq,
_check_context_queue(drawctxt, count),
msecs_to_jiffies(_context_queue_wait));
spin_lock(&drawctxt->lock);
trace_adreno_drawctxt_wake(drawctxt);
/*
* Account for the possibility that the context got invalidated
* while we were sleeping
*/
if (ret > 0)
ret = kgsl_check_context_state(&drawctxt->base);
else if (ret == 0)
ret = -ETIMEDOUT;
}
return ret;
}
static unsigned int _check_context_state_to_queue_cmds(
struct adreno_context *drawctxt, u32 count)
{
int ret = kgsl_check_context_state(&drawctxt->base);
if (ret)
return ret;
return _wait_for_room_in_context_queue(drawctxt, count);
}
static void _queue_drawobj(struct adreno_context *drawctxt,
struct kgsl_drawobj *drawobj)
{
struct kgsl_context *context = drawobj->context;
/* Put the command into the queue */
drawctxt->drawqueue[drawctxt->drawqueue_tail] = drawobj;
drawctxt->drawqueue_tail = (drawctxt->drawqueue_tail + 1) %
ADRENO_CONTEXT_DRAWQUEUE_SIZE;
drawctxt->queued++;
msm_perf_events_update(MSM_PERF_GFX, MSM_PERF_QUEUE,
pid_nr(context->proc_priv->pid),
context->id, drawobj->timestamp,
!!(drawobj->flags & KGSL_DRAWOBJ_END_OF_FRAME));
trace_adreno_cmdbatch_queued(drawobj, drawctxt->queued);
}
static int _queue_cmdobj(struct adreno_device *adreno_dev,
struct adreno_context *drawctxt, struct kgsl_drawobj_cmd *cmdobj,
uint32_t *timestamp, unsigned int user_ts)
{
struct kgsl_drawobj *drawobj = DRAWOBJ(cmdobj);
u32 j;
int ret;
ret = get_timestamp(drawctxt, drawobj, timestamp, user_ts);
if (ret)
return ret;
/*
* If this is a real command then we need to force any markers
* queued before it to dispatch to keep time linear - set the
* skip bit so the commands get NOPed.
*/
j = drawctxt->drawqueue_head;
while (j != drawctxt->drawqueue_tail) {
if (drawctxt->drawqueue[j]->type == MARKEROBJ_TYPE) {
struct kgsl_drawobj_cmd *markerobj =
CMDOBJ(drawctxt->drawqueue[j]);
set_bit(CMDOBJ_SKIP, &markerobj->priv);
}
j = DRAWQUEUE_NEXT(j, ADRENO_CONTEXT_DRAWQUEUE_SIZE);
}
drawctxt->queued_timestamp = *timestamp;
_queue_drawobj(drawctxt, drawobj);
return 0;
}
static void _queue_syncobj(struct adreno_context *drawctxt,
struct kgsl_drawobj_sync *syncobj, uint32_t *timestamp)
{
struct kgsl_drawobj *drawobj = DRAWOBJ(syncobj);
*timestamp = 0;
drawobj->timestamp = 0;
_queue_drawobj(drawctxt, drawobj);
}
static int _queue_markerobj(struct adreno_device *adreno_dev,
struct adreno_context *drawctxt, struct kgsl_drawobj_cmd *markerobj,
u32 *timestamp, u32 user_ts)
{
struct kgsl_drawobj *drawobj = DRAWOBJ(markerobj);
int ret;
ret = get_timestamp(drawctxt, drawobj, timestamp, user_ts);
if (ret)
return ret;
/*
* See if we can fastpath this thing - if nothing is queued
* and nothing is inflight retire without bothering the GPU
*/
if (!drawctxt->queued && kgsl_check_timestamp(drawobj->device,
drawobj->context, drawctxt->queued_timestamp)) {
_retire_timestamp(drawobj);
return 1;
}
/*
* Remember the last queued timestamp - the marker will block
* until that timestamp is expired (unless another command
* comes along and forces the marker to execute)
*/
markerobj->marker_timestamp = drawctxt->queued_timestamp;
drawctxt->queued_timestamp = *timestamp;
_queue_drawobj(drawctxt, drawobj);
return 0;
}
static int _queue_bindobj(struct adreno_context *drawctxt,
struct kgsl_drawobj *drawobj, u32 *timestamp, u32 user_ts)
{
int ret;
ret = get_timestamp(drawctxt, drawobj, timestamp, user_ts);
if (ret)
return ret;
drawctxt->queued_timestamp = *timestamp;
_queue_drawobj(drawctxt, drawobj);
return 0;
}
static void _queue_timelineobj(struct adreno_context *drawctxt,
struct kgsl_drawobj *drawobj)
{
/*
* This drawobj is not submitted to the GPU so use a timestamp of 0.
* Update the timestamp through a subsequent marker to keep userspace
* happy.
*/
drawobj->timestamp = 0;
_queue_drawobj(drawctxt, drawobj);
}
static int adreno_hwsched_queue_cmds(struct kgsl_device_private *dev_priv,
struct kgsl_context *context, struct kgsl_drawobj *drawobj[],
u32 count, u32 *timestamp)
{
struct kgsl_device *device = dev_priv->device;
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
struct adreno_context *drawctxt = ADRENO_CONTEXT(context);
struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
struct adreno_dispatch_job *job;
int ret;
unsigned int i, user_ts;
/*
* There is always a possibility that dispatcher may end up pushing
* the last popped draw object back to the context drawqueue. Hence,
* we can only queue up to _context_drawqueue_size - 1 here to make
* sure we never let drawqueue->queued exceed _context_drawqueue_size.
*/
if (!count || count > _context_drawqueue_size - 1)
return -EINVAL;
for (i = 0; i < count; i++) {
struct kgsl_drawobj_cmd *cmdobj;
struct kgsl_memobj_node *ib;
if (!is_cmdobj(drawobj[i]))
continue;
cmdobj = CMDOBJ(drawobj[i]);
list_for_each_entry(ib, &cmdobj->cmdlist, node)
cmdobj->numibs++;
if (cmdobj->numibs > HWSCHED_MAX_IBS)
return -EINVAL;
}
ret = kgsl_check_context_state(&drawctxt->base);
if (ret)
return ret;
ret = adreno_verify_cmdobj(dev_priv, context, drawobj, count);
if (ret)
return ret;
/* wait for the suspend gate */
wait_for_completion(&device->halt_gate);
job = kmem_cache_alloc(jobs_cache, GFP_KERNEL);
if (!job)
return -ENOMEM;
job->drawctxt = drawctxt;
spin_lock(&drawctxt->lock);
ret = _check_context_state_to_queue_cmds(drawctxt, count);
if (ret) {
spin_unlock(&drawctxt->lock);
kmem_cache_free(jobs_cache, job);
return ret;
}
user_ts = *timestamp;
/*
* If there is only one drawobj in the array and it is of
* type SYNCOBJ_TYPE, skip comparing user_ts as it can be 0
*/
if (!(count == 1 && drawobj[0]->type == SYNCOBJ_TYPE) &&
(drawctxt->base.flags & KGSL_CONTEXT_USER_GENERATED_TS)) {
/*
* User specified timestamps need to be greater than the last
* issued timestamp in the context
*/
if (timestamp_cmp(drawctxt->timestamp, user_ts) >= 0) {
spin_unlock(&drawctxt->lock);
kmem_cache_free(jobs_cache, job);
return -ERANGE;
}
}
for (i = 0; i < count; i++) {
switch (drawobj[i]->type) {
case MARKEROBJ_TYPE:
ret = _queue_markerobj(adreno_dev, drawctxt,
CMDOBJ(drawobj[i]),
timestamp, user_ts);
if (ret == 1) {
spin_unlock(&drawctxt->lock);
kmem_cache_free(jobs_cache, job);
return 0;
} else if (ret) {
spin_unlock(&drawctxt->lock);
kmem_cache_free(jobs_cache, job);
return ret;
}
break;
case CMDOBJ_TYPE:
ret = _queue_cmdobj(adreno_dev, drawctxt,
CMDOBJ(drawobj[i]),
timestamp, user_ts);
if (ret) {
spin_unlock(&drawctxt->lock);
kmem_cache_free(jobs_cache, job);
return ret;
}
break;
case SYNCOBJ_TYPE:
_queue_syncobj(drawctxt, SYNCOBJ(drawobj[i]),
timestamp);
break;
case BINDOBJ_TYPE:
ret = _queue_bindobj(drawctxt, drawobj[i], timestamp,
user_ts);
if (ret) {
spin_unlock(&drawctxt->lock);
kmem_cache_free(jobs_cache, job);
return ret;
}
break;
case TIMELINEOBJ_TYPE:
_queue_timelineobj(drawctxt, drawobj[i]);
break;
default:
spin_unlock(&drawctxt->lock);
kmem_cache_free(jobs_cache, job);
return -EINVAL;
}
}
adreno_track_context(adreno_dev, NULL, drawctxt);
spin_unlock(&drawctxt->lock);
/* Add the context to the dispatcher pending list */
if (_kgsl_context_get(&drawctxt->base)) {
trace_dispatch_queue_context(drawctxt);
llist_add(&job->node, &hwsched->jobs[drawctxt->base.priority]);
adreno_hwsched_issuecmds(adreno_dev);
} else
kmem_cache_free(jobs_cache, job);
return 0;
}
void adreno_hwsched_retire_cmdobj(struct adreno_hwsched *hwsched,
struct kgsl_drawobj_cmd *cmdobj)
{
struct kgsl_drawobj *drawobj = DRAWOBJ(cmdobj);
struct kgsl_mem_entry *entry;
struct kgsl_drawobj_profiling_buffer *profile_buffer;
struct kgsl_context *context = drawobj->context;
msm_perf_events_update(MSM_PERF_GFX, MSM_PERF_RETIRED,
pid_nr(context->proc_priv->pid),
context->id, drawobj->timestamp,
!!(drawobj->flags & KGSL_DRAWOBJ_END_OF_FRAME));
if (drawobj->flags & KGSL_DRAWOBJ_END_OF_FRAME) {
atomic64_inc(&drawobj->context->proc_priv->frame_count);
atomic_inc(&drawobj->context->proc_priv->period->frames);
}
entry = cmdobj->profiling_buf_entry;
if (entry) {
profile_buffer = kgsl_gpuaddr_to_vaddr(&entry->memdesc,
cmdobj->profiling_buffer_gpuaddr);
if (profile_buffer == NULL)
return;
kgsl_memdesc_unmap(&entry->memdesc);
}
trace_adreno_cmdbatch_done(drawobj->context->id,
drawobj->context->priority, drawobj->timestamp);
if (hwsched->big_cmdobj == cmdobj) {
hwsched->big_cmdobj = NULL;
kgsl_drawobj_put(drawobj);
}
kgsl_drawobj_destroy(drawobj);
}
static bool drawobj_retired(struct adreno_device *adreno_dev,
struct kgsl_drawobj *drawobj)
{
struct adreno_context *drawctxt = ADRENO_CONTEXT(drawobj->context);
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct kgsl_drawobj_cmd *cmdobj;
struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
if ((drawobj->type & SYNCOBJ_TYPE) != 0) {
struct gmu_context_queue_header *hdr =
drawctxt->gmu_context_queue.hostptr;
if (timestamp_cmp(drawobj->timestamp, hdr->sync_obj_ts) > 0)
return false;
trace_adreno_syncobj_retired(drawobj->context->id, drawobj->timestamp);
kgsl_drawobj_destroy(drawobj);
return true;
}
cmdobj = CMDOBJ(drawobj);
if (!kgsl_check_timestamp(device, drawobj->context,
drawobj->timestamp))
return false;
adreno_hwsched_retire_cmdobj(hwsched, cmdobj);
return true;
}
static void retire_drawobj_list(struct adreno_device *adreno_dev)
{
struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
struct cmd_list_obj *obj, *tmp;
list_for_each_entry_safe(obj, tmp, &hwsched->cmd_list, node) {
struct kgsl_drawobj *drawobj = obj->drawobj;
if (!drawobj_retired(adreno_dev, drawobj))
continue;
list_del_init(&obj->node);
kmem_cache_free(obj_cache, obj);
hwsched->inflight--;
}
}
/* Take down the dispatcher and release any power states */
static void hwsched_power_down(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
mutex_lock(&device->mutex);
if (test_and_clear_bit(ADRENO_HWSCHED_ACTIVE, &hwsched->flags))
complete_all(&hwsched->idle_gate);
if (test_bit(ADRENO_HWSCHED_POWER, &hwsched->flags)) {
adreno_active_count_put(adreno_dev);
clear_bit(ADRENO_HWSCHED_POWER, &hwsched->flags);
}
mutex_unlock(&device->mutex);
}
static void adreno_hwsched_queue_context(struct adreno_device *adreno_dev,
struct adreno_context *drawctxt)
{
hwsched_queue_context(adreno_dev, drawctxt);
adreno_hwsched_trigger(adreno_dev);
}
void adreno_hwsched_start(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
complete_all(&device->halt_gate);
adreno_hwsched_trigger(adreno_dev);
}
static void change_preemption(struct adreno_device *adreno_dev, void *priv)
{
change_bit(ADRENO_DEVICE_PREEMPTION, &adreno_dev->priv);
}
static int _preemption_store(struct adreno_device *adreno_dev, bool val)
{
if (!adreno_preemption_feature_set(adreno_dev) ||
(test_bit(ADRENO_DEVICE_PREEMPTION, &adreno_dev->priv) == val))
return 0;
return adreno_power_cycle(adreno_dev, change_preemption, NULL);
}
static bool _preemption_show(struct adreno_device *adreno_dev)
{
return adreno_is_preemption_enabled(adreno_dev);
}
static unsigned int _preempt_count_show(struct adreno_device *adreno_dev)
{
const struct adreno_hwsched_ops *hwsched_ops =
adreno_dev->hwsched.hwsched_ops;
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
u32 count;
mutex_lock(&device->mutex);
count = hwsched_ops->preempt_count(adreno_dev);
mutex_unlock(&device->mutex);
return count;
}
static int _ft_long_ib_detect_store(struct adreno_device *adreno_dev, bool val)
{
return adreno_power_cycle_bool(adreno_dev, &adreno_dev->long_ib_detect,
val);
}
static bool _ft_long_ib_detect_show(struct adreno_device *adreno_dev)
{
return adreno_dev->long_ib_detect;
}
static ADRENO_SYSFS_BOOL(preemption);
static ADRENO_SYSFS_RO_U32(preempt_count);
static ADRENO_SYSFS_BOOL(ft_long_ib_detect);
static const struct attribute *_hwsched_attr_list[] = {
&adreno_attr_preemption.attr.attr,
&adreno_attr_preempt_count.attr.attr,
&adreno_attr_ft_long_ib_detect.attr.attr,
NULL,
};
void adreno_hwsched_deregister_hw_fence(struct adreno_device *adreno_dev)
{
struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
struct adreno_hw_fence *hw_fence = &hwsched->hw_fence;
if (!test_bit(ADRENO_HWSCHED_HW_FENCE, &hwsched->flags))
return;
msm_hw_fence_deregister(hwsched->hw_fence.handle);
if (hw_fence->memdesc.sgt)
sg_free_table(hw_fence->memdesc.sgt);
memset(&hw_fence->memdesc, 0x0, sizeof(hw_fence->memdesc));
kmem_cache_destroy(hwsched->hw_fence_cache);
clear_bit(ADRENO_HWSCHED_HW_FENCE, &hwsched->flags);
}
static void adreno_hwsched_dispatcher_close(struct adreno_device *adreno_dev)
{
struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
if (!IS_ERR_OR_NULL(hwsched->worker))
kthread_destroy_worker(hwsched->worker);
adreno_set_dispatch_ops(adreno_dev, NULL);
kmem_cache_destroy(jobs_cache);
kmem_cache_destroy(obj_cache);
sysfs_remove_files(&device->dev->kobj, _hwsched_attr_list);
kfree(hwsched->ctxt_bad);
adreno_hwsched_deregister_hw_fence(adreno_dev);
if (hwsched->global_ctxtq.hostptr)
kgsl_sharedmem_free(&hwsched->global_ctxtq);
}
static void force_retire_timestamp(struct kgsl_device *device,
struct kgsl_drawobj *drawobj)
{
kgsl_sharedmem_writel(device->memstore,
KGSL_MEMSTORE_OFFSET(drawobj->context->id, soptimestamp),
drawobj->timestamp);
kgsl_sharedmem_writel(device->memstore,
KGSL_MEMSTORE_OFFSET(drawobj->context->id, eoptimestamp),
drawobj->timestamp);
}
/* Return true if drawobj needs to replayed, false otherwise */
static bool drawobj_replay(struct adreno_device *adreno_dev,
struct kgsl_drawobj *drawobj)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct kgsl_drawobj_cmd *cmdobj;
struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
if ((drawobj->type & SYNCOBJ_TYPE) != 0) {
if (kgsl_drawobj_events_pending(SYNCOBJ(drawobj)))
return true;
trace_adreno_syncobj_retired(drawobj->context->id, drawobj->timestamp);
kgsl_drawobj_destroy(drawobj);
return false;
}
cmdobj = CMDOBJ(drawobj);
if (kgsl_check_timestamp(device, drawobj->context,
drawobj->timestamp) || kgsl_context_is_bad(drawobj->context)) {
adreno_hwsched_retire_cmdobj(hwsched, cmdobj);
return false;
}
return true;
}
void adreno_hwsched_replay(struct adreno_device *adreno_dev)
{
struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
const struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev);
struct cmd_list_obj *obj, *tmp;
u32 retired = 0;
list_for_each_entry_safe(obj, tmp, &hwsched->cmd_list, node) {
struct kgsl_drawobj *drawobj = obj->drawobj;
/*
* Get rid of retired objects or objects that belong to detached
* or invalidated contexts
*/
if (drawobj_replay(adreno_dev, drawobj)) {
hwsched->hwsched_ops->submit_drawobj(adreno_dev, drawobj);
continue;
}
retired++;
list_del_init(&obj->node);
kmem_cache_free(obj_cache, obj);
hwsched->inflight--;
}
if (hwsched->recurring_cmdobj) {
u32 event;
if (kgsl_context_invalid(
hwsched->recurring_cmdobj->base.context)) {
clear_bit(CMDOBJ_RECURRING_START,
&hwsched->recurring_cmdobj->priv);
set_bit(CMDOBJ_RECURRING_STOP,
&hwsched->recurring_cmdobj->priv);
event = GPU_SSR_FATAL;
} else {
event = GPU_SSR_END;
}
gpudev->send_recurring_cmdobj(adreno_dev,
hwsched->recurring_cmdobj);
srcu_notifier_call_chain(&device->nh, event, NULL);
}
/* Signal fences */
if (retired)
kgsl_process_event_groups(device);
}
static void do_fault_header(struct adreno_device *adreno_dev,
struct kgsl_drawobj *drawobj, int fault)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
const struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev);
struct adreno_context *drawctxt;
u32 status = 0, rptr = 0, wptr = 0, ib1sz = 0, ib2sz = 0;
u64 ib1base = 0, ib2base = 0;
bool gx_on = adreno_gx_is_on(adreno_dev);
u32 ctxt_id = 0, ts = 0;
int rb_id = -1;
dev_err(device->dev, "Fault id:%d and GX is %s\n", fault, gx_on ? "ON" : "OFF");
if (!gx_on && !drawobj)
return;
if (gpudev->fault_header)
return gpudev->fault_header(adreno_dev, drawobj);
if (gx_on) {
adreno_readreg(adreno_dev, ADRENO_REG_RBBM_STATUS, &status);
adreno_readreg(adreno_dev, ADRENO_REG_CP_RB_RPTR, &rptr);
adreno_readreg(adreno_dev, ADRENO_REG_CP_RB_WPTR, &wptr);
adreno_readreg64(adreno_dev, ADRENO_REG_CP_IB1_BASE,
ADRENO_REG_CP_IB1_BASE_HI, &ib1base);
adreno_readreg(adreno_dev, ADRENO_REG_CP_IB1_BUFSZ, &ib1sz);
adreno_readreg64(adreno_dev, ADRENO_REG_CP_IB2_BASE,
ADRENO_REG_CP_IB2_BASE_HI, &ib2base);
adreno_readreg(adreno_dev, ADRENO_REG_CP_IB2_BUFSZ, &ib2sz);
dev_err(device->dev,
"status %8.8X rb %4.4x/%4.4x ib1 %16.16llX/%4.4x ib2 %16.16llX/%4.4x\n",
status, rptr, wptr, ib1base, ib1sz, ib2base, ib2sz);
}
if (drawobj) {
drawctxt = ADRENO_CONTEXT(drawobj->context);
drawobj->context->last_faulted_cmd_ts = drawobj->timestamp;
drawobj->context->total_fault_count++;
ctxt_id = drawobj->context->id;
ts = drawobj->timestamp;
rb_id = adreno_get_level(drawobj->context);
pr_context(device, drawobj->context,
"ctx %u ctx_type %s ts %u policy %lX dispatch_queue=%d\n",
drawobj->context->id, kgsl_context_type(drawctxt->type),
drawobj->timestamp, CMDOBJ(drawobj)->fault_recovery,
drawobj->context->gmu_dispatch_queue);
pr_context(device, drawobj->context,
"cmdline: %s\n", drawctxt->base.proc_priv->cmdline);
}
trace_adreno_gpu_fault(ctxt_id, ts, status, rptr, wptr, ib1base, ib1sz,
ib2base, ib2sz, rb_id);
}
static struct cmd_list_obj *get_active_cmdobj_lpac(
struct adreno_device *adreno_dev)
{
struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
struct cmd_list_obj *obj, *tmp, *active_obj = NULL;
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
u32 consumed = 0, retired = 0;
struct kgsl_drawobj *drawobj = NULL;
list_for_each_entry_safe(obj, tmp, &hwsched->cmd_list, node) {
drawobj = obj->drawobj;
if (!(kgsl_context_is_lpac(drawobj->context)))
continue;
kgsl_readtimestamp(device, drawobj->context,
KGSL_TIMESTAMP_CONSUMED, &consumed);
kgsl_readtimestamp(device, drawobj->context,
KGSL_TIMESTAMP_RETIRED, &retired);
if (!consumed)
continue;
if (consumed == retired)
continue;
/*
* Find the first submission that started but didn't finish
* We only care about one ringbuffer for LPAC so just look for the
* first unfinished submission
*/
if (!active_obj)
active_obj = obj;
}
if (active_obj) {
drawobj = active_obj->drawobj;
if (kref_get_unless_zero(&drawobj->refcount)) {
struct kgsl_drawobj_cmd *cmdobj = CMDOBJ(drawobj);
set_bit(CMDOBJ_FAULT, &cmdobj->priv);
return active_obj;
}
}
return NULL;
}
static struct cmd_list_obj *get_active_cmdobj(
struct adreno_device *adreno_dev)
{
struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
struct cmd_list_obj *obj, *tmp, *active_obj = NULL;
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
u32 consumed = 0, retired = 0, prio = UINT_MAX;
struct kgsl_drawobj *drawobj = NULL;
list_for_each_entry_safe(obj, tmp, &hwsched->cmd_list, node) {
drawobj = obj->drawobj;
/* We track LPAC separately */
if (!is_cmdobj(drawobj) || kgsl_context_is_lpac(drawobj->context))
continue;
kgsl_readtimestamp(device, drawobj->context,
KGSL_TIMESTAMP_CONSUMED, &consumed);
kgsl_readtimestamp(device, drawobj->context,
KGSL_TIMESTAMP_RETIRED, &retired);
if (!consumed)
continue;
if (consumed == retired)
continue;
/* Find the first submission that started but didn't finish */
if (!active_obj) {
active_obj = obj;
prio = adreno_get_level(drawobj->context);
continue;
}
/* Find the highest priority active submission */
if (adreno_get_level(drawobj->context) < prio) {
active_obj = obj;
prio = adreno_get_level(drawobj->context);
}
}
if (active_obj) {
struct kgsl_drawobj_cmd *cmdobj;
drawobj = active_obj->drawobj;
cmdobj = CMDOBJ(drawobj);
if (kref_get_unless_zero(&drawobj->refcount)) {
set_bit(CMDOBJ_FAULT, &cmdobj->priv);
return active_obj;
}
}
return NULL;
}
static struct cmd_list_obj *get_fault_cmdobj(struct adreno_device *adreno_dev,
u32 ctxt_id, u32 ts)
{
struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
struct cmd_list_obj *obj, *tmp;
list_for_each_entry_safe(obj, tmp, &hwsched->cmd_list, node) {
struct kgsl_drawobj *drawobj = obj->drawobj;
if (!is_cmdobj(drawobj))
continue;
if ((ctxt_id == drawobj->context->id) &&
(ts == drawobj->timestamp)) {
if (kref_get_unless_zero(&drawobj->refcount)) {
struct kgsl_drawobj_cmd *cmdobj = CMDOBJ(drawobj);
set_bit(CMDOBJ_FAULT, &cmdobj->priv);
return obj;
}
}
}
return NULL;
}
static bool context_is_throttled(struct kgsl_device *device,
struct kgsl_context *context)
{
if (ktime_ms_delta(ktime_get(), context->fault_time) >
_fault_throttle_time) {
context->fault_time = ktime_get();
context->fault_count = 1;
return false;
}
context->fault_count++;
if (context->fault_count > _fault_throttle_burst) {
pr_context(device, context,
"gpu fault threshold exceeded %d faults in %d msecs\n",
_fault_throttle_burst, _fault_throttle_time);
return true;
}
return false;
}
static void _print_syncobj(struct adreno_device *adreno_dev, struct kgsl_drawobj *drawobj)
{
int i, j, fence_index = 0;
struct kgsl_drawobj_sync *syncobj = SYNCOBJ(drawobj);
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
for (i = 0; i < syncobj->numsyncs; i++) {
struct kgsl_drawobj_sync_event *event = &syncobj->synclist[i];
struct kgsl_sync_fence_cb *kcb = event->handle;
struct dma_fence **fences;
struct dma_fence_array *array;
u32 num_fences;
array = to_dma_fence_array(kcb->fence);
if (array != NULL) {
num_fences = array->num_fences;
fences = array->fences;
} else {
num_fences = 1;
fences = &kcb->fence;
}
for (j = 0; j < num_fences; j++, fence_index++) {
bool kgsl = is_kgsl_fence(fences[j]);
bool signaled = test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fences[j]->flags);
char value[32] = "unknown";
if (fences[j]->ops->timeline_value_str)
fences[j]->ops->timeline_value_str(fences[j], value, sizeof(value));
dev_err(device->dev,
"dma fence[%d] signaled:%d kgsl:%d ctx:%llu seqno:%llu value:%s\n",
fence_index, signaled, kgsl, fences[j]->context, fences[j]->seqno,
value);
}
}
}
static void print_fault_syncobj(struct adreno_device *adreno_dev,
u32 ctxt_id, u32 ts)
{
struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
struct cmd_list_obj *obj;
list_for_each_entry(obj, &hwsched->cmd_list, node) {
struct kgsl_drawobj *drawobj = obj->drawobj;
if (drawobj->type == SYNCOBJ_TYPE) {
if ((ctxt_id == drawobj->context->id) &&
(ts == drawobj->timestamp))
_print_syncobj(adreno_dev, drawobj);
}
}
}
static void adreno_hwsched_reset_and_snapshot_legacy(struct adreno_device *adreno_dev, int fault)
{
struct kgsl_drawobj *drawobj = NULL;
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct kgsl_context *context = NULL;
struct cmd_list_obj *obj;
const struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev);
struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
struct hfi_context_bad_cmd_legacy *cmd = hwsched->ctxt_bad;
if (device->state != KGSL_STATE_ACTIVE)
return;
if (hwsched->recurring_cmdobj)
srcu_notifier_call_chain(&device->nh, GPU_SSR_BEGIN, NULL);
if (cmd->error == GMU_SYNCOBJ_TIMEOUT_ERROR) {
print_fault_syncobj(adreno_dev, cmd->ctxt_id, cmd->ts);
gmu_core_fault_snapshot(device);
goto done;
}
/*
* First, try to see if the faulted command object is marked
* in case there was a context bad hfi. But, with stall-on-fault,
* we know that GMU cannot send context bad hfi. Hence, attempt
* to walk the list of active submissions to find the one that
* faulted.
*/
obj = get_fault_cmdobj(adreno_dev, cmd->ctxt_id, cmd->ts);
if (!obj && (fault & ADRENO_IOMMU_PAGE_FAULT))
obj = get_active_cmdobj(adreno_dev);
if (obj) {
drawobj = obj->drawobj;
trace_adreno_cmdbatch_fault(CMDOBJ(drawobj), fault);
} else if (hwsched->recurring_cmdobj &&
hwsched->recurring_cmdobj->base.context->id == cmd->ctxt_id) {
drawobj = DRAWOBJ(hwsched->recurring_cmdobj);
trace_adreno_cmdbatch_fault(hwsched->recurring_cmdobj, fault);
if (!kref_get_unless_zero(&drawobj->refcount))
drawobj = NULL;
}
if (!drawobj) {
if (fault & ADRENO_GMU_FAULT)
gmu_core_fault_snapshot(device);
else
kgsl_device_snapshot(device, NULL, NULL, false);
goto done;
}
context = drawobj->context;
do_fault_header(adreno_dev, drawobj, fault);
kgsl_device_snapshot(device, context, NULL, false);
force_retire_timestamp(device, drawobj);
if ((context->flags & KGSL_CONTEXT_INVALIDATE_ON_FAULT) ||
(context->flags & KGSL_CONTEXT_NO_FAULT_TOLERANCE) ||
(cmd->error == GMU_GPU_SW_HANG) ||
(cmd->error == GMU_GPU_SW_FUSE_VIOLATION) ||
context_is_throttled(device, context)) {
adreno_drawctxt_set_guilty(device, context);
}
/*
* Put back the reference which we incremented while trying to find
* faulted command object
*/
kgsl_drawobj_put(drawobj);
done:
memset(hwsched->ctxt_bad, 0x0, HFI_MAX_MSG_SIZE);
gpudev->reset(adreno_dev);
}
static void adreno_hwsched_reset_and_snapshot(struct adreno_device *adreno_dev, int fault)
{
struct kgsl_drawobj *drawobj = NULL;
struct kgsl_drawobj *drawobj_lpac = NULL;
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct kgsl_context *context = NULL;
struct kgsl_context *context_lpac = NULL;
struct cmd_list_obj *obj;
struct cmd_list_obj *obj_lpac;
const struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev);
struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
struct hfi_context_bad_cmd *cmd = hwsched->ctxt_bad;
if (device->state != KGSL_STATE_ACTIVE)
return;
if (hwsched->recurring_cmdobj)
srcu_notifier_call_chain(&device->nh, GPU_SSR_BEGIN, NULL);
if (cmd->error == GMU_SYNCOBJ_TIMEOUT_ERROR) {
print_fault_syncobj(adreno_dev, cmd->gc.ctxt_id, cmd->gc.ts);
gmu_core_fault_snapshot(device);
goto done;
}
/*
* First, try to see if the faulted command object is marked
* in case there was a context bad hfi. But, with stall-on-fault,
* we know that GMU cannot send context bad hfi. Hence, attempt
* to walk the list of active submissions to find the one that
* faulted.
*/
obj = get_fault_cmdobj(adreno_dev, cmd->gc.ctxt_id, cmd->gc.ts);
obj_lpac = get_fault_cmdobj(adreno_dev, cmd->lpac.ctxt_id, cmd->lpac.ts);
if (!obj && (fault & ADRENO_IOMMU_PAGE_FAULT))
obj = get_active_cmdobj(adreno_dev);
if (obj) {
drawobj = obj->drawobj;
CMDOBJ(drawobj)->fault_recovery = cmd->gc.policy;
} else if (hwsched->recurring_cmdobj &&
hwsched->recurring_cmdobj->base.context->id == cmd->gc.ctxt_id) {
drawobj = DRAWOBJ(hwsched->recurring_cmdobj);
CMDOBJ(drawobj)->fault_recovery = cmd->gc.policy;
if (!kref_get_unless_zero(&drawobj->refcount))
drawobj = NULL;
}
do_fault_header(adreno_dev, drawobj, fault);
if (!obj_lpac && (fault & ADRENO_IOMMU_PAGE_FAULT))
obj_lpac = get_active_cmdobj_lpac(adreno_dev);
if (!obj && !obj_lpac) {
if (fault & ADRENO_GMU_FAULT)
gmu_core_fault_snapshot(device);
else
kgsl_device_snapshot(device, NULL, NULL, false);
goto done;
}
if (obj)
context = drawobj->context;
if (obj_lpac) {
drawobj_lpac = obj_lpac->drawobj;
CMDOBJ(drawobj_lpac)->fault_recovery = cmd->lpac.policy;
context_lpac = drawobj_lpac->context;
if (gpudev->lpac_fault_header)
gpudev->lpac_fault_header(adreno_dev, drawobj_lpac);
}
kgsl_device_snapshot(device, context, context_lpac, false);
if (drawobj) {
force_retire_timestamp(device, drawobj);
if (context && ((context->flags & KGSL_CONTEXT_INVALIDATE_ON_FAULT) ||
(context->flags & KGSL_CONTEXT_NO_FAULT_TOLERANCE) ||
(cmd->error == GMU_GPU_SW_HANG) ||
(cmd->error == GMU_GPU_SW_FUSE_VIOLATION) ||
context_is_throttled(device, context)))
adreno_drawctxt_set_guilty(device, context);
/*
* Put back the reference which we incremented while trying to find
* faulted command object
*/
kgsl_drawobj_put(drawobj);
}
if (drawobj_lpac) {
force_retire_timestamp(device, drawobj_lpac);
if (context_lpac && ((context_lpac->flags & KGSL_CONTEXT_INVALIDATE_ON_FAULT) ||
(context_lpac->flags & KGSL_CONTEXT_NO_FAULT_TOLERANCE) ||
(cmd->error == GMU_GPU_SW_HANG) ||
(cmd->error == GMU_GPU_SW_FUSE_VIOLATION) ||
context_is_throttled(device, context_lpac)))
adreno_drawctxt_set_guilty(device, context_lpac);
/*
* Put back the reference which we incremented while trying to find
* faulted command object
*/
kgsl_drawobj_put(drawobj_lpac);
}
done:
memset(hwsched->ctxt_bad, 0x0, HFI_MAX_MSG_SIZE);
gpudev->reset(adreno_dev);
}
static bool adreno_hwsched_do_fault(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
int fault;
fault = atomic_xchg(&hwsched->fault, 0);
if (fault == 0)
return false;
mutex_lock(&device->mutex);
if (test_bit(ADRENO_HWSCHED_CTX_BAD_LEGACY, &hwsched->flags))
adreno_hwsched_reset_and_snapshot_legacy(adreno_dev, fault);
else
adreno_hwsched_reset_and_snapshot(adreno_dev, fault);
adreno_hwsched_trigger(adreno_dev);
mutex_unlock(&device->mutex);
return true;
}
static void adreno_hwsched_work(struct kthread_work *work)
{
struct adreno_hwsched *hwsched = container_of(work,
struct adreno_hwsched, work);
struct adreno_device *adreno_dev = container_of(hwsched,
struct adreno_device, hwsched);
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
mutex_lock(&hwsched->mutex);
if (adreno_hwsched_do_fault(adreno_dev)) {
mutex_unlock(&hwsched->mutex);
return;
}
/*
* As long as there are inflight commands, process retired comamnds from
* all drawqueues
*/
retire_drawobj_list(adreno_dev);
/* Signal fences */
kgsl_process_event_groups(device);
/* Run the scheduler for to dispatch new commands */
hwsched_issuecmds(adreno_dev);
if (hwsched->inflight == 0) {
hwsched_power_down(adreno_dev);
} else {
mutex_lock(&device->mutex);
kgsl_pwrscale_update(device);
kgsl_start_idle_timer(device);
mutex_unlock(&device->mutex);
}
mutex_unlock(&hwsched->mutex);
}
void adreno_hwsched_fault(struct adreno_device *adreno_dev,
u32 fault)
{
struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
u32 curr = atomic_read(&hwsched->fault);
atomic_set(&hwsched->fault, curr | fault);
/* make sure fault is written before triggering dispatcher */
smp_wmb();
adreno_hwsched_trigger(adreno_dev);
}
void adreno_hwsched_clear_fault(struct adreno_device *adreno_dev)
{
atomic_set(&adreno_dev->hwsched.fault, 0);
/* make sure other CPUs see the update */
smp_wmb();
}
static bool is_tx_slot_available(struct adreno_device *adreno_dev)
{
struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
void *ptr = hwsched->hw_fence.mem_descriptor.virtual_addr;
struct msm_hw_fence_hfi_queue_header *hdr = (struct msm_hw_fence_hfi_queue_header *)
(ptr + sizeof(struct msm_hw_fence_hfi_queue_table_header));
u32 queue_size_dwords = hdr->queue_size / sizeof(u32);
u32 payload_size_dwords = hdr->pkt_size / sizeof(u32);
u32 free_dwords, write_idx = hdr->write_index, read_idx = hdr->read_index;
u32 reserved_dwords = atomic_read(&hwsched->hw_fence_count) * payload_size_dwords;
free_dwords = read_idx <= write_idx ?
queue_size_dwords - (write_idx - read_idx) :
read_idx - write_idx;
if (free_dwords - reserved_dwords <= payload_size_dwords)
return false;
return true;
}
static void adreno_hwsched_create_hw_fence(struct adreno_device *adreno_dev,
struct kgsl_sync_fence *kfence)
{
struct kgsl_sync_timeline *ktimeline = kfence->parent;
struct kgsl_context *context = ktimeline->context;
const struct adreno_hwsched_ops *hwsched_ops =
adreno_dev->hwsched.hwsched_ops;
if (!test_bit(ADRENO_HWSCHED_HW_FENCE, &adreno_dev->hwsched.flags))
return;
/* Do not create a hardware backed fence, if this context is bad or going away */
if (kgsl_context_is_bad(context))
return;
if (!is_tx_slot_available(adreno_dev))
return;
hwsched_ops->create_hw_fence(adreno_dev, kfence);
}
static const struct adreno_dispatch_ops hwsched_ops = {
.close = adreno_hwsched_dispatcher_close,
.queue_cmds = adreno_hwsched_queue_cmds,
.queue_context = adreno_hwsched_queue_context,
.fault = adreno_hwsched_fault,
.create_hw_fence = adreno_hwsched_create_hw_fence,
.get_fault = adreno_hwsched_gpu_fault,
};
static void hwsched_lsr_check(struct work_struct *work)
{
struct adreno_hwsched *hwsched = container_of(work,
struct adreno_hwsched, lsr_check_ws);
struct adreno_device *adreno_dev = container_of(hwsched,
struct adreno_device, hwsched);
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
mutex_lock(&device->mutex);
kgsl_pwrscale_update_stats(device);
kgsl_pwrscale_update(device);
mutex_unlock(&device->mutex);
mod_timer(&hwsched->lsr_timer, jiffies + msecs_to_jiffies(10));
}
static void hwsched_lsr_timer(struct timer_list *t)
{
struct adreno_hwsched *hwsched = container_of(t, struct adreno_hwsched,
lsr_timer);
kgsl_schedule_work(&hwsched->lsr_check_ws);
}
int adreno_hwsched_init(struct adreno_device *adreno_dev,
const struct adreno_hwsched_ops *target_hwsched_ops)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
int i;
memset(hwsched, 0, sizeof(*hwsched));
hwsched->ctxt_bad = kzalloc(HFI_MAX_MSG_SIZE, GFP_KERNEL);
if (!hwsched->ctxt_bad)
return -ENOMEM;
hwsched->worker = kthread_create_worker(0, "kgsl_hwsched");
if (IS_ERR(hwsched->worker)) {
kfree(hwsched->ctxt_bad);
return PTR_ERR(hwsched->worker);
}
mutex_init(&hwsched->mutex);
kthread_init_work(&hwsched->work, adreno_hwsched_work);
jobs_cache = KMEM_CACHE(adreno_dispatch_job, 0);
obj_cache = KMEM_CACHE(cmd_list_obj, 0);
INIT_LIST_HEAD(&hwsched->cmd_list);
for (i = 0; i < ARRAY_SIZE(hwsched->jobs); i++) {
init_llist_head(&hwsched->jobs[i]);
init_llist_head(&hwsched->requeue[i]);
}
sched_set_fifo(hwsched->worker->task);
WARN_ON(sysfs_create_files(&device->dev->kobj, _hwsched_attr_list));
adreno_set_dispatch_ops(adreno_dev, &hwsched_ops);
hwsched->hwsched_ops = target_hwsched_ops;
init_completion(&hwsched->idle_gate);
complete_all(&hwsched->idle_gate);
if (ADRENO_FEATURE(adreno_dev, ADRENO_LSR)) {
INIT_WORK(&hwsched->lsr_check_ws, hwsched_lsr_check);
timer_setup(&hwsched->lsr_timer, hwsched_lsr_timer, 0);
}
return 0;
}
void adreno_hwsched_parse_fault_cmdobj(struct adreno_device *adreno_dev,
struct kgsl_snapshot *snapshot)
{
struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
struct cmd_list_obj *obj, *tmp;
/*
* During IB parse, vmalloc is called which can sleep and
* should not be called from atomic context. Since IBs are not
* dumped during atomic snapshot, there is no need to parse it.
*/
if (adreno_dev->dev.snapshot_atomic)
return;
list_for_each_entry_safe(obj, tmp, &hwsched->cmd_list, node) {
struct kgsl_drawobj *drawobj = obj->drawobj;
struct kgsl_drawobj_cmd *cmdobj;
if (!is_cmdobj(drawobj))
continue;
cmdobj = CMDOBJ(drawobj);
if (test_bit(CMDOBJ_FAULT, &cmdobj->priv)) {
struct kgsl_memobj_node *ib;
list_for_each_entry(ib, &cmdobj->cmdlist, node) {
if (drawobj->context->flags & KGSL_CONTEXT_LPAC)
adreno_parse_ib_lpac(KGSL_DEVICE(adreno_dev),
snapshot, snapshot->process_lpac,
ib->gpuaddr, ib->size >> 2);
else
adreno_parse_ib(KGSL_DEVICE(adreno_dev),
snapshot, snapshot->process,
ib->gpuaddr, ib->size >> 2);
}
clear_bit(CMDOBJ_FAULT, &cmdobj->priv);
}
}
}
static int unregister_context(int id, void *ptr, void *data)
{
struct kgsl_context *context = ptr;
struct adreno_context *drawctxt = ADRENO_CONTEXT(context);
if (drawctxt->gmu_context_queue.gmuaddr != 0) {
struct gmu_context_queue_header *header = drawctxt->gmu_context_queue.hostptr;
header->read_index = header->write_index;
/* This is to make sure GMU sees the correct indices after recovery */
mb();
}
/*
* We don't need to send the unregister hfi packet because
* we are anyway going to lose the gmu state of registered
* contexts. So just reset the flag so that the context
* registers with gmu on its first submission post slumber.
*/
context->gmu_registered = false;
/*
* Consider the scenario where non-recurring submissions were made
* by a context. Here internal_timestamp of context would be non
* zero. After slumber, last retired timestamp is not held by GMU.
* If this context submits a recurring workload, the context is
* registered again, but the internal timestamp is not updated. When
* the context is unregistered in send_context_unregister_hfi(),
* we could be waiting on old internal_timestamp which is not held by
* GMU. This can result in GMU errors. Hence set internal_timestamp
* to zero when entering slumber.
*/
drawctxt->internal_timestamp = 0;
return 0;
}
void adreno_hwsched_unregister_contexts(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
read_lock(&device->context_lock);
idr_for_each(&device->context_idr, unregister_context, NULL);
read_unlock(&device->context_lock);
if (hwsched->global_ctxtq.hostptr) {
struct gmu_context_queue_header *header = hwsched->global_ctxtq.hostptr;
header->read_index = header->write_index;
/* This is to make sure GMU sees the correct indices after recovery */
mb();
}
hwsched->global_ctxt_gmu_registered = false;
}
static int hwsched_idle(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
int ret;
/* Block any new submissions from being submitted */
adreno_get_gpu_halt(adreno_dev);
mutex_unlock(&device->mutex);
/*
* Flush the worker to make sure all executing
* or pending dispatcher works on worker are
* finished
*/
adreno_hwsched_flush(adreno_dev);
ret = wait_for_completion_timeout(&hwsched->idle_gate,
msecs_to_jiffies(ADRENO_IDLE_TIMEOUT));
if (ret == 0) {
ret = -ETIMEDOUT;
WARN(1, "hwsched halt timeout\n");
} else if (ret < 0) {
dev_err(device->dev, "hwsched halt failed %d\n", ret);
} else {
ret = 0;
}
mutex_lock(&device->mutex);
/*
* This will allow the dispatcher to start submitting to
* hardware once device mutex is released
*/
adreno_put_gpu_halt(adreno_dev);
/*
* Requeue dispatcher work to resubmit pending commands
* that may have been blocked due to this idling request
*/
adreno_hwsched_trigger(adreno_dev);
return ret;
}
int adreno_hwsched_idle(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
unsigned long wait = jiffies + msecs_to_jiffies(ADRENO_IDLE_TIMEOUT);
const struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev);
int ret;
if (WARN_ON(!mutex_is_locked(&device->mutex)))
return -EDEADLK;
if (!kgsl_state_is_awake(device))
return 0;
ret = hwsched_idle(adreno_dev);
if (ret)
return ret;
do {
if (adreno_hwsched_gpu_fault(adreno_dev))
return -EDEADLK;
if (gpudev->hw_isidle(adreno_dev))
return 0;
} while (time_before(jiffies, wait));
/*
* Under rare conditions, preemption can cause the while loop to exit
* without checking if the gpu is idle. check one last time before we
* return failure.
*/
if (adreno_hwsched_gpu_fault(adreno_dev))
return -EDEADLK;
if (gpudev->hw_isidle(adreno_dev))
return 0;
return -ETIMEDOUT;
}
void adreno_hwsched_register_hw_fence(struct adreno_device *adreno_dev)
{
struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
struct adreno_hw_fence *hw_fence = &hwsched->hw_fence;
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
int ret;
if (!ADRENO_FEATURE(adreno_dev, ADRENO_HW_FENCE))
return;
/* Enable hardware fences only if context queues are enabled */
if (!adreno_hwsched_context_queue_enabled(adreno_dev))
return;
if (test_bit(ADRENO_HWSCHED_HW_FENCE, &hwsched->flags))
return;
hw_fence->handle = msm_hw_fence_register(HW_FENCE_CLIENT_ID_CTX0,
&hw_fence->mem_descriptor);
if (IS_ERR_OR_NULL(hw_fence->handle)) {
dev_err(device->dev, "HW fences not supported: %d\n",
PTR_ERR_OR_ZERO(hw_fence->handle));
hw_fence->handle = NULL;
return;
}
/*
* We need to set up the memory descriptor with the physical address of the Tx/Rx Queues so
* that these buffers can be imported in to GMU VA space
*/
kgsl_memdesc_init(device, &hw_fence->memdesc, 0);
hw_fence->memdesc.physaddr = hw_fence->mem_descriptor.device_addr;
hw_fence->memdesc.size = hw_fence->mem_descriptor.size;
hw_fence->memdesc.hostptr = hw_fence->mem_descriptor.virtual_addr;
ret = kgsl_memdesc_sg_dma(&hw_fence->memdesc, hw_fence->memdesc.physaddr,
hw_fence->memdesc.size);
if (ret) {
dev_err(device->dev, "Failed to setup HW fences memdesc: %d\n",
ret);
msm_hw_fence_deregister(hw_fence->handle);
hw_fence->handle = NULL;
memset(&hw_fence->memdesc, 0x0, sizeof(hw_fence->memdesc));
return;
}
hwsched->hw_fence_cache = KMEM_CACHE(adreno_hw_fence_entry, 0);
set_bit(ADRENO_HWSCHED_HW_FENCE, &hwsched->flags);
}
int adreno_hwsched_wait_ack_completion(struct adreno_device *adreno_dev,
struct device *dev, struct pending_cmd *ack,
void (*process_msgq)(struct adreno_device *adreno_dev))
{
int rc;
/* Only allow a single log in a second */
static DEFINE_RATELIMIT_STATE(_rs, HZ, 1);
static u32 unprocessed, processed;
const struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev);
u64 start, end;
start = gpudev->read_alwayson(adreno_dev);
rc = wait_for_completion_timeout(&ack->complete,
msecs_to_jiffies(HFI_RSP_TIMEOUT));
/*
* A non-zero return value means the completion is complete, whereas zero indicates
* timeout
*/
if (rc) {
/*
* If an ack goes unprocessed, keep track of processed and unprocessed acks
* because we may not log each unprocessed ack due to ratelimiting
*/
if (unprocessed)
processed++;
return 0;
}
/*
* It is possible the ack came, but due to HLOS latencies in processing hfi interrupt
* and/or the f2h daemon, the ack isn't processed yet. Hence, process the msgq one last
* time.
*/
process_msgq(adreno_dev);
end = gpudev->read_alwayson(adreno_dev);
if (completion_done(&ack->complete)) {
unprocessed++;
if (__ratelimit(&_rs))
dev_err(dev, "Ack unprocessed for id:%d sequence=%d count=%d/%d ticks=%llu/%llu\n",
MSG_HDR_GET_ID(ack->sent_hdr), MSG_HDR_GET_SEQNUM(ack->sent_hdr),
unprocessed, processed, start, end);
return 0;
}
dev_err(dev, "Ack timeout for id:%d sequence=%d ticks=%llu/%llu\n",
MSG_HDR_GET_ID(ack->sent_hdr), MSG_HDR_GET_SEQNUM(ack->sent_hdr), start, end);
gmu_core_fault_snapshot(KGSL_DEVICE(adreno_dev));
return -ETIMEDOUT;
}
int adreno_hwsched_ctxt_unregister_wait_completion(
struct adreno_device *adreno_dev,
struct device *dev, struct pending_cmd *ack,
void (*process_msgq)(struct adreno_device *adreno_dev),
struct hfi_unregister_ctxt_cmd *cmd)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
const struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev);
int ret;
u64 start, end;
start = gpudev->read_alwayson(adreno_dev);
mutex_unlock(&device->mutex);
ret = wait_for_completion_timeout(&ack->complete,
msecs_to_jiffies(msecs_to_jiffies(30 * 1000)));
mutex_lock(&device->mutex);
if (ret)
return 0;
/*
* It is possible the ack came, but due to HLOS latencies in processing hfi interrupt
* and/or the f2h daemon, the ack isn't processed yet. Hence, process the msgq one last
* time.
*/
process_msgq(adreno_dev);
end = gpudev->read_alwayson(adreno_dev);
if (completion_done(&ack->complete)) {
dev_err_ratelimited(dev,
"Ack unprocessed for context unregister seq: %d ctx: %u ts: %u ticks=%llu/%llu\n",
MSG_HDR_GET_SEQNUM(ack->sent_hdr), cmd->ctxt_id,
cmd->ts, start, end);
return 0;
}
dev_err_ratelimited(dev,
"Ack timeout for context unregister seq: %d ctx: %u ts: %u ticks=%llu/%llu\n",
MSG_HDR_GET_SEQNUM(ack->sent_hdr), cmd->ctxt_id, cmd->ts, start, end);
return -ETIMEDOUT;
}
u32 adreno_hwsched_parse_payload(struct payload_section *payload, u32 key)
{
u32 i;
/* Each key-value pair is 2 dwords */
for (i = 0; i < payload->dwords; i += 2) {
if (payload->data[i] == key)
return payload->data[i + 1];
}
return 0;
}
static void adreno_hwsched_lookup_key_value(struct adreno_device *adreno_dev,
u32 type, u32 key, u32 *ptr, u32 num_values)
{
struct hfi_context_bad_cmd *cmd = adreno_dev->hwsched.ctxt_bad;
u32 i = 0, payload_bytes;
void *start;
if (!cmd->hdr)
return;
payload_bytes = (MSG_HDR_GET_SIZE(cmd->hdr) << 2) -
offsetof(struct hfi_context_bad_cmd, payload);
start = &cmd->payload[0];
while (i < payload_bytes) {
struct payload_section *payload = start + i;
/* key-value pair is 'num_values + 1' dwords */
if ((payload->type == type) && (payload->data[i] == key)) {
u32 j = 1;
do {
ptr[j - 1] = payload->data[i + j];
j++;
} while (num_values--);
break;
}
i += struct_size(payload, data, payload->dwords);
}
}
bool adreno_hwsched_log_nonfatal_gpu_fault(struct adreno_device *adreno_dev,
struct device *dev, u32 error)
{
bool non_fatal = true;
switch (error) {
case GMU_CP_AHB_ERROR: {
u32 err_details[2];
adreno_hwsched_lookup_key_value(adreno_dev, PAYLOAD_FAULT_REGS,
KEY_CP_AHB_ERROR, err_details, 2);
dev_crit_ratelimited(dev,
"CP: AHB bus error, CP_RL_ERROR_DETAILS_0:0x%x CP_RL_ERROR_DETAILS_1:0x%x\n",
err_details[0], err_details[1]);
break;
}
case GMU_ATB_ASYNC_FIFO_OVERFLOW:
dev_crit_ratelimited(dev, "RBBM: ATB ASYNC overflow\n");
break;
case GMU_RBBM_ATB_BUF_OVERFLOW:
dev_crit_ratelimited(dev, "RBBM: ATB bus overflow\n");
break;
case GMU_UCHE_OOB_ACCESS:
dev_crit_ratelimited(dev, "UCHE: Out of bounds access\n");
break;
case GMU_UCHE_TRAP_INTR:
dev_crit_ratelimited(dev, "UCHE: Trap interrupt\n");
break;
case GMU_TSB_WRITE_ERROR: {
u32 addr[2];
adreno_hwsched_lookup_key_value(adreno_dev, PAYLOAD_FAULT_REGS,
KEY_TSB_WRITE_ERROR, addr, 2);
dev_crit_ratelimited(dev, "TSB: Write error interrupt: Address: 0x%lx MID: %lu\n",
FIELD_GET(GENMASK(16, 0), addr[1]) << 32 | addr[0],
FIELD_GET(GENMASK(31, 23), addr[1]));
break;
}
default:
non_fatal = false;
break;
}
return non_fatal;
}
Reference in New Issue View Git Blame Copy Permalink