android/android_kernel_samsung_sm8650
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

io_uring: modify io_get_cqe for CQE32

Browse Source 
Modify accesses to the CQE array to take large CQE's into account. The
index needs to be shifted by one for large CQE's.

Signed-off-by: Stefan Roesch <shr@fb.com>
Reviewed-by: Kanchan Joshi <joshi.k@samsung.com>
Link: https://lore.kernel.org/r/20220426182134.136504-7-shr@fb.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Stefan Roesch 2022-04-26 11:21:28 -07:00 committed by Jens Axboe
parent effcf8bdeb
commit 2fee6bc640
1 changed files with 17 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
fs/io_uring.c
View File

@ -2069,8 +2069,12 @@ static noinline struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx)
{ {
struct io_rings *rings = ctx->rings; struct io_rings *rings = ctx->rings;
unsigned int off = ctx->cached_cq_tail & (ctx->cq_entries - 1); unsigned int off = ctx->cached_cq_tail & (ctx->cq_entries - 1);
unsigned int shift = 0;
unsigned int free, queued, len; unsigned int free, queued, len;
if (ctx->flags & IORING_SETUP_CQE32)
shift = 1;
/* userspace may cheat modifying the tail, be safe and do min */ /* userspace may cheat modifying the tail, be safe and do min */
queued = min(__io_cqring_events(ctx), ctx->cq_entries); queued = min(__io_cqring_events(ctx), ctx->cq_entries);
free = ctx->cq_entries - queued; free = ctx->cq_entries - queued;
@ -2082,15 +2086,26 @@ static noinline struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx)
ctx->cached_cq_tail++; ctx->cached_cq_tail++;
ctx->cqe_cached = &rings->cqes[off]; ctx->cqe_cached = &rings->cqes[off];
ctx->cqe_sentinel = ctx->cqe_cached + len; ctx->cqe_sentinel = ctx->cqe_cached + len;
return ctx->cqe_cached++; ctx->cqe_cached++;
return &rings->cqes[off << shift];
} }
static inline struct io_uring_cqe *io_get_cqe(struct io_ring_ctx *ctx) static inline struct io_uring_cqe *io_get_cqe(struct io_ring_ctx *ctx)
{ {
if (likely(ctx->cqe_cached < ctx->cqe_sentinel)) { if (likely(ctx->cqe_cached < ctx->cqe_sentinel)) {
struct io_uring_cqe *cqe = ctx->cqe_cached;
if (ctx->flags & IORING_SETUP_CQE32) {
unsigned int off = ctx->cqe_cached - ctx->rings->cqes;
cqe += off;
}
ctx->cached_cq_tail++; ctx->cached_cq_tail++;
return ctx->cqe_cached++; ctx->cqe_cached++;
return cqe;
} }
return __io_get_cqe(ctx); return __io_get_cqe(ctx);
} }