android/android_kernel_samsung_sm8650
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

io_uring: use kvmalloc for fixed files

Browse Source 
Instead of hand-coded two-level tables for registered files, allocate
them with kvmalloc(). In many cases small enough tables are enough, and
so can be kmalloc()'ed removing an extra memory load and a bunch of bit
logic instructions from the hot path. If the table is larger, we trade
off all the pros with a TLB-assisted memory lookup.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/280421d3b48775dabab773006bb5588c7b2dabc0.1628471125.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Pavel Begunkov
2021-08-09 13:04:01 +01:00
committed by Jens Axboe
parent 5fd4617840
commit 042b0d85ea
1 changed files with 10 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
fs/io_uring.c
View File

@ -92,13 +92,8 @@
#define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES) #define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES)
#define IORING_SQPOLL_CAP_ENTRIES_VALUE 8 #define IORING_SQPOLL_CAP_ENTRIES_VALUE 8
/* /* 512 entries per page on 64-bit archs, 64 pages max */
* Shift of 9 is 512 entries, or exactly one page on 64-bit archs #define IORING_MAX_FIXED_FILES (1U << 15)
*/
#define IORING_FILE_TABLE_SHIFT 9
#define IORING_MAX_FILES_TABLE (1U << IORING_FILE_TABLE_SHIFT)
#define IORING_FILE_TABLE_MASK (IORING_MAX_FILES_TABLE - 1)
#define IORING_MAX_FIXED_FILES (64 * IORING_MAX_FILES_TABLE)
#define IORING_MAX_RESTRICTIONS (IORING_RESTRICTION_LAST + \ #define IORING_MAX_RESTRICTIONS (IORING_RESTRICTION_LAST + \
IORING_REGISTER_LAST + IORING_OP_LAST) IORING_REGISTER_LAST + IORING_OP_LAST)
@ -235,8 +230,7 @@ struct io_rsrc_put {
}; };
struct io_file_table { struct io_file_table {
/* two level table */ struct io_fixed_file *files;
struct io_fixed_file **files;
}; };
struct io_rsrc_node { struct io_rsrc_node {
@ -6333,12 +6327,9 @@ static void io_wq_submit_work(struct io_wq_work *work)
#define FFS_MASK ~(FFS_ASYNC_READ|FFS_ASYNC_WRITE|FFS_ISREG) #define FFS_MASK ~(FFS_ASYNC_READ|FFS_ASYNC_WRITE|FFS_ISREG)
static inline struct io_fixed_file *io_fixed_file_slot(struct io_file_table *table, static inline struct io_fixed_file *io_fixed_file_slot(struct io_file_table *table,
unsigned i) unsigned i)
{ {
struct io_fixed_file *table_l2; return &table->files[i];
table_l2 = table->files[i >> IORING_FILE_TABLE_SHIFT];
return &table_l2[i & IORING_FILE_TABLE_MASK];
} }
static inline struct file *io_file_from_index(struct io_ring_ctx *ctx, static inline struct file *io_file_from_index(struct io_ring_ctx *ctx,
@ -7276,17 +7267,13 @@ fail:
static bool io_alloc_file_tables(struct io_file_table *table, unsigned nr_files) static bool io_alloc_file_tables(struct io_file_table *table, unsigned nr_files)
{ {
size_t size = nr_files * sizeof(struct io_fixed_file); table->files = kvcalloc(nr_files, sizeof(table->files[0]), GFP_KERNEL);
table->files = (struct io_fixed_file **)io_alloc_page_table(size);
return !!table->files; return !!table->files;
} }
static void io_free_file_tables(struct io_file_table *table, unsigned nr_files) static void io_free_file_tables(struct io_file_table *table)
{ {
size_t size = nr_files * sizeof(struct io_fixed_file); kvfree(table->files);
io_free_page_table((void **)table->files, size);
table->files = NULL; table->files = NULL;
} }
@ -7311,7 +7298,7 @@ static void __io_sqe_files_unregister(struct io_ring_ctx *ctx)
fput(file); fput(file);
} }
#endif #endif
io_free_file_tables(&ctx->file_table, ctx->nr_user_files); io_free_file_tables(&ctx->file_table);
io_rsrc_data_free(ctx->file_data); io_rsrc_data_free(ctx->file_data);
ctx->file_data = NULL; ctx->file_data = NULL;
ctx->nr_user_files = 0; ctx->nr_user_files = 0;
@ -7779,7 +7766,7 @@ out_fput:
if (file) if (file)
fput(file); fput(file);
} }
io_free_file_tables(&ctx->file_table, nr_args); io_free_file_tables(&ctx->file_table);
ctx->nr_user_files = 0; ctx->nr_user_files = 0;
out_free: out_free:
io_rsrc_data_free(ctx->file_data); io_rsrc_data_free(ctx->file_data);