b2efa05265
cic is association between io_context and request_queue. A cic is linked from both ioc and q and should be destroyed when either one goes away. As ioc and q both have their own locks, locking becomes a bit complex - both orders work for removal from one but not from the other. Currently, cfq tries to circumvent this locking order issue with RCU. ioc->lock nests inside queue_lock but the radix tree and cic's are also protected by RCU allowing either side to walk their lists without grabbing lock. This rather unconventional use of RCU quickly devolves into extremely fragile convolution. e.g. The following is from cfqd going away too soon after ioc and q exits raced. general protection fault: 0000 [#1] PREEMPT SMP CPU 2 Modules linked in: [ 88.503444] Pid: 599, comm: hexdump Not tainted 3.1.0-rc10-work+ #158 Bochs Bochs RIP: 0010:[<ffffffff81397628>] [<ffffffff81397628>] cfq_exit_single_io_context+0x58/0xf0 ... Call Trace: [<ffffffff81395a4a>] call_for_each_cic+0x5a/0x90 [<ffffffff81395ab5>] cfq_exit_io_context+0x15/0x20 [<ffffffff81389130>] exit_io_context+0x100/0x140 [<ffffffff81098a29>] do_exit+0x579/0x850 [<ffffffff81098d5b>] do_group_exit+0x5b/0xd0 [<ffffffff81098de7>] sys_exit_group+0x17/0x20 [<ffffffff81b02f2b>] system_call_fastpath+0x16/0x1b The only real hot path here is cic lookup during request initialization and avoiding extra locking requires very confined use of RCU. This patch makes cic removal from both ioc and request_queue perform double-locking and unlink immediately. * From q side, the change is almost trivial as ioc->lock nests inside queue_lock. It just needs to grab each ioc->lock as it walks cic_list and unlink it. * From ioc side, it's a bit more difficult because of inversed lock order. ioc needs its lock to walk its cic_list but can't grab the matching queue_lock and needs to perform unlock-relock dancing. Unlinking is now wholly done from put_io_context() and fast path is optimized by using the queue_lock the caller already holds, which is by far the most common case. If the ioc accessed multiple devices, it tries with trylock. In unlikely cases of fast path failure, it falls back to full double-locking dance from workqueue. Double-locking isn't the prettiest thing in the world but it's *far* simpler and more understandable than RCU trick without adding any meaningful overhead. This still leaves a lot of now unnecessary RCU logics. Future patches will trim them. -v2: Vivek pointed out that cic->q was being dereferenced after cic->release() was called. Updated to use local variable @this_q instead. Signed-off-by: Tejun Heo <tj@kernel.org> Cc: Vivek Goyal <vgoyal@redhat.com> Signed-off-by: Jens Axboe <axboe@kernel.dk>
99 lines
2.1 KiB
C
99 lines
2.1 KiB
C
#ifndef IOCONTEXT_H
|
|
#define IOCONTEXT_H
|
|
|
|
#include <linux/radix-tree.h>
|
|
#include <linux/rcupdate.h>
|
|
#include <linux/workqueue.h>
|
|
|
|
struct cfq_queue;
|
|
struct cfq_ttime {
|
|
unsigned long last_end_request;
|
|
|
|
unsigned long ttime_total;
|
|
unsigned long ttime_samples;
|
|
unsigned long ttime_mean;
|
|
};
|
|
|
|
enum {
|
|
CIC_IOPRIO_CHANGED,
|
|
CIC_CGROUP_CHANGED,
|
|
};
|
|
|
|
struct cfq_io_context {
|
|
void *key;
|
|
struct request_queue *q;
|
|
|
|
struct cfq_queue *cfqq[2];
|
|
|
|
struct io_context *ioc;
|
|
|
|
struct cfq_ttime ttime;
|
|
|
|
struct list_head queue_list;
|
|
struct hlist_node cic_list;
|
|
|
|
unsigned long changed;
|
|
|
|
void (*exit)(struct cfq_io_context *);
|
|
void (*release)(struct cfq_io_context *);
|
|
|
|
struct rcu_head rcu_head;
|
|
};
|
|
|
|
/*
|
|
* I/O subsystem state of the associated processes. It is refcounted
|
|
* and kmalloc'ed. These could be shared between processes.
|
|
*/
|
|
struct io_context {
|
|
atomic_long_t refcount;
|
|
atomic_t nr_tasks;
|
|
|
|
/* all the fields below are protected by this lock */
|
|
spinlock_t lock;
|
|
|
|
unsigned short ioprio;
|
|
|
|
/*
|
|
* For request batching
|
|
*/
|
|
int nr_batch_requests; /* Number of requests left in the batch */
|
|
unsigned long last_waited; /* Time last woken after wait for request */
|
|
|
|
struct radix_tree_root radix_root;
|
|
struct hlist_head cic_list;
|
|
void __rcu *ioc_data;
|
|
|
|
struct work_struct release_work;
|
|
};
|
|
|
|
static inline struct io_context *ioc_task_link(struct io_context *ioc)
|
|
{
|
|
/*
|
|
* if ref count is zero, don't allow sharing (ioc is going away, it's
|
|
* a race).
|
|
*/
|
|
if (ioc && atomic_long_inc_not_zero(&ioc->refcount)) {
|
|
atomic_inc(&ioc->nr_tasks);
|
|
return ioc;
|
|
}
|
|
|
|
return NULL;
|
|
}
|
|
|
|
struct task_struct;
|
|
#ifdef CONFIG_BLOCK
|
|
void put_io_context(struct io_context *ioc, struct request_queue *locked_q);
|
|
void exit_io_context(struct task_struct *task);
|
|
struct io_context *get_task_io_context(struct task_struct *task,
|
|
gfp_t gfp_flags, int node);
|
|
void ioc_ioprio_changed(struct io_context *ioc, int ioprio);
|
|
void ioc_cgroup_changed(struct io_context *ioc);
|
|
#else
|
|
struct io_context;
|
|
static inline void put_io_context(struct io_context *ioc,
|
|
struct request_queue *locked_q) { }
|
|
static inline void exit_io_context(struct task_struct *task) { }
|
|
#endif
|
|
|
|
#endif
|