From d732580b4eb31553c63744a47d590f770cafb8f0 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 5 Mar 2012 13:14:58 -0800 Subject: block: implement blk_queue_bypass_start/end() Rename and extend elv_queisce_start/end() to blk_queue_bypass_start/end() which are exported and supports nesting via @q->bypass_depth. Also add blk_queue_bypass() to test bypass state. This will be further extended and used for blkio_group management. Signed-off-by: Tejun Heo Cc: Vivek Goyal Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'include/linux/blkdev.h') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 606cf339bb56..315db1d91bc4 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -389,6 +389,8 @@ struct request_queue { struct mutex sysfs_lock; + int bypass_depth; + #if defined(CONFIG_BLK_DEV_BSG) bsg_job_fn *bsg_job_fn; int bsg_job_size; @@ -406,7 +408,7 @@ struct request_queue { #define QUEUE_FLAG_SYNCFULL 3 /* read queue has been filled */ #define QUEUE_FLAG_ASYNCFULL 4 /* write queue has been filled */ #define QUEUE_FLAG_DEAD 5 /* queue being torn down */ -#define QUEUE_FLAG_ELVSWITCH 6 /* don't use elevator, just do FIFO */ +#define QUEUE_FLAG_BYPASS 6 /* act as dumb FIFO queue */ #define QUEUE_FLAG_BIDI 7 /* queue supports bidi requests */ #define QUEUE_FLAG_NOMERGES 8 /* disable merge attempts */ #define QUEUE_FLAG_SAME_COMP 9 /* complete on same CPU-group */ @@ -494,6 +496,7 @@ static inline void queue_flag_clear(unsigned int flag, struct request_queue *q) #define blk_queue_tagged(q) test_bit(QUEUE_FLAG_QUEUED, &(q)->queue_flags) #define blk_queue_stopped(q) test_bit(QUEUE_FLAG_STOPPED, &(q)->queue_flags) #define blk_queue_dead(q) test_bit(QUEUE_FLAG_DEAD, &(q)->queue_flags) +#define blk_queue_bypass(q) test_bit(QUEUE_FLAG_BYPASS, &(q)->queue_flags) #define blk_queue_nomerges(q) test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags) #define blk_queue_noxmerges(q) \ test_bit(QUEUE_FLAG_NOXMERGES, &(q)->queue_flags) -- cgit v1.2.3-59-g8ed1b From 923adde1be1df57cebd80c563058e503376645e8 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 5 Mar 2012 13:15:13 -0800 Subject: blkcg: clear all request_queues on blkcg policy [un]registrations Keep track of all request_queues which have blkcg initialized and turn on bypass and invoke blkcg_clear_queue() on all before making changes to blkcg policies. This is to prepare for moving blkg management into blkcg core. Note that this uses more brute force than necessary. Finer grained shoot down will be implemented later and given that policy [un]registration almost never happens on running systems (blk-throtl can't be built as a module and cfq usually is the builtin default iosched), this shouldn't be a problem for the time being. Signed-off-by: Tejun Heo Cc: Vivek Goyal Signed-off-by: Jens Axboe --- block/blk-cgroup.c | 48 +++++++++++++++++++++++++++++++++++++++++++++++- include/linux/blkdev.h | 3 +++ 2 files changed, 50 insertions(+), 1 deletion(-) (limited to 'include/linux/blkdev.h') diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index b302ce1d662b..266c0707d588 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -27,6 +27,9 @@ static DEFINE_SPINLOCK(blkio_list_lock); static LIST_HEAD(blkio_list); +static DEFINE_MUTEX(all_q_mutex); +static LIST_HEAD(all_q_list); + struct blkio_cgroup blkio_root_cgroup = { .weight = 2*BLKIO_WEIGHT_DEFAULT }; EXPORT_SYMBOL_GPL(blkio_root_cgroup); @@ -1472,9 +1475,20 @@ done: */ int blkcg_init_queue(struct request_queue *q) { + int ret; + might_sleep(); - return blk_throtl_init(q); + ret = blk_throtl_init(q); + if (ret) + return ret; + + mutex_lock(&all_q_mutex); + INIT_LIST_HEAD(&q->all_q_node); + list_add_tail(&q->all_q_node, &all_q_list); + mutex_unlock(&all_q_mutex); + + return 0; } /** @@ -1498,6 +1512,10 @@ void blkcg_drain_queue(struct request_queue *q) */ void blkcg_exit_queue(struct request_queue *q) { + mutex_lock(&all_q_mutex); + list_del_init(&q->all_q_node); + mutex_unlock(&all_q_mutex); + blk_throtl_exit(q); } @@ -1543,8 +1561,33 @@ static void blkiocg_attach(struct cgroup_subsys *ss, struct cgroup *cgrp, } } +static void blkcg_bypass_start(void) + __acquires(&all_q_mutex) +{ + struct request_queue *q; + + mutex_lock(&all_q_mutex); + + list_for_each_entry(q, &all_q_list, all_q_node) { + blk_queue_bypass_start(q); + blkg_destroy_all(q); + } +} + +static void blkcg_bypass_end(void) + __releases(&all_q_mutex) +{ + struct request_queue *q; + + list_for_each_entry(q, &all_q_list, all_q_node) + blk_queue_bypass_end(q); + + mutex_unlock(&all_q_mutex); +} + void blkio_policy_register(struct blkio_policy_type *blkiop) { + blkcg_bypass_start(); spin_lock(&blkio_list_lock); BUG_ON(blkio_policy[blkiop->plid]); @@ -1552,11 +1595,13 @@ void blkio_policy_register(struct blkio_policy_type *blkiop) list_add_tail(&blkiop->list, &blkio_list); spin_unlock(&blkio_list_lock); + blkcg_bypass_end(); } EXPORT_SYMBOL_GPL(blkio_policy_register); void blkio_policy_unregister(struct blkio_policy_type *blkiop) { + blkcg_bypass_start(); spin_lock(&blkio_list_lock); BUG_ON(blkio_policy[blkiop->plid] != blkiop); @@ -1564,5 +1609,6 @@ void blkio_policy_unregister(struct blkio_policy_type *blkiop) list_del_init(&blkiop->list); spin_unlock(&blkio_list_lock); + blkcg_bypass_end(); } EXPORT_SYMBOL_GPL(blkio_policy_unregister); diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 315db1d91bc4..e8c0bbd06b9a 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -397,6 +397,9 @@ struct request_queue { struct bsg_class_device bsg_dev; #endif +#ifdef CONFIG_BLK_CGROUP + struct list_head all_q_node; +#endif #ifdef CONFIG_BLK_DEV_THROTTLING /* Throttle data */ struct throtl_data *td; -- cgit v1.2.3-59-g8ed1b From 4eef3049986e8397d5003916aed8cad6567a5e02 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 5 Mar 2012 13:15:18 -0800 Subject: blkcg: move per-queue blkg list heads and counters to queue and blkg Currently, specific policy implementations are responsible for maintaining list and number of blkgs. This duplicates code unnecessarily, and hinders factoring common code and providing blkcg API with better defined semantics. After this patch, request_queue hosts list heads and counters and blkg has list nodes for both policies. This patch only relocates the necessary fields and the next patch will actually move management code into blkcg core. Note that request_queue->blkg_list[] and ->nr_blkgs[] are hardcoded to have 2 elements. This is to avoid include dependency and will be removed by the next patch. This patch doesn't introduce any behavior change. -v2: Now unnecessary conditional on CONFIG_BLK_CGROUP_MODULE removed as pointed out by Vivek. Signed-off-by: Tejun Heo Cc: Vivek Goyal Signed-off-by: Jens Axboe --- block/blk-cgroup.c | 2 ++ block/blk-cgroup.h | 1 + block/blk-core.c | 4 ++++ block/blk-throttle.c | 49 +++++++++++++++++++++++-------------------------- block/cfq-iosched.c | 47 +++++++++++++++++++---------------------------- include/linux/blkdev.h | 5 +++++ 6 files changed, 54 insertions(+), 54 deletions(-) (limited to 'include/linux/blkdev.h') diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index 91f9824be5cc..e940972ccd66 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -499,6 +499,8 @@ static struct blkio_group *blkg_alloc(struct blkio_cgroup *blkcg, spin_lock_init(&blkg->stats_lock); rcu_assign_pointer(blkg->q, q); + INIT_LIST_HEAD(&blkg->q_node[0]); + INIT_LIST_HEAD(&blkg->q_node[1]); blkg->blkcg = blkcg; blkg->plid = pol->plid; blkg->refcnt = 1; diff --git a/block/blk-cgroup.h b/block/blk-cgroup.h index 60e96b4be4ce..ae96f196d469 100644 --- a/block/blk-cgroup.h +++ b/block/blk-cgroup.h @@ -178,6 +178,7 @@ struct blkg_policy_data { struct blkio_group { /* Pointer to the associated request_queue, RCU protected */ struct request_queue __rcu *q; + struct list_head q_node[BLKIO_NR_POLICIES]; struct hlist_node blkcg_node; struct blkio_cgroup *blkcg; /* Store cgroup path */ diff --git a/block/blk-core.c b/block/blk-core.c index c3434c6395b9..83a47fcf5946 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -547,6 +547,10 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id) INIT_LIST_HEAD(&q->queue_head); INIT_LIST_HEAD(&q->timeout_list); INIT_LIST_HEAD(&q->icq_list); +#ifdef CONFIG_BLK_CGROUP + INIT_LIST_HEAD(&q->blkg_list[0]); + INIT_LIST_HEAD(&q->blkg_list[1]); +#endif INIT_LIST_HEAD(&q->flush_queue[0]); INIT_LIST_HEAD(&q->flush_queue[1]); INIT_LIST_HEAD(&q->flush_data_in_flight); diff --git a/block/blk-throttle.c b/block/blk-throttle.c index b2fddaf20b98..c15d38307e1d 100644 --- a/block/blk-throttle.c +++ b/block/blk-throttle.c @@ -41,9 +41,6 @@ struct throtl_rb_root { #define rb_entry_tg(node) rb_entry((node), struct throtl_grp, rb_node) struct throtl_grp { - /* List of throtl groups on the request queue*/ - struct hlist_node tg_node; - /* active throtl group service_tree member */ struct rb_node rb_node; @@ -83,9 +80,6 @@ struct throtl_grp { struct throtl_data { - /* List of throtl groups */ - struct hlist_head tg_list; - /* service tree for active throtl groups */ struct throtl_rb_root tg_service_tree; @@ -152,7 +146,6 @@ static void throtl_init_blkio_group(struct blkio_group *blkg) { struct throtl_grp *tg = blkg_to_tg(blkg); - INIT_HLIST_NODE(&tg->tg_node); RB_CLEAR_NODE(&tg->rb_node); bio_list_init(&tg->bio_lists[0]); bio_list_init(&tg->bio_lists[1]); @@ -167,11 +160,9 @@ static void throtl_init_blkio_group(struct blkio_group *blkg) static void throtl_link_blkio_group(struct request_queue *q, struct blkio_group *blkg) { - struct throtl_data *td = q->td; - struct throtl_grp *tg = blkg_to_tg(blkg); - - hlist_add_head(&tg->tg_node, &td->tg_list); - td->nr_undestroyed_grps++; + list_add(&blkg->q_node[BLKIO_POLICY_THROTL], + &q->blkg_list[BLKIO_POLICY_THROTL]); + q->nr_blkgs[BLKIO_POLICY_THROTL]++; } static struct @@ -711,8 +702,8 @@ static int throtl_select_dispatch(struct throtl_data *td, struct bio_list *bl) static void throtl_process_limit_change(struct throtl_data *td) { - struct throtl_grp *tg; - struct hlist_node *pos, *n; + struct request_queue *q = td->queue; + struct blkio_group *blkg, *n; if (!td->limits_changed) return; @@ -721,7 +712,10 @@ static void throtl_process_limit_change(struct throtl_data *td) throtl_log(td, "limits changed"); - hlist_for_each_entry_safe(tg, pos, n, &td->tg_list, tg_node) { + list_for_each_entry_safe(blkg, n, &q->blkg_list[BLKIO_POLICY_THROTL], + q_node[BLKIO_POLICY_THROTL]) { + struct throtl_grp *tg = blkg_to_tg(blkg); + if (!tg->limits_changed) continue; @@ -822,26 +816,31 @@ throtl_schedule_delayed_work(struct throtl_data *td, unsigned long delay) static void throtl_destroy_tg(struct throtl_data *td, struct throtl_grp *tg) { + struct blkio_group *blkg = tg_to_blkg(tg); + /* Something wrong if we are trying to remove same group twice */ - BUG_ON(hlist_unhashed(&tg->tg_node)); + WARN_ON_ONCE(list_empty(&blkg->q_node[BLKIO_POLICY_THROTL])); - hlist_del_init(&tg->tg_node); + list_del_init(&blkg->q_node[BLKIO_POLICY_THROTL]); /* * Put the reference taken at the time of creation so that when all * queues are gone, group can be destroyed. */ blkg_put(tg_to_blkg(tg)); - td->nr_undestroyed_grps--; + td->queue->nr_blkgs[BLKIO_POLICY_THROTL]--; } static bool throtl_release_tgs(struct throtl_data *td, bool release_root) { - struct hlist_node *pos, *n; - struct throtl_grp *tg; + struct request_queue *q = td->queue; + struct blkio_group *blkg, *n; bool empty = true; - hlist_for_each_entry_safe(tg, pos, n, &td->tg_list, tg_node) { + list_for_each_entry_safe(blkg, n, &q->blkg_list[BLKIO_POLICY_THROTL], + q_node[BLKIO_POLICY_THROTL]) { + struct throtl_grp *tg = blkg_to_tg(blkg); + /* skip root? */ if (!release_root && tg == td->root_tg) continue; @@ -851,7 +850,7 @@ static bool throtl_release_tgs(struct throtl_data *td, bool release_root) * it from cgroup list, then it will take care of destroying * cfqg also. */ - if (!blkiocg_del_blkio_group(tg_to_blkg(tg))) + if (!blkiocg_del_blkio_group(blkg)) throtl_destroy_tg(td, tg); else empty = false; @@ -1114,7 +1113,6 @@ int blk_throtl_init(struct request_queue *q) if (!td) return -ENOMEM; - INIT_HLIST_HEAD(&td->tg_list); td->tg_service_tree = THROTL_RB_ROOT; td->limits_changed = false; INIT_DELAYED_WORK(&td->throtl_work, blk_throtl_work); @@ -1144,7 +1142,7 @@ int blk_throtl_init(struct request_queue *q) void blk_throtl_exit(struct request_queue *q) { struct throtl_data *td = q->td; - bool wait = false; + bool wait; BUG_ON(!td); @@ -1154,8 +1152,7 @@ void blk_throtl_exit(struct request_queue *q) throtl_release_tgs(td, true); /* If there are other groups */ - if (td->nr_undestroyed_grps > 0) - wait = true; + wait = q->nr_blkgs[BLKIO_POLICY_THROTL]; spin_unlock_irq(q->queue_lock); diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c index 11dd9d7f2edb..e846803280a6 100644 --- a/block/cfq-iosched.c +++ b/block/cfq-iosched.c @@ -208,9 +208,7 @@ struct cfq_group { unsigned long saved_workload_slice; enum wl_type_t saved_workload; enum wl_prio_t saved_serving_prio; -#ifdef CONFIG_CFQ_GROUP_IOSCHED - struct hlist_node cfqd_node; -#endif + /* number of requests that are on the dispatch list or inside driver */ int dispatched; struct cfq_ttime ttime; @@ -302,12 +300,6 @@ struct cfq_data { struct cfq_queue oom_cfqq; unsigned long last_delayed_sync; - - /* List of cfq groups being managed on this device*/ - struct hlist_head cfqg_list; - - /* Number of groups which are on blkcg->blkg_list */ - unsigned int nr_blkcg_linked_grps; }; static inline struct cfq_group *blkg_to_cfqg(struct blkio_group *blkg) @@ -1056,13 +1048,9 @@ static void cfq_update_blkio_group_weight(struct request_queue *q, static void cfq_link_blkio_group(struct request_queue *q, struct blkio_group *blkg) { - struct cfq_data *cfqd = q->elevator->elevator_data; - struct cfq_group *cfqg = blkg_to_cfqg(blkg); - - cfqd->nr_blkcg_linked_grps++; - - /* Add group on cfqd list */ - hlist_add_head(&cfqg->cfqd_node, &cfqd->cfqg_list); + list_add(&blkg->q_node[BLKIO_POLICY_PROP], + &q->blkg_list[BLKIO_POLICY_PROP]); + q->nr_blkgs[BLKIO_POLICY_PROP]++; } static void cfq_init_blkio_group(struct blkio_group *blkg) @@ -1110,13 +1098,15 @@ static void cfq_link_cfqq_cfqg(struct cfq_queue *cfqq, struct cfq_group *cfqg) static void cfq_destroy_cfqg(struct cfq_data *cfqd, struct cfq_group *cfqg) { + struct blkio_group *blkg = cfqg_to_blkg(cfqg); + /* Something wrong if we are trying to remove same group twice */ - BUG_ON(hlist_unhashed(&cfqg->cfqd_node)); + BUG_ON(list_empty(&blkg->q_node[BLKIO_POLICY_PROP])); - hlist_del_init(&cfqg->cfqd_node); + list_del_init(&blkg->q_node[BLKIO_POLICY_PROP]); - BUG_ON(cfqd->nr_blkcg_linked_grps <= 0); - cfqd->nr_blkcg_linked_grps--; + BUG_ON(cfqd->queue->nr_blkgs[BLKIO_POLICY_PROP] <= 0); + cfqd->queue->nr_blkgs[BLKIO_POLICY_PROP]--; /* * Put the reference taken at the time of creation so that when all @@ -1127,18 +1117,19 @@ static void cfq_destroy_cfqg(struct cfq_data *cfqd, struct cfq_group *cfqg) static bool cfq_release_cfq_groups(struct cfq_data *cfqd) { - struct hlist_node *pos, *n; - struct cfq_group *cfqg; + struct request_queue *q = cfqd->queue; + struct blkio_group *blkg, *n; bool empty = true; - hlist_for_each_entry_safe(cfqg, pos, n, &cfqd->cfqg_list, cfqd_node) { + list_for_each_entry_safe(blkg, n, &q->blkg_list[BLKIO_POLICY_PROP], + q_node[BLKIO_POLICY_PROP]) { /* * If cgroup removal path got to blk_group first and removed * it from cgroup list, then it will take care of destroying * cfqg also. */ - if (!cfq_blkiocg_del_blkio_group(cfqg_to_blkg(cfqg))) - cfq_destroy_cfqg(cfqd, cfqg); + if (!cfq_blkiocg_del_blkio_group(blkg)) + cfq_destroy_cfqg(cfqd, blkg_to_cfqg(blkg)); else empty = false; } @@ -3558,13 +3549,13 @@ static void cfq_exit_queue(struct elevator_queue *e) cfq_put_async_queues(cfqd); cfq_release_cfq_groups(cfqd); +#ifdef CONFIG_BLK_CGROUP /* * If there are groups which we could not unlink from blkcg list, * wait for a rcu period for them to be freed. */ - if (cfqd->nr_blkcg_linked_grps) - wait = true; - + wait = q->nr_blkgs[BLKIO_POLICY_PROP]; +#endif spin_unlock_irq(q->queue_lock); cfq_shutdown_timer_wq(cfqd); diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index e8c0bbd06b9a..f4e35edea70f 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -362,6 +362,11 @@ struct request_queue { struct list_head timeout_list; struct list_head icq_list; +#ifdef CONFIG_BLK_CGROUP + /* XXX: array size hardcoded to avoid include dependency (temporary) */ + struct list_head blkg_list[2]; + int nr_blkgs[2]; +#endif struct queue_limits limits; -- cgit v1.2.3-59-g8ed1b From 03aa264ac15637b6f98374270bcdf31400965505 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 5 Mar 2012 13:15:19 -0800 Subject: blkcg: let blkcg core manage per-queue blkg list and counter With the previous patch to move blkg list heads and counters to request_queue and blkg, logic to manage them in both policies are almost identical and can be moved to blkcg core. This patch moves blkg link logic into blkg_lookup_create(), implements common blkg unlink code in blkg_destroy(), and updates blkg_destory_all() so that it's policy specific and can skip root group. The updated blkg_destroy_all() is now used to both clear queue for bypassing and elv switching, and release all blkgs on q exit. This patch introduces a race window where policy [de]registration may race against queue blkg clearing. This can only be a problem on cfq unload and shouldn't be a real problem in practice (and we have many other places where this race already exists). Future patches will remove these unlikely races. Signed-off-by: Tejun Heo Cc: Vivek Goyal Signed-off-by: Jens Axboe --- block/blk-cgroup.c | 72 +++++++++++++++++++++++++++-------- block/blk-cgroup.h | 15 +++----- block/blk-throttle.c | 99 +----------------------------------------------- block/cfq-iosched.c | 100 +++---------------------------------------------- block/elevator.c | 5 ++- include/linux/blkdev.h | 4 +- 6 files changed, 74 insertions(+), 221 deletions(-) (limited to 'include/linux/blkdev.h') diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index e940972ccd66..2ca9a15db0f7 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -596,8 +596,11 @@ struct blkio_group *blkg_lookup_create(struct blkio_cgroup *blkcg, /* insert */ spin_lock(&blkcg->lock); swap(blkg, new_blkg); + hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list); - pol->ops.blkio_link_group_fn(q, blkg); + list_add(&blkg->q_node[plid], &q->blkg_list[plid]); + q->nr_blkgs[plid]++; + spin_unlock(&blkcg->lock); out: blkg_free(new_blkg); @@ -646,36 +649,69 @@ struct blkio_group *blkg_lookup(struct blkio_cgroup *blkcg, } EXPORT_SYMBOL_GPL(blkg_lookup); -void blkg_destroy_all(struct request_queue *q) +static void blkg_destroy(struct blkio_group *blkg, enum blkio_policy_id plid) +{ + struct request_queue *q = blkg->q; + + lockdep_assert_held(q->queue_lock); + + /* Something wrong if we are trying to remove same group twice */ + WARN_ON_ONCE(list_empty(&blkg->q_node[plid])); + list_del_init(&blkg->q_node[plid]); + + WARN_ON_ONCE(q->nr_blkgs[plid] <= 0); + q->nr_blkgs[plid]--; + + /* + * Put the reference taken at the time of creation so that when all + * queues are gone, group can be destroyed. + */ + blkg_put(blkg); +} + +void blkg_destroy_all(struct request_queue *q, enum blkio_policy_id plid, + bool destroy_root) { - struct blkio_policy_type *pol; + struct blkio_group *blkg, *n; while (true) { bool done = true; - spin_lock(&blkio_list_lock); spin_lock_irq(q->queue_lock); - /* - * clear_queue_fn() might return with non-empty group list - * if it raced cgroup removal and lost. cgroup removal is - * guaranteed to make forward progress and retrying after a - * while is enough. This ugliness is scheduled to be - * removed after locking update. - */ - list_for_each_entry(pol, &blkio_list, list) - if (!pol->ops.blkio_clear_queue_fn(q)) + list_for_each_entry_safe(blkg, n, &q->blkg_list[plid], + q_node[plid]) { + /* skip root? */ + if (!destroy_root && blkg->blkcg == &blkio_root_cgroup) + continue; + + /* + * If cgroup removal path got to blk_group first + * and removed it from cgroup list, then it will + * take care of destroying cfqg also. + */ + if (!blkiocg_del_blkio_group(blkg)) + blkg_destroy(blkg, plid); + else done = false; + } spin_unlock_irq(q->queue_lock); - spin_unlock(&blkio_list_lock); + /* + * Group list may not be empty if we raced cgroup removal + * and lost. cgroup removal is guaranteed to make forward + * progress and retrying after a while is enough. This + * ugliness is scheduled to be removed after locking + * update. + */ if (done) break; msleep(10); /* just some random duration I like */ } } +EXPORT_SYMBOL_GPL(blkg_destroy_all); static void blkg_rcu_free(struct rcu_head *rcu_head) { @@ -1549,11 +1585,13 @@ static int blkiocg_pre_destroy(struct cgroup_subsys *subsys, * this event. */ spin_lock(&blkio_list_lock); + spin_lock_irqsave(q->queue_lock, flags); list_for_each_entry(blkiop, &blkio_list, list) { if (blkiop->plid != blkg->plid) continue; - blkiop->ops.blkio_unlink_group_fn(q, blkg); + blkg_destroy(blkg, blkiop->plid); } + spin_unlock_irqrestore(q->queue_lock, flags); spin_unlock(&blkio_list_lock); } while (1); @@ -1695,12 +1733,14 @@ static void blkcg_bypass_start(void) __acquires(&all_q_mutex) { struct request_queue *q; + int i; mutex_lock(&all_q_mutex); list_for_each_entry(q, &all_q_list, all_q_node) { blk_queue_bypass_start(q); - blkg_destroy_all(q); + for (i = 0; i < BLKIO_NR_POLICIES; i++) + blkg_destroy_all(q, i, false); } } diff --git a/block/blk-cgroup.h b/block/blk-cgroup.h index ae96f196d469..83ce5fa0a604 100644 --- a/block/blk-cgroup.h +++ b/block/blk-cgroup.h @@ -196,11 +196,6 @@ struct blkio_group { }; typedef void (blkio_init_group_fn)(struct blkio_group *blkg); -typedef void (blkio_link_group_fn)(struct request_queue *q, - struct blkio_group *blkg); -typedef void (blkio_unlink_group_fn)(struct request_queue *q, - struct blkio_group *blkg); -typedef bool (blkio_clear_queue_fn)(struct request_queue *q); typedef void (blkio_update_group_weight_fn)(struct request_queue *q, struct blkio_group *blkg, unsigned int weight); typedef void (blkio_update_group_read_bps_fn)(struct request_queue *q, @@ -214,9 +209,6 @@ typedef void (blkio_update_group_write_iops_fn)(struct request_queue *q, struct blkio_policy_ops { blkio_init_group_fn *blkio_init_group_fn; - blkio_link_group_fn *blkio_link_group_fn; - blkio_unlink_group_fn *blkio_unlink_group_fn; - blkio_clear_queue_fn *blkio_clear_queue_fn; blkio_update_group_weight_fn *blkio_update_group_weight_fn; blkio_update_group_read_bps_fn *blkio_update_group_read_bps_fn; blkio_update_group_write_bps_fn *blkio_update_group_write_bps_fn; @@ -238,7 +230,8 @@ extern void blkcg_exit_queue(struct request_queue *q); /* Blkio controller policy registration */ extern void blkio_policy_register(struct blkio_policy_type *); extern void blkio_policy_unregister(struct blkio_policy_type *); -extern void blkg_destroy_all(struct request_queue *q); +extern void blkg_destroy_all(struct request_queue *q, + enum blkio_policy_id plid, bool destroy_root); /** * blkg_to_pdata - get policy private data @@ -319,7 +312,9 @@ static inline void blkcg_drain_queue(struct request_queue *q) { } static inline void blkcg_exit_queue(struct request_queue *q) { } static inline void blkio_policy_register(struct blkio_policy_type *blkiop) { } static inline void blkio_policy_unregister(struct blkio_policy_type *blkiop) { } -static inline void blkg_destroy_all(struct request_queue *q) { } +static inline void blkg_destroy_all(struct request_queue *q, + enum blkio_policy_id plid, + bool destory_root) { } static inline void *blkg_to_pdata(struct blkio_group *blkg, struct blkio_policy_type *pol) { return NULL; } diff --git a/block/blk-throttle.c b/block/blk-throttle.c index c15d38307e1d..132941260e58 100644 --- a/block/blk-throttle.c +++ b/block/blk-throttle.c @@ -157,14 +157,6 @@ static void throtl_init_blkio_group(struct blkio_group *blkg) tg->iops[WRITE] = -1; } -static void throtl_link_blkio_group(struct request_queue *q, - struct blkio_group *blkg) -{ - list_add(&blkg->q_node[BLKIO_POLICY_THROTL], - &q->blkg_list[BLKIO_POLICY_THROTL]); - q->nr_blkgs[BLKIO_POLICY_THROTL]++; -} - static struct throtl_grp *throtl_lookup_tg(struct throtl_data *td, struct blkio_cgroup *blkcg) { @@ -813,89 +805,6 @@ throtl_schedule_delayed_work(struct throtl_data *td, unsigned long delay) } } -static void -throtl_destroy_tg(struct throtl_data *td, struct throtl_grp *tg) -{ - struct blkio_group *blkg = tg_to_blkg(tg); - - /* Something wrong if we are trying to remove same group twice */ - WARN_ON_ONCE(list_empty(&blkg->q_node[BLKIO_POLICY_THROTL])); - - list_del_init(&blkg->q_node[BLKIO_POLICY_THROTL]); - - /* - * Put the reference taken at the time of creation so that when all - * queues are gone, group can be destroyed. - */ - blkg_put(tg_to_blkg(tg)); - td->queue->nr_blkgs[BLKIO_POLICY_THROTL]--; -} - -static bool throtl_release_tgs(struct throtl_data *td, bool release_root) -{ - struct request_queue *q = td->queue; - struct blkio_group *blkg, *n; - bool empty = true; - - list_for_each_entry_safe(blkg, n, &q->blkg_list[BLKIO_POLICY_THROTL], - q_node[BLKIO_POLICY_THROTL]) { - struct throtl_grp *tg = blkg_to_tg(blkg); - - /* skip root? */ - if (!release_root && tg == td->root_tg) - continue; - - /* - * If cgroup removal path got to blk_group first and removed - * it from cgroup list, then it will take care of destroying - * cfqg also. - */ - if (!blkiocg_del_blkio_group(blkg)) - throtl_destroy_tg(td, tg); - else - empty = false; - } - return empty; -} - -/* - * Blk cgroup controller notification saying that blkio_group object is being - * delinked as associated cgroup object is going away. That also means that - * no new IO will come in this group. So get rid of this group as soon as - * any pending IO in the group is finished. - * - * This function is called under rcu_read_lock(). @q is the rcu protected - * pointer. That means @q is a valid request_queue pointer as long as we - * are rcu read lock. - * - * @q was fetched from blkio_group under blkio_cgroup->lock. That means - * it should not be NULL as even if queue was going away, cgroup deltion - * path got to it first. - */ -void throtl_unlink_blkio_group(struct request_queue *q, - struct blkio_group *blkg) -{ - unsigned long flags; - - spin_lock_irqsave(q->queue_lock, flags); - throtl_destroy_tg(q->td, blkg_to_tg(blkg)); - spin_unlock_irqrestore(q->queue_lock, flags); -} - -static bool throtl_clear_queue(struct request_queue *q) -{ - lockdep_assert_held(q->queue_lock); - - /* - * Clear tgs but leave the root one alone. This is necessary - * because root_tg is expected to be persistent and safe because - * blk-throtl can never be disabled while @q is alive. This is a - * kludge to prepare for unified blkg. This whole function will be - * removed soon. - */ - return throtl_release_tgs(q->td, false); -} - static void throtl_update_blkio_group_common(struct throtl_data *td, struct throtl_grp *tg) { @@ -960,9 +869,6 @@ static void throtl_shutdown_wq(struct request_queue *q) static struct blkio_policy_type blkio_policy_throtl = { .ops = { .blkio_init_group_fn = throtl_init_blkio_group, - .blkio_link_group_fn = throtl_link_blkio_group, - .blkio_unlink_group_fn = throtl_unlink_blkio_group, - .blkio_clear_queue_fn = throtl_clear_queue, .blkio_update_group_read_bps_fn = throtl_update_blkio_group_read_bps, .blkio_update_group_write_bps_fn = @@ -1148,12 +1054,11 @@ void blk_throtl_exit(struct request_queue *q) throtl_shutdown_wq(q); - spin_lock_irq(q->queue_lock); - throtl_release_tgs(td, true); + blkg_destroy_all(q, BLKIO_POLICY_THROTL, true); /* If there are other groups */ + spin_lock_irq(q->queue_lock); wait = q->nr_blkgs[BLKIO_POLICY_THROTL]; - spin_unlock_irq(q->queue_lock); /* diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c index e846803280a6..dc73690dec44 100644 --- a/block/cfq-iosched.c +++ b/block/cfq-iosched.c @@ -1045,14 +1045,6 @@ static void cfq_update_blkio_group_weight(struct request_queue *q, cfqg->needs_update = true; } -static void cfq_link_blkio_group(struct request_queue *q, - struct blkio_group *blkg) -{ - list_add(&blkg->q_node[BLKIO_POLICY_PROP], - &q->blkg_list[BLKIO_POLICY_PROP]); - q->nr_blkgs[BLKIO_POLICY_PROP]++; -} - static void cfq_init_blkio_group(struct blkio_group *blkg) { struct cfq_group *cfqg = blkg_to_cfqg(blkg); @@ -1096,84 +1088,6 @@ static void cfq_link_cfqq_cfqg(struct cfq_queue *cfqq, struct cfq_group *cfqg) blkg_get(cfqg_to_blkg(cfqg)); } -static void cfq_destroy_cfqg(struct cfq_data *cfqd, struct cfq_group *cfqg) -{ - struct blkio_group *blkg = cfqg_to_blkg(cfqg); - - /* Something wrong if we are trying to remove same group twice */ - BUG_ON(list_empty(&blkg->q_node[BLKIO_POLICY_PROP])); - - list_del_init(&blkg->q_node[BLKIO_POLICY_PROP]); - - BUG_ON(cfqd->queue->nr_blkgs[BLKIO_POLICY_PROP] <= 0); - cfqd->queue->nr_blkgs[BLKIO_POLICY_PROP]--; - - /* - * Put the reference taken at the time of creation so that when all - * queues are gone, group can be destroyed. - */ - blkg_put(cfqg_to_blkg(cfqg)); -} - -static bool cfq_release_cfq_groups(struct cfq_data *cfqd) -{ - struct request_queue *q = cfqd->queue; - struct blkio_group *blkg, *n; - bool empty = true; - - list_for_each_entry_safe(blkg, n, &q->blkg_list[BLKIO_POLICY_PROP], - q_node[BLKIO_POLICY_PROP]) { - /* - * If cgroup removal path got to blk_group first and removed - * it from cgroup list, then it will take care of destroying - * cfqg also. - */ - if (!cfq_blkiocg_del_blkio_group(blkg)) - cfq_destroy_cfqg(cfqd, blkg_to_cfqg(blkg)); - else - empty = false; - } - return empty; -} - -/* - * Blk cgroup controller notification saying that blkio_group object is being - * delinked as associated cgroup object is going away. That also means that - * no new IO will come in this group. So get rid of this group as soon as - * any pending IO in the group is finished. - * - * This function is called under rcu_read_lock(). key is the rcu protected - * pointer. That means @q is a valid request_queue pointer as long as we - * are rcu read lock. - * - * @q was fetched from blkio_group under blkio_cgroup->lock. That means - * it should not be NULL as even if elevator was exiting, cgroup deltion - * path got to it first. - */ -static void cfq_unlink_blkio_group(struct request_queue *q, - struct blkio_group *blkg) -{ - struct cfq_data *cfqd = q->elevator->elevator_data; - unsigned long flags; - - spin_lock_irqsave(q->queue_lock, flags); - cfq_destroy_cfqg(cfqd, blkg_to_cfqg(blkg)); - spin_unlock_irqrestore(q->queue_lock, flags); -} - -static struct elevator_type iosched_cfq; - -static bool cfq_clear_queue(struct request_queue *q) -{ - lockdep_assert_held(q->queue_lock); - - /* shoot down blkgs iff the current elevator is cfq */ - if (!q->elevator || q->elevator->type != &iosched_cfq) - return true; - - return cfq_release_cfq_groups(q->elevator->elevator_data); -} - #else /* GROUP_IOSCHED */ static struct cfq_group *cfq_lookup_create_cfqg(struct cfq_data *cfqd, struct blkio_cgroup *blkcg) @@ -1186,8 +1100,6 @@ cfq_link_cfqq_cfqg(struct cfq_queue *cfqq, struct cfq_group *cfqg) { cfqq->cfqg = cfqg; } -static void cfq_release_cfq_groups(struct cfq_data *cfqd) {} - #endif /* GROUP_IOSCHED */ /* @@ -3547,17 +3459,20 @@ static void cfq_exit_queue(struct elevator_queue *e) __cfq_slice_expired(cfqd, cfqd->active_queue, 0); cfq_put_async_queues(cfqd); - cfq_release_cfq_groups(cfqd); + + spin_unlock_irq(q->queue_lock); + + blkg_destroy_all(q, BLKIO_POLICY_PROP, true); #ifdef CONFIG_BLK_CGROUP /* * If there are groups which we could not unlink from blkcg list, * wait for a rcu period for them to be freed. */ + spin_lock_irq(q->queue_lock); wait = q->nr_blkgs[BLKIO_POLICY_PROP]; -#endif spin_unlock_irq(q->queue_lock); - +#endif cfq_shutdown_timer_wq(cfqd); /* @@ -3794,9 +3709,6 @@ static struct elevator_type iosched_cfq = { static struct blkio_policy_type blkio_policy_cfq = { .ops = { .blkio_init_group_fn = cfq_init_blkio_group, - .blkio_link_group_fn = cfq_link_blkio_group, - .blkio_unlink_group_fn = cfq_unlink_blkio_group, - .blkio_clear_queue_fn = cfq_clear_queue, .blkio_update_group_weight_fn = cfq_update_blkio_group_weight, }, .plid = BLKIO_POLICY_PROP, diff --git a/block/elevator.c b/block/elevator.c index 8c7561fd2c79..d4d39dab841a 100644 --- a/block/elevator.c +++ b/block/elevator.c @@ -876,7 +876,7 @@ static int elevator_switch(struct request_queue *q, struct elevator_type *new_e) { struct elevator_queue *old = q->elevator; bool registered = old->registered; - int err; + int i, err; /* * Turn on BYPASS and drain all requests w/ elevator private data. @@ -895,7 +895,8 @@ static int elevator_switch(struct request_queue *q, struct elevator_type *new_e) ioc_clear_queue(q); spin_unlock_irq(q->queue_lock); - blkg_destroy_all(q); + for (i = 0; i < BLKIO_NR_POLICIES; i++) + blkg_destroy_all(q, i, false); /* allocate, init and register new elevator */ err = -ENOMEM; diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index f4e35edea70f..b4d1d4bfc168 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -364,8 +364,8 @@ struct request_queue { struct list_head icq_list; #ifdef CONFIG_BLK_CGROUP /* XXX: array size hardcoded to avoid include dependency (temporary) */ - struct list_head blkg_list[2]; - int nr_blkgs[2]; + struct list_head blkg_list; + int nr_blkgs; #endif struct queue_limits limits; -- cgit v1.2.3-59-g8ed1b From c875f4d0250a1f070fa26087a73bdd8f54c48100 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 5 Mar 2012 13:15:22 -0800 Subject: blkcg: drop unnecessary RCU locking Now that blkg additions / removals are always done under both q and blkcg locks, the only places RCU locking is necessary are blkg_lookup[_create]() for lookup w/o blkcg lock. This patch drops unncessary RCU locking replacing it with plain blkcg locking as necessary. * blkiocg_pre_destroy() already perform proper locking and don't need RCU. Dropped. * blkio_read_blkg_stats() now uses blkcg->lock instead of RCU read lock. This isn't a hot path. * Now unnecessary synchronize_rcu() from queue exit paths removed. This makes q->nr_blkgs unnecessary. Dropped. * RCU annotation on blkg->q removed. -v2: Vivek pointed out that blkg_lookup_create() still needs to be called under rcu_read_lock(). Updated. -v3: After the update, stats_lock locking in blkio_read_blkg_stats() shouldn't be using _irq variant as it otherwise ends up enabling irq while blkcg->lock is locked. Fixed. Signed-off-by: Tejun Heo Cc: Vivek Goyal Signed-off-by: Jens Axboe --- block/blk-cgroup.c | 24 +++++++++--------------- block/blk-cgroup.h | 4 ++-- block/blk-throttle.c | 33 +-------------------------------- block/cfq-iosched.c | 24 ------------------------ include/linux/blkdev.h | 1 - 5 files changed, 12 insertions(+), 74 deletions(-) (limited to 'include/linux/blkdev.h') diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index e9e3b038c702..27d39a810cb6 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -500,7 +500,7 @@ static struct blkio_group *blkg_alloc(struct blkio_cgroup *blkcg, return NULL; spin_lock_init(&blkg->stats_lock); - rcu_assign_pointer(blkg->q, q); + blkg->q = q; INIT_LIST_HEAD(&blkg->q_node); blkg->blkcg = blkcg; blkg->refcnt = 1; @@ -611,7 +611,6 @@ struct blkio_group *blkg_lookup_create(struct blkio_cgroup *blkcg, hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list); list_add(&blkg->q_node, &q->blkg_list); - q->nr_blkgs++; spin_unlock(&blkcg->lock); out: @@ -648,9 +647,6 @@ static void blkg_destroy(struct blkio_group *blkg) list_del_init(&blkg->q_node); hlist_del_init_rcu(&blkg->blkcg_node); - WARN_ON_ONCE(q->nr_blkgs <= 0); - q->nr_blkgs--; - /* * Put the reference taken at the time of creation so that when all * queues are gone, group can be destroyed. @@ -1232,8 +1228,9 @@ static int blkio_read_blkg_stats(struct blkio_cgroup *blkcg, struct hlist_node *n; uint64_t cgroup_total = 0; - rcu_read_lock(); - hlist_for_each_entry_rcu(blkg, n, &blkcg->blkg_list, blkcg_node) { + spin_lock_irq(&blkcg->lock); + + hlist_for_each_entry(blkg, n, &blkcg->blkg_list, blkcg_node) { const char *dname = blkg_dev_name(blkg); int plid = BLKIOFILE_POLICY(cft->private); @@ -1243,15 +1240,16 @@ static int blkio_read_blkg_stats(struct blkio_cgroup *blkcg, cgroup_total += blkio_get_stat_cpu(blkg, plid, cb, dname, type); } else { - spin_lock_irq(&blkg->stats_lock); + spin_lock(&blkg->stats_lock); cgroup_total += blkio_get_stat(blkg, plid, cb, dname, type); - spin_unlock_irq(&blkg->stats_lock); + spin_unlock(&blkg->stats_lock); } } if (show_total) cb->fill(cb, "Total", cgroup_total); - rcu_read_unlock(); + + spin_unlock_irq(&blkcg->lock); return 0; } @@ -1583,28 +1581,24 @@ static int blkiocg_pre_destroy(struct cgroup_subsys *subsys, { struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgroup); - rcu_read_lock(); spin_lock_irq(&blkcg->lock); while (!hlist_empty(&blkcg->blkg_list)) { struct blkio_group *blkg = hlist_entry(blkcg->blkg_list.first, struct blkio_group, blkcg_node); - struct request_queue *q = rcu_dereference(blkg->q); + struct request_queue *q = blkg->q; if (spin_trylock(q->queue_lock)) { blkg_destroy(blkg); spin_unlock(q->queue_lock); } else { spin_unlock_irq(&blkcg->lock); - rcu_read_unlock(); cpu_relax(); - rcu_read_lock(); spin_lock(&blkcg->lock); } } spin_unlock_irq(&blkcg->lock); - rcu_read_unlock(); return 0; } diff --git a/block/blk-cgroup.h b/block/blk-cgroup.h index df73040a6a5f..66eaefefcbd2 100644 --- a/block/blk-cgroup.h +++ b/block/blk-cgroup.h @@ -176,8 +176,8 @@ struct blkg_policy_data { }; struct blkio_group { - /* Pointer to the associated request_queue, RCU protected */ - struct request_queue __rcu *q; + /* Pointer to the associated request_queue */ + struct request_queue *q; struct list_head q_node; struct hlist_node blkcg_node; struct blkio_cgroup *blkcg; diff --git a/block/blk-throttle.c b/block/blk-throttle.c index e35ee7aeea69..bfa5168249eb 100644 --- a/block/blk-throttle.c +++ b/block/blk-throttle.c @@ -1046,39 +1046,8 @@ int blk_throtl_init(struct request_queue *q) void blk_throtl_exit(struct request_queue *q) { - struct throtl_data *td = q->td; - bool wait; - - BUG_ON(!td); - + BUG_ON(!q->td); throtl_shutdown_wq(q); - - /* If there are other groups */ - spin_lock_irq(q->queue_lock); - wait = q->nr_blkgs; - spin_unlock_irq(q->queue_lock); - - /* - * Wait for tg_to_blkg(tg)->q accessors to exit their grace periods. - * Do this wait only if there are other undestroyed groups out - * there (other than root group). This can happen if cgroup deletion - * path claimed the responsibility of cleaning up a group before - * queue cleanup code get to the group. - * - * Do not call synchronize_rcu() unconditionally as there are drivers - * which create/delete request queue hundreds of times during scan/boot - * and synchronize_rcu() can take significant time and slow down boot. - */ - if (wait) - synchronize_rcu(); - - /* - * Just being safe to make sure after previous flush if some body did - * update limits through cgroup and another work got queued, cancel - * it. - */ - throtl_shutdown_wq(q); - kfree(q->td); } diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c index 393eaa59913b..9e386d9bcb79 100644 --- a/block/cfq-iosched.c +++ b/block/cfq-iosched.c @@ -3449,7 +3449,6 @@ static void cfq_exit_queue(struct elevator_queue *e) { struct cfq_data *cfqd = e->elevator_data; struct request_queue *q = cfqd->queue; - bool wait = false; cfq_shutdown_timer_wq(cfqd); @@ -3462,31 +3461,8 @@ static void cfq_exit_queue(struct elevator_queue *e) spin_unlock_irq(q->queue_lock); -#ifdef CONFIG_BLK_CGROUP - /* - * If there are groups which we could not unlink from blkcg list, - * wait for a rcu period for them to be freed. - */ - spin_lock_irq(q->queue_lock); - wait = q->nr_blkgs; - spin_unlock_irq(q->queue_lock); -#endif cfq_shutdown_timer_wq(cfqd); - /* - * Wait for cfqg->blkg->key accessors to exit their grace periods. - * Do this wait only if there are other unlinked groups out - * there. This can happen if cgroup deletion path claimed the - * responsibility of cleaning up a group before queue cleanup code - * get to the group. - * - * Do not call synchronize_rcu() unconditionally as there are drivers - * which create/delete request queue hundreds of times during scan/boot - * and synchronize_rcu() can take significant time and slow down boot. - */ - if (wait) - synchronize_rcu(); - #ifndef CONFIG_CFQ_GROUP_IOSCHED kfree(cfqd->root_group); #endif diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index b4d1d4bfc168..33f1b29e53f4 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -365,7 +365,6 @@ struct request_queue { #ifdef CONFIG_BLK_CGROUP /* XXX: array size hardcoded to avoid include dependency (temporary) */ struct list_head blkg_list; - int nr_blkgs; #endif struct queue_limits limits; -- cgit v1.2.3-59-g8ed1b From 8bd435b30ecacb69bbb8b2d3e251f770b807c5b2 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 13 Apr 2012 13:11:28 -0700 Subject: blkcg: remove static policy ID enums Remove BLKIO_POLICY_* enums and let blkio_policy_register() allocate @pol->plid dynamically on registration. The maximum number of blkcg policies which can be registered at the same time is defined by BLKCG_MAX_POLS constant added to include/linux/blkdev.h. Note that blkio_policy_register() now may fail. Policy init functions updated accordingly and unnecessary ifdefs removed from cfq_init(). Signed-off-by: Tejun Heo Cc: Vivek Goyal Signed-off-by: Jens Axboe --- block/blk-cgroup.c | 59 +++++++++++++++++++++++++++++++++++++------------- block/blk-cgroup.h | 15 ++++--------- block/blk-throttle.c | 4 +--- block/cfq-iosched.c | 25 +++++++++++---------- include/linux/blkdev.h | 7 +++++- 5 files changed, 69 insertions(+), 41 deletions(-) (limited to 'include/linux/blkdev.h') diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index b1231524a097..2d4d7d6d9ae9 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -31,7 +31,7 @@ static LIST_HEAD(all_q_list); struct blkio_cgroup blkio_root_cgroup = { .cfq_weight = 2 * CFQ_WEIGHT_DEFAULT }; EXPORT_SYMBOL_GPL(blkio_root_cgroup); -static struct blkio_policy_type *blkio_policy[BLKIO_NR_POLICIES]; +static struct blkio_policy_type *blkio_policy[BLKCG_MAX_POLS]; struct blkio_cgroup *cgroup_to_blkio_cgroup(struct cgroup *cgroup) { @@ -67,7 +67,7 @@ static void blkg_free(struct blkio_group *blkg) if (!blkg) return; - for (i = 0; i < BLKIO_NR_POLICIES; i++) { + for (i = 0; i < BLKCG_MAX_POLS; i++) { struct blkio_policy_type *pol = blkio_policy[i]; struct blkg_policy_data *pd = blkg->pd[i]; @@ -107,7 +107,7 @@ static struct blkio_group *blkg_alloc(struct blkio_cgroup *blkcg, blkg->refcnt = 1; cgroup_path(blkcg->css.cgroup, blkg->path, sizeof(blkg->path)); - for (i = 0; i < BLKIO_NR_POLICIES; i++) { + for (i = 0; i < BLKCG_MAX_POLS; i++) { struct blkio_policy_type *pol = blkio_policy[i]; struct blkg_policy_data *pd; @@ -127,7 +127,7 @@ static struct blkio_group *blkg_alloc(struct blkio_cgroup *blkcg, } /* invoke per-policy init */ - for (i = 0; i < BLKIO_NR_POLICIES; i++) { + for (i = 0; i < BLKCG_MAX_POLS; i++) { struct blkio_policy_type *pol = blkio_policy[i]; if (pol) @@ -320,7 +320,7 @@ blkiocg_reset_stats(struct cgroup *cgroup, struct cftype *cftype, u64 val) * anyway. If you get hit by a race, retry. */ hlist_for_each_entry(blkg, n, &blkcg->blkg_list, blkcg_node) { - for (i = 0; i < BLKIO_NR_POLICIES; i++) { + for (i = 0; i < BLKCG_MAX_POLS; i++) { struct blkio_policy_type *pol = blkio_policy[i]; if (pol && pol->ops.blkio_reset_group_stats_fn) @@ -729,46 +729,75 @@ struct cgroup_subsys blkio_subsys = { }; EXPORT_SYMBOL_GPL(blkio_subsys); -void blkio_policy_register(struct blkio_policy_type *blkiop) +/** + * blkio_policy_register - register a blkcg policy + * @blkiop: blkcg policy to register + * + * Register @blkiop with blkcg core. Might sleep and @blkiop may be + * modified on successful registration. Returns 0 on success and -errno on + * failure. + */ +int blkio_policy_register(struct blkio_policy_type *blkiop) { struct request_queue *q; + int i, ret; mutex_lock(&blkcg_pol_mutex); - blkcg_bypass_start(); + /* find an empty slot */ + ret = -ENOSPC; + for (i = 0; i < BLKCG_MAX_POLS; i++) + if (!blkio_policy[i]) + break; + if (i >= BLKCG_MAX_POLS) + goto out_unlock; - BUG_ON(blkio_policy[blkiop->plid]); - blkio_policy[blkiop->plid] = blkiop; + /* register and update blkgs */ + blkiop->plid = i; + blkio_policy[i] = blkiop; + + blkcg_bypass_start(); list_for_each_entry(q, &all_q_list, all_q_node) update_root_blkg_pd(q, blkiop); - blkcg_bypass_end(); + /* everything is in place, add intf files for the new policy */ if (blkiop->cftypes) WARN_ON(cgroup_add_cftypes(&blkio_subsys, blkiop->cftypes)); - + ret = 0; +out_unlock: mutex_unlock(&blkcg_pol_mutex); + return ret; } EXPORT_SYMBOL_GPL(blkio_policy_register); +/** + * blkiop_policy_unregister - unregister a blkcg policy + * @blkiop: blkcg policy to unregister + * + * Undo blkio_policy_register(@blkiop). Might sleep. + */ void blkio_policy_unregister(struct blkio_policy_type *blkiop) { struct request_queue *q; mutex_lock(&blkcg_pol_mutex); + if (WARN_ON(blkio_policy[blkiop->plid] != blkiop)) + goto out_unlock; + + /* kill the intf files first */ if (blkiop->cftypes) cgroup_rm_cftypes(&blkio_subsys, blkiop->cftypes); - blkcg_bypass_start(); - - BUG_ON(blkio_policy[blkiop->plid] != blkiop); + /* unregister and update blkgs */ blkio_policy[blkiop->plid] = NULL; + blkcg_bypass_start(); list_for_each_entry(q, &all_q_list, all_q_node) update_root_blkg_pd(q, blkiop); blkcg_bypass_end(); - +out_unlock: mutex_unlock(&blkcg_pol_mutex); } EXPORT_SYMBOL_GPL(blkio_policy_unregister); diff --git a/block/blk-cgroup.h b/block/blk-cgroup.h index 26949731108f..be80d6eb6531 100644 --- a/block/blk-cgroup.h +++ b/block/blk-cgroup.h @@ -17,13 +17,6 @@ #include #include -enum blkio_policy_id { - BLKIO_POLICY_PROP = 0, /* Proportional Bandwidth division */ - BLKIO_POLICY_THROTL, /* Throttling */ - - BLKIO_NR_POLICIES, -}; - /* Max limits for throttle policy */ #define THROTL_IOPS_MAX UINT_MAX @@ -86,7 +79,7 @@ struct blkio_group { /* reference count */ int refcnt; - struct blkg_policy_data *pd[BLKIO_NR_POLICIES]; + struct blkg_policy_data *pd[BLKCG_MAX_POLS]; struct rcu_head rcu_head; }; @@ -103,7 +96,7 @@ struct blkio_policy_ops { struct blkio_policy_type { struct blkio_policy_ops ops; - enum blkio_policy_id plid; + int plid; size_t pdata_size; /* policy specific private data size */ struct cftype *cftypes; /* cgroup files for the policy */ }; @@ -113,7 +106,7 @@ extern void blkcg_drain_queue(struct request_queue *q); extern void blkcg_exit_queue(struct request_queue *q); /* Blkio controller policy registration */ -extern void blkio_policy_register(struct blkio_policy_type *); +extern int blkio_policy_register(struct blkio_policy_type *); extern void blkio_policy_unregister(struct blkio_policy_type *); extern void blkg_destroy_all(struct request_queue *q, bool destroy_root); extern void update_root_blkg_pd(struct request_queue *q, @@ -329,7 +322,7 @@ struct blkio_policy_type { static inline int blkcg_init_queue(struct request_queue *q) { return 0; } static inline void blkcg_drain_queue(struct request_queue *q) { } static inline void blkcg_exit_queue(struct request_queue *q) { } -static inline void blkio_policy_register(struct blkio_policy_type *blkiop) { } +static inline int blkio_policy_register(struct blkio_policy_type *blkiop) { return 0; } static inline void blkio_policy_unregister(struct blkio_policy_type *blkiop) { } static inline void blkg_destroy_all(struct request_queue *q, bool destory_root) { } diff --git a/block/blk-throttle.c b/block/blk-throttle.c index 07c17c27a628..0dc4645aa7fe 100644 --- a/block/blk-throttle.c +++ b/block/blk-throttle.c @@ -1089,7 +1089,6 @@ static struct blkio_policy_type blkio_policy_throtl = { .blkio_exit_group_fn = throtl_exit_blkio_group, .blkio_reset_group_stats_fn = throtl_reset_group_stats, }, - .plid = BLKIO_POLICY_THROTL, .pdata_size = sizeof(struct throtl_grp), .cftypes = throtl_files, }; @@ -1271,8 +1270,7 @@ static int __init throtl_init(void) if (!kthrotld_workqueue) panic("Failed to create kthrotld\n"); - blkio_policy_register(&blkio_policy_throtl); - return 0; + return blkio_policy_register(&blkio_policy_throtl); } module_init(throtl_init); diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c index d02f0ae9637f..08db2fc70c29 100644 --- a/block/cfq-iosched.c +++ b/block/cfq-iosched.c @@ -4157,7 +4157,6 @@ static struct blkio_policy_type blkio_policy_cfq = { .blkio_init_group_fn = cfq_init_blkio_group, .blkio_reset_group_stats_fn = cfqg_stats_reset, }, - .plid = BLKIO_POLICY_PROP, .pdata_size = sizeof(struct cfq_group), .cftypes = cfq_blkcg_files, }; @@ -4181,27 +4180,31 @@ static int __init cfq_init(void) #else cfq_group_idle = 0; #endif + + ret = blkio_policy_register(&blkio_policy_cfq); + if (ret) + return ret; + cfq_pool = KMEM_CACHE(cfq_queue, 0); if (!cfq_pool) - return -ENOMEM; + goto err_pol_unreg; ret = elv_register(&iosched_cfq); - if (ret) { - kmem_cache_destroy(cfq_pool); - return ret; - } + if (ret) + goto err_free_pool; -#ifdef CONFIG_CFQ_GROUP_IOSCHED - blkio_policy_register(&blkio_policy_cfq); -#endif return 0; + +err_free_pool: + kmem_cache_destroy(cfq_pool); +err_pol_unreg: + blkio_policy_unregister(&blkio_policy_cfq); + return ret; } static void __exit cfq_exit(void) { -#ifdef CONFIG_CFQ_GROUP_IOSCHED blkio_policy_unregister(&blkio_policy_cfq); -#endif elv_unregister(&iosched_cfq); kmem_cache_destroy(cfq_pool); } diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 33f1b29e53f4..d2c69f8c188a 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -35,6 +35,12 @@ struct bsg_job; #define BLKDEV_MIN_RQ 4 #define BLKDEV_MAX_RQ 128 /* Default maximum */ +/* + * Maximum number of blkcg policies allowed to be registered concurrently. + * Defined here to simplify include dependency. + */ +#define BLKCG_MAX_POLS 2 + struct request; typedef void (rq_end_io_fn)(struct request *, int); @@ -363,7 +369,6 @@ struct request_queue { struct list_head icq_list; #ifdef CONFIG_BLK_CGROUP - /* XXX: array size hardcoded to avoid include dependency (temporary) */ struct list_head blkg_list; #endif -- cgit v1.2.3-59-g8ed1b From 03d8e11142a893ad322285d3c8a08e88b570cda1 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 13 Apr 2012 13:11:32 -0700 Subject: blkcg: add request_queue->root_blkg With per-queue policy activation, root blkg creation will be moved to blkcg core. Add q->root_blkg in preparation. For blk-throtl, this replaces throtl_data->root_tg; however, cfq needs to keep cfqd->root_group for !CONFIG_CFQ_GROUP_IOSCHED. This is to prepare for per-queue policy activation and doesn't cause any functional difference. Signed-off-by: Tejun Heo Cc: Vivek Goyal Signed-off-by: Jens Axboe --- block/blk-throttle.c | 16 ++++++++++------ block/cfq-iosched.c | 4 +++- include/linux/blkdev.h | 2 ++ 3 files changed, 15 insertions(+), 7 deletions(-) (limited to 'include/linux/blkdev.h') diff --git a/block/blk-throttle.c b/block/blk-throttle.c index 6f1bfdf9a1b7..8c520fad6885 100644 --- a/block/blk-throttle.c +++ b/block/blk-throttle.c @@ -97,7 +97,6 @@ struct throtl_data /* service tree for active throtl groups */ struct throtl_rb_root tg_service_tree; - struct throtl_grp *root_tg; struct request_queue *queue; /* Total Number of queued bios on READ and WRITE lists */ @@ -131,6 +130,11 @@ static inline struct blkio_group *tg_to_blkg(struct throtl_grp *tg) return pdata_to_blkg(tg); } +static inline struct throtl_grp *td_root_tg(struct throtl_data *td) +{ + return blkg_to_tg(td->queue->root_blkg); +} + enum tg_state_flags { THROTL_TG_FLAG_on_rr = 0, /* on round-robin busy list */ }; @@ -261,7 +265,7 @@ throtl_grp *throtl_lookup_tg(struct throtl_data *td, struct blkio_cgroup *blkcg) * Avoid lookup in this case */ if (blkcg == &blkio_root_cgroup) - return td->root_tg; + return td_root_tg(td); return blkg_to_tg(blkg_lookup(blkcg, td->queue)); } @@ -277,7 +281,7 @@ static struct throtl_grp *throtl_lookup_create_tg(struct throtl_data *td, * Avoid lookup in this case */ if (blkcg == &blkio_root_cgroup) { - tg = td->root_tg; + tg = td_root_tg(td); } else { struct blkio_group *blkg; @@ -287,7 +291,7 @@ static struct throtl_grp *throtl_lookup_create_tg(struct throtl_data *td, if (!IS_ERR(blkg)) tg = blkg_to_tg(blkg); else if (!blk_queue_dead(q)) - tg = td->root_tg; + tg = td_root_tg(td); } return tg; @@ -1245,12 +1249,12 @@ int blk_throtl_init(struct request_queue *q) blkg = blkg_lookup_create(&blkio_root_cgroup, q, true); if (!IS_ERR(blkg)) - td->root_tg = blkg_to_tg(blkg); + q->root_blkg = blkg; spin_unlock_irq(q->queue_lock); rcu_read_unlock(); - if (!td->root_tg) { + if (!q->root_blkg) { kfree(td); return -ENOMEM; } diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c index de95f9a2acf8..86440e04f3ee 100644 --- a/block/cfq-iosched.c +++ b/block/cfq-iosched.c @@ -3964,8 +3964,10 @@ static int cfq_init_queue(struct request_queue *q) spin_lock_irq(q->queue_lock); blkg = blkg_lookup_create(&blkio_root_cgroup, q, true); - if (!IS_ERR(blkg)) + if (!IS_ERR(blkg)) { + q->root_blkg = blkg; cfqd->root_group = blkg_to_cfqg(blkg); + } spin_unlock_irq(q->queue_lock); rcu_read_unlock(); diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index d2c69f8c188a..b01c377fd739 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -31,6 +31,7 @@ struct blk_trace; struct request; struct sg_io_hdr; struct bsg_job; +struct blkio_group; #define BLKDEV_MIN_RQ 4 #define BLKDEV_MAX_RQ 128 /* Default maximum */ @@ -369,6 +370,7 @@ struct request_queue { struct list_head icq_list; #ifdef CONFIG_BLK_CGROUP + struct blkio_group *root_blkg; struct list_head blkg_list; #endif -- cgit v1.2.3-59-g8ed1b From a2b1693bac45ea3fe3ba612fd22c45f17449f610 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 13 Apr 2012 13:11:33 -0700 Subject: blkcg: implement per-queue policy activation All blkcg policies were assumed to be enabled on all request_queues. Due to various implementation obstacles, during the recent blkcg core updates, this was temporarily implemented as shooting down all !root blkgs on elevator switch and policy [de]registration combined with half-broken in-place root blkg updates. In addition to being buggy and racy, this meant losing all blkcg configurations across those events. Now that blkcg is cleaned up enough, this patch replaces the temporary implementation with proper per-queue policy activation. Each blkcg policy should call the new blkcg_[de]activate_policy() to enable and disable the policy on a specific queue. blkcg_activate_policy() allocates and installs policy data for the policy for all existing blkgs. blkcg_deactivate_policy() does the reverse. If a policy is not enabled for a given queue, blkg printing / config functions skip the respective blkg for the queue. blkcg_activate_policy() also takes care of root blkg creation, and cfq_init_queue() and blk_throtl_init() are updated accordingly. This replaces blkcg_bypass_{start|end}() and update_root_blkg_pd() unnecessary. Dropped. v2: cfq_init_queue() was returning uninitialized @ret on root_group alloc failure if !CONFIG_CFQ_GROUP_IOSCHED. Fixed. Signed-off-by: Tejun Heo Cc: Vivek Goyal Signed-off-by: Jens Axboe --- block/blk-cgroup.c | 228 +++++++++++++++++++++++++++++++++---------------- block/blk-cgroup.h | 15 +++- block/blk-throttle.c | 52 +++++------ block/cfq-iosched.c | 37 ++++---- block/elevator.c | 2 - include/linux/blkdev.h | 1 + 6 files changed, 201 insertions(+), 134 deletions(-) (limited to 'include/linux/blkdev.h') diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index d6e4555c982f..d6d59ad105b4 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -54,6 +54,17 @@ struct blkio_cgroup *bio_blkio_cgroup(struct bio *bio) } EXPORT_SYMBOL_GPL(bio_blkio_cgroup); +static bool blkcg_policy_enabled(struct request_queue *q, + const struct blkio_policy_type *pol) +{ + return pol && test_bit(pol->plid, q->blkcg_pols); +} + +static size_t blkg_pd_size(const struct blkio_policy_type *pol) +{ + return sizeof(struct blkg_policy_data) + pol->pdata_size; +} + /** * blkg_free - free a blkg * @blkg: blkg to free @@ -111,12 +122,11 @@ static struct blkio_group *blkg_alloc(struct blkio_cgroup *blkcg, struct blkio_policy_type *pol = blkio_policy[i]; struct blkg_policy_data *pd; - if (!pol) + if (!blkcg_policy_enabled(q, pol)) continue; /* alloc per-policy data and attach it to blkg */ - pd = kzalloc_node(sizeof(*pd) + pol->pdata_size, GFP_ATOMIC, - q->node); + pd = kzalloc_node(blkg_pd_size(pol), GFP_ATOMIC, q->node); if (!pd) { blkg_free(blkg); return NULL; @@ -130,7 +140,7 @@ static struct blkio_group *blkg_alloc(struct blkio_cgroup *blkcg, for (i = 0; i < BLKCG_MAX_POLS; i++) { struct blkio_policy_type *pol = blkio_policy[i]; - if (pol) + if (blkcg_policy_enabled(blkg->q, pol)) pol->ops.blkio_init_group_fn(blkg); } @@ -236,36 +246,6 @@ static void blkg_destroy(struct blkio_group *blkg) blkg_put(blkg); } -/* - * XXX: This updates blkg policy data in-place for root blkg, which is - * necessary across elevator switch and policy registration as root blkgs - * aren't shot down. This broken and racy implementation is temporary. - * Eventually, blkg shoot down will be replaced by proper in-place update. - */ -void update_root_blkg_pd(struct request_queue *q, - const struct blkio_policy_type *pol) -{ - struct blkio_group *blkg = blkg_lookup(&blkio_root_cgroup, q); - struct blkg_policy_data *pd; - - if (!blkg) - return; - - kfree(blkg->pd[pol->plid]); - blkg->pd[pol->plid] = NULL; - - if (!pol) - return; - - pd = kzalloc(sizeof(*pd) + pol->pdata_size, GFP_KERNEL); - WARN_ON_ONCE(!pd); - - blkg->pd[pol->plid] = pd; - pd->blkg = blkg; - pol->ops.blkio_init_group_fn(blkg); -} -EXPORT_SYMBOL_GPL(update_root_blkg_pd); - /** * blkg_destroy_all - destroy all blkgs associated with a request_queue * @q: request_queue of interest @@ -339,7 +319,8 @@ blkiocg_reset_stats(struct cgroup *cgroup, struct cftype *cftype, u64 val) for (i = 0; i < BLKCG_MAX_POLS; i++) { struct blkio_policy_type *pol = blkio_policy[i]; - if (pol && pol->ops.blkio_reset_group_stats_fn) + if (blkcg_policy_enabled(blkg->q, pol) && + pol->ops.blkio_reset_group_stats_fn) pol->ops.blkio_reset_group_stats_fn(blkg); } } @@ -385,7 +366,7 @@ void blkcg_print_blkgs(struct seq_file *sf, struct blkio_cgroup *blkcg, spin_lock_irq(&blkcg->lock); hlist_for_each_entry(blkg, n, &blkcg->blkg_list, blkcg_node) - if (blkg->pd[pol->plid]) + if (blkcg_policy_enabled(blkg->q, pol)) total += prfill(sf, blkg->pd[pol->plid]->pdata, data); spin_unlock_irq(&blkcg->lock); @@ -510,7 +491,10 @@ int blkg_conf_prep(struct blkio_cgroup *blkcg, rcu_read_lock(); spin_lock_irq(disk->queue->queue_lock); - blkg = blkg_lookup_create(blkcg, disk->queue, false); + if (blkcg_policy_enabled(disk->queue, pol)) + blkg = blkg_lookup_create(blkcg, disk->queue, false); + else + blkg = ERR_PTR(-EINVAL); if (IS_ERR(blkg)) { ret = PTR_ERR(blkg); @@ -712,30 +696,6 @@ static int blkiocg_can_attach(struct cgroup *cgrp, struct cgroup_taskset *tset) return ret; } -static void blkcg_bypass_start(void) - __acquires(&all_q_mutex) -{ - struct request_queue *q; - - mutex_lock(&all_q_mutex); - - list_for_each_entry(q, &all_q_list, all_q_node) { - blk_queue_bypass_start(q); - blkg_destroy_all(q, false); - } -} - -static void blkcg_bypass_end(void) - __releases(&all_q_mutex) -{ - struct request_queue *q; - - list_for_each_entry(q, &all_q_list, all_q_node) - blk_queue_bypass_end(q); - - mutex_unlock(&all_q_mutex); -} - struct cgroup_subsys blkio_subsys = { .name = "blkio", .create = blkiocg_create, @@ -748,6 +708,139 @@ struct cgroup_subsys blkio_subsys = { }; EXPORT_SYMBOL_GPL(blkio_subsys); +/** + * blkcg_activate_policy - activate a blkcg policy on a request_queue + * @q: request_queue of interest + * @pol: blkcg policy to activate + * + * Activate @pol on @q. Requires %GFP_KERNEL context. @q goes through + * bypass mode to populate its blkgs with policy_data for @pol. + * + * Activation happens with @q bypassed, so nobody would be accessing blkgs + * from IO path. Update of each blkg is protected by both queue and blkcg + * locks so that holding either lock and testing blkcg_policy_enabled() is + * always enough for dereferencing policy data. + * + * The caller is responsible for synchronizing [de]activations and policy + * [un]registerations. Returns 0 on success, -errno on failure. + */ +int blkcg_activate_policy(struct request_queue *q, + const struct blkio_policy_type *pol) +{ + LIST_HEAD(pds); + struct blkio_group *blkg; + struct blkg_policy_data *pd, *n; + int cnt = 0, ret; + + if (blkcg_policy_enabled(q, pol)) + return 0; + + blk_queue_bypass_start(q); + + /* make sure the root blkg exists and count the existing blkgs */ + spin_lock_irq(q->queue_lock); + + rcu_read_lock(); + blkg = blkg_lookup_create(&blkio_root_cgroup, q, true); + rcu_read_unlock(); + + if (IS_ERR(blkg)) { + ret = PTR_ERR(blkg); + goto out_unlock; + } + q->root_blkg = blkg; + + list_for_each_entry(blkg, &q->blkg_list, q_node) + cnt++; + + spin_unlock_irq(q->queue_lock); + + /* allocate policy_data for all existing blkgs */ + while (cnt--) { + pd = kzalloc_node(blkg_pd_size(pol), GFP_KERNEL, q->node); + if (!pd) { + ret = -ENOMEM; + goto out_free; + } + list_add_tail(&pd->alloc_node, &pds); + } + + /* + * Install the allocated pds. With @q bypassing, no new blkg + * should have been created while the queue lock was dropped. + */ + spin_lock_irq(q->queue_lock); + + list_for_each_entry(blkg, &q->blkg_list, q_node) { + if (WARN_ON(list_empty(&pds))) { + /* umm... this shouldn't happen, just abort */ + ret = -ENOMEM; + goto out_unlock; + } + pd = list_first_entry(&pds, struct blkg_policy_data, alloc_node); + list_del_init(&pd->alloc_node); + + /* grab blkcg lock too while installing @pd on @blkg */ + spin_lock(&blkg->blkcg->lock); + + blkg->pd[pol->plid] = pd; + pd->blkg = blkg; + pol->ops.blkio_init_group_fn(blkg); + + spin_unlock(&blkg->blkcg->lock); + } + + __set_bit(pol->plid, q->blkcg_pols); + ret = 0; +out_unlock: + spin_unlock_irq(q->queue_lock); +out_free: + blk_queue_bypass_end(q); + list_for_each_entry_safe(pd, n, &pds, alloc_node) + kfree(pd); + return ret; +} +EXPORT_SYMBOL_GPL(blkcg_activate_policy); + +/** + * blkcg_deactivate_policy - deactivate a blkcg policy on a request_queue + * @q: request_queue of interest + * @pol: blkcg policy to deactivate + * + * Deactivate @pol on @q. Follows the same synchronization rules as + * blkcg_activate_policy(). + */ +void blkcg_deactivate_policy(struct request_queue *q, + const struct blkio_policy_type *pol) +{ + struct blkio_group *blkg; + + if (!blkcg_policy_enabled(q, pol)) + return; + + blk_queue_bypass_start(q); + spin_lock_irq(q->queue_lock); + + __clear_bit(pol->plid, q->blkcg_pols); + + list_for_each_entry(blkg, &q->blkg_list, q_node) { + /* grab blkcg lock too while removing @pd from @blkg */ + spin_lock(&blkg->blkcg->lock); + + if (pol->ops.blkio_exit_group_fn) + pol->ops.blkio_exit_group_fn(blkg); + + kfree(blkg->pd[pol->plid]); + blkg->pd[pol->plid] = NULL; + + spin_unlock(&blkg->blkcg->lock); + } + + spin_unlock_irq(q->queue_lock); + blk_queue_bypass_end(q); +} +EXPORT_SYMBOL_GPL(blkcg_deactivate_policy); + /** * blkio_policy_register - register a blkcg policy * @blkiop: blkcg policy to register @@ -758,7 +851,6 @@ EXPORT_SYMBOL_GPL(blkio_subsys); */ int blkio_policy_register(struct blkio_policy_type *blkiop) { - struct request_queue *q; int i, ret; mutex_lock(&blkcg_pol_mutex); @@ -775,11 +867,6 @@ int blkio_policy_register(struct blkio_policy_type *blkiop) blkiop->plid = i; blkio_policy[i] = blkiop; - blkcg_bypass_start(); - list_for_each_entry(q, &all_q_list, all_q_node) - update_root_blkg_pd(q, blkiop); - blkcg_bypass_end(); - /* everything is in place, add intf files for the new policy */ if (blkiop->cftypes) WARN_ON(cgroup_add_cftypes(&blkio_subsys, blkiop->cftypes)); @@ -798,8 +885,6 @@ EXPORT_SYMBOL_GPL(blkio_policy_register); */ void blkio_policy_unregister(struct blkio_policy_type *blkiop) { - struct request_queue *q; - mutex_lock(&blkcg_pol_mutex); if (WARN_ON(blkio_policy[blkiop->plid] != blkiop)) @@ -811,11 +896,6 @@ void blkio_policy_unregister(struct blkio_policy_type *blkiop) /* unregister and update blkgs */ blkio_policy[blkiop->plid] = NULL; - - blkcg_bypass_start(); - list_for_each_entry(q, &all_q_list, all_q_node) - update_root_blkg_pd(q, blkiop); - blkcg_bypass_end(); out_unlock: mutex_unlock(&blkcg_pol_mutex); } diff --git a/block/blk-cgroup.h b/block/blk-cgroup.h index df1c7b290c22..66253a7c8ff4 100644 --- a/block/blk-cgroup.h +++ b/block/blk-cgroup.h @@ -64,6 +64,9 @@ struct blkg_policy_data { /* the blkg this per-policy data belongs to */ struct blkio_group *blkg; + /* used during policy activation */ + struct list_head alloc_node; + /* pol->pdata_size bytes of private data used by policy impl */ char pdata[] __aligned(__alignof__(unsigned long long)); }; @@ -108,9 +111,11 @@ extern void blkcg_exit_queue(struct request_queue *q); /* Blkio controller policy registration */ extern int blkio_policy_register(struct blkio_policy_type *); extern void blkio_policy_unregister(struct blkio_policy_type *); +extern int blkcg_activate_policy(struct request_queue *q, + const struct blkio_policy_type *pol); +extern void blkcg_deactivate_policy(struct request_queue *q, + const struct blkio_policy_type *pol); extern void blkg_destroy_all(struct request_queue *q, bool destroy_root); -extern void update_root_blkg_pd(struct request_queue *q, - const struct blkio_policy_type *pol); void blkcg_print_blkgs(struct seq_file *sf, struct blkio_cgroup *blkcg, u64 (*prfill)(struct seq_file *, void *, int), @@ -325,10 +330,12 @@ static inline void blkcg_drain_queue(struct request_queue *q) { } static inline void blkcg_exit_queue(struct request_queue *q) { } static inline int blkio_policy_register(struct blkio_policy_type *blkiop) { return 0; } static inline void blkio_policy_unregister(struct blkio_policy_type *blkiop) { } +static inline int blkcg_activate_policy(struct request_queue *q, + const struct blkio_policy_type *pol) { return 0; } +static inline void blkcg_deactivate_policy(struct request_queue *q, + const struct blkio_policy_type *pol) { } static inline void blkg_destroy_all(struct request_queue *q, bool destory_root) { } -static inline void update_root_blkg_pd(struct request_queue *q, - const struct blkio_policy_type *pol) { } static inline void *blkg_to_pdata(struct blkio_group *blkg, struct blkio_policy_type *pol) { return NULL; } diff --git a/block/blk-throttle.c b/block/blk-throttle.c index 8c520fad6885..2fc964e06ea4 100644 --- a/block/blk-throttle.c +++ b/block/blk-throttle.c @@ -995,35 +995,31 @@ static int tg_set_conf(struct cgroup *cgrp, struct cftype *cft, const char *buf, struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgrp); struct blkg_conf_ctx ctx; struct throtl_grp *tg; + struct throtl_data *td; int ret; ret = blkg_conf_prep(blkcg, &blkio_policy_throtl, buf, &ctx); if (ret) return ret; - ret = -EINVAL; tg = blkg_to_tg(ctx.blkg); - if (tg) { - struct throtl_data *td = ctx.blkg->q->td; - - if (!ctx.v) - ctx.v = -1; + td = ctx.blkg->q->td; - if (is_u64) - *(u64 *)((void *)tg + cft->private) = ctx.v; - else - *(unsigned int *)((void *)tg + cft->private) = ctx.v; + if (!ctx.v) + ctx.v = -1; - /* XXX: we don't need the following deferred processing */ - xchg(&tg->limits_changed, true); - xchg(&td->limits_changed, true); - throtl_schedule_delayed_work(td, 0); + if (is_u64) + *(u64 *)((void *)tg + cft->private) = ctx.v; + else + *(unsigned int *)((void *)tg + cft->private) = ctx.v; - ret = 0; - } + /* XXX: we don't need the following deferred processing */ + xchg(&tg->limits_changed, true); + xchg(&td->limits_changed, true); + throtl_schedule_delayed_work(td, 0); blkg_conf_finish(&ctx); - return ret; + return 0; } static int tg_set_conf_u64(struct cgroup *cgrp, struct cftype *cft, @@ -1230,7 +1226,7 @@ void blk_throtl_drain(struct request_queue *q) int blk_throtl_init(struct request_queue *q) { struct throtl_data *td; - struct blkio_group *blkg; + int ret; td = kzalloc_node(sizeof(*td), GFP_KERNEL, q->node); if (!td) @@ -1243,28 +1239,18 @@ int blk_throtl_init(struct request_queue *q) q->td = td; td->queue = q; - /* alloc and init root group. */ - rcu_read_lock(); - spin_lock_irq(q->queue_lock); - - blkg = blkg_lookup_create(&blkio_root_cgroup, q, true); - if (!IS_ERR(blkg)) - q->root_blkg = blkg; - - spin_unlock_irq(q->queue_lock); - rcu_read_unlock(); - - if (!q->root_blkg) { + /* activate policy */ + ret = blkcg_activate_policy(q, &blkio_policy_throtl); + if (ret) kfree(td); - return -ENOMEM; - } - return 0; + return ret; } void blk_throtl_exit(struct request_queue *q) { BUG_ON(!q->td); throtl_shutdown_wq(q); + blkcg_deactivate_policy(q, &blkio_policy_throtl); kfree(q->td); } diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c index 86440e04f3ee..0203652e1f34 100644 --- a/block/cfq-iosched.c +++ b/block/cfq-iosched.c @@ -1406,8 +1406,7 @@ static int cfqg_set_weight_device(struct cgroup *cgrp, struct cftype *cft, ret = -EINVAL; cfqg = blkg_to_cfqg(ctx.blkg); - if (cfqg && (!ctx.v || (ctx.v >= CFQ_WEIGHT_MIN && - ctx.v <= CFQ_WEIGHT_MAX))) { + if (!ctx.v || (ctx.v >= CFQ_WEIGHT_MIN && ctx.v <= CFQ_WEIGHT_MAX)) { cfqg->dev_weight = ctx.v; cfqg->new_weight = cfqg->dev_weight ?: blkcg->cfq_weight; ret = 0; @@ -3938,7 +3937,7 @@ static void cfq_exit_queue(struct elevator_queue *e) #ifndef CONFIG_CFQ_GROUP_IOSCHED kfree(cfqd->root_group); #endif - update_root_blkg_pd(q, &blkio_policy_cfq); + blkcg_deactivate_policy(q, &blkio_policy_cfq); kfree(cfqd); } @@ -3946,7 +3945,7 @@ static int cfq_init_queue(struct request_queue *q) { struct cfq_data *cfqd; struct blkio_group *blkg __maybe_unused; - int i; + int i, ret; cfqd = kmalloc_node(sizeof(*cfqd), GFP_KERNEL | __GFP_ZERO, q->node); if (!cfqd) @@ -3960,28 +3959,20 @@ static int cfq_init_queue(struct request_queue *q) /* Init root group and prefer root group over other groups by default */ #ifdef CONFIG_CFQ_GROUP_IOSCHED - rcu_read_lock(); - spin_lock_irq(q->queue_lock); - - blkg = blkg_lookup_create(&blkio_root_cgroup, q, true); - if (!IS_ERR(blkg)) { - q->root_blkg = blkg; - cfqd->root_group = blkg_to_cfqg(blkg); - } + ret = blkcg_activate_policy(q, &blkio_policy_cfq); + if (ret) + goto out_free; - spin_unlock_irq(q->queue_lock); - rcu_read_unlock(); + cfqd->root_group = blkg_to_cfqg(q->root_blkg); #else + ret = -ENOMEM; cfqd->root_group = kzalloc_node(sizeof(*cfqd->root_group), GFP_KERNEL, cfqd->queue->node); - if (cfqd->root_group) - cfq_init_cfqg_base(cfqd->root_group); -#endif - if (!cfqd->root_group) { - kfree(cfqd); - return -ENOMEM; - } + if (!cfqd->root_group) + goto out_free; + cfq_init_cfqg_base(cfqd->root_group); +#endif cfqd->root_group->weight = 2 * CFQ_WEIGHT_DEFAULT; /* @@ -4031,6 +4022,10 @@ static int cfq_init_queue(struct request_queue *q) */ cfqd->last_delayed_sync = jiffies - HZ; return 0; + +out_free: + kfree(cfqd); + return ret; } /* diff --git a/block/elevator.c b/block/elevator.c index be3ab6df0fea..6a55d418896f 100644 --- a/block/elevator.c +++ b/block/elevator.c @@ -896,8 +896,6 @@ static int elevator_switch(struct request_queue *q, struct elevator_type *new_e) ioc_clear_queue(q); spin_unlock_irq(q->queue_lock); - blkg_destroy_all(q, false); - /* allocate, init and register new elevator */ err = -ENOMEM; q->elevator = elevator_alloc(q, new_e); diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index b01c377fd739..68720ab275d4 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -370,6 +370,7 @@ struct request_queue { struct list_head icq_list; #ifdef CONFIG_BLK_CGROUP + DECLARE_BITMAP (blkcg_pols, BLKCG_MAX_POLS); struct blkio_group *root_blkg; struct list_head blkg_list; #endif -- cgit v1.2.3-59-g8ed1b From 3c798398e393e5f9502dbab2b51e6c25e2e8f2ac Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 16 Apr 2012 13:57:25 -0700 Subject: blkcg: mass rename of blkcg API During the recent blkcg cleanup, most of blkcg API has changed to such extent that mass renaming wouldn't cause any noticeable pain. Take the chance and cleanup the naming. * Rename blkio_cgroup to blkcg. * Drop blkio / blkiocg prefixes and consistently use blkcg. * Rename blkio_group to blkcg_gq, which is consistent with io_cq but keep the blkg prefix / variable name. * Rename policy method type and field names to signify they're dealing with policy data. * Rename blkio_policy_type to blkcg_policy. This patch doesn't cause any functional change. Signed-off-by: Tejun Heo Cc: Vivek Goyal Signed-off-by: Jens Axboe --- block/blk-cgroup.c | 202 ++++++++++++++++++++++++------------------------- block/blk-cgroup.h | 109 +++++++++++++------------- block/blk-throttle.c | 72 +++++++++--------- block/cfq-iosched.c | 78 +++++++++---------- include/linux/blkdev.h | 4 +- 5 files changed, 230 insertions(+), 235 deletions(-) (limited to 'include/linux/blkdev.h') diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index 63337024e4d7..997570329517 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -26,39 +26,39 @@ static DEFINE_MUTEX(blkcg_pol_mutex); -struct blkio_cgroup blkio_root_cgroup = { .cfq_weight = 2 * CFQ_WEIGHT_DEFAULT }; -EXPORT_SYMBOL_GPL(blkio_root_cgroup); +struct blkcg blkcg_root = { .cfq_weight = 2 * CFQ_WEIGHT_DEFAULT }; +EXPORT_SYMBOL_GPL(blkcg_root); -static struct blkio_policy_type *blkio_policy[BLKCG_MAX_POLS]; +static struct blkcg_policy *blkcg_policy[BLKCG_MAX_POLS]; -struct blkio_cgroup *cgroup_to_blkio_cgroup(struct cgroup *cgroup) +struct blkcg *cgroup_to_blkcg(struct cgroup *cgroup) { return container_of(cgroup_subsys_state(cgroup, blkio_subsys_id), - struct blkio_cgroup, css); + struct blkcg, css); } -EXPORT_SYMBOL_GPL(cgroup_to_blkio_cgroup); +EXPORT_SYMBOL_GPL(cgroup_to_blkcg); -static struct blkio_cgroup *task_blkio_cgroup(struct task_struct *tsk) +static struct blkcg *task_blkcg(struct task_struct *tsk) { return container_of(task_subsys_state(tsk, blkio_subsys_id), - struct blkio_cgroup, css); + struct blkcg, css); } -struct blkio_cgroup *bio_blkio_cgroup(struct bio *bio) +struct blkcg *bio_blkcg(struct bio *bio) { if (bio && bio->bi_css) - return container_of(bio->bi_css, struct blkio_cgroup, css); - return task_blkio_cgroup(current); + return container_of(bio->bi_css, struct blkcg, css); + return task_blkcg(current); } -EXPORT_SYMBOL_GPL(bio_blkio_cgroup); +EXPORT_SYMBOL_GPL(bio_blkcg); static bool blkcg_policy_enabled(struct request_queue *q, - const struct blkio_policy_type *pol) + const struct blkcg_policy *pol) { return pol && test_bit(pol->plid, q->blkcg_pols); } -static size_t blkg_pd_size(const struct blkio_policy_type *pol) +static size_t blkg_pd_size(const struct blkcg_policy *pol) { return sizeof(struct blkg_policy_data) + pol->pdata_size; } @@ -69,7 +69,7 @@ static size_t blkg_pd_size(const struct blkio_policy_type *pol) * * Free @blkg which may be partially allocated. */ -static void blkg_free(struct blkio_group *blkg) +static void blkg_free(struct blkcg_gq *blkg) { int i; @@ -77,14 +77,14 @@ static void blkg_free(struct blkio_group *blkg) return; for (i = 0; i < BLKCG_MAX_POLS; i++) { - struct blkio_policy_type *pol = blkio_policy[i]; + struct blkcg_policy *pol = blkcg_policy[i]; struct blkg_policy_data *pd = blkg->pd[i]; if (!pd) continue; - if (pol && pol->ops.blkio_exit_group_fn) - pol->ops.blkio_exit_group_fn(blkg); + if (pol && pol->ops.pd_exit_fn) + pol->ops.pd_exit_fn(blkg); kfree(pd); } @@ -99,10 +99,9 @@ static void blkg_free(struct blkio_group *blkg) * * Allocate a new blkg assocating @blkcg and @q. */ -static struct blkio_group *blkg_alloc(struct blkio_cgroup *blkcg, - struct request_queue *q) +static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q) { - struct blkio_group *blkg; + struct blkcg_gq *blkg; int i; /* alloc and init base part */ @@ -116,7 +115,7 @@ static struct blkio_group *blkg_alloc(struct blkio_cgroup *blkcg, blkg->refcnt = 1; for (i = 0; i < BLKCG_MAX_POLS; i++) { - struct blkio_policy_type *pol = blkio_policy[i]; + struct blkcg_policy *pol = blkcg_policy[i]; struct blkg_policy_data *pd; if (!blkcg_policy_enabled(q, pol)) @@ -135,19 +134,19 @@ static struct blkio_group *blkg_alloc(struct blkio_cgroup *blkcg, /* invoke per-policy init */ for (i = 0; i < BLKCG_MAX_POLS; i++) { - struct blkio_policy_type *pol = blkio_policy[i]; + struct blkcg_policy *pol = blkcg_policy[i]; if (blkcg_policy_enabled(blkg->q, pol)) - pol->ops.blkio_init_group_fn(blkg); + pol->ops.pd_init_fn(blkg); } return blkg; } -static struct blkio_group *__blkg_lookup(struct blkio_cgroup *blkcg, - struct request_queue *q) +static struct blkcg_gq *__blkg_lookup(struct blkcg *blkcg, + struct request_queue *q) { - struct blkio_group *blkg; + struct blkcg_gq *blkg; struct hlist_node *n; hlist_for_each_entry_rcu(blkg, n, &blkcg->blkg_list, blkcg_node) @@ -165,8 +164,7 @@ static struct blkio_group *__blkg_lookup(struct blkio_cgroup *blkcg, * under RCU read lock and is guaranteed to return %NULL if @q is bypassing * - see blk_queue_bypass_start() for details. */ -struct blkio_group *blkg_lookup(struct blkio_cgroup *blkcg, - struct request_queue *q) +struct blkcg_gq *blkg_lookup(struct blkcg *blkcg, struct request_queue *q) { WARN_ON_ONCE(!rcu_read_lock_held()); @@ -176,11 +174,11 @@ struct blkio_group *blkg_lookup(struct blkio_cgroup *blkcg, } EXPORT_SYMBOL_GPL(blkg_lookup); -static struct blkio_group *__blkg_lookup_create(struct blkio_cgroup *blkcg, - struct request_queue *q) +static struct blkcg_gq *__blkg_lookup_create(struct blkcg *blkcg, + struct request_queue *q) __releases(q->queue_lock) __acquires(q->queue_lock) { - struct blkio_group *blkg; + struct blkcg_gq *blkg; WARN_ON_ONCE(!rcu_read_lock_held()); lockdep_assert_held(q->queue_lock); @@ -213,8 +211,8 @@ out: return blkg; } -struct blkio_group *blkg_lookup_create(struct blkio_cgroup *blkcg, - struct request_queue *q) +struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg, + struct request_queue *q) { /* * This could be the first entry point of blkcg implementation and @@ -226,10 +224,10 @@ struct blkio_group *blkg_lookup_create(struct blkio_cgroup *blkcg, } EXPORT_SYMBOL_GPL(blkg_lookup_create); -static void blkg_destroy(struct blkio_group *blkg) +static void blkg_destroy(struct blkcg_gq *blkg) { struct request_queue *q = blkg->q; - struct blkio_cgroup *blkcg = blkg->blkcg; + struct blkcg *blkcg = blkg->blkcg; lockdep_assert_held(q->queue_lock); lockdep_assert_held(&blkcg->lock); @@ -255,12 +253,12 @@ static void blkg_destroy(struct blkio_group *blkg) */ static void blkg_destroy_all(struct request_queue *q) { - struct blkio_group *blkg, *n; + struct blkcg_gq *blkg, *n; lockdep_assert_held(q->queue_lock); list_for_each_entry_safe(blkg, n, &q->blkg_list, q_node) { - struct blkio_cgroup *blkcg = blkg->blkcg; + struct blkcg *blkcg = blkg->blkcg; spin_lock(&blkcg->lock); blkg_destroy(blkg); @@ -270,10 +268,10 @@ static void blkg_destroy_all(struct request_queue *q) static void blkg_rcu_free(struct rcu_head *rcu_head) { - blkg_free(container_of(rcu_head, struct blkio_group, rcu_head)); + blkg_free(container_of(rcu_head, struct blkcg_gq, rcu_head)); } -void __blkg_release(struct blkio_group *blkg) +void __blkg_release(struct blkcg_gq *blkg) { /* release the extra blkcg reference this blkg has been holding */ css_put(&blkg->blkcg->css); @@ -291,11 +289,11 @@ void __blkg_release(struct blkio_group *blkg) } EXPORT_SYMBOL_GPL(__blkg_release); -static int -blkiocg_reset_stats(struct cgroup *cgroup, struct cftype *cftype, u64 val) +static int blkcg_reset_stats(struct cgroup *cgroup, struct cftype *cftype, + u64 val) { - struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgroup); - struct blkio_group *blkg; + struct blkcg *blkcg = cgroup_to_blkcg(cgroup); + struct blkcg_gq *blkg; struct hlist_node *n; int i; @@ -309,11 +307,11 @@ blkiocg_reset_stats(struct cgroup *cgroup, struct cftype *cftype, u64 val) */ hlist_for_each_entry(blkg, n, &blkcg->blkg_list, blkcg_node) { for (i = 0; i < BLKCG_MAX_POLS; i++) { - struct blkio_policy_type *pol = blkio_policy[i]; + struct blkcg_policy *pol = blkcg_policy[i]; if (blkcg_policy_enabled(blkg->q, pol) && - pol->ops.blkio_reset_group_stats_fn) - pol->ops.blkio_reset_group_stats_fn(blkg); + pol->ops.pd_reset_stats_fn) + pol->ops.pd_reset_stats_fn(blkg); } } @@ -322,7 +320,7 @@ blkiocg_reset_stats(struct cgroup *cgroup, struct cftype *cftype, u64 val) return 0; } -static const char *blkg_dev_name(struct blkio_group *blkg) +static const char *blkg_dev_name(struct blkcg_gq *blkg) { /* some drivers (floppy) instantiate a queue w/o disk registered */ if (blkg->q->backing_dev_info.dev) @@ -347,12 +345,12 @@ static const char *blkg_dev_name(struct blkio_group *blkg) * This is to be used to construct print functions for * cftype->read_seq_string method. */ -void blkcg_print_blkgs(struct seq_file *sf, struct blkio_cgroup *blkcg, +void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg, u64 (*prfill)(struct seq_file *, void *, int), - const struct blkio_policy_type *pol, int data, + const struct blkcg_policy *pol, int data, bool show_total) { - struct blkio_group *blkg; + struct blkcg_gq *blkg; struct hlist_node *n; u64 total = 0; @@ -462,13 +460,12 @@ EXPORT_SYMBOL_GPL(blkg_prfill_rwstat); * value. This function returns with RCU read lock and queue lock held and * must be paired with blkg_conf_finish(). */ -int blkg_conf_prep(struct blkio_cgroup *blkcg, - const struct blkio_policy_type *pol, const char *input, - struct blkg_conf_ctx *ctx) +int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol, + const char *input, struct blkg_conf_ctx *ctx) __acquires(rcu) __acquires(disk->queue->queue_lock) { struct gendisk *disk; - struct blkio_group *blkg; + struct blkcg_gq *blkg; unsigned int major, minor; unsigned long long v; int part, ret; @@ -529,16 +526,16 @@ void blkg_conf_finish(struct blkg_conf_ctx *ctx) } EXPORT_SYMBOL_GPL(blkg_conf_finish); -struct cftype blkio_files[] = { +struct cftype blkcg_files[] = { { .name = "reset_stats", - .write_u64 = blkiocg_reset_stats, + .write_u64 = blkcg_reset_stats, }, { } /* terminate */ }; /** - * blkiocg_pre_destroy - cgroup pre_destroy callback + * blkcg_pre_destroy - cgroup pre_destroy callback * @cgroup: cgroup of interest * * This function is called when @cgroup is about to go away and responsible @@ -548,15 +545,15 @@ struct cftype blkio_files[] = { * * This is the blkcg counterpart of ioc_release_fn(). */ -static int blkiocg_pre_destroy(struct cgroup *cgroup) +static int blkcg_pre_destroy(struct cgroup *cgroup) { - struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgroup); + struct blkcg *blkcg = cgroup_to_blkcg(cgroup); spin_lock_irq(&blkcg->lock); while (!hlist_empty(&blkcg->blkg_list)) { - struct blkio_group *blkg = hlist_entry(blkcg->blkg_list.first, - struct blkio_group, blkcg_node); + struct blkcg_gq *blkg = hlist_entry(blkcg->blkg_list.first, + struct blkcg_gq, blkcg_node); struct request_queue *q = blkg->q; if (spin_trylock(q->queue_lock)) { @@ -573,22 +570,22 @@ static int blkiocg_pre_destroy(struct cgroup *cgroup) return 0; } -static void blkiocg_destroy(struct cgroup *cgroup) +static void blkcg_destroy(struct cgroup *cgroup) { - struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgroup); + struct blkcg *blkcg = cgroup_to_blkcg(cgroup); - if (blkcg != &blkio_root_cgroup) + if (blkcg != &blkcg_root) kfree(blkcg); } -static struct cgroup_subsys_state *blkiocg_create(struct cgroup *cgroup) +static struct cgroup_subsys_state *blkcg_create(struct cgroup *cgroup) { static atomic64_t id_seq = ATOMIC64_INIT(0); - struct blkio_cgroup *blkcg; + struct blkcg *blkcg; struct cgroup *parent = cgroup->parent; if (!parent) { - blkcg = &blkio_root_cgroup; + blkcg = &blkcg_root; goto done; } @@ -656,7 +653,7 @@ void blkcg_exit_queue(struct request_queue *q) * of the main cic data structures. For now we allow a task to change * its cgroup only if it's the only owner of its ioc. */ -static int blkiocg_can_attach(struct cgroup *cgrp, struct cgroup_taskset *tset) +static int blkcg_can_attach(struct cgroup *cgrp, struct cgroup_taskset *tset) { struct task_struct *task; struct io_context *ioc; @@ -677,12 +674,12 @@ static int blkiocg_can_attach(struct cgroup *cgrp, struct cgroup_taskset *tset) struct cgroup_subsys blkio_subsys = { .name = "blkio", - .create = blkiocg_create, - .can_attach = blkiocg_can_attach, - .pre_destroy = blkiocg_pre_destroy, - .destroy = blkiocg_destroy, + .create = blkcg_create, + .can_attach = blkcg_can_attach, + .pre_destroy = blkcg_pre_destroy, + .destroy = blkcg_destroy, .subsys_id = blkio_subsys_id, - .base_cftypes = blkio_files, + .base_cftypes = blkcg_files, .module = THIS_MODULE, }; EXPORT_SYMBOL_GPL(blkio_subsys); @@ -704,10 +701,10 @@ EXPORT_SYMBOL_GPL(blkio_subsys); * [un]registerations. Returns 0 on success, -errno on failure. */ int blkcg_activate_policy(struct request_queue *q, - const struct blkio_policy_type *pol) + const struct blkcg_policy *pol) { LIST_HEAD(pds); - struct blkio_group *blkg; + struct blkcg_gq *blkg; struct blkg_policy_data *pd, *n; int cnt = 0, ret; @@ -720,7 +717,7 @@ int blkcg_activate_policy(struct request_queue *q, spin_lock_irq(q->queue_lock); rcu_read_lock(); - blkg = __blkg_lookup_create(&blkio_root_cgroup, q); + blkg = __blkg_lookup_create(&blkcg_root, q); rcu_read_unlock(); if (IS_ERR(blkg)) { @@ -764,7 +761,7 @@ int blkcg_activate_policy(struct request_queue *q, blkg->pd[pol->plid] = pd; pd->blkg = blkg; - pol->ops.blkio_init_group_fn(blkg); + pol->ops.pd_init_fn(blkg); spin_unlock(&blkg->blkcg->lock); } @@ -790,9 +787,9 @@ EXPORT_SYMBOL_GPL(blkcg_activate_policy); * blkcg_activate_policy(). */ void blkcg_deactivate_policy(struct request_queue *q, - const struct blkio_policy_type *pol) + const struct blkcg_policy *pol) { - struct blkio_group *blkg; + struct blkcg_gq *blkg; if (!blkcg_policy_enabled(q, pol)) return; @@ -810,8 +807,8 @@ void blkcg_deactivate_policy(struct request_queue *q, /* grab blkcg lock too while removing @pd from @blkg */ spin_lock(&blkg->blkcg->lock); - if (pol->ops.blkio_exit_group_fn) - pol->ops.blkio_exit_group_fn(blkg); + if (pol->ops.pd_exit_fn) + pol->ops.pd_exit_fn(blkg); kfree(blkg->pd[pol->plid]); blkg->pd[pol->plid] = NULL; @@ -825,14 +822,13 @@ void blkcg_deactivate_policy(struct request_queue *q, EXPORT_SYMBOL_GPL(blkcg_deactivate_policy); /** - * blkio_policy_register - register a blkcg policy - * @blkiop: blkcg policy to register + * blkcg_policy_register - register a blkcg policy + * @pol: blkcg policy to register * - * Register @blkiop with blkcg core. Might sleep and @blkiop may be - * modified on successful registration. Returns 0 on success and -errno on - * failure. + * Register @pol with blkcg core. Might sleep and @pol may be modified on + * successful registration. Returns 0 on success and -errno on failure. */ -int blkio_policy_register(struct blkio_policy_type *blkiop) +int blkcg_policy_register(struct blkcg_policy *pol) { int i, ret; @@ -841,45 +837,45 @@ int blkio_policy_register(struct blkio_policy_type *blkiop) /* find an empty slot */ ret = -ENOSPC; for (i = 0; i < BLKCG_MAX_POLS; i++) - if (!blkio_policy[i]) + if (!blkcg_policy[i]) break; if (i >= BLKCG_MAX_POLS) goto out_unlock; /* register and update blkgs */ - blkiop->plid = i; - blkio_policy[i] = blkiop; + pol->plid = i; + blkcg_policy[i] = pol; /* everything is in place, add intf files for the new policy */ - if (blkiop->cftypes) - WARN_ON(cgroup_add_cftypes(&blkio_subsys, blkiop->cftypes)); + if (pol->cftypes) + WARN_ON(cgroup_add_cftypes(&blkio_subsys, pol->cftypes)); ret = 0; out_unlock: mutex_unlock(&blkcg_pol_mutex); return ret; } -EXPORT_SYMBOL_GPL(blkio_policy_register); +EXPORT_SYMBOL_GPL(blkcg_policy_register); /** - * blkiop_policy_unregister - unregister a blkcg policy - * @blkiop: blkcg policy to unregister + * blkcg_policy_unregister - unregister a blkcg policy + * @pol: blkcg policy to unregister * - * Undo blkio_policy_register(@blkiop). Might sleep. + * Undo blkcg_policy_register(@pol). Might sleep. */ -void blkio_policy_unregister(struct blkio_policy_type *blkiop) +void blkcg_policy_unregister(struct blkcg_policy *pol) { mutex_lock(&blkcg_pol_mutex); - if (WARN_ON(blkio_policy[blkiop->plid] != blkiop)) + if (WARN_ON(blkcg_policy[pol->plid] != pol)) goto out_unlock; /* kill the intf files first */ - if (blkiop->cftypes) - cgroup_rm_cftypes(&blkio_subsys, blkiop->cftypes); + if (pol->cftypes) + cgroup_rm_cftypes(&blkio_subsys, pol->cftypes); /* unregister and update blkgs */ - blkio_policy[blkiop->plid] = NULL; + blkcg_policy[pol->plid] = NULL; out_unlock: mutex_unlock(&blkcg_pol_mutex); } -EXPORT_SYMBOL_GPL(blkio_policy_unregister); +EXPORT_SYMBOL_GPL(blkcg_policy_unregister); diff --git a/block/blk-cgroup.h b/block/blk-cgroup.h index b347aa08d166..a443b84d2c16 100644 --- a/block/blk-cgroup.h +++ b/block/blk-cgroup.h @@ -37,7 +37,7 @@ enum blkg_rwstat_type { BLKG_RWSTAT_TOTAL = BLKG_RWSTAT_NR, }; -struct blkio_cgroup { +struct blkcg { struct cgroup_subsys_state css; spinlock_t lock; struct hlist_head blkg_list; @@ -45,7 +45,7 @@ struct blkio_cgroup { /* for policies to test whether associated blkcg has changed */ uint64_t id; - /* TODO: per-policy storage in blkio_cgroup */ + /* TODO: per-policy storage in blkcg */ unsigned int cfq_weight; /* belongs to cfq */ }; @@ -62,7 +62,7 @@ struct blkg_rwstat { /* per-blkg per-policy data */ struct blkg_policy_data { /* the blkg this per-policy data belongs to */ - struct blkio_group *blkg; + struct blkcg_gq *blkg; /* used during policy activation */ struct list_head alloc_node; @@ -71,12 +71,13 @@ struct blkg_policy_data { char pdata[] __aligned(__alignof__(unsigned long long)); }; -struct blkio_group { +/* association between a blk cgroup and a request queue */ +struct blkcg_gq { /* Pointer to the associated request_queue */ struct request_queue *q; struct list_head q_node; struct hlist_node blkcg_node; - struct blkio_cgroup *blkcg; + struct blkcg *blkcg; /* reference count */ int refcnt; @@ -85,18 +86,18 @@ struct blkio_group { struct rcu_head rcu_head; }; -typedef void (blkio_init_group_fn)(struct blkio_group *blkg); -typedef void (blkio_exit_group_fn)(struct blkio_group *blkg); -typedef void (blkio_reset_group_stats_fn)(struct blkio_group *blkg); +typedef void (blkcg_pol_init_pd_fn)(struct blkcg_gq *blkg); +typedef void (blkcg_pol_exit_pd_fn)(struct blkcg_gq *blkg); +typedef void (blkcg_pol_reset_pd_stats_fn)(struct blkcg_gq *blkg); -struct blkio_policy_ops { - blkio_init_group_fn *blkio_init_group_fn; - blkio_exit_group_fn *blkio_exit_group_fn; - blkio_reset_group_stats_fn *blkio_reset_group_stats_fn; +struct blkcg_policy_ops { + blkcg_pol_init_pd_fn *pd_init_fn; + blkcg_pol_exit_pd_fn *pd_exit_fn; + blkcg_pol_reset_pd_stats_fn *pd_reset_stats_fn; }; -struct blkio_policy_type { - struct blkio_policy_ops ops; +struct blkcg_policy { + struct blkcg_policy_ops ops; int plid; /* policy specific private data size */ size_t pdata_size; @@ -104,29 +105,28 @@ struct blkio_policy_type { struct cftype *cftypes; }; -extern struct blkio_cgroup blkio_root_cgroup; +extern struct blkcg blkcg_root; -struct blkio_cgroup *cgroup_to_blkio_cgroup(struct cgroup *cgroup); -struct blkio_cgroup *bio_blkio_cgroup(struct bio *bio); -struct blkio_group *blkg_lookup(struct blkio_cgroup *blkcg, - struct request_queue *q); -struct blkio_group *blkg_lookup_create(struct blkio_cgroup *blkcg, - struct request_queue *q); +struct blkcg *cgroup_to_blkcg(struct cgroup *cgroup); +struct blkcg *bio_blkcg(struct bio *bio); +struct blkcg_gq *blkg_lookup(struct blkcg *blkcg, struct request_queue *q); +struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg, + struct request_queue *q); int blkcg_init_queue(struct request_queue *q); void blkcg_drain_queue(struct request_queue *q); void blkcg_exit_queue(struct request_queue *q); /* Blkio controller policy registration */ -int blkio_policy_register(struct blkio_policy_type *); -void blkio_policy_unregister(struct blkio_policy_type *); +int blkcg_policy_register(struct blkcg_policy *pol); +void blkcg_policy_unregister(struct blkcg_policy *pol); int blkcg_activate_policy(struct request_queue *q, - const struct blkio_policy_type *pol); + const struct blkcg_policy *pol); void blkcg_deactivate_policy(struct request_queue *q, - const struct blkio_policy_type *pol); + const struct blkcg_policy *pol); -void blkcg_print_blkgs(struct seq_file *sf, struct blkio_cgroup *blkcg, +void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg, u64 (*prfill)(struct seq_file *, void *, int), - const struct blkio_policy_type *pol, int data, + const struct blkcg_policy *pol, int data, bool show_total); u64 __blkg_prfill_u64(struct seq_file *sf, void *pdata, u64 v); u64 __blkg_prfill_rwstat(struct seq_file *sf, void *pdata, @@ -136,13 +136,12 @@ u64 blkg_prfill_rwstat(struct seq_file *sf, void *pdata, int off); struct blkg_conf_ctx { struct gendisk *disk; - struct blkio_group *blkg; + struct blkcg_gq *blkg; u64 v; }; -int blkg_conf_prep(struct blkio_cgroup *blkcg, - const struct blkio_policy_type *pol, const char *input, - struct blkg_conf_ctx *ctx); +int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol, + const char *input, struct blkg_conf_ctx *ctx); void blkg_conf_finish(struct blkg_conf_ctx *ctx); @@ -153,8 +152,8 @@ void blkg_conf_finish(struct blkg_conf_ctx *ctx); * * Return pointer to private data associated with the @blkg-@pol pair. */ -static inline void *blkg_to_pdata(struct blkio_group *blkg, - struct blkio_policy_type *pol) +static inline void *blkg_to_pdata(struct blkcg_gq *blkg, + struct blkcg_policy *pol) { return blkg ? blkg->pd[pol->plid]->pdata : NULL; } @@ -165,7 +164,7 @@ static inline void *blkg_to_pdata(struct blkio_group *blkg, * * @pdata is policy private data. Determine the blkg it's associated with. */ -static inline struct blkio_group *pdata_to_blkg(void *pdata) +static inline struct blkcg_gq *pdata_to_blkg(void *pdata) { if (pdata) { struct blkg_policy_data *pd = @@ -183,7 +182,7 @@ static inline struct blkio_group *pdata_to_blkg(void *pdata) * * Format the path of the cgroup of @blkg into @buf. */ -static inline int blkg_path(struct blkio_group *blkg, char *buf, int buflen) +static inline int blkg_path(struct blkcg_gq *blkg, char *buf, int buflen) { int ret; @@ -201,14 +200,14 @@ static inline int blkg_path(struct blkio_group *blkg, char *buf, int buflen) * * The caller should be holding queue_lock and an existing reference. */ -static inline void blkg_get(struct blkio_group *blkg) +static inline void blkg_get(struct blkcg_gq *blkg) { lockdep_assert_held(blkg->q->queue_lock); WARN_ON_ONCE(!blkg->refcnt); blkg->refcnt++; } -void __blkg_release(struct blkio_group *blkg); +void __blkg_release(struct blkcg_gq *blkg); /** * blkg_put - put a blkg reference @@ -216,7 +215,7 @@ void __blkg_release(struct blkio_group *blkg); * * The caller should be holding queue_lock. */ -static inline void blkg_put(struct blkio_group *blkg) +static inline void blkg_put(struct blkcg_gq *blkg) { lockdep_assert_held(blkg->q->queue_lock); WARN_ON_ONCE(blkg->refcnt <= 0); @@ -343,32 +342,32 @@ static inline void blkg_rwstat_reset(struct blkg_rwstat *rwstat) struct cgroup; -struct blkio_group { +struct blkcg_gq { }; -struct blkio_policy_type { +struct blkcg_policy { }; -static inline struct blkio_cgroup *cgroup_to_blkio_cgroup(struct cgroup *cgroup) { return NULL; } -static inline struct blkio_cgroup *bio_blkio_cgroup(struct bio *bio) { return NULL; } -static inline struct blkio_group *blkg_lookup(struct blkio_cgroup *blkcg, void *key) { return NULL; } +static inline struct blkcg *cgroup_to_blkcg(struct cgroup *cgroup) { return NULL; } +static inline struct blkcg *bio_blkcg(struct bio *bio) { return NULL; } +static inline struct blkcg_gq *blkg_lookup(struct blkcg *blkcg, void *key) { return NULL; } static inline int blkcg_init_queue(struct request_queue *q) { return 0; } static inline void blkcg_drain_queue(struct request_queue *q) { } static inline void blkcg_exit_queue(struct request_queue *q) { } -static inline int blkio_policy_register(struct blkio_policy_type *blkiop) { return 0; } -static inline void blkio_policy_unregister(struct blkio_policy_type *blkiop) { } +static inline int blkcg_policy_register(struct blkcg_policy *pol) { return 0; } +static inline void blkcg_policy_unregister(struct blkcg_policy *pol) { } static inline int blkcg_activate_policy(struct request_queue *q, - const struct blkio_policy_type *pol) { return 0; } + const struct blkcg_policy *pol) { return 0; } static inline void blkcg_deactivate_policy(struct request_queue *q, - const struct blkio_policy_type *pol) { } - -static inline void *blkg_to_pdata(struct blkio_group *blkg, - struct blkio_policy_type *pol) { return NULL; } -static inline struct blkio_group *pdata_to_blkg(void *pdata, - struct blkio_policy_type *pol) { return NULL; } -static inline char *blkg_path(struct blkio_group *blkg) { return NULL; } -static inline void blkg_get(struct blkio_group *blkg) { } -static inline void blkg_put(struct blkio_group *blkg) { } + const struct blkcg_policy *pol) { } + +static inline void *blkg_to_pdata(struct blkcg_gq *blkg, + struct blkcg_policy *pol) { return NULL; } +static inline struct blkcg_gq *pdata_to_blkg(void *pdata, + struct blkcg_policy *pol) { return NULL; } +static inline char *blkg_path(struct blkcg_gq *blkg) { return NULL; } +static inline void blkg_get(struct blkcg_gq *blkg) { } +static inline void blkg_put(struct blkcg_gq *blkg) { } #endif /* CONFIG_BLK_CGROUP */ #endif /* _BLK_CGROUP_H */ diff --git a/block/blk-throttle.c b/block/blk-throttle.c index e9b7a47f6da0..00c7eff66ecf 100644 --- a/block/blk-throttle.c +++ b/block/blk-throttle.c @@ -21,7 +21,7 @@ static int throtl_quantum = 32; /* Throttling is performed over 100ms slice and after that slice is renewed */ static unsigned long throtl_slice = HZ/10; /* 100 ms */ -static struct blkio_policy_type blkio_policy_throtl; +static struct blkcg_policy blkcg_policy_throtl; /* A workqueue to queue throttle related work */ static struct workqueue_struct *kthrotld_workqueue; @@ -120,12 +120,12 @@ static LIST_HEAD(tg_stats_alloc_list); static void tg_stats_alloc_fn(struct work_struct *); static DECLARE_DELAYED_WORK(tg_stats_alloc_work, tg_stats_alloc_fn); -static inline struct throtl_grp *blkg_to_tg(struct blkio_group *blkg) +static inline struct throtl_grp *blkg_to_tg(struct blkcg_gq *blkg) { - return blkg_to_pdata(blkg, &blkio_policy_throtl); + return blkg_to_pdata(blkg, &blkcg_policy_throtl); } -static inline struct blkio_group *tg_to_blkg(struct throtl_grp *tg) +static inline struct blkcg_gq *tg_to_blkg(struct throtl_grp *tg) { return pdata_to_blkg(tg); } @@ -208,7 +208,7 @@ alloc_stats: goto alloc_stats; } -static void throtl_init_blkio_group(struct blkio_group *blkg) +static void throtl_pd_init(struct blkcg_gq *blkg) { struct throtl_grp *tg = blkg_to_tg(blkg); @@ -233,7 +233,7 @@ static void throtl_init_blkio_group(struct blkio_group *blkg) spin_unlock(&tg_stats_alloc_lock); } -static void throtl_exit_blkio_group(struct blkio_group *blkg) +static void throtl_pd_exit(struct blkcg_gq *blkg) { struct throtl_grp *tg = blkg_to_tg(blkg); @@ -244,7 +244,7 @@ static void throtl_exit_blkio_group(struct blkio_group *blkg) free_percpu(tg->stats_cpu); } -static void throtl_reset_group_stats(struct blkio_group *blkg) +static void throtl_pd_reset_stats(struct blkcg_gq *blkg) { struct throtl_grp *tg = blkg_to_tg(blkg); int cpu; @@ -260,33 +260,33 @@ static void throtl_reset_group_stats(struct blkio_group *blkg) } } -static struct -throtl_grp *throtl_lookup_tg(struct throtl_data *td, struct blkio_cgroup *blkcg) +static struct throtl_grp *throtl_lookup_tg(struct throtl_data *td, + struct blkcg *blkcg) { /* - * This is the common case when there are no blkio cgroups. - * Avoid lookup in this case + * This is the common case when there are no blkcgs. Avoid lookup + * in this case */ - if (blkcg == &blkio_root_cgroup) + if (blkcg == &blkcg_root) return td_root_tg(td); return blkg_to_tg(blkg_lookup(blkcg, td->queue)); } static struct throtl_grp *throtl_lookup_create_tg(struct throtl_data *td, - struct blkio_cgroup *blkcg) + struct blkcg *blkcg) { struct request_queue *q = td->queue; struct throtl_grp *tg = NULL; /* - * This is the common case when there are no blkio cgroups. - * Avoid lookup in this case + * This is the common case when there are no blkcgs. Avoid lookup + * in this case */ - if (blkcg == &blkio_root_cgroup) { + if (blkcg == &blkcg_root) { tg = td_root_tg(td); } else { - struct blkio_group *blkg; + struct blkcg_gq *blkg; blkg = blkg_lookup_create(blkcg, q); @@ -665,7 +665,7 @@ static bool tg_may_dispatch(struct throtl_data *td, struct throtl_grp *tg, return 0; } -static void throtl_update_dispatch_stats(struct blkio_group *blkg, u64 bytes, +static void throtl_update_dispatch_stats(struct blkcg_gq *blkg, u64 bytes, int rw) { struct throtl_grp *tg = blkg_to_tg(blkg); @@ -822,7 +822,7 @@ static int throtl_select_dispatch(struct throtl_data *td, struct bio_list *bl) static void throtl_process_limit_change(struct throtl_data *td) { struct request_queue *q = td->queue; - struct blkio_group *blkg, *n; + struct blkcg_gq *blkg, *n; if (!td->limits_changed) return; @@ -951,9 +951,9 @@ static u64 tg_prfill_cpu_rwstat(struct seq_file *sf, void *pdata, int off) static int tg_print_cpu_rwstat(struct cgroup *cgrp, struct cftype *cft, struct seq_file *sf) { - struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgrp); + struct blkcg *blkcg = cgroup_to_blkcg(cgrp); - blkcg_print_blkgs(sf, blkcg, tg_prfill_cpu_rwstat, &blkio_policy_throtl, + blkcg_print_blkgs(sf, blkcg, tg_prfill_cpu_rwstat, &blkcg_policy_throtl, cft->private, true); return 0; } @@ -979,29 +979,29 @@ static u64 tg_prfill_conf_uint(struct seq_file *sf, void *pdata, int off) static int tg_print_conf_u64(struct cgroup *cgrp, struct cftype *cft, struct seq_file *sf) { - blkcg_print_blkgs(sf, cgroup_to_blkio_cgroup(cgrp), tg_prfill_conf_u64, - &blkio_policy_throtl, cft->private, false); + blkcg_print_blkgs(sf, cgroup_to_blkcg(cgrp), tg_prfill_conf_u64, + &blkcg_policy_throtl, cft->private, false); return 0; } static int tg_print_conf_uint(struct cgroup *cgrp, struct cftype *cft, struct seq_file *sf) { - blkcg_print_blkgs(sf, cgroup_to_blkio_cgroup(cgrp), tg_prfill_conf_uint, - &blkio_policy_throtl, cft->private, false); + blkcg_print_blkgs(sf, cgroup_to_blkcg(cgrp), tg_prfill_conf_uint, + &blkcg_policy_throtl, cft->private, false); return 0; } static int tg_set_conf(struct cgroup *cgrp, struct cftype *cft, const char *buf, bool is_u64) { - struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgrp); + struct blkcg *blkcg = cgroup_to_blkcg(cgrp); struct blkg_conf_ctx ctx; struct throtl_grp *tg; struct throtl_data *td; int ret; - ret = blkg_conf_prep(blkcg, &blkio_policy_throtl, buf, &ctx); + ret = blkg_conf_prep(blkcg, &blkcg_policy_throtl, buf, &ctx); if (ret) return ret; @@ -1086,11 +1086,11 @@ static void throtl_shutdown_wq(struct request_queue *q) cancel_delayed_work_sync(&td->throtl_work); } -static struct blkio_policy_type blkio_policy_throtl = { +static struct blkcg_policy blkcg_policy_throtl = { .ops = { - .blkio_init_group_fn = throtl_init_blkio_group, - .blkio_exit_group_fn = throtl_exit_blkio_group, - .blkio_reset_group_stats_fn = throtl_reset_group_stats, + .pd_init_fn = throtl_pd_init, + .pd_exit_fn = throtl_pd_exit, + .pd_reset_stats_fn = throtl_pd_reset_stats, }, .pdata_size = sizeof(struct throtl_grp), .cftypes = throtl_files, @@ -1101,7 +1101,7 @@ bool blk_throtl_bio(struct request_queue *q, struct bio *bio) struct throtl_data *td = q->td; struct throtl_grp *tg; bool rw = bio_data_dir(bio), update_disptime = true; - struct blkio_cgroup *blkcg; + struct blkcg *blkcg; bool throttled = false; if (bio->bi_rw & REQ_THROTTLED) { @@ -1118,7 +1118,7 @@ bool blk_throtl_bio(struct request_queue *q, struct bio *bio) * just update the dispatch stats in lockless manner and return. */ rcu_read_lock(); - blkcg = bio_blkio_cgroup(bio); + blkcg = bio_blkcg(bio); tg = throtl_lookup_tg(td, blkcg); if (tg) { if (tg_no_rule_group(tg, rw)) { @@ -1243,7 +1243,7 @@ int blk_throtl_init(struct request_queue *q) td->queue = q; /* activate policy */ - ret = blkcg_activate_policy(q, &blkio_policy_throtl); + ret = blkcg_activate_policy(q, &blkcg_policy_throtl); if (ret) kfree(td); return ret; @@ -1253,7 +1253,7 @@ void blk_throtl_exit(struct request_queue *q) { BUG_ON(!q->td); throtl_shutdown_wq(q); - blkcg_deactivate_policy(q, &blkio_policy_throtl); + blkcg_deactivate_policy(q, &blkcg_policy_throtl); kfree(q->td); } @@ -1263,7 +1263,7 @@ static int __init throtl_init(void) if (!kthrotld_workqueue) panic("Failed to create kthrotld\n"); - return blkio_policy_register(&blkio_policy_throtl); + return blkcg_policy_register(&blkcg_policy_throtl); } module_init(throtl_init); diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c index 901286b5f5cb..792218281d91 100644 --- a/block/cfq-iosched.c +++ b/block/cfq-iosched.c @@ -17,7 +17,7 @@ #include "blk.h" #include "blk-cgroup.h" -static struct blkio_policy_type blkio_policy_cfq __maybe_unused; +static struct blkcg_policy blkcg_policy_cfq __maybe_unused; /* * tunables @@ -202,7 +202,7 @@ struct cfqg_stats { struct blkg_stat dequeue; /* total time spent waiting for it to be assigned a timeslice. */ struct blkg_stat group_wait_time; - /* time spent idling for this blkio_group */ + /* time spent idling for this blkcg_gq */ struct blkg_stat idle_time; /* total time with empty current active q with other requests queued */ struct blkg_stat empty_time; @@ -553,12 +553,12 @@ static inline void cfqg_stats_update_avg_queue_size(struct cfq_group *cfqg) { } #ifdef CONFIG_CFQ_GROUP_IOSCHED -static inline struct cfq_group *blkg_to_cfqg(struct blkio_group *blkg) +static inline struct cfq_group *blkg_to_cfqg(struct blkcg_gq *blkg) { - return blkg_to_pdata(blkg, &blkio_policy_cfq); + return blkg_to_pdata(blkg, &blkcg_policy_cfq); } -static inline struct blkio_group *cfqg_to_blkg(struct cfq_group *cfqg) +static inline struct blkcg_gq *cfqg_to_blkg(struct cfq_group *cfqg) { return pdata_to_blkg(cfqg); } @@ -637,7 +637,7 @@ static inline void cfqg_stats_update_completion(struct cfq_group *cfqg, io_start_time - start_time); } -static void cfqg_stats_reset(struct blkio_group *blkg) +static void cfq_pd_reset_stats(struct blkcg_gq *blkg) { struct cfq_group *cfqg = blkg_to_cfqg(blkg); struct cfqg_stats *stats = &cfqg->stats; @@ -662,8 +662,8 @@ static void cfqg_stats_reset(struct blkio_group *blkg) #else /* CONFIG_CFQ_GROUP_IOSCHED */ -static inline struct cfq_group *blkg_to_cfqg(struct blkio_group *blkg) { return NULL; } -static inline struct blkio_group *cfqg_to_blkg(struct cfq_group *cfqg) { return NULL; } +static inline struct cfq_group *blkg_to_cfqg(struct blkcg_gq *blkg) { return NULL; } +static inline struct blkcg_gq *cfqg_to_blkg(struct cfq_group *cfqg) { return NULL; } static inline void cfqg_get(struct cfq_group *cfqg) { } static inline void cfqg_put(struct cfq_group *cfqg) { } @@ -1331,7 +1331,7 @@ static void cfq_init_cfqg_base(struct cfq_group *cfqg) } #ifdef CONFIG_CFQ_GROUP_IOSCHED -static void cfq_init_blkio_group(struct blkio_group *blkg) +static void cfq_pd_init(struct blkcg_gq *blkg) { struct cfq_group *cfqg = blkg_to_cfqg(blkg); @@ -1344,16 +1344,16 @@ static void cfq_init_blkio_group(struct blkio_group *blkg) * be held. */ static struct cfq_group *cfq_lookup_create_cfqg(struct cfq_data *cfqd, - struct blkio_cgroup *blkcg) + struct blkcg *blkcg) { struct request_queue *q = cfqd->queue; struct cfq_group *cfqg = NULL; - /* avoid lookup for the common case where there's no blkio cgroup */ - if (blkcg == &blkio_root_cgroup) { + /* avoid lookup for the common case where there's no blkcg */ + if (blkcg == &blkcg_root) { cfqg = cfqd->root_group; } else { - struct blkio_group *blkg; + struct blkcg_gq *blkg; blkg = blkg_lookup_create(blkcg, q); if (!IS_ERR(blkg)) @@ -1386,8 +1386,8 @@ static u64 cfqg_prfill_weight_device(struct seq_file *sf, void *pdata, int off) static int cfqg_print_weight_device(struct cgroup *cgrp, struct cftype *cft, struct seq_file *sf) { - blkcg_print_blkgs(sf, cgroup_to_blkio_cgroup(cgrp), - cfqg_prfill_weight_device, &blkio_policy_cfq, 0, + blkcg_print_blkgs(sf, cgroup_to_blkcg(cgrp), + cfqg_prfill_weight_device, &blkcg_policy_cfq, 0, false); return 0; } @@ -1395,19 +1395,19 @@ static int cfqg_print_weight_device(struct cgroup *cgrp, struct cftype *cft, static int cfq_print_weight(struct cgroup *cgrp, struct cftype *cft, struct seq_file *sf) { - seq_printf(sf, "%u\n", cgroup_to_blkio_cgroup(cgrp)->cfq_weight); + seq_printf(sf, "%u\n", cgroup_to_blkcg(cgrp)->cfq_weight); return 0; } static int cfqg_set_weight_device(struct cgroup *cgrp, struct cftype *cft, const char *buf) { - struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgrp); + struct blkcg *blkcg = cgroup_to_blkcg(cgrp); struct blkg_conf_ctx ctx; struct cfq_group *cfqg; int ret; - ret = blkg_conf_prep(blkcg, &blkio_policy_cfq, buf, &ctx); + ret = blkg_conf_prep(blkcg, &blkcg_policy_cfq, buf, &ctx); if (ret) return ret; @@ -1425,8 +1425,8 @@ static int cfqg_set_weight_device(struct cgroup *cgrp, struct cftype *cft, static int cfq_set_weight(struct cgroup *cgrp, struct cftype *cft, u64 val) { - struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgrp); - struct blkio_group *blkg; + struct blkcg *blkcg = cgroup_to_blkcg(cgrp); + struct blkcg_gq *blkg; struct hlist_node *n; if (val < CFQ_WEIGHT_MIN || val > CFQ_WEIGHT_MAX) @@ -1449,9 +1449,9 @@ static int cfq_set_weight(struct cgroup *cgrp, struct cftype *cft, u64 val) static int cfqg_print_stat(struct cgroup *cgrp, struct cftype *cft, struct seq_file *sf) { - struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgrp); + struct blkcg *blkcg = cgroup_to_blkcg(cgrp); - blkcg_print_blkgs(sf, blkcg, blkg_prfill_stat, &blkio_policy_cfq, + blkcg_print_blkgs(sf, blkcg, blkg_prfill_stat, &blkcg_policy_cfq, cft->private, false); return 0; } @@ -1459,9 +1459,9 @@ static int cfqg_print_stat(struct cgroup *cgrp, struct cftype *cft, static int cfqg_print_rwstat(struct cgroup *cgrp, struct cftype *cft, struct seq_file *sf) { - struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgrp); + struct blkcg *blkcg = cgroup_to_blkcg(cgrp); - blkcg_print_blkgs(sf, blkcg, blkg_prfill_rwstat, &blkio_policy_cfq, + blkcg_print_blkgs(sf, blkcg, blkg_prfill_rwstat, &blkcg_policy_cfq, cft->private, true); return 0; } @@ -1485,10 +1485,10 @@ static u64 cfqg_prfill_avg_queue_size(struct seq_file *sf, void *pdata, int off) static int cfqg_print_avg_queue_size(struct cgroup *cgrp, struct cftype *cft, struct seq_file *sf) { - struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgrp); + struct blkcg *blkcg = cgroup_to_blkcg(cgrp); blkcg_print_blkgs(sf, blkcg, cfqg_prfill_avg_queue_size, - &blkio_policy_cfq, 0, false); + &blkcg_policy_cfq, 0, false); return 0; } #endif /* CONFIG_DEBUG_BLK_CGROUP */ @@ -1580,7 +1580,7 @@ static struct cftype cfq_blkcg_files[] = { }; #else /* GROUP_IOSCHED */ static struct cfq_group *cfq_lookup_create_cfqg(struct cfq_data *cfqd, - struct blkio_cgroup *blkcg) + struct blkcg *blkcg) { return cfqd->root_group; } @@ -3135,7 +3135,7 @@ static void check_blkcg_changed(struct cfq_io_cq *cic, struct bio *bio) uint64_t id; rcu_read_lock(); - id = bio_blkio_cgroup(bio)->id; + id = bio_blkcg(bio)->id; rcu_read_unlock(); /* @@ -3166,14 +3166,14 @@ static struct cfq_queue * cfq_find_alloc_queue(struct cfq_data *cfqd, bool is_sync, struct cfq_io_cq *cic, struct bio *bio, gfp_t gfp_mask) { - struct blkio_cgroup *blkcg; + struct blkcg *blkcg; struct cfq_queue *cfqq, *new_cfqq = NULL; struct cfq_group *cfqg; retry: rcu_read_lock(); - blkcg = bio_blkio_cgroup(bio); + blkcg = bio_blkcg(bio); cfqg = cfq_lookup_create_cfqg(cfqd, blkcg); cfqq = cic_to_cfqq(cic, is_sync); @@ -3944,14 +3944,14 @@ static void cfq_exit_queue(struct elevator_queue *e) #ifndef CONFIG_CFQ_GROUP_IOSCHED kfree(cfqd->root_group); #endif - blkcg_deactivate_policy(q, &blkio_policy_cfq); + blkcg_deactivate_policy(q, &blkcg_policy_cfq); kfree(cfqd); } static int cfq_init_queue(struct request_queue *q) { struct cfq_data *cfqd; - struct blkio_group *blkg __maybe_unused; + struct blkcg_gq *blkg __maybe_unused; int i, ret; cfqd = kmalloc_node(sizeof(*cfqd), GFP_KERNEL | __GFP_ZERO, q->node); @@ -3966,7 +3966,7 @@ static int cfq_init_queue(struct request_queue *q) /* Init root group and prefer root group over other groups by default */ #ifdef CONFIG_CFQ_GROUP_IOSCHED - ret = blkcg_activate_policy(q, &blkio_policy_cfq); + ret = blkcg_activate_policy(q, &blkcg_policy_cfq); if (ret) goto out_free; @@ -4156,10 +4156,10 @@ static struct elevator_type iosched_cfq = { }; #ifdef CONFIG_CFQ_GROUP_IOSCHED -static struct blkio_policy_type blkio_policy_cfq = { +static struct blkcg_policy blkcg_policy_cfq = { .ops = { - .blkio_init_group_fn = cfq_init_blkio_group, - .blkio_reset_group_stats_fn = cfqg_stats_reset, + .pd_init_fn = cfq_pd_init, + .pd_reset_stats_fn = cfq_pd_reset_stats, }, .pdata_size = sizeof(struct cfq_group), .cftypes = cfq_blkcg_files, @@ -4185,7 +4185,7 @@ static int __init cfq_init(void) cfq_group_idle = 0; #endif - ret = blkio_policy_register(&blkio_policy_cfq); + ret = blkcg_policy_register(&blkcg_policy_cfq); if (ret) return ret; @@ -4202,13 +4202,13 @@ static int __init cfq_init(void) err_free_pool: kmem_cache_destroy(cfq_pool); err_pol_unreg: - blkio_policy_unregister(&blkio_policy_cfq); + blkcg_policy_unregister(&blkcg_policy_cfq); return ret; } static void __exit cfq_exit(void) { - blkio_policy_unregister(&blkio_policy_cfq); + blkcg_policy_unregister(&blkcg_policy_cfq); elv_unregister(&iosched_cfq); kmem_cache_destroy(cfq_pool); } diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 68720ab275d4..af33fb1adfee 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -31,7 +31,7 @@ struct blk_trace; struct request; struct sg_io_hdr; struct bsg_job; -struct blkio_group; +struct blkcg_gq; #define BLKDEV_MIN_RQ 4 #define BLKDEV_MAX_RQ 128 /* Default maximum */ @@ -371,7 +371,7 @@ struct request_queue { struct list_head icq_list; #ifdef CONFIG_BLK_CGROUP DECLARE_BITMAP (blkcg_pols, BLKCG_MAX_POLS); - struct blkio_group *root_blkg; + struct blkcg_gq *root_blkg; struct list_head blkg_list; #endif -- cgit v1.2.3-59-g8ed1b