aboutsummaryrefslogtreecommitdiffstats
path: root/block
diff options
context:
space:
mode:
Diffstat (limited to 'block')
-rw-r--r--block/blk-cgroup.c52
-rw-r--r--block/blk-cgroup.h60
-rw-r--r--block/blk-throttle.c37
-rw-r--r--block/cfq-iosched.c46
4 files changed, 112 insertions, 83 deletions
diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c
index 997570329517..3d495528a765 100644
--- a/block/blk-cgroup.c
+++ b/block/blk-cgroup.c
@@ -58,11 +58,6 @@ static bool blkcg_policy_enabled(struct request_queue *q,
return pol && test_bit(pol->plid, q->blkcg_pols);
}
-static size_t blkg_pd_size(const struct blkcg_policy *pol)
-{
- return sizeof(struct blkg_policy_data) + pol->pdata_size;
-}
-
/**
* blkg_free - free a blkg
* @blkg: blkg to free
@@ -122,7 +117,7 @@ static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q)
continue;
/* alloc per-policy data and attach it to blkg */
- pd = kzalloc_node(blkg_pd_size(pol), GFP_ATOMIC, q->node);
+ pd = kzalloc_node(pol->pd_size, GFP_ATOMIC, q->node);
if (!pd) {
blkg_free(blkg);
return NULL;
@@ -346,7 +341,8 @@ static const char *blkg_dev_name(struct blkcg_gq *blkg)
* cftype->read_seq_string method.
*/
void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg,
- u64 (*prfill)(struct seq_file *, void *, int),
+ u64 (*prfill)(struct seq_file *,
+ struct blkg_policy_data *, int),
const struct blkcg_policy *pol, int data,
bool show_total)
{
@@ -357,7 +353,7 @@ void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg,
spin_lock_irq(&blkcg->lock);
hlist_for_each_entry(blkg, n, &blkcg->blkg_list, blkcg_node)
if (blkcg_policy_enabled(blkg->q, pol))
- total += prfill(sf, blkg->pd[pol->plid]->pdata, data);
+ total += prfill(sf, blkg->pd[pol->plid], data);
spin_unlock_irq(&blkcg->lock);
if (show_total)
@@ -368,14 +364,14 @@ EXPORT_SYMBOL_GPL(blkcg_print_blkgs);
/**
* __blkg_prfill_u64 - prfill helper for a single u64 value
* @sf: seq_file to print to
- * @pdata: policy private data of interest
+ * @pd: policy private data of interest
* @v: value to print
*
- * Print @v to @sf for the device assocaited with @pdata.
+ * Print @v to @sf for the device assocaited with @pd.
*/
-u64 __blkg_prfill_u64(struct seq_file *sf, void *pdata, u64 v)
+u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v)
{
- const char *dname = blkg_dev_name(pdata_to_blkg(pdata));
+ const char *dname = blkg_dev_name(pd->blkg);
if (!dname)
return 0;
@@ -388,12 +384,12 @@ EXPORT_SYMBOL_GPL(__blkg_prfill_u64);
/**
* __blkg_prfill_rwstat - prfill helper for a blkg_rwstat
* @sf: seq_file to print to
- * @pdata: policy private data of interest
+ * @pd: policy private data of interest
* @rwstat: rwstat to print
*
- * Print @rwstat to @sf for the device assocaited with @pdata.
+ * Print @rwstat to @sf for the device assocaited with @pd.
*/
-u64 __blkg_prfill_rwstat(struct seq_file *sf, void *pdata,
+u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
const struct blkg_rwstat *rwstat)
{
static const char *rwstr[] = {
@@ -402,7 +398,7 @@ u64 __blkg_prfill_rwstat(struct seq_file *sf, void *pdata,
[BLKG_RWSTAT_SYNC] = "Sync",
[BLKG_RWSTAT_ASYNC] = "Async",
};
- const char *dname = blkg_dev_name(pdata_to_blkg(pdata));
+ const char *dname = blkg_dev_name(pd->blkg);
u64 v;
int i;
@@ -421,30 +417,31 @@ u64 __blkg_prfill_rwstat(struct seq_file *sf, void *pdata,
/**
* blkg_prfill_stat - prfill callback for blkg_stat
* @sf: seq_file to print to
- * @pdata: policy private data of interest
- * @off: offset to the blkg_stat in @pdata
+ * @pd: policy private data of interest
+ * @off: offset to the blkg_stat in @pd
*
* prfill callback for printing a blkg_stat.
*/
-u64 blkg_prfill_stat(struct seq_file *sf, void *pdata, int off)
+u64 blkg_prfill_stat(struct seq_file *sf, struct blkg_policy_data *pd, int off)
{
- return __blkg_prfill_u64(sf, pdata, blkg_stat_read(pdata + off));
+ return __blkg_prfill_u64(sf, pd, blkg_stat_read((void *)pd + off));
}
EXPORT_SYMBOL_GPL(blkg_prfill_stat);
/**
* blkg_prfill_rwstat - prfill callback for blkg_rwstat
* @sf: seq_file to print to
- * @pdata: policy private data of interest
- * @off: offset to the blkg_rwstat in @pdata
+ * @pd: policy private data of interest
+ * @off: offset to the blkg_rwstat in @pd
*
* prfill callback for printing a blkg_rwstat.
*/
-u64 blkg_prfill_rwstat(struct seq_file *sf, void *pdata, int off)
+u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
+ int off)
{
- struct blkg_rwstat rwstat = blkg_rwstat_read(pdata + off);
+ struct blkg_rwstat rwstat = blkg_rwstat_read((void *)pd + off);
- return __blkg_prfill_rwstat(sf, pdata, &rwstat);
+ return __blkg_prfill_rwstat(sf, pd, &rwstat);
}
EXPORT_SYMBOL_GPL(blkg_prfill_rwstat);
@@ -733,7 +730,7 @@ int blkcg_activate_policy(struct request_queue *q,
/* allocate policy_data for all existing blkgs */
while (cnt--) {
- pd = kzalloc_node(blkg_pd_size(pol), GFP_KERNEL, q->node);
+ pd = kzalloc_node(pol->pd_size, GFP_KERNEL, q->node);
if (!pd) {
ret = -ENOMEM;
goto out_free;
@@ -832,6 +829,9 @@ int blkcg_policy_register(struct blkcg_policy *pol)
{
int i, ret;
+ if (WARN_ON(pol->pd_size < sizeof(struct blkg_policy_data)))
+ return -EINVAL;
+
mutex_lock(&blkcg_pol_mutex);
/* find an empty slot */
diff --git a/block/blk-cgroup.h b/block/blk-cgroup.h
index a443b84d2c16..18b021e1c05f 100644
--- a/block/blk-cgroup.h
+++ b/block/blk-cgroup.h
@@ -59,16 +59,25 @@ struct blkg_rwstat {
uint64_t cnt[BLKG_RWSTAT_NR];
};
-/* per-blkg per-policy data */
+/*
+ * A blkcg_gq (blkg) is association between a block cgroup (blkcg) and a
+ * request_queue (q). This is used by blkcg policies which need to track
+ * information per blkcg - q pair.
+ *
+ * There can be multiple active blkcg policies and each has its private
+ * data on each blkg, the size of which is determined by
+ * blkcg_policy->pd_size. blkcg core allocates and frees such areas
+ * together with blkg and invokes pd_init/exit_fn() methods.
+ *
+ * Such private data must embed struct blkg_policy_data (pd) at the
+ * beginning and pd_size can't be smaller than pd.
+ */
struct blkg_policy_data {
/* the blkg this per-policy data belongs to */
struct blkcg_gq *blkg;
/* used during policy activation */
struct list_head alloc_node;
-
- /* pol->pdata_size bytes of private data used by policy impl */
- char pdata[] __aligned(__alignof__(unsigned long long));
};
/* association between a blk cgroup and a request queue */
@@ -100,7 +109,7 @@ struct blkcg_policy {
struct blkcg_policy_ops ops;
int plid;
/* policy specific private data size */
- size_t pdata_size;
+ size_t pd_size;
/* cgroup files for the policy */
struct cftype *cftypes;
};
@@ -125,14 +134,16 @@ void blkcg_deactivate_policy(struct request_queue *q,
const struct blkcg_policy *pol);
void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg,
- u64 (*prfill)(struct seq_file *, void *, int),
+ u64 (*prfill)(struct seq_file *,
+ struct blkg_policy_data *, int),
const struct blkcg_policy *pol, int data,
bool show_total);
-u64 __blkg_prfill_u64(struct seq_file *sf, void *pdata, u64 v);
-u64 __blkg_prfill_rwstat(struct seq_file *sf, void *pdata,
+u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v);
+u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
const struct blkg_rwstat *rwstat);
-u64 blkg_prfill_stat(struct seq_file *sf, void *pdata, int off);
-u64 blkg_prfill_rwstat(struct seq_file *sf, void *pdata, int off);
+u64 blkg_prfill_stat(struct seq_file *sf, struct blkg_policy_data *pd, int off);
+u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
+ int off);
struct blkg_conf_ctx {
struct gendisk *disk;
@@ -152,26 +163,21 @@ void blkg_conf_finish(struct blkg_conf_ctx *ctx);
*
* Return pointer to private data associated with the @blkg-@pol pair.
*/
-static inline void *blkg_to_pdata(struct blkcg_gq *blkg,
- struct blkcg_policy *pol)
+static inline struct blkg_policy_data *blkg_to_pd(struct blkcg_gq *blkg,
+ struct blkcg_policy *pol)
{
- return blkg ? blkg->pd[pol->plid]->pdata : NULL;
+ return blkg ? blkg->pd[pol->plid] : NULL;
}
/**
* pdata_to_blkg - get blkg associated with policy private data
- * @pdata: policy private data of interest
+ * @pd: policy private data of interest
*
- * @pdata is policy private data. Determine the blkg it's associated with.
+ * @pd is policy private data. Determine the blkg it's associated with.
*/
-static inline struct blkcg_gq *pdata_to_blkg(void *pdata)
+static inline struct blkcg_gq *pd_to_blkg(struct blkg_policy_data *pd)
{
- if (pdata) {
- struct blkg_policy_data *pd =
- container_of(pdata, struct blkg_policy_data, pdata);
- return pd->blkg;
- }
- return NULL;
+ return pd ? pd->blkg : NULL;
}
/**
@@ -342,6 +348,9 @@ static inline void blkg_rwstat_reset(struct blkg_rwstat *rwstat)
struct cgroup;
+struct blkg_policy_data {
+};
+
struct blkcg_gq {
};
@@ -361,10 +370,9 @@ static inline int blkcg_activate_policy(struct request_queue *q,
static inline void blkcg_deactivate_policy(struct request_queue *q,
const struct blkcg_policy *pol) { }
-static inline void *blkg_to_pdata(struct blkcg_gq *blkg,
- struct blkcg_policy *pol) { return NULL; }
-static inline struct blkcg_gq *pdata_to_blkg(void *pdata,
- struct blkcg_policy *pol) { return NULL; }
+static inline struct blkg_policy_data *blkg_to_pd(struct blkcg_gq *blkg,
+ struct blkcg_policy *pol) { return NULL; }
+static inline struct blkcg_gq *pd_to_blkg(struct blkg_policy_data *pd) { return NULL; }
static inline char *blkg_path(struct blkcg_gq *blkg) { return NULL; }
static inline void blkg_get(struct blkcg_gq *blkg) { }
static inline void blkg_put(struct blkcg_gq *blkg) { }
diff --git a/block/blk-throttle.c b/block/blk-throttle.c
index 00c7eff66ecf..6a0a17a83862 100644
--- a/block/blk-throttle.c
+++ b/block/blk-throttle.c
@@ -49,6 +49,9 @@ struct tg_stats_cpu {
};
struct throtl_grp {
+ /* must be the first member */
+ struct blkg_policy_data pd;
+
/* active throtl group service_tree member */
struct rb_node rb_node;
@@ -120,14 +123,19 @@ static LIST_HEAD(tg_stats_alloc_list);
static void tg_stats_alloc_fn(struct work_struct *);
static DECLARE_DELAYED_WORK(tg_stats_alloc_work, tg_stats_alloc_fn);
+static inline struct throtl_grp *pd_to_tg(struct blkg_policy_data *pd)
+{
+ return pd ? container_of(pd, struct throtl_grp, pd) : NULL;
+}
+
static inline struct throtl_grp *blkg_to_tg(struct blkcg_gq *blkg)
{
- return blkg_to_pdata(blkg, &blkcg_policy_throtl);
+ return pd_to_tg(blkg_to_pd(blkg, &blkcg_policy_throtl));
}
static inline struct blkcg_gq *tg_to_blkg(struct throtl_grp *tg)
{
- return pdata_to_blkg(tg);
+ return pd_to_blkg(&tg->pd);
}
static inline struct throtl_grp *td_root_tg(struct throtl_data *td)
@@ -931,9 +939,10 @@ throtl_schedule_delayed_work(struct throtl_data *td, unsigned long delay)
}
}
-static u64 tg_prfill_cpu_rwstat(struct seq_file *sf, void *pdata, int off)
+static u64 tg_prfill_cpu_rwstat(struct seq_file *sf,
+ struct blkg_policy_data *pd, int off)
{
- struct throtl_grp *tg = pdata;
+ struct throtl_grp *tg = pd_to_tg(pd);
struct blkg_rwstat rwstat = { }, tmp;
int i, cpu;
@@ -945,7 +954,7 @@ static u64 tg_prfill_cpu_rwstat(struct seq_file *sf, void *pdata, int off)
rwstat.cnt[i] += tmp.cnt[i];
}
- return __blkg_prfill_rwstat(sf, pdata, &rwstat);
+ return __blkg_prfill_rwstat(sf, pd, &rwstat);
}
static int tg_print_cpu_rwstat(struct cgroup *cgrp, struct cftype *cft,
@@ -958,22 +967,26 @@ static int tg_print_cpu_rwstat(struct cgroup *cgrp, struct cftype *cft,
return 0;
}
-static u64 tg_prfill_conf_u64(struct seq_file *sf, void *pdata, int off)
+static u64 tg_prfill_conf_u64(struct seq_file *sf, struct blkg_policy_data *pd,
+ int off)
{
- u64 v = *(u64 *)(pdata + off);
+ struct throtl_grp *tg = pd_to_tg(pd);
+ u64 v = *(u64 *)((void *)tg + off);
if (v == -1)
return 0;
- return __blkg_prfill_u64(sf, pdata, v);
+ return __blkg_prfill_u64(sf, pd, v);
}
-static u64 tg_prfill_conf_uint(struct seq_file *sf, void *pdata, int off)
+static u64 tg_prfill_conf_uint(struct seq_file *sf, struct blkg_policy_data *pd,
+ int off)
{
- unsigned int v = *(unsigned int *)(pdata + off);
+ struct throtl_grp *tg = pd_to_tg(pd);
+ unsigned int v = *(unsigned int *)((void *)tg + off);
if (v == -1)
return 0;
- return __blkg_prfill_u64(sf, pdata, v);
+ return __blkg_prfill_u64(sf, pd, v);
}
static int tg_print_conf_u64(struct cgroup *cgrp, struct cftype *cft,
@@ -1092,7 +1105,7 @@ static struct blkcg_policy blkcg_policy_throtl = {
.pd_exit_fn = throtl_pd_exit,
.pd_reset_stats_fn = throtl_pd_reset_stats,
},
- .pdata_size = sizeof(struct throtl_grp),
+ .pd_size = sizeof(struct throtl_grp),
.cftypes = throtl_files,
};
diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c
index 792218281d91..7865cc38ea77 100644
--- a/block/cfq-iosched.c
+++ b/block/cfq-iosched.c
@@ -217,6 +217,9 @@ struct cfqg_stats {
/* This is per cgroup per device grouping structure */
struct cfq_group {
+ /* must be the first member */
+ struct blkg_policy_data pd;
+
/* group service_tree member */
struct rb_node rb_node;
@@ -409,6 +412,21 @@ CFQ_CFQQ_FNS(deep);
CFQ_CFQQ_FNS(wait_busy);
#undef CFQ_CFQQ_FNS
+static inline struct cfq_group *pd_to_cfqg(struct blkg_policy_data *pd)
+{
+ return pd ? container_of(pd, struct cfq_group, pd) : NULL;
+}
+
+static inline struct cfq_group *blkg_to_cfqg(struct blkcg_gq *blkg)
+{
+ return pd_to_cfqg(blkg_to_pd(blkg, &blkcg_policy_cfq));
+}
+
+static inline struct blkcg_gq *cfqg_to_blkg(struct cfq_group *cfqg)
+{
+ return pd_to_blkg(&cfqg->pd);
+}
+
#if defined(CONFIG_CFQ_GROUP_IOSCHED) && defined(CONFIG_DEBUG_BLK_CGROUP)
/* cfqg stats flags */
@@ -553,16 +571,6 @@ static inline void cfqg_stats_update_avg_queue_size(struct cfq_group *cfqg) { }
#ifdef CONFIG_CFQ_GROUP_IOSCHED
-static inline struct cfq_group *blkg_to_cfqg(struct blkcg_gq *blkg)
-{
- return blkg_to_pdata(blkg, &blkcg_policy_cfq);
-}
-
-static inline struct blkcg_gq *cfqg_to_blkg(struct cfq_group *cfqg)
-{
- return pdata_to_blkg(cfqg);
-}
-
static inline void cfqg_get(struct cfq_group *cfqg)
{
return blkg_get(cfqg_to_blkg(cfqg));
@@ -662,8 +670,6 @@ static void cfq_pd_reset_stats(struct blkcg_gq *blkg)
#else /* CONFIG_CFQ_GROUP_IOSCHED */
-static inline struct cfq_group *blkg_to_cfqg(struct blkcg_gq *blkg) { return NULL; }
-static inline struct blkcg_gq *cfqg_to_blkg(struct cfq_group *cfqg) { return NULL; }
static inline void cfqg_get(struct cfq_group *cfqg) { }
static inline void cfqg_put(struct cfq_group *cfqg) { }
@@ -1374,13 +1380,14 @@ static void cfq_link_cfqq_cfqg(struct cfq_queue *cfqq, struct cfq_group *cfqg)
cfqg_get(cfqg);
}
-static u64 cfqg_prfill_weight_device(struct seq_file *sf, void *pdata, int off)
+static u64 cfqg_prfill_weight_device(struct seq_file *sf,
+ struct blkg_policy_data *pd, int off)
{
- struct cfq_group *cfqg = pdata;
+ struct cfq_group *cfqg = pd_to_cfqg(pd);
if (!cfqg->dev_weight)
return 0;
- return __blkg_prfill_u64(sf, pdata, cfqg->dev_weight);
+ return __blkg_prfill_u64(sf, pd, cfqg->dev_weight);
}
static int cfqg_print_weight_device(struct cgroup *cgrp, struct cftype *cft,
@@ -1467,9 +1474,10 @@ static int cfqg_print_rwstat(struct cgroup *cgrp, struct cftype *cft,
}
#ifdef CONFIG_DEBUG_BLK_CGROUP
-static u64 cfqg_prfill_avg_queue_size(struct seq_file *sf, void *pdata, int off)
+static u64 cfqg_prfill_avg_queue_size(struct seq_file *sf,
+ struct blkg_policy_data *pd, int off)
{
- struct cfq_group *cfqg = pdata;
+ struct cfq_group *cfqg = pd_to_cfqg(pd);
u64 samples = blkg_stat_read(&cfqg->stats.avg_queue_size_samples);
u64 v = 0;
@@ -1477,7 +1485,7 @@ static u64 cfqg_prfill_avg_queue_size(struct seq_file *sf, void *pdata, int off)
v = blkg_stat_read(&cfqg->stats.avg_queue_size_sum);
do_div(v, samples);
}
- __blkg_prfill_u64(sf, pdata, v);
+ __blkg_prfill_u64(sf, pd, v);
return 0;
}
@@ -4161,7 +4169,7 @@ static struct blkcg_policy blkcg_policy_cfq = {
.pd_init_fn = cfq_pd_init,
.pd_reset_stats_fn = cfq_pd_reset_stats,
},
- .pdata_size = sizeof(struct cfq_group),
+ .pd_size = sizeof(struct cfq_group),
.cftypes = cfq_blkcg_files,
};
#endif
Copyright © 1996 – 2023 Jason A. Donenfeld. All Rights Reverse Engineered.