diff options
| -rw-r--r-- | block/blk-cgroup.c | 92 | ||||
| -rw-r--r-- | block/blk-cgroup.h | 40 | ||||
| -rw-r--r-- | block/cfq-iosched.c | 70 |
3 files changed, 173 insertions, 29 deletions
diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index 0ac817b750db..6e43fa355e71 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -9,6 +9,10 @@ * * Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com> * Nauman Rafique <nauman@google.com> + * + * For policy-specific per-blkcg data: + * Copyright (C) 2015 Paolo Valente <paolo.valente@unimore.it> + * Arianna Avanzini <avanzini.arianna@gmail.com> */ #include <linux/ioprio.h> #include <linux/kdev_t.h> @@ -26,8 +30,7 @@ static DEFINE_MUTEX(blkcg_pol_mutex); -struct blkcg blkcg_root = { .cfq_weight = 2 * CFQ_WEIGHT_DEFAULT, - .cfq_leaf_weight = 2 * CFQ_WEIGHT_DEFAULT, }; +struct blkcg blkcg_root; EXPORT_SYMBOL_GPL(blkcg_root); static struct blkcg_policy *blkcg_policy[BLKCG_MAX_POLS]; @@ -823,6 +826,8 @@ static struct cgroup_subsys_state * blkcg_css_alloc(struct cgroup_subsys_state *parent_css) { struct blkcg *blkcg; + struct cgroup_subsys_state *ret; + int i; if (!parent_css) { blkcg = &blkcg_root; @@ -830,17 +835,49 @@ blkcg_css_alloc(struct cgroup_subsys_state *parent_css) } blkcg = kzalloc(sizeof(*blkcg), GFP_KERNEL); - if (!blkcg) - return ERR_PTR(-ENOMEM); + if (!blkcg) { + ret = ERR_PTR(-ENOMEM); + goto free_blkcg; + } + + for (i = 0; i < BLKCG_MAX_POLS ; i++) { + struct blkcg_policy *pol = blkcg_policy[i]; + struct blkcg_policy_data *cpd; + + /* + * If the policy hasn't been attached yet, wait for it + * to be attached before doing anything else. Otherwise, + * check if the policy requires any specific per-cgroup + * data: if it does, allocate and initialize it. + */ + if (!pol || !pol->cpd_size) + continue; + + BUG_ON(blkcg->pd[i]); + cpd = kzalloc(pol->cpd_size, GFP_KERNEL); + if (!cpd) { + ret = ERR_PTR(-ENOMEM); + goto free_pd_blkcg; + } + blkcg->pd[i] = cpd; + cpd->plid = i; + pol->cpd_init_fn(blkcg); + } - blkcg->cfq_weight = CFQ_WEIGHT_DEFAULT; - blkcg->cfq_leaf_weight = CFQ_WEIGHT_DEFAULT; done: spin_lock_init(&blkcg->lock); INIT_RADIX_TREE(&blkcg->blkg_tree, GFP_ATOMIC); INIT_HLIST_HEAD(&blkcg->blkg_list); return &blkcg->css; + +free_pd_blkcg: + for (i--; i >= 0; i--) + kfree(blkcg->pd[i]); + +free_blkcg: + kfree(blkcg); + return ret; } /** @@ -958,8 +995,10 @@ int blkcg_activate_policy(struct request_queue *q, const struct blkcg_policy *pol) { LIST_HEAD(pds); + LIST_HEAD(cpds); struct blkcg_gq *blkg, *new_blkg; - struct blkg_policy_data *pd, *n; + struct blkg_policy_data *pd, *nd; + struct blkcg_policy_data *cpd, *cnd; int cnt = 0, ret; bool preloaded; @@ -1003,7 +1042,10 @@ int blkcg_activate_policy(struct request_queue *q, spin_unlock_irq(q->queue_lock); - /* allocate policy_data for all existing blkgs */ + /* + * Allocate per-blkg and per-blkcg policy data + * for all existing blkgs. + */ while (cnt--) { pd = kzalloc_node(pol->pd_size, GFP_KERNEL, q->node); if (!pd) { @@ -1011,26 +1053,50 @@ int blkcg_activate_policy(struct request_queue *q, goto out_free; } list_add_tail(&pd->alloc_node, &pds); + + if (!pol->cpd_size) + continue; + cpd = kzalloc_node(pol->cpd_size, GFP_KERNEL, q->node); + if (!cpd) { + ret = -ENOMEM; + goto out_free; + } + list_add_tail(&cpd->alloc_node, &cpds); } /* - * Install the allocated pds. With @q bypassing, no new blkg + * Install the allocated pds and cpds. With @q bypassing, no new blkg * should have been created while the queue lock was dropped. */ spin_lock_irq(q->queue_lock); list_for_each_entry(blkg, &q->blkg_list, q_node) { - if (WARN_ON(list_empty(&pds))) { + if (WARN_ON(list_empty(&pds)) || + WARN_ON(pol->cpd_size && list_empty(&cpds))) { /* umm... this shouldn't happen, just abort */ ret = -ENOMEM; goto out_unlock; } + cpd = list_first_entry(&cpds, struct blkcg_policy_data, + alloc_node); + list_del_init(&cpd->alloc_node); pd = list_first_entry(&pds, struct blkg_policy_data, alloc_node); list_del_init(&pd->alloc_node); /* grab blkcg lock too while installing @pd on @blkg */ spin_lock(&blkg->blkcg->lock); + if (!pol->cpd_size) + goto no_cpd; + if (!blkg->blkcg->pd[pol->plid]) { + /* Per-policy per-blkcg data */ + blkg->blkcg->pd[pol->plid] = cpd; + cpd->plid = pol->plid; + pol->cpd_init_fn(blkg->blkcg); + } else { /* must free it as it has already been extracted */ + kfree(cpd); + } +no_cpd: blkg->pd[pol->plid] = pd; pd->blkg = blkg; pd->plid = pol->plid; @@ -1045,8 +1111,10 @@ out_unlock: spin_unlock_irq(q->queue_lock); out_free: blk_queue_bypass_end(q); - list_for_each_entry_safe(pd, n, &pds, alloc_node) + list_for_each_entry_safe(pd, nd, &pds, alloc_node) kfree(pd); + list_for_each_entry_safe(cpd, cnd, &cpds, alloc_node) + kfree(cpd); return ret; } EXPORT_SYMBOL_GPL(blkcg_activate_policy); @@ -1087,6 +1155,8 @@ void blkcg_deactivate_policy(struct request_queue *q, kfree(blkg->pd[pol->plid]); blkg->pd[pol->plid] = NULL; + kfree(blkg->blkcg->pd[pol->plid]); + blkg->blkcg->pd[pol->plid] = NULL; spin_unlock(&blkg->blkcg->lock); } diff --git a/block/blk-cgroup.h b/block/blk-cgroup.h index c567865b5f1d..74296a78bba1 100644 --- a/block/blk-cgroup.h +++ b/block/blk-cgroup.h @@ -23,11 +23,6 @@ /* Max limits for throttle policy */ #define THROTL_IOPS_MAX UINT_MAX -/* CFQ specific, out here for blkcg->cfq_weight */ -#define CFQ_WEIGHT_MIN 10 -#define CFQ_WEIGHT_MAX 1000 -#define CFQ_WEIGHT_DEFAULT 500 - #ifdef CONFIG_BLK_CGROUP enum blkg_rwstat_type { @@ -50,9 +45,7 @@ struct blkcg { struct blkcg_gq *blkg_hint; struct hlist_head blkg_list; - /* TODO: per-policy storage in blkcg */ - unsigned int cfq_weight; /* belongs to cfq */ - unsigned int cfq_leaf_weight; + struct blkcg_policy_data *pd[BLKCG_MAX_POLS]; }; struct blkg_stat { @@ -87,6 +80,24 @@ struct blkg_policy_data { struct list_head alloc_node; }; +/* + * Policies that need to keep per-blkcg data which is independent + * from any request_queue associated to it must specify its size + * with the cpd_size field of the blkcg_policy structure and + * embed a blkcg_policy_data in it. blkcg core allocates + * policy-specific per-blkcg structures lazily the first time + * they are actually needed, so it handles them together with + * blkgs. cpd_init() is invoked to let each policy handle + * per-blkcg data. + */ +struct blkcg_policy_data { + /* the policy id this per-policy data belongs to */ + int plid; + + /* used during policy activation */ + struct list_head alloc_node; +}; + /* association between a blk cgroup and a request queue */ struct blkcg_gq { /* Pointer to the associated request_queue */ @@ -112,6 +123,7 @@ struct blkcg_gq { struct rcu_head rcu_head; }; +typedef void (blkcg_pol_init_cpd_fn)(const struct blkcg *blkcg); typedef void (blkcg_pol_init_pd_fn)(struct blkcg_gq *blkg); typedef void (blkcg_pol_online_pd_fn)(struct blkcg_gq *blkg); typedef void (blkcg_pol_offline_pd_fn)(struct blkcg_gq *blkg); @@ -122,10 +134,13 @@ struct blkcg_policy { int plid; /* policy specific private data size */ size_t pd_size; + /* policy specific per-blkcg data size */ + size_t cpd_size; /* cgroup files for the policy */ struct cftype *cftypes; /* operations */ + blkcg_pol_init_cpd_fn *cpd_init_fn; blkcg_pol_init_pd_fn *pd_init_fn; blkcg_pol_online_pd_fn *pd_online_fn; blkcg_pol_offline_pd_fn *pd_offline_fn; @@ -218,6 +233,12 @@ static inline struct blkg_policy_data *blkg_to_pd(struct blkcg_gq *blkg, return blkg ? blkg->pd[pol->plid] : NULL; } +static inline struct blkcg_policy_data *blkcg_to_cpd(struct blkcg *blkcg, + struct blkcg_policy *pol) +{ + return blkcg ? blkcg->pd[pol->plid] : NULL; +} + /** * pdata_to_blkg - get blkg associated with policy private data * @pd: policy private data of interest @@ -564,6 +585,9 @@ struct blkcg; struct blkg_policy_data { }; +struct blkcg_policy_data { +}; + struct blkcg_gq { }; diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c index 402be0139122..c808ad87652d 100644 --- a/block/cfq-iosched.c +++ b/block/cfq-iosched.c @@ -67,6 +67,11 @@ static struct kmem_cache *cfq_pool; #define sample_valid(samples) ((samples) > 80) #define rb_entry_cfqg(node) rb_entry((node), struct cfq_group, rb_node) +/* blkio-related constants */ +#define CFQ_WEIGHT_MIN 10 +#define CFQ_WEIGHT_MAX 1000 +#define CFQ_WEIGHT_DEFAULT 500 + struct cfq_ttime { unsigned long last_end_request; @@ -212,6 +217,15 @@ struct cfqg_stats { #endif /* CONFIG_CFQ_GROUP_IOSCHED */ }; +/* Per-cgroup data */ +struct cfq_group_data { + /* must be the first member */ + struct blkcg_policy_data pd; + + unsigned int weight; + unsigned int leaf_weight; +}; + /* This is per cgroup per device grouping structure */ struct cfq_group { /* must be the first member */ @@ -451,6 +465,12 @@ static inline struct cfq_group *pd_to_cfqg(struct blkg_policy_data *pd) return pd ? container_of(pd, struct cfq_group, pd) : NULL; } +static struct cfq_group_data +*cpd_to_cfqgd(struct blkcg_policy_data *cpd) +{ + return cpd ? container_of(cpd, struct cfq_group_data, pd) : NULL; +} + static inline struct blkcg_gq *cfqg_to_blkg(struct cfq_group *cfqg) { return pd_to_blkg(&cfqg->pd); @@ -607,6 +627,11 @@ static inline struct cfq_group *blkg_to_cfqg(struct blkcg_gq *blkg) return pd_to_cfqg(blkg_to_pd(blkg, &blkcg_policy_cfq)); } +static struct cfq_group_data *blkcg_to_cfqgd(struct blkcg *blkcg) +{ + return cpd_to_cfqgd(blkcg_to_cpd(blkcg, &blkcg_policy_cfq)); +} + static inline struct cfq_group *cfqg_parent(struct cfq_group *cfqg) { struct blkcg_gq *pblkg = cfqg_to_blkg(cfqg)->parent; @@ -1544,13 +1569,28 @@ static void cfqg_stats_init(struct cfqg_stats *stats) #endif } +static void cfq_cpd_init(const struct blkcg *blkcg) +{ + struct cfq_group_data *cgd = + cpd_to_cfqgd(blkcg->pd[blkcg_policy_cfq.plid]); + + if (blkcg == &blkcg_root) { + cgd->weight = 2 * CFQ_WEIGHT_DEFAULT; + cgd->leaf_weight = 2 * CFQ_WEIGHT_DEFAULT; + } else { + cgd->weight = CFQ_WEIGHT_DEFAULT; + cgd->leaf_weight = CFQ_WEIGHT_DEFAULT; + } +} + static void cfq_pd_init(struct blkcg_gq *blkg) { struct cfq_group *cfqg = blkg_to_cfqg(blkg); + struct cfq_group_data *cgd = blkcg_to_cfqgd(blkg->blkcg); cfq_init_cfqg_base(cfqg); - cfqg->weight = blkg->blkcg->cfq_weight; - cfqg->leaf_weight = blkg->blkcg->cfq_leaf_weight; + cfqg->weight = cgd->weight; + cfqg->leaf_weight = cgd->leaf_weight; cfqg_stats_init(&cfqg->stats); cfqg_stats_init(&cfqg->dead_stats); } @@ -1673,13 +1713,17 @@ static int cfqg_print_leaf_weight_device(struct seq_file *sf, void *v) static int cfq_print_weight(struct seq_file *sf, void *v) { - seq_printf(sf, "%u\n", css_to_blkcg(seq_css(sf))->cfq_weight); + struct blkcg *blkcg = css_to_blkcg(seq_css(sf)); + + seq_printf(sf, "%u\n", blkcg_to_cfqgd(blkcg)->weight); return 0; } static int cfq_print_leaf_weight(struct seq_file *sf, void *v) { - seq_printf(sf, "%u\n", css_to_blkcg(seq_css(sf))->cfq_leaf_weight); + struct blkcg *blkcg = css_to_blkcg(seq_css(sf)); + + seq_printf(sf, "%u\n", blkcg_to_cfqgd(blkcg)->leaf_weight); return 0; } @@ -1690,6 +1734,7 @@ static ssize_t __cfqg_set_weight_device(struct kernfs_open_file *of, struct blkcg *blkcg = css_to_blkcg(of_css(of)); struct blkg_conf_ctx ctx; struct cfq_group *cfqg; + struct cfq_group_data *cfqgd; int ret; ret = blkg_conf_prep(blkcg, &blkcg_policy_cfq, buf, &ctx); @@ -1698,13 +1743,14 @@ static ssize_t __cfqg_set_weight_device(struct kernfs_open_file *of, ret = -EINVAL; cfqg = blkg_to_cfqg(ctx.blkg); + cfqgd = blkcg_to_cfqgd(blkcg); if (!ctx.v || (ctx.v >= CFQ_WEIGHT_MIN && ctx.v <= CFQ_WEIGHT_MAX)) { if (!is_leaf_weight) { cfqg->dev_weight = ctx.v; - cfqg->new_weight = ctx.v ?: blkcg->cfq_weight; + cfqg->new_weight = ctx.v ?: cfqgd->weight; } else { cfqg->dev_leaf_weight = ctx.v; - cfqg->new_leaf_weight = ctx.v ?: blkcg->cfq_leaf_weight; + cfqg->new_leaf_weight = ctx.v ?: cfqgd->leaf_weight; } ret = 0; } @@ -1730,16 +1776,18 @@ static int __cfq_set_weight(struct cgroup_subsys_state *css, struct cftype *cft, { struct blkcg *blkcg = css_to_blkcg(css); struct blkcg_gq *blkg; + struct cfq_group_data *cfqgd; if (val < CFQ_WEIGHT_MIN || val > CFQ_WEIGHT_MAX) return -EINVAL; spin_lock_irq(&blkcg->lock); + cfqgd = blkcg_to_cfqgd(blkcg); if (!is_leaf_weight) - blkcg->cfq_weight = val; + cfqgd->weight = val; else - blkcg->cfq_leaf_weight = val; + cfqgd->leaf_weight = val; hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) { struct cfq_group *cfqg = blkg_to_cfqg(blkg); @@ -1749,10 +1797,10 @@ static int __cfq_set_weight(struct cgroup_subsys_state *css, struct cftype *cft, if (!is_leaf_weight) { if (!cfqg->dev_weight) - cfqg->new_weight = blkcg->cfq_weight; + cfqg->new_weight = cfqgd->weight; } else { if (!cfqg->dev_leaf_weight) - cfqg->new_leaf_weight = blkcg->cfq_leaf_weight; + cfqg->new_leaf_weight = cfqgd->leaf_weight; } } @@ -4603,8 +4651,10 @@ static struct elevator_type iosched_cfq = { #ifdef CONFIG_CFQ_GROUP_IOSCHED static struct blkcg_policy blkcg_policy_cfq = { .pd_size = sizeof(struct cfq_group), + .cpd_size = sizeof(struct cfq_group_data), .cftypes = cfq_blkcg_files, + .cpd_init_fn = cfq_cpd_init, .pd_init_fn = cfq_pd_init, .pd_offline_fn = cfq_pd_offline, .pd_reset_stats_fn = cfq_pd_reset_stats, |