From eb571eeade2598635f813b3284d02c13a380301e Mon Sep 17 00:00:00 2001 From: Joe Lawrence Date: Wed, 2 Jul 2014 15:35:16 -0400 Subject: block,scsi: verify return pointer from blk_get_request The blk-core dead queue checks introduce an error scenario to blk_get_request that returns NULL if the request queue has been shutdown. This affects the behavior for __GFP_WAIT callers, who should verify the return value before dereferencing. Signed-off-by: Joe Lawrence Acked-by: Jiri Kosina [for pktdvd] Reviewed-by: Jeff Moyer Signed-off-by: Jens Axboe --- block/scsi_ioctl.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'block') diff --git a/block/scsi_ioctl.c b/block/scsi_ioctl.c index 51bf5155ee75..29d056782833 100644 --- a/block/scsi_ioctl.c +++ b/block/scsi_ioctl.c @@ -448,6 +448,10 @@ int sg_scsi_ioctl(struct request_queue *q, struct gendisk *disk, fmode_t mode, } rq = blk_get_request(q, in_len ? WRITE : READ, __GFP_WAIT); + if (!rq) { + err = -ENODEV; + goto error_free_buffer; + } cmdlen = COMMAND_SIZE(opcode); @@ -520,8 +524,9 @@ out: } error: - kfree(buffer); blk_put_request(rq); +error_free_buffer: + kfree(buffer); return err; } EXPORT_SYMBOL_GPL(sg_scsi_ioctl); @@ -534,6 +539,8 @@ static int __blk_send_generic(struct request_queue *q, struct gendisk *bd_disk, int err; rq = blk_get_request(q, WRITE, __GFP_WAIT); + if (!rq) + return -ENODEV; blk_rq_set_block_pc(rq); rq->timeout = BLK_DEFAULT_SG_TIMEOUT; rq->cmd[0] = cmd; -- cgit v1.2.3 From a492f075450f3ba87de36e5ffe92a9d0c7af9723 Mon Sep 17 00:00:00 2001 From: Joe Lawrence Date: Thu, 28 Aug 2014 08:15:21 -0600 Subject: block,scsi: fixup blk_get_request dead queue scenarios The blk_get_request function may fail in low-memory conditions or during device removal (even if __GFP_WAIT is set). To distinguish between these errors, modify the blk_get_request call stack to return the appropriate ERR_PTR. Verify that all callers check the return status and consider IS_ERR instead of a simple NULL pointer check. For consistency, make a similar change to the blk_mq_alloc_request leg of blk_get_request. It may fail if the queue is dead, or the caller was unwilling to wait. Signed-off-by: Joe Lawrence Acked-by: Jiri Kosina [for pktdvd] Acked-by: Boaz Harrosh [for osd] Reviewed-by: Jeff Moyer Signed-off-by: Jens Axboe --- block/blk-core.c | 34 +++++++++++++++++----------------- block/blk-mq.c | 8 ++++++-- block/bsg.c | 8 ++++---- block/scsi_ioctl.c | 12 ++++++------ 4 files changed, 33 insertions(+), 29 deletions(-) (limited to 'block') diff --git a/block/blk-core.c b/block/blk-core.c index c359d72e9d76..93603e6ff479 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -933,9 +933,9 @@ static struct io_context *rq_ioc(struct bio *bio) * Get a free request from @q. This function may fail under memory * pressure or if @q is dead. * - * Must be callled with @q->queue_lock held and, - * Returns %NULL on failure, with @q->queue_lock held. - * Returns !%NULL on success, with @q->queue_lock *not held*. + * Must be called with @q->queue_lock held and, + * Returns ERR_PTR on failure, with @q->queue_lock held. + * Returns request pointer on success, with @q->queue_lock *not held*. */ static struct request *__get_request(struct request_list *rl, int rw_flags, struct bio *bio, gfp_t gfp_mask) @@ -949,7 +949,7 @@ static struct request *__get_request(struct request_list *rl, int rw_flags, int may_queue; if (unlikely(blk_queue_dying(q))) - return NULL; + return ERR_PTR(-ENODEV); may_queue = elv_may_queue(q, rw_flags); if (may_queue == ELV_MQUEUE_NO) @@ -974,7 +974,7 @@ static struct request *__get_request(struct request_list *rl, int rw_flags, * process is not a "batcher", and not * exempted by the IO scheduler */ - return NULL; + return ERR_PTR(-ENOMEM); } } } @@ -992,7 +992,7 @@ static struct request *__get_request(struct request_list *rl, int rw_flags, * allocated with any setting of ->nr_requests */ if (rl->count[is_sync] >= (3 * q->nr_requests / 2)) - return NULL; + return ERR_PTR(-ENOMEM); q->nr_rqs[is_sync]++; rl->count[is_sync]++; @@ -1097,7 +1097,7 @@ fail_alloc: rq_starved: if (unlikely(rl->count[is_sync] == 0)) rl->starved[is_sync] = 1; - return NULL; + return ERR_PTR(-ENOMEM); } /** @@ -1110,9 +1110,9 @@ rq_starved: * Get a free request from @q. If %__GFP_WAIT is set in @gfp_mask, this * function keeps retrying under memory pressure and fails iff @q is dead. * - * Must be callled with @q->queue_lock held and, - * Returns %NULL on failure, with @q->queue_lock held. - * Returns !%NULL on success, with @q->queue_lock *not held*. + * Must be called with @q->queue_lock held and, + * Returns ERR_PTR on failure, with @q->queue_lock held. + * Returns request pointer on success, with @q->queue_lock *not held*. */ static struct request *get_request(struct request_queue *q, int rw_flags, struct bio *bio, gfp_t gfp_mask) @@ -1125,12 +1125,12 @@ static struct request *get_request(struct request_queue *q, int rw_flags, rl = blk_get_rl(q, bio); /* transferred to @rq on success */ retry: rq = __get_request(rl, rw_flags, bio, gfp_mask); - if (rq) + if (!IS_ERR(rq)) return rq; if (!(gfp_mask & __GFP_WAIT) || unlikely(blk_queue_dying(q))) { blk_put_rl(rl); - return NULL; + return rq; } /* wait on @rl and retry */ @@ -1167,7 +1167,7 @@ static struct request *blk_old_get_request(struct request_queue *q, int rw, spin_lock_irq(q->queue_lock); rq = get_request(q, rw, NULL, gfp_mask); - if (!rq) + if (IS_ERR(rq)) spin_unlock_irq(q->queue_lock); /* q->queue_lock is unlocked at this point */ @@ -1219,8 +1219,8 @@ struct request *blk_make_request(struct request_queue *q, struct bio *bio, { struct request *rq = blk_get_request(q, bio_data_dir(bio), gfp_mask); - if (unlikely(!rq)) - return ERR_PTR(-ENOMEM); + if (IS_ERR(rq)) + return rq; blk_rq_set_block_pc(rq); @@ -1615,8 +1615,8 @@ get_rq: * Returns with the queue unlocked. */ req = get_request(q, rw_flags, bio, GFP_NOIO); - if (unlikely(!req)) { - bio_endio(bio, -ENODEV); /* @q is dead */ + if (IS_ERR(req)) { + bio_endio(bio, PTR_ERR(req)); /* @q is dead */ goto out_unlock; } diff --git a/block/blk-mq.c b/block/blk-mq.c index 5189cb1e478a..940aa8a34b70 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -218,9 +218,11 @@ struct request *blk_mq_alloc_request(struct request_queue *q, int rw, gfp_t gfp, struct blk_mq_hw_ctx *hctx; struct request *rq; struct blk_mq_alloc_data alloc_data; + int ret; - if (blk_mq_queue_enter(q)) - return NULL; + ret = blk_mq_queue_enter(q); + if (ret) + return ERR_PTR(ret); ctx = blk_mq_get_ctx(q); hctx = q->mq_ops->map_queue(q, ctx->cpu); @@ -240,6 +242,8 @@ struct request *blk_mq_alloc_request(struct request_queue *q, int rw, gfp_t gfp, ctx = alloc_data.ctx; } blk_mq_put_ctx(ctx); + if (!rq) + return ERR_PTR(-EWOULDBLOCK); return rq; } EXPORT_SYMBOL(blk_mq_alloc_request); diff --git a/block/bsg.c b/block/bsg.c index ff46addde5d8..73c78fd12cc1 100644 --- a/block/bsg.c +++ b/block/bsg.c @@ -270,8 +270,8 @@ bsg_map_hdr(struct bsg_device *bd, struct sg_io_v4 *hdr, fmode_t has_write_perm, * map scatter-gather elements separately and string them to request */ rq = blk_get_request(q, rw, GFP_KERNEL); - if (!rq) - return ERR_PTR(-ENOMEM); + if (IS_ERR(rq)) + return rq; blk_rq_set_block_pc(rq); ret = blk_fill_sgv4_hdr_rq(q, rq, hdr, bd, has_write_perm); @@ -285,8 +285,8 @@ bsg_map_hdr(struct bsg_device *bd, struct sg_io_v4 *hdr, fmode_t has_write_perm, } next_rq = blk_get_request(q, READ, GFP_KERNEL); - if (!next_rq) { - ret = -ENOMEM; + if (IS_ERR(next_rq)) { + ret = PTR_ERR(next_rq); goto out; } rq->next_rq = next_rq; diff --git a/block/scsi_ioctl.c b/block/scsi_ioctl.c index 29d056782833..a8b0d0208448 100644 --- a/block/scsi_ioctl.c +++ b/block/scsi_ioctl.c @@ -318,8 +318,8 @@ static int sg_io(struct request_queue *q, struct gendisk *bd_disk, at_head = 1; rq = blk_get_request(q, writing ? WRITE : READ, GFP_KERNEL); - if (!rq) - return -ENOMEM; + if (IS_ERR(rq)) + return PTR_ERR(rq); blk_rq_set_block_pc(rq); if (blk_fill_sghdr_rq(q, rq, hdr, mode)) { @@ -448,8 +448,8 @@ int sg_scsi_ioctl(struct request_queue *q, struct gendisk *disk, fmode_t mode, } rq = blk_get_request(q, in_len ? WRITE : READ, __GFP_WAIT); - if (!rq) { - err = -ENODEV; + if (IS_ERR(rq)) { + err = PTR_ERR(rq); goto error_free_buffer; } @@ -539,8 +539,8 @@ static int __blk_send_generic(struct request_queue *q, struct gendisk *bd_disk, int err; rq = blk_get_request(q, WRITE, __GFP_WAIT); - if (!rq) - return -ENODEV; + if (IS_ERR(rq)) + return PTR_ERR(rq); blk_rq_set_block_pc(rq); rq->timeout = BLK_DEFAULT_SG_TIMEOUT; rq->cmd[0] = cmd; -- cgit v1.2.3 From 55872c5a3c01f0fe7b5298d19e24e237f5b5ff06 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 28 Aug 2014 15:05:40 -0600 Subject: bsg: fix potential error pointer dereference Dan writes: block/bsg.c:327 bsg_map_hdr() error: 'next_rq' dereferencing possible ERR_PTR(). Fix this by setting next_rq to NULL, for the case where it can be != NULL but an error pointer. Reported-by: Dan Carpenter Signed-off-by: Jens Axboe --- block/bsg.c | 1 + 1 file changed, 1 insertion(+) (limited to 'block') diff --git a/block/bsg.c b/block/bsg.c index 73c78fd12cc1..276e869e686c 100644 --- a/block/bsg.c +++ b/block/bsg.c @@ -287,6 +287,7 @@ bsg_map_hdr(struct bsg_device *bd, struct sg_io_v4 *hdr, fmode_t has_write_perm, next_rq = blk_get_request(q, READ, GFP_KERNEL); if (IS_ERR(next_rq)) { ret = PTR_ERR(next_rq); + next_rq = NULL; goto out; } rq->next_rq = next_rq; -- cgit v1.2.3 From f4da80727cfbc3590d95ff17ef8db96e6f1483a4 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 8 Sep 2014 08:15:20 +0900 Subject: blkcg: remove blkcg->id blkcg->id is a unique id given to each blkcg; however, the cgroup_subsys_state which each blkcg embeds already has ->serial_nr which can be used for the same purpose. Drop blkcg->id and replace its uses with blkcg->css.serial_nr. Rename cfq_cgroup->blkcg_id to ->blkcg_serial_nr and @id in check_blkcg_changed() to @serial_nr for consistency. Signed-off-by: Tejun Heo Acked-by: Vivek Goyal Signed-off-by: Jens Axboe --- block/blk-cgroup.c | 2 -- block/blk-cgroup.h | 3 --- block/cfq-iosched.c | 10 +++++----- 3 files changed, 5 insertions(+), 10 deletions(-) (limited to 'block') diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index e17da947f6bd..0ac817b750db 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -822,7 +822,6 @@ static void blkcg_css_free(struct cgroup_subsys_state *css) static struct cgroup_subsys_state * blkcg_css_alloc(struct cgroup_subsys_state *parent_css) { - static atomic64_t id_seq = ATOMIC64_INIT(0); struct blkcg *blkcg; if (!parent_css) { @@ -836,7 +835,6 @@ blkcg_css_alloc(struct cgroup_subsys_state *parent_css) blkcg->cfq_weight = CFQ_WEIGHT_DEFAULT; blkcg->cfq_leaf_weight = CFQ_WEIGHT_DEFAULT; - blkcg->id = atomic64_inc_return(&id_seq); /* root is 0, start from 1 */ done: spin_lock_init(&blkcg->lock); INIT_RADIX_TREE(&blkcg->blkg_tree, GFP_ATOMIC); diff --git a/block/blk-cgroup.h b/block/blk-cgroup.h index d3fd7aa3d2a3..c567865b5f1d 100644 --- a/block/blk-cgroup.h +++ b/block/blk-cgroup.h @@ -50,9 +50,6 @@ struct blkcg { struct blkcg_gq *blkg_hint; struct hlist_head blkg_list; - /* for policies to test whether associated blkcg has changed */ - uint64_t id; - /* TODO: per-policy storage in blkcg */ unsigned int cfq_weight; /* belongs to cfq */ unsigned int cfq_leaf_weight; diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c index cadc37841744..900f569afcc5 100644 --- a/block/cfq-iosched.c +++ b/block/cfq-iosched.c @@ -299,7 +299,7 @@ struct cfq_io_cq { struct cfq_ttime ttime; int ioprio; /* the current ioprio */ #ifdef CONFIG_CFQ_GROUP_IOSCHED - uint64_t blkcg_id; /* the current blkcg ID */ + uint64_t blkcg_serial_nr; /* the current blkcg serial */ #endif }; @@ -3534,17 +3534,17 @@ static void check_blkcg_changed(struct cfq_io_cq *cic, struct bio *bio) { struct cfq_data *cfqd = cic_to_cfqd(cic); struct cfq_queue *sync_cfqq; - uint64_t id; + uint64_t serial_nr; rcu_read_lock(); - id = bio_blkcg(bio)->id; + serial_nr = bio_blkcg(bio)->css.serial_nr; rcu_read_unlock(); /* * Check whether blkcg has changed. The condition may trigger * spuriously on a newly created cic but there's no harm. */ - if (unlikely(!cfqd) || likely(cic->blkcg_id == id)) + if (unlikely(!cfqd) || likely(cic->blkcg_serial_nr == serial_nr)) return; sync_cfqq = cic_to_cfqq(cic, 1); @@ -3558,7 +3558,7 @@ static void check_blkcg_changed(struct cfq_io_cq *cic, struct bio *bio) cfq_put_queue(sync_cfqq); } - cic->blkcg_id = id; + cic->blkcg_serial_nr = serial_nr; } #else static inline void check_blkcg_changed(struct cfq_io_cq *cic, struct bio *bio) { } -- cgit v1.2.3 From ff9ea323816dc1c8ac7144afd4eab3ac97704430 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 8 Sep 2014 08:03:56 +0900 Subject: block, bdi: an active gendisk always has a request_queue associated with it bdev_get_queue() returns the request_queue associated with the specified block_device. blk_get_backing_dev_info() makes use of bdev_get_queue() to determine the associated bdi given a block_device. All the callers of bdev_get_queue() including blk_get_backing_dev_info() assume that bdev_get_queue() may return NULL and implement NULL handling; however, bdev_get_queue() requires the passed in block_device is opened and attached to its gendisk. Because an active gendisk always has a valid request_queue associated with it, bdev_get_queue() can never return NULL and neither can blk_get_backing_dev_info(). Make it clear that neither of the two functions can return NULL and remove NULL handling from all the callers. Signed-off-by: Tejun Heo Cc: Chris Mason Cc: Dave Chinner Signed-off-by: Jens Axboe --- block/blk-core.c | 10 +++------- block/compat_ioctl.c | 4 ---- block/ioctl.c | 4 ---- 3 files changed, 3 insertions(+), 15 deletions(-) (limited to 'block') diff --git a/block/blk-core.c b/block/blk-core.c index 93603e6ff479..817446175489 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -83,18 +83,14 @@ void blk_queue_congestion_threshold(struct request_queue *q) * @bdev: device * * Locates the passed device's request queue and returns the address of its - * backing_dev_info - * - * Will return NULL if the request queue cannot be located. + * backing_dev_info. This function can only be called if @bdev is opened + * and the return value is never NULL. */ struct backing_dev_info *blk_get_backing_dev_info(struct block_device *bdev) { - struct backing_dev_info *ret = NULL; struct request_queue *q = bdev_get_queue(bdev); - if (q) - ret = &q->backing_dev_info; - return ret; + return &q->backing_dev_info; } EXPORT_SYMBOL(blk_get_backing_dev_info); diff --git a/block/compat_ioctl.c b/block/compat_ioctl.c index 18b282ce361e..f678c733df40 100644 --- a/block/compat_ioctl.c +++ b/block/compat_ioctl.c @@ -709,8 +709,6 @@ long compat_blkdev_ioctl(struct file *file, unsigned cmd, unsigned long arg) if (!arg) return -EINVAL; bdi = blk_get_backing_dev_info(bdev); - if (bdi == NULL) - return -ENOTTY; return compat_put_long(arg, (bdi->ra_pages * PAGE_CACHE_SIZE) / 512); case BLKROGET: /* compatible */ @@ -731,8 +729,6 @@ long compat_blkdev_ioctl(struct file *file, unsigned cmd, unsigned long arg) if (!capable(CAP_SYS_ADMIN)) return -EACCES; bdi = blk_get_backing_dev_info(bdev); - if (bdi == NULL) - return -ENOTTY; bdi->ra_pages = (arg * 512) / PAGE_CACHE_SIZE; return 0; case BLKGETSIZE: diff --git a/block/ioctl.c b/block/ioctl.c index d6cda8147c91..6c7bf903742f 100644 --- a/block/ioctl.c +++ b/block/ioctl.c @@ -356,8 +356,6 @@ int blkdev_ioctl(struct block_device *bdev, fmode_t mode, unsigned cmd, if (!arg) return -EINVAL; bdi = blk_get_backing_dev_info(bdev); - if (bdi == NULL) - return -ENOTTY; return put_long(arg, (bdi->ra_pages * PAGE_CACHE_SIZE) / 512); case BLKROGET: return put_int(arg, bdev_read_only(bdev) != 0); @@ -386,8 +384,6 @@ int blkdev_ioctl(struct block_device *bdev, fmode_t mode, unsigned cmd, if(!capable(CAP_SYS_ADMIN)) return -EACCES; bdi = blk_get_backing_dev_info(bdev); - if (bdi == NULL) - return -ENOTTY; bdi->ra_pages = (arg * 512) / PAGE_CACHE_SIZE; return 0; case BLKBSZSET: -- cgit v1.2.3 From bf57229745f849e500ba69ff91e35bc8160a7373 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 13 Sep 2014 16:40:08 -0700 Subject: blk-mq: remove REQ_END Pass an explicit parameter for the last request in a batch to ->queue_rq instead of using a request flag. Besides being a cleaner and non-stateful interface this is also required for the next patch, which fixes the blk-mq I/O submission code to not start a time too early. Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- block/blk-mq.c | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) (limited to 'block') diff --git a/block/blk-mq.c b/block/blk-mq.c index e743d28620b2..32b4797f4186 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -384,7 +384,7 @@ void blk_mq_complete_request(struct request *rq) } EXPORT_SYMBOL(blk_mq_complete_request); -static void blk_mq_start_request(struct request *rq, bool last) +static void blk_mq_start_request(struct request *rq) { struct request_queue *q = rq->q; @@ -421,16 +421,6 @@ static void blk_mq_start_request(struct request *rq, bool last) */ rq->nr_phys_segments++; } - - /* - * Flag the last request in the series so that drivers know when IO - * should be kicked off, if they don't do it on a per-request basis. - * - * Note: the flag isn't the only condition drivers should do kick off. - * If drive is busy, the last request might not have the bit set. - */ - if (last) - rq->cmd_flags |= REQ_END; } static void __blk_mq_requeue_request(struct request *rq) @@ -440,8 +430,6 @@ static void __blk_mq_requeue_request(struct request *rq) trace_block_rq_requeue(q, rq); clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags); - rq->cmd_flags &= ~REQ_END; - if (q->dma_drain_size && blk_rq_bytes(rq)) rq->nr_phys_segments--; } @@ -755,9 +743,9 @@ static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx) rq = list_first_entry(&rq_list, struct request, queuelist); list_del_init(&rq->queuelist); - blk_mq_start_request(rq, list_empty(&rq_list)); + blk_mq_start_request(rq); - ret = q->mq_ops->queue_rq(hctx, rq); + ret = q->mq_ops->queue_rq(hctx, rq, list_empty(&rq_list)); switch (ret) { case BLK_MQ_RQ_QUEUE_OK: queued++; @@ -1198,14 +1186,14 @@ static void blk_mq_make_request(struct request_queue *q, struct bio *bio) int ret; blk_mq_bio_to_request(rq, bio); - blk_mq_start_request(rq, true); + blk_mq_start_request(rq); /* * For OK queue, we are done. For error, kill it. Any other * error (busy), just add it to our list as we previously * would have done */ - ret = q->mq_ops->queue_rq(data.hctx, rq); + ret = q->mq_ops->queue_rq(data.hctx, rq, true); if (ret == BLK_MQ_RQ_QUEUE_OK) goto done; else { -- cgit v1.2.3 From e2490073cd7c3d6f6ef6e029a208edd4d38efac4 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 13 Sep 2014 16:40:09 -0700 Subject: blk-mq: call blk_mq_start_request from ->queue_rq When we call blk_mq_start_request from the core blk-mq code before calling into ->queue_rq there is a racy window where the timeout handler can hit before we've fully set up the driver specific part of the command. Move the call to blk_mq_start_request into the driver so the driver can start the request only once it is fully set up. Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- block/blk-mq.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) (limited to 'block') diff --git a/block/blk-mq.c b/block/blk-mq.c index 32b4797f4186..141f2e06803a 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -384,7 +384,7 @@ void blk_mq_complete_request(struct request *rq) } EXPORT_SYMBOL(blk_mq_complete_request); -static void blk_mq_start_request(struct request *rq) +void blk_mq_start_request(struct request *rq) { struct request_queue *q = rq->q; @@ -422,16 +422,18 @@ static void blk_mq_start_request(struct request *rq) rq->nr_phys_segments++; } } +EXPORT_SYMBOL(blk_mq_start_request); static void __blk_mq_requeue_request(struct request *rq) { struct request_queue *q = rq->q; trace_block_rq_requeue(q, rq); - clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags); - if (q->dma_drain_size && blk_rq_bytes(rq)) - rq->nr_phys_segments--; + if (test_and_clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) { + if (q->dma_drain_size && blk_rq_bytes(rq)) + rq->nr_phys_segments--; + } } void blk_mq_requeue_request(struct request *rq) @@ -743,8 +745,6 @@ static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx) rq = list_first_entry(&rq_list, struct request, queuelist); list_del_init(&rq->queuelist); - blk_mq_start_request(rq); - ret = q->mq_ops->queue_rq(hctx, rq, list_empty(&rq_list)); switch (ret) { case BLK_MQ_RQ_QUEUE_OK: @@ -1186,7 +1186,6 @@ static void blk_mq_make_request(struct request_queue *q, struct bio *bio) int ret; blk_mq_bio_to_request(rq, bio); - blk_mq_start_request(rq); /* * For OK queue, we are done. For error, kill it. Any other -- cgit v1.2.3 From c8a446ad695ada43a885ec12b38411dbd190a11b Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 13 Sep 2014 16:40:10 -0700 Subject: blk-mq: rename blk_mq_end_io to blk_mq_end_request Now that we've changed the driver API on the submission side use the opportunity to fix up the name on the completion side to fit into the general scheme. Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- block/blk-flush.c | 4 ++-- block/blk-mq.c | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) (limited to 'block') diff --git a/block/blk-flush.c b/block/blk-flush.c index 3cb5e9e7108a..698e6926388c 100644 --- a/block/blk-flush.c +++ b/block/blk-flush.c @@ -202,7 +202,7 @@ static bool blk_flush_complete_seq(struct request *rq, unsigned int seq, list_del_init(&rq->flush.list); blk_flush_restore_request(rq); if (q->mq_ops) - blk_mq_end_io(rq, error); + blk_mq_end_request(rq, error); else __blk_end_request_all(rq, error); break; @@ -378,7 +378,7 @@ void blk_insert_flush(struct request *rq) */ if (!policy) { if (q->mq_ops) - blk_mq_end_io(rq, 0); + blk_mq_end_request(rq, 0); else __blk_end_bidi_request(rq, 0, 0, 0); return; diff --git a/block/blk-mq.c b/block/blk-mq.c index 141f2e06803a..1713686f5c2f 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -300,7 +300,7 @@ void blk_mq_clone_flush_request(struct request *flush_rq, hctx->cmd_size); } -inline void __blk_mq_end_io(struct request *rq, int error) +inline void __blk_mq_end_request(struct request *rq, int error) { blk_account_io_done(rq); @@ -312,15 +312,15 @@ inline void __blk_mq_end_io(struct request *rq, int error) blk_mq_free_request(rq); } } -EXPORT_SYMBOL(__blk_mq_end_io); +EXPORT_SYMBOL(__blk_mq_end_request); -void blk_mq_end_io(struct request *rq, int error) +void blk_mq_end_request(struct request *rq, int error) { if (blk_update_request(rq, error, blk_rq_bytes(rq))) BUG(); - __blk_mq_end_io(rq, error); + __blk_mq_end_request(rq, error); } -EXPORT_SYMBOL(blk_mq_end_io); +EXPORT_SYMBOL(blk_mq_end_request); static void __blk_mq_complete_request_remote(void *data) { @@ -360,7 +360,7 @@ void __blk_mq_complete_request(struct request *rq) struct request_queue *q = rq->q; if (!q->softirq_done_fn) - blk_mq_end_io(rq, rq->errors); + blk_mq_end_request(rq, rq->errors); else blk_mq_ipi_complete_request(rq); } @@ -758,7 +758,7 @@ static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx) pr_err("blk-mq: bad return on queue: %d\n", ret); case BLK_MQ_RQ_QUEUE_ERROR: rq->errors = -EIO; - blk_mq_end_io(rq, rq->errors); + blk_mq_end_request(rq, rq->errors); break; } @@ -1200,7 +1200,7 @@ static void blk_mq_make_request(struct request_queue *q, struct bio *bio) if (ret == BLK_MQ_RQ_QUEUE_ERROR) { rq->errors = -EIO; - blk_mq_end_io(rq, rq->errors); + blk_mq_end_request(rq, rq->errors); goto done; } } -- cgit v1.2.3 From 81481eb423c295c5480a3fab9bb961cf286c91e7 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 13 Sep 2014 16:40:11 -0700 Subject: blk-mq: fix and simplify tag iteration for the timeout handler Don't do a kmalloc from timer to handle timeouts, chances are we could be under heavy load or similar and thus just miss out on the timeouts. Fortunately it is very easy to just iterate over all in use tags, and doing this properly actually cleans up the blk_mq_busy_iter API as well, and prepares us for the next patch by passing a reserved argument to the iterator. Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- block/blk-mq-tag.c | 44 ++++++++++++---------------- block/blk-mq.c | 85 ++++++++++++++++-------------------------------------- 2 files changed, 43 insertions(+), 86 deletions(-) (limited to 'block') diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c index c1b92426c95e..b08788086414 100644 --- a/block/blk-mq-tag.c +++ b/block/blk-mq-tag.c @@ -392,45 +392,37 @@ void blk_mq_put_tag(struct blk_mq_hw_ctx *hctx, unsigned int tag, __blk_mq_put_reserved_tag(tags, tag); } -static void bt_for_each_free(struct blk_mq_bitmap_tags *bt, - unsigned long *free_map, unsigned int off) +static void bt_for_each(struct blk_mq_hw_ctx *hctx, + struct blk_mq_bitmap_tags *bt, unsigned int off, + busy_iter_fn *fn, void *data, bool reserved) { - int i; + struct request *rq; + int bit, i; for (i = 0; i < bt->map_nr; i++) { struct blk_align_bitmap *bm = &bt->map[i]; - int bit = 0; - - do { - bit = find_next_zero_bit(&bm->word, bm->depth, bit); - if (bit >= bm->depth) - break; - __set_bit(bit + off, free_map); - bit++; - } while (1); + for (bit = find_first_bit(&bm->word, bm->depth); + bit < bm->depth; + bit = find_next_bit(&bm->word, bm->depth, bit + 1)) { + rq = blk_mq_tag_to_rq(hctx->tags, off + bit); + if (rq->q == hctx->queue) + fn(hctx, rq, data, reserved); + } off += (1 << bt->bits_per_word); } } -void blk_mq_tag_busy_iter(struct blk_mq_tags *tags, - void (*fn)(void *, unsigned long *), void *data) +void blk_mq_tag_busy_iter(struct blk_mq_hw_ctx *hctx, busy_iter_fn *fn, + void *priv) { - unsigned long *tag_map; - size_t map_size; - - map_size = ALIGN(tags->nr_tags, BITS_PER_LONG) / BITS_PER_LONG; - tag_map = kzalloc(map_size * sizeof(unsigned long), GFP_ATOMIC); - if (!tag_map) - return; + struct blk_mq_tags *tags = hctx->tags; - bt_for_each_free(&tags->bitmap_tags, tag_map, tags->nr_reserved_tags); if (tags->nr_reserved_tags) - bt_for_each_free(&tags->breserved_tags, tag_map, 0); - - fn(data, tag_map); - kfree(tag_map); + bt_for_each(hctx, &tags->breserved_tags, 0, fn, priv, true); + bt_for_each(hctx, &tags->bitmap_tags, tags->nr_reserved_tags, fn, priv, + false); } EXPORT_SYMBOL(blk_mq_tag_busy_iter); diff --git a/block/blk-mq.c b/block/blk-mq.c index 1713686f5c2f..3baebcaf36db 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -525,58 +525,6 @@ struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag) } EXPORT_SYMBOL(blk_mq_tag_to_rq); -struct blk_mq_timeout_data { - struct blk_mq_hw_ctx *hctx; - unsigned long *next; - unsigned int *next_set; -}; - -static void blk_mq_timeout_check(void *__data, unsigned long *free_tags) -{ - struct blk_mq_timeout_data *data = __data; - struct blk_mq_hw_ctx *hctx = data->hctx; - unsigned int tag; - - /* It may not be in flight yet (this is where - * the REQ_ATOMIC_STARTED flag comes in). The requests are - * statically allocated, so we know it's always safe to access the - * memory associated with a bit offset into ->rqs[]. - */ - tag = 0; - do { - struct request *rq; - - tag = find_next_zero_bit(free_tags, hctx->tags->nr_tags, tag); - if (tag >= hctx->tags->nr_tags) - break; - - rq = blk_mq_tag_to_rq(hctx->tags, tag++); - if (rq->q != hctx->queue) - continue; - if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) - continue; - - blk_rq_check_expired(rq, data->next, data->next_set); - } while (1); -} - -static void blk_mq_hw_ctx_check_timeout(struct blk_mq_hw_ctx *hctx, - unsigned long *next, - unsigned int *next_set) -{ - struct blk_mq_timeout_data data = { - .hctx = hctx, - .next = next, - .next_set = next_set, - }; - - /* - * Ask the tagging code to iterate busy requests, so we can - * check them for timeout. - */ - blk_mq_tag_busy_iter(hctx->tags, blk_mq_timeout_check, &data); -} - static enum blk_eh_timer_return blk_mq_rq_timed_out(struct request *rq) { struct request_queue *q = rq->q; @@ -598,13 +546,30 @@ static enum blk_eh_timer_return blk_mq_rq_timed_out(struct request *rq) return q->mq_ops->timeout(rq); } + +struct blk_mq_timeout_data { + unsigned long next; + unsigned int next_set; +}; + +static void blk_mq_check_expired(struct blk_mq_hw_ctx *hctx, + struct request *rq, void *priv, bool reserved) +{ + struct blk_mq_timeout_data *data = priv; + + if (test_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) + blk_rq_check_expired(rq, &data->next, &data->next_set); +} -static void blk_mq_rq_timer(unsigned long data) +static void blk_mq_rq_timer(unsigned long priv) { - struct request_queue *q = (struct request_queue *) data; + struct request_queue *q = (struct request_queue *)priv; + struct blk_mq_timeout_data data = { + .next = 0, + .next_set = 0, + }; struct blk_mq_hw_ctx *hctx; - unsigned long next = 0; - int i, next_set = 0; + int i; queue_for_each_hw_ctx(q, hctx, i) { /* @@ -614,12 +579,12 @@ static void blk_mq_rq_timer(unsigned long data) if (!hctx->nr_ctx || !hctx->tags) continue; - blk_mq_hw_ctx_check_timeout(hctx, &next, &next_set); + blk_mq_tag_busy_iter(hctx, blk_mq_check_expired, &data); } - if (next_set) { - next = blk_rq_timeout(round_jiffies_up(next)); - mod_timer(&q->timeout, next); + if (data.next_set) { + data.next = blk_rq_timeout(round_jiffies_up(data.next)); + mod_timer(&q->timeout, data.next); } else { queue_for_each_hw_ctx(q, hctx, i) blk_mq_tag_idle(hctx); -- cgit v1.2.3 From 46f92d42ee37e10970e33891b7b61a342bd97aeb Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 13 Sep 2014 16:40:12 -0700 Subject: blk-mq: unshared timeout handler Duplicate the (small) timeout handler in blk-mq so that we can pass arguments more easily to the driver timeout handler. This enables the next patch. Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- block/blk-mq.c | 53 +++++++++++++++++++++++++++++++++++++---------------- block/blk-timeout.c | 8 ++------ block/blk.h | 2 -- 3 files changed, 39 insertions(+), 24 deletions(-) (limited to 'block') diff --git a/block/blk-mq.c b/block/blk-mq.c index 3baebcaf36db..298d6e360661 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -525,9 +525,15 @@ struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag) } EXPORT_SYMBOL(blk_mq_tag_to_rq); -static enum blk_eh_timer_return blk_mq_rq_timed_out(struct request *rq) +struct blk_mq_timeout_data { + unsigned long next; + unsigned int next_set; +}; + +static void blk_mq_rq_timed_out(struct request *req) { - struct request_queue *q = rq->q; + struct blk_mq_ops *ops = req->q->mq_ops; + enum blk_eh_timer_return ret = BLK_EH_RESET_TIMER; /* * We know that complete is set at this point. If STARTED isn't set @@ -538,27 +544,43 @@ static enum blk_eh_timer_return blk_mq_rq_timed_out(struct request *rq) * we both flags will get cleared. So check here again, and ignore * a timeout event with a request that isn't active. */ - if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) - return BLK_EH_NOT_HANDLED; - - if (!q->mq_ops->timeout) - return BLK_EH_RESET_TIMER; + if (!test_bit(REQ_ATOM_STARTED, &req->atomic_flags)) + return; - return q->mq_ops->timeout(rq); + if (ops->timeout) + ret = ops->timeout(req); + + switch (ret) { + case BLK_EH_HANDLED: + __blk_mq_complete_request(req); + break; + case BLK_EH_RESET_TIMER: + blk_add_timer(req); + blk_clear_rq_complete(req); + break; + case BLK_EH_NOT_HANDLED: + break; + default: + printk(KERN_ERR "block: bad eh return: %d\n", ret); + break; + } } -struct blk_mq_timeout_data { - unsigned long next; - unsigned int next_set; -}; - static void blk_mq_check_expired(struct blk_mq_hw_ctx *hctx, struct request *rq, void *priv, bool reserved) { struct blk_mq_timeout_data *data = priv; - if (test_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) - blk_rq_check_expired(rq, &data->next, &data->next_set); + if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) + return; + + if (time_after_eq(jiffies, rq->deadline)) { + if (!blk_mark_rq_complete(rq)) + blk_mq_rq_timed_out(rq); + } else if (!data->next_set || time_after(data->next, rq->deadline)) { + data->next = rq->deadline; + data->next_set = 1; + } } static void blk_mq_rq_timer(unsigned long priv) @@ -1781,7 +1803,6 @@ struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set) else blk_queue_make_request(q, blk_sq_make_request); - blk_queue_rq_timed_out(q, blk_mq_rq_timed_out); if (set->timeout) blk_queue_rq_timeout(q, set->timeout); diff --git a/block/blk-timeout.c b/block/blk-timeout.c index 95a09590ccfd..4d448259e622 100644 --- a/block/blk-timeout.c +++ b/block/blk-timeout.c @@ -7,7 +7,6 @@ #include #include "blk.h" -#include "blk-mq.h" #ifdef CONFIG_FAIL_IO_TIMEOUT @@ -90,10 +89,7 @@ static void blk_rq_timed_out(struct request *req) switch (ret) { case BLK_EH_HANDLED: /* Can we use req->errors here? */ - if (q->mq_ops) - __blk_mq_complete_request(req); - else - __blk_complete_request(req); + __blk_complete_request(req); break; case BLK_EH_RESET_TIMER: blk_add_timer(req); @@ -113,7 +109,7 @@ static void blk_rq_timed_out(struct request *req) } } -void blk_rq_check_expired(struct request *rq, unsigned long *next_timeout, +static void blk_rq_check_expired(struct request *rq, unsigned long *next_timeout, unsigned int *next_set) { if (time_after_eq(jiffies, rq->deadline)) { diff --git a/block/blk.h b/block/blk.h index 6748c4f8d7a1..e515a285d4c9 100644 --- a/block/blk.h +++ b/block/blk.h @@ -38,8 +38,6 @@ bool __blk_end_bidi_request(struct request *rq, int error, unsigned int nr_bytes, unsigned int bidi_bytes); void blk_rq_timed_out_timer(unsigned long data); -void blk_rq_check_expired(struct request *rq, unsigned long *next_timeout, - unsigned int *next_set); unsigned long blk_rq_timeout(unsigned long timeout); void blk_add_timer(struct request *req); void blk_delete_timer(struct request *); -- cgit v1.2.3 From 0152fb6b57c4fae769ee75ea2ae670f4ff39fba9 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 13 Sep 2014 16:40:13 -0700 Subject: blk-mq: pass a reserved argument to the timeout handler Allow blk-mq to pass an argument to the timeout handler to indicate if we're timing out a reserved or regular command. For many drivers those need to be handled different. Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- block/blk-mq.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'block') diff --git a/block/blk-mq.c b/block/blk-mq.c index 298d6e360661..d12f1983d493 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -530,7 +530,7 @@ struct blk_mq_timeout_data { unsigned int next_set; }; -static void blk_mq_rq_timed_out(struct request *req) +static void blk_mq_rq_timed_out(struct request *req, bool reserved) { struct blk_mq_ops *ops = req->q->mq_ops; enum blk_eh_timer_return ret = BLK_EH_RESET_TIMER; @@ -548,7 +548,7 @@ static void blk_mq_rq_timed_out(struct request *req) return; if (ops->timeout) - ret = ops->timeout(req); + ret = ops->timeout(req, reserved); switch (ret) { case BLK_EH_HANDLED: @@ -576,7 +576,7 @@ static void blk_mq_check_expired(struct blk_mq_hw_ctx *hctx, if (time_after_eq(jiffies, rq->deadline)) { if (!blk_mark_rq_complete(rq)) - blk_mq_rq_timed_out(rq); + blk_mq_rq_timed_out(rq, reserved); } else if (!data->next_set || time_after(data->next, rq->deadline)) { data->next = rq->deadline; data->next_set = 1; -- cgit v1.2.3 From 2edd2c740b2918eb0a9a1fe1b69678b903769ec2 Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Wed, 17 Sep 2014 17:47:58 +0800 Subject: blk-mq: remove unnecessary blk_clear_rq_complete() This patch removes two unnecessary blk_clear_rq_complete(), the REQ_ATOM_COMPLETE flag is cleared inside blk_mq_start_request(), so: - The blk_clear_rq_complete() in blk_flush_restore_request() needn't because the request will be freed later, and clearing it here may open a small race window with timeout. - The blk_clear_rq_complete() in blk_mq_requeue_request() isn't necessary too, even though REQ_ATOM_STARTED is cleared in __blk_mq_requeue_request(), in theory it still may cause a small race window with timeout since the two clear_bit() may be reordered. Signed-off-by: Ming Lei Signed-off-by: Jens Axboe --- block/blk-flush.c | 2 -- block/blk-mq.c | 1 - 2 files changed, 3 deletions(-) (limited to 'block') diff --git a/block/blk-flush.c b/block/blk-flush.c index 698e6926388c..c8e25768f2e1 100644 --- a/block/blk-flush.c +++ b/block/blk-flush.c @@ -126,8 +126,6 @@ static void blk_flush_restore_request(struct request *rq) /* make @rq a normal request */ rq->cmd_flags &= ~REQ_FLUSH_SEQ; rq->end_io = rq->flush.saved_end_io; - - blk_clear_rq_complete(rq); } static bool blk_flush_queue_rq(struct request *rq, bool add_front) diff --git a/block/blk-mq.c b/block/blk-mq.c index d12f1983d493..3b277b4eaa95 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -439,7 +439,6 @@ static void __blk_mq_requeue_request(struct request *rq) void blk_mq_requeue_request(struct request *rq) { __blk_mq_requeue_request(rq); - blk_clear_rq_complete(rq); BUG_ON(blk_queued_rq(rq)); blk_mq_add_to_requeue_list(rq, true); -- cgit v1.2.3 From aedcd72f6c283dffefbb8b808ae67bdd2c6eb11a Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 17 Sep 2014 08:27:03 -0600 Subject: blk-mq: limit memory consumption if a crash dump is active It's not uncommon for crash dump kernels to be limited to 128MB or something low in that area. This is normally not a problem for devices as we don't use that much memory, but for some shared SCSI setups with huge queue depths, it can potentially fill most of memory with tons of request allocations. blk-mq does scale back when it fails to allocate memory, but it scales back just enough so that blk-mq succeeds. This could still leave the system with not enough memory to make any real progress. Check if we are in a kdump environment and limit the hardware queues and tag depth. Signed-off-by: Jens Axboe --- block/blk-mq.c | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'block') diff --git a/block/blk-mq.c b/block/blk-mq.c index 3b277b4eaa95..c5345a951820 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -20,6 +20,7 @@ #include #include #include +#include #include @@ -1742,6 +1743,16 @@ struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set) if (!ctx) return ERR_PTR(-ENOMEM); + /* + * If a crashdump is active, then we are potentially in a very + * memory constrained environment. Limit us to 1 queue and + * 64 tags to prevent using too much memory. + */ + if (is_kdump_kernel()) { + set->nr_hw_queues = 1; + set->queue_depth = min(64U, set->queue_depth); + } + hctxs = kmalloc_node(set->nr_hw_queues * sizeof(*hctxs), GFP_KERNEL, set->numa_node); -- cgit v1.2.3 From 5e940aaa597c15e916618240ae5838864f36c91e Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Fri, 19 Sep 2014 21:53:46 +0800 Subject: blk-timeout: fix blk_add_timer Commit 8cb34819cdd5d(blk-mq: unshared timeout handler) introduces blk-mq's own timeout handler, and removes following line: blk_queue_rq_timed_out(q, blk_mq_rq_timed_out); which then causes blk_add_timer() to bypass adding the timer, since blk-mq no longer has q->rq_timed_out_fn defined. This patch fixes the problem by bypassing the check for blk-mq, so that both request deadlines are still set and the rolling timer updated. Signed-off-by: Ming Lei Signed-off-by: Jens Axboe --- block/blk-timeout.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'block') diff --git a/block/blk-timeout.c b/block/blk-timeout.c index 4d448259e622..8bae410b8a1b 100644 --- a/block/blk-timeout.c +++ b/block/blk-timeout.c @@ -186,7 +186,8 @@ void blk_add_timer(struct request *req) struct request_queue *q = req->q; unsigned long expiry; - if (!q->rq_timed_out_fn) + /* blk-mq has its own handler, so we don't need ->rq_timed_out_fn */ + if (!q->mq_ops && !q->rq_timed_out_fn) return; BUG_ON(!list_empty(&req->timeout_list)); -- cgit v1.2.3 From 90415837659fec54f33584b423dab250eb1e8432 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 22 Sep 2014 10:21:48 -0600 Subject: block: fix blk_abort_request on blk-mq Signed-off-by: Christoph Hellwig Moved blk_mq_rq_timed_out() definition to the private blk-mq.h header. Signed-off-by: Jens Axboe --- block/blk-mq.c | 2 +- block/blk-mq.h | 2 ++ block/blk-timeout.c | 6 +++++- 3 files changed, 8 insertions(+), 2 deletions(-) (limited to 'block') diff --git a/block/blk-mq.c b/block/blk-mq.c index c5345a951820..a3a80884ed95 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -530,7 +530,7 @@ struct blk_mq_timeout_data { unsigned int next_set; }; -static void blk_mq_rq_timed_out(struct request *req, bool reserved) +void blk_mq_rq_timed_out(struct request *req, bool reserved) { struct blk_mq_ops *ops = req->q->mq_ops; enum blk_eh_timer_return ret = BLK_EH_RESET_TIMER; diff --git a/block/blk-mq.h b/block/blk-mq.h index ca4964a6295d..a3c613a9df2d 100644 --- a/block/blk-mq.h +++ b/block/blk-mq.h @@ -60,6 +60,8 @@ extern int blk_mq_hw_queue_to_node(unsigned int *map, unsigned int); extern int blk_mq_sysfs_register(struct request_queue *q); extern void blk_mq_sysfs_unregister(struct request_queue *q); +extern void blk_mq_rq_timed_out(struct request *req, bool reserved); + /* * Basic implementation of sparser bitmap, allowing the user to spread * the bits over more cachelines. diff --git a/block/blk-timeout.c b/block/blk-timeout.c index 8bae410b8a1b..56c025894cdf 100644 --- a/block/blk-timeout.c +++ b/block/blk-timeout.c @@ -7,6 +7,7 @@ #include #include "blk.h" +#include "blk-mq.h" #ifdef CONFIG_FAIL_IO_TIMEOUT @@ -158,7 +159,10 @@ void blk_abort_request(struct request *req) if (blk_mark_rq_complete(req)) return; blk_delete_timer(req); - blk_rq_timed_out(req); + if (req->q->mq_ops) + blk_mq_rq_timed_out(req, false); + else + blk_rq_timed_out(req); } EXPORT_SYMBOL_GPL(blk_abort_request); -- cgit v1.2.3 From 08e98fc6016c890c2f4ffba6decc0ca9d2d5d7f8 Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Thu, 25 Sep 2014 23:23:38 +0800 Subject: blk-mq: handle failure path for initializing hctx Failure of initializing one hctx isn't handled, so this patch introduces blk_mq_init_hctx() and its pair to handle it explicitly. Also this patch makes code cleaner. Reviewed-by: Christoph Hellwig Signed-off-by: Ming Lei Signed-off-by: Jens Axboe --- block/blk-mq.c | 114 ++++++++++++++++++++++++++++++++++----------------------- 1 file changed, 69 insertions(+), 45 deletions(-) (limited to 'block') diff --git a/block/blk-mq.c b/block/blk-mq.c index a3a80884ed95..66ef1fb79326 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -1509,6 +1509,20 @@ static int blk_mq_hctx_notify(void *data, unsigned long action, return NOTIFY_OK; } +static void blk_mq_exit_hctx(struct request_queue *q, + struct blk_mq_tag_set *set, + struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx) +{ + blk_mq_tag_idle(hctx); + + if (set->ops->exit_hctx) + set->ops->exit_hctx(hctx, hctx_idx); + + blk_mq_unregister_cpu_notifier(&hctx->cpu_notifier); + kfree(hctx->ctxs); + blk_mq_free_bitmap(&hctx->ctx_map); +} + static void blk_mq_exit_hw_queues(struct request_queue *q, struct blk_mq_tag_set *set, int nr_queue) { @@ -1518,17 +1532,8 @@ static void blk_mq_exit_hw_queues(struct request_queue *q, queue_for_each_hw_ctx(q, hctx, i) { if (i == nr_queue) break; - - blk_mq_tag_idle(hctx); - - if (set->ops->exit_hctx) - set->ops->exit_hctx(hctx, i); - - blk_mq_unregister_cpu_notifier(&hctx->cpu_notifier); - kfree(hctx->ctxs); - blk_mq_free_bitmap(&hctx->ctx_map); + blk_mq_exit_hctx(q, set, hctx, i); } - } static void blk_mq_free_hw_queues(struct request_queue *q, @@ -1543,53 +1548,72 @@ static void blk_mq_free_hw_queues(struct request_queue *q, } } -static int blk_mq_init_hw_queues(struct request_queue *q, - struct blk_mq_tag_set *set) +static int blk_mq_init_hctx(struct request_queue *q, + struct blk_mq_tag_set *set, + struct blk_mq_hw_ctx *hctx, unsigned hctx_idx) { - struct blk_mq_hw_ctx *hctx; - unsigned int i; + int node; + + node = hctx->numa_node; + if (node == NUMA_NO_NODE) + node = hctx->numa_node = set->numa_node; + + INIT_DELAYED_WORK(&hctx->run_work, blk_mq_run_work_fn); + INIT_DELAYED_WORK(&hctx->delay_work, blk_mq_delay_work_fn); + spin_lock_init(&hctx->lock); + INIT_LIST_HEAD(&hctx->dispatch); + hctx->queue = q; + hctx->queue_num = hctx_idx; + hctx->flags = set->flags; + hctx->cmd_size = set->cmd_size; + + blk_mq_init_cpu_notifier(&hctx->cpu_notifier, + blk_mq_hctx_notify, hctx); + blk_mq_register_cpu_notifier(&hctx->cpu_notifier); + + hctx->tags = set->tags[hctx_idx]; /* - * Initialize hardware queues + * Allocate space for all possible cpus to avoid allocation at + * runtime */ - queue_for_each_hw_ctx(q, hctx, i) { - int node; + hctx->ctxs = kmalloc_node(nr_cpu_ids * sizeof(void *), + GFP_KERNEL, node); + if (!hctx->ctxs) + goto unregister_cpu_notifier; - node = hctx->numa_node; - if (node == NUMA_NO_NODE) - node = hctx->numa_node = set->numa_node; + if (blk_mq_alloc_bitmap(&hctx->ctx_map, node)) + goto free_ctxs; - INIT_DELAYED_WORK(&hctx->run_work, blk_mq_run_work_fn); - INIT_DELAYED_WORK(&hctx->delay_work, blk_mq_delay_work_fn); - spin_lock_init(&hctx->lock); - INIT_LIST_HEAD(&hctx->dispatch); - hctx->queue = q; - hctx->queue_num = i; - hctx->flags = set->flags; - hctx->cmd_size = set->cmd_size; + hctx->nr_ctx = 0; - blk_mq_init_cpu_notifier(&hctx->cpu_notifier, - blk_mq_hctx_notify, hctx); - blk_mq_register_cpu_notifier(&hctx->cpu_notifier); + if (set->ops->init_hctx && + set->ops->init_hctx(hctx, set->driver_data, hctx_idx)) + goto free_bitmap; - hctx->tags = set->tags[i]; + return 0; - /* - * Allocate space for all possible cpus to avoid allocation at - * runtime - */ - hctx->ctxs = kmalloc_node(nr_cpu_ids * sizeof(void *), - GFP_KERNEL, node); - if (!hctx->ctxs) - break; + free_bitmap: + blk_mq_free_bitmap(&hctx->ctx_map); + free_ctxs: + kfree(hctx->ctxs); + unregister_cpu_notifier: + blk_mq_unregister_cpu_notifier(&hctx->cpu_notifier); - if (blk_mq_alloc_bitmap(&hctx->ctx_map, node)) - break; + return -1; +} - hctx->nr_ctx = 0; +static int blk_mq_init_hw_queues(struct request_queue *q, + struct blk_mq_tag_set *set) +{ + struct blk_mq_hw_ctx *hctx; + unsigned int i; - if (set->ops->init_hctx && - set->ops->init_hctx(hctx, set->driver_data, i)) + /* + * Initialize hardware queues + */ + queue_for_each_hw_ctx(q, hctx, i) { + if (blk_mq_init_hctx(q, set, hctx, i)) break; } -- cgit v1.2.3 From 1bcb1eada4f11a713cbe586d1b5a5d93a48277cb Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Thu, 25 Sep 2014 23:23:39 +0800 Subject: blk-mq: allocate flush_rq in blk_mq_init_flush() It is reasonable to allocate flush req in blk_mq_init_flush(). Reviewed-by: Christoph Hellwig Signed-off-by: Ming Lei Signed-off-by: Jens Axboe --- block/blk-flush.c | 11 ++++++++++- block/blk-mq.c | 16 ++++++---------- block/blk-mq.h | 2 +- 3 files changed, 17 insertions(+), 12 deletions(-) (limited to 'block') diff --git a/block/blk-flush.c b/block/blk-flush.c index c8e25768f2e1..55028a707927 100644 --- a/block/blk-flush.c +++ b/block/blk-flush.c @@ -472,7 +472,16 @@ int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask, } EXPORT_SYMBOL(blkdev_issue_flush); -void blk_mq_init_flush(struct request_queue *q) +int blk_mq_init_flush(struct request_queue *q) { + struct blk_mq_tag_set *set = q->tag_set; + spin_lock_init(&q->mq_flush_lock); + + q->flush_rq = kzalloc(round_up(sizeof(struct request) + + set->cmd_size, cache_line_size()), + GFP_KERNEL); + if (!q->flush_rq) + return -ENOMEM; + return 0; } diff --git a/block/blk-mq.c b/block/blk-mq.c index 66ef1fb79326..78bcf8bfb22a 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -1848,17 +1848,10 @@ struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set) if (set->ops->complete) blk_queue_softirq_done(q, set->ops->complete); - blk_mq_init_flush(q); blk_mq_init_cpu_queues(q, set->nr_hw_queues); - q->flush_rq = kzalloc(round_up(sizeof(struct request) + - set->cmd_size, cache_line_size()), - GFP_KERNEL); - if (!q->flush_rq) - goto err_hw; - if (blk_mq_init_hw_queues(q, set)) - goto err_flush_rq; + goto err_hw; mutex_lock(&all_q_mutex); list_add_tail(&q->all_q_node, &all_q_list); @@ -1866,12 +1859,15 @@ struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set) blk_mq_add_queue_tag_set(set, q); + if (blk_mq_init_flush(q)) + goto err_hw_queues; + blk_mq_map_swqueue(q); return q; -err_flush_rq: - kfree(q->flush_rq); +err_hw_queues: + blk_mq_exit_hw_queues(q, set, set->nr_hw_queues); err_hw: blk_cleanup_queue(q); err_hctxs: diff --git a/block/blk-mq.h b/block/blk-mq.h index a3c613a9df2d..ecac69c08937 100644 --- a/block/blk-mq.h +++ b/block/blk-mq.h @@ -27,7 +27,7 @@ struct blk_mq_ctx { void __blk_mq_complete_request(struct request *rq); void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async); -void blk_mq_init_flush(struct request_queue *q); +int blk_mq_init_flush(struct request_queue *q); void blk_mq_freeze_queue(struct request_queue *q); void blk_mq_free_queue(struct request_queue *q); void blk_mq_clone_flush_request(struct request *flush_rq, -- cgit v1.2.3 From f355265571440a7db16e784b6edf4e7d26971a03 Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Thu, 25 Sep 2014 23:23:40 +0800 Subject: block: introduce blk_init_flush and its pair These two temporary functions are introduced for holding flush initialization and de-initialization, so that we can introduce 'flush queue' easier in the following patch. And once 'flush queue' and its allocation/free functions are ready, they will be removed for sake of code readability. Reviewed-by: Christoph Hellwig Signed-off-by: Ming Lei Signed-off-by: Jens Axboe --- block/blk-core.c | 5 ++--- block/blk-flush.c | 19 ++++++++++++++++++- block/blk-mq.c | 2 +- block/blk-mq.h | 1 - block/blk-sysfs.c | 4 ++-- block/blk.h | 3 +++ 6 files changed, 26 insertions(+), 8 deletions(-) (limited to 'block') diff --git a/block/blk-core.c b/block/blk-core.c index 6946a4275e6f..0a9d17269957 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -705,8 +705,7 @@ blk_init_allocated_queue(struct request_queue *q, request_fn_proc *rfn, if (!q) return NULL; - q->flush_rq = kzalloc(sizeof(struct request), GFP_KERNEL); - if (!q->flush_rq) + if (blk_init_flush(q)) return NULL; if (blk_init_rl(&q->root_rl, q, GFP_KERNEL)) @@ -742,7 +741,7 @@ blk_init_allocated_queue(struct request_queue *q, request_fn_proc *rfn, return q; fail: - kfree(q->flush_rq); + blk_exit_flush(q); return NULL; } EXPORT_SYMBOL(blk_init_allocated_queue); diff --git a/block/blk-flush.c b/block/blk-flush.c index 55028a707927..c72ab32fd8eb 100644 --- a/block/blk-flush.c +++ b/block/blk-flush.c @@ -472,7 +472,7 @@ int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask, } EXPORT_SYMBOL(blkdev_issue_flush); -int blk_mq_init_flush(struct request_queue *q) +static int blk_mq_init_flush(struct request_queue *q) { struct blk_mq_tag_set *set = q->tag_set; @@ -485,3 +485,20 @@ int blk_mq_init_flush(struct request_queue *q) return -ENOMEM; return 0; } + +int blk_init_flush(struct request_queue *q) +{ + if (q->mq_ops) + return blk_mq_init_flush(q); + + q->flush_rq = kzalloc(sizeof(struct request), GFP_KERNEL); + if (!q->flush_rq) + return -ENOMEM; + + return 0; +} + +void blk_exit_flush(struct request_queue *q) +{ + kfree(q->flush_rq); +} diff --git a/block/blk-mq.c b/block/blk-mq.c index 78bcf8bfb22a..2758cdf2de94 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -1859,7 +1859,7 @@ struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set) blk_mq_add_queue_tag_set(set, q); - if (blk_mq_init_flush(q)) + if (blk_init_flush(q)) goto err_hw_queues; blk_mq_map_swqueue(q); diff --git a/block/blk-mq.h b/block/blk-mq.h index ecac69c08937..d567d5283ffa 100644 --- a/block/blk-mq.h +++ b/block/blk-mq.h @@ -27,7 +27,6 @@ struct blk_mq_ctx { void __blk_mq_complete_request(struct request *rq); void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async); -int blk_mq_init_flush(struct request_queue *q); void blk_mq_freeze_queue(struct request_queue *q); void blk_mq_free_queue(struct request_queue *q); void blk_mq_clone_flush_request(struct request *flush_rq, diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c index 17f5c84ce7bf..949075952119 100644 --- a/block/blk-sysfs.c +++ b/block/blk-sysfs.c @@ -517,11 +517,11 @@ static void blk_release_queue(struct kobject *kobj) if (q->queue_tags) __blk_queue_free_tags(q); + blk_exit_flush(q); + if (q->mq_ops) blk_mq_free_queue(q); - kfree(q->flush_rq); - blk_trace_shutdown(q); bdi_destroy(&q->backing_dev_info); diff --git a/block/blk.h b/block/blk.h index e515a285d4c9..c6fa3d4c6a89 100644 --- a/block/blk.h +++ b/block/blk.h @@ -22,6 +22,9 @@ static inline void __blk_get_queue(struct request_queue *q) kobject_get(&q->kobj); } +int blk_init_flush(struct request_queue *q); +void blk_exit_flush(struct request_queue *q); + int blk_init_rl(struct request_list *rl, struct request_queue *q, gfp_t gfp_mask); void blk_exit_rl(struct request_list *rl); -- cgit v1.2.3 From 3c09676c12b1dabf84acbb5849bfc54acadaf092 Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Thu, 25 Sep 2014 23:23:41 +0800 Subject: block: move flush initialization to blk_flush_init These fields are always used with the flush request, so initialize them together. Reviewed-by: Christoph Hellwig Signed-off-by: Ming Lei Signed-off-by: Jens Axboe --- block/blk-core.c | 3 --- block/blk-flush.c | 4 ++++ 2 files changed, 4 insertions(+), 3 deletions(-) (limited to 'block') diff --git a/block/blk-core.c b/block/blk-core.c index 0a9d17269957..222fe84d6ac4 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -600,9 +600,6 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id) #ifdef CONFIG_BLK_CGROUP INIT_LIST_HEAD(&q->blkg_list); #endif - INIT_LIST_HEAD(&q->flush_queue[0]); - INIT_LIST_HEAD(&q->flush_queue[1]); - INIT_LIST_HEAD(&q->flush_data_in_flight); INIT_DELAYED_WORK(&q->delay_work, blk_delay_work); kobject_init(&q->kobj, &blk_queue_ktype); diff --git a/block/blk-flush.c b/block/blk-flush.c index c72ab32fd8eb..a49ffbdcfcdc 100644 --- a/block/blk-flush.c +++ b/block/blk-flush.c @@ -488,6 +488,10 @@ static int blk_mq_init_flush(struct request_queue *q) int blk_init_flush(struct request_queue *q) { + INIT_LIST_HEAD(&q->flush_queue[0]); + INIT_LIST_HEAD(&q->flush_queue[1]); + INIT_LIST_HEAD(&q->flush_data_in_flight); + if (q->mq_ops) return blk_mq_init_flush(q); -- cgit v1.2.3 From 7ddab5de5b80d3111f9e6765714e728b2c4f1c07 Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Thu, 25 Sep 2014 23:23:42 +0800 Subject: block: avoid to use q->flush_rq directly This patch trys to use local variable to access flush request, so that we can convert to per-queue flush machinery a bit easier. Reviewed-by: Christoph Hellwig Signed-off-by: Ming Lei Signed-off-by: Jens Axboe --- block/blk-flush.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) (limited to 'block') diff --git a/block/blk-flush.c b/block/blk-flush.c index a49ffbdcfcdc..caf44756d329 100644 --- a/block/blk-flush.c +++ b/block/blk-flush.c @@ -223,7 +223,7 @@ static void flush_end_io(struct request *flush_rq, int error) if (q->mq_ops) { spin_lock_irqsave(&q->mq_flush_lock, flags); - q->flush_rq->tag = -1; + flush_rq->tag = -1; } running = &q->flush_queue[q->flush_running_idx]; @@ -281,6 +281,7 @@ static bool blk_kick_flush(struct request_queue *q) struct list_head *pending = &q->flush_queue[q->flush_pending_idx]; struct request *first_rq = list_first_entry(pending, struct request, flush.list); + struct request *flush_rq = q->flush_rq; /* C1 described at the top of this file */ if (q->flush_pending_idx != q->flush_running_idx || list_empty(pending)) @@ -298,16 +299,16 @@ static bool blk_kick_flush(struct request_queue *q) */ q->flush_pending_idx ^= 1; - blk_rq_init(q, q->flush_rq); + blk_rq_init(q, flush_rq); if (q->mq_ops) - blk_mq_clone_flush_request(q->flush_rq, first_rq); + blk_mq_clone_flush_request(flush_rq, first_rq); - q->flush_rq->cmd_type = REQ_TYPE_FS; - q->flush_rq->cmd_flags = WRITE_FLUSH | REQ_FLUSH_SEQ; - q->flush_rq->rq_disk = first_rq->rq_disk; - q->flush_rq->end_io = flush_end_io; + flush_rq->cmd_type = REQ_TYPE_FS; + flush_rq->cmd_flags = WRITE_FLUSH | REQ_FLUSH_SEQ; + flush_rq->rq_disk = first_rq->rq_disk; + flush_rq->end_io = flush_end_io; - return blk_flush_queue_rq(q->flush_rq, false); + return blk_flush_queue_rq(flush_rq, false); } static void flush_data_end_io(struct request *rq, int error) -- cgit v1.2.3 From 7c94e1c157a227837b04f02f5edeff8301410ba2 Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Thu, 25 Sep 2014 23:23:43 +0800 Subject: block: introduce blk_flush_queue to drive flush machinery This patch introduces 'struct blk_flush_queue' and puts all flush machinery related fields into this structure, so that - flush implementation details aren't exposed to driver - it is easy to convert to per dispatch-queue flush machinery This patch is basically a mechanical replacement. Reviewed-by: Christoph Hellwig Signed-off-by: Ming Lei Signed-off-by: Jens Axboe --- block/blk-core.c | 4 +- block/blk-flush.c | 109 ++++++++++++++++++++++++++++++++++-------------------- block/blk-mq.c | 10 +++-- block/blk.h | 22 ++++++++++- 4 files changed, 97 insertions(+), 48 deletions(-) (limited to 'block') diff --git a/block/blk-core.c b/block/blk-core.c index 222fe84d6ac4..cfaca8ca6cc4 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -390,11 +390,13 @@ static void __blk_drain_queue(struct request_queue *q, bool drain_all) * be drained. Check all the queues and counters. */ if (drain_all) { + struct blk_flush_queue *fq = blk_get_flush_queue(q); drain |= !list_empty(&q->queue_head); for (i = 0; i < 2; i++) { drain |= q->nr_rqs[i]; drain |= q->in_flight[i]; - drain |= !list_empty(&q->flush_queue[i]); + if (fq) + drain |= !list_empty(&fq->flush_queue[i]); } } diff --git a/block/blk-flush.c b/block/blk-flush.c index caf44756d329..b01a86d6bf86 100644 --- a/block/blk-flush.c +++ b/block/blk-flush.c @@ -28,7 +28,7 @@ * * The actual execution of flush is double buffered. Whenever a request * needs to execute PRE or POSTFLUSH, it queues at - * q->flush_queue[q->flush_pending_idx]. Once certain criteria are met, a + * fq->flush_queue[fq->flush_pending_idx]. Once certain criteria are met, a * flush is issued and the pending_idx is toggled. When the flush * completes, all the requests which were pending are proceeded to the next * step. This allows arbitrary merging of different types of FLUSH/FUA @@ -155,7 +155,7 @@ static bool blk_flush_queue_rq(struct request *rq, bool add_front) * completion and trigger the next step. * * CONTEXT: - * spin_lock_irq(q->queue_lock or q->mq_flush_lock) + * spin_lock_irq(q->queue_lock or fq->mq_flush_lock) * * RETURNS: * %true if requests were added to the dispatch queue, %false otherwise. @@ -164,7 +164,8 @@ static bool blk_flush_complete_seq(struct request *rq, unsigned int seq, int error) { struct request_queue *q = rq->q; - struct list_head *pending = &q->flush_queue[q->flush_pending_idx]; + struct blk_flush_queue *fq = blk_get_flush_queue(q); + struct list_head *pending = &fq->flush_queue[fq->flush_pending_idx]; bool queued = false, kicked; BUG_ON(rq->flush.seq & seq); @@ -180,12 +181,12 @@ static bool blk_flush_complete_seq(struct request *rq, unsigned int seq, case REQ_FSEQ_POSTFLUSH: /* queue for flush */ if (list_empty(pending)) - q->flush_pending_since = jiffies; + fq->flush_pending_since = jiffies; list_move_tail(&rq->flush.list, pending); break; case REQ_FSEQ_DATA: - list_move_tail(&rq->flush.list, &q->flush_data_in_flight); + list_move_tail(&rq->flush.list, &fq->flush_data_in_flight); queued = blk_flush_queue_rq(rq, true); break; @@ -220,17 +221,18 @@ static void flush_end_io(struct request *flush_rq, int error) bool queued = false; struct request *rq, *n; unsigned long flags = 0; + struct blk_flush_queue *fq = blk_get_flush_queue(q); if (q->mq_ops) { - spin_lock_irqsave(&q->mq_flush_lock, flags); + spin_lock_irqsave(&fq->mq_flush_lock, flags); flush_rq->tag = -1; } - running = &q->flush_queue[q->flush_running_idx]; - BUG_ON(q->flush_pending_idx == q->flush_running_idx); + running = &fq->flush_queue[fq->flush_running_idx]; + BUG_ON(fq->flush_pending_idx == fq->flush_running_idx); /* account completion of the flush request */ - q->flush_running_idx ^= 1; + fq->flush_running_idx ^= 1; if (!q->mq_ops) elv_completed_request(q, flush_rq); @@ -254,13 +256,13 @@ static void flush_end_io(struct request *flush_rq, int error) * directly into request_fn may confuse the driver. Always use * kblockd. */ - if (queued || q->flush_queue_delayed) { + if (queued || fq->flush_queue_delayed) { WARN_ON(q->mq_ops); blk_run_queue_async(q); } - q->flush_queue_delayed = 0; + fq->flush_queue_delayed = 0; if (q->mq_ops) - spin_unlock_irqrestore(&q->mq_flush_lock, flags); + spin_unlock_irqrestore(&fq->mq_flush_lock, flags); } /** @@ -271,33 +273,34 @@ static void flush_end_io(struct request *flush_rq, int error) * Please read the comment at the top of this file for more info. * * CONTEXT: - * spin_lock_irq(q->queue_lock or q->mq_flush_lock) + * spin_lock_irq(q->queue_lock or fq->mq_flush_lock) * * RETURNS: * %true if flush was issued, %false otherwise. */ static bool blk_kick_flush(struct request_queue *q) { - struct list_head *pending = &q->flush_queue[q->flush_pending_idx]; + struct blk_flush_queue *fq = blk_get_flush_queue(q); + struct list_head *pending = &fq->flush_queue[fq->flush_pending_idx]; struct request *first_rq = list_first_entry(pending, struct request, flush.list); - struct request *flush_rq = q->flush_rq; + struct request *flush_rq = fq->flush_rq; /* C1 described at the top of this file */ - if (q->flush_pending_idx != q->flush_running_idx || list_empty(pending)) + if (fq->flush_pending_idx != fq->flush_running_idx || list_empty(pending)) return false; /* C2 and C3 */ - if (!list_empty(&q->flush_data_in_flight) && + if (!list_empty(&fq->flush_data_in_flight) && time_before(jiffies, - q->flush_pending_since + FLUSH_PENDING_TIMEOUT)) + fq->flush_pending_since + FLUSH_PENDING_TIMEOUT)) return false; /* * Issue flush and toggle pending_idx. This makes pending_idx * different from running_idx, which means flush is in flight. */ - q->flush_pending_idx ^= 1; + fq->flush_pending_idx ^= 1; blk_rq_init(q, flush_rq); if (q->mq_ops) @@ -329,6 +332,7 @@ static void mq_flush_data_end_io(struct request *rq, int error) struct blk_mq_hw_ctx *hctx; struct blk_mq_ctx *ctx; unsigned long flags; + struct blk_flush_queue *fq = blk_get_flush_queue(q); ctx = rq->mq_ctx; hctx = q->mq_ops->map_queue(q, ctx->cpu); @@ -337,10 +341,10 @@ static void mq_flush_data_end_io(struct request *rq, int error) * After populating an empty queue, kick it to avoid stall. Read * the comment in flush_end_io(). */ - spin_lock_irqsave(&q->mq_flush_lock, flags); + spin_lock_irqsave(&fq->mq_flush_lock, flags); if (blk_flush_complete_seq(rq, REQ_FSEQ_DATA, error)) blk_mq_run_hw_queue(hctx, true); - spin_unlock_irqrestore(&q->mq_flush_lock, flags); + spin_unlock_irqrestore(&fq->mq_flush_lock, flags); } /** @@ -408,11 +412,13 @@ void blk_insert_flush(struct request *rq) rq->cmd_flags |= REQ_FLUSH_SEQ; rq->flush.saved_end_io = rq->end_io; /* Usually NULL */ if (q->mq_ops) { + struct blk_flush_queue *fq = blk_get_flush_queue(q); + rq->end_io = mq_flush_data_end_io; - spin_lock_irq(&q->mq_flush_lock); + spin_lock_irq(&fq->mq_flush_lock); blk_flush_complete_seq(rq, REQ_FSEQ_ACTIONS & ~policy, 0); - spin_unlock_irq(&q->mq_flush_lock); + spin_unlock_irq(&fq->mq_flush_lock); return; } rq->end_io = flush_data_end_io; @@ -473,31 +479,52 @@ int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask, } EXPORT_SYMBOL(blkdev_issue_flush); -static int blk_mq_init_flush(struct request_queue *q) +static struct blk_flush_queue *blk_alloc_flush_queue( + struct request_queue *q) { - struct blk_mq_tag_set *set = q->tag_set; + struct blk_flush_queue *fq; + int rq_sz = sizeof(struct request); - spin_lock_init(&q->mq_flush_lock); + fq = kzalloc(sizeof(*fq), GFP_KERNEL); + if (!fq) + goto fail; - q->flush_rq = kzalloc(round_up(sizeof(struct request) + - set->cmd_size, cache_line_size()), - GFP_KERNEL); - if (!q->flush_rq) - return -ENOMEM; - return 0; + if (q->mq_ops) { + spin_lock_init(&fq->mq_flush_lock); + rq_sz = round_up(rq_sz + q->tag_set->cmd_size, + cache_line_size()); + } + + fq->flush_rq = kzalloc(rq_sz, GFP_KERNEL); + if (!fq->flush_rq) + goto fail_rq; + + INIT_LIST_HEAD(&fq->flush_queue[0]); + INIT_LIST_HEAD(&fq->flush_queue[1]); + INIT_LIST_HEAD(&fq->flush_data_in_flight); + + return fq; + + fail_rq: + kfree(fq); + fail: + return NULL; } -int blk_init_flush(struct request_queue *q) +static void blk_free_flush_queue(struct blk_flush_queue *fq) { - INIT_LIST_HEAD(&q->flush_queue[0]); - INIT_LIST_HEAD(&q->flush_queue[1]); - INIT_LIST_HEAD(&q->flush_data_in_flight); + /* bio based request queue hasn't flush queue */ + if (!fq) + return; - if (q->mq_ops) - return blk_mq_init_flush(q); + kfree(fq->flush_rq); + kfree(fq); +} - q->flush_rq = kzalloc(sizeof(struct request), GFP_KERNEL); - if (!q->flush_rq) +int blk_init_flush(struct request_queue *q) +{ + q->fq = blk_alloc_flush_queue(q); + if (!q->fq) return -ENOMEM; return 0; @@ -505,5 +532,5 @@ int blk_init_flush(struct request_queue *q) void blk_exit_flush(struct request_queue *q) { - kfree(q->flush_rq); + blk_free_flush_queue(q->fq); } diff --git a/block/blk-mq.c b/block/blk-mq.c index 2758cdf2de94..d39e8a5eaeaa 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -508,20 +508,22 @@ void blk_mq_kick_requeue_list(struct request_queue *q) } EXPORT_SYMBOL(blk_mq_kick_requeue_list); -static inline bool is_flush_request(struct request *rq, unsigned int tag) +static inline bool is_flush_request(struct request *rq, + struct blk_flush_queue *fq, unsigned int tag) { return ((rq->cmd_flags & REQ_FLUSH_SEQ) && - rq->q->flush_rq->tag == tag); + fq->flush_rq->tag == tag); } struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag) { struct request *rq = tags->rqs[tag]; + struct blk_flush_queue *fq = blk_get_flush_queue(rq->q); - if (!is_flush_request(rq, tag)) + if (!is_flush_request(rq, fq, tag)) return rq; - return rq->q->flush_rq; + return fq->flush_rq; } EXPORT_SYMBOL(blk_mq_tag_to_rq); diff --git a/block/blk.h b/block/blk.h index c6fa3d4c6a89..833c4ac6c4eb 100644 --- a/block/blk.h +++ b/block/blk.h @@ -12,11 +12,28 @@ /* Max future timer expiry for timeouts */ #define BLK_MAX_TIMEOUT (5 * HZ) +struct blk_flush_queue { + unsigned int flush_queue_delayed:1; + unsigned int flush_pending_idx:1; + unsigned int flush_running_idx:1; + unsigned long flush_pending_since; + struct list_head flush_queue[2]; + struct list_head flush_data_in_flight; + struct request *flush_rq; + spinlock_t mq_flush_lock; +}; + extern struct kmem_cache *blk_requestq_cachep; extern struct kmem_cache *request_cachep; extern struct kobj_type blk_queue_ktype; extern struct ida blk_queue_ida; +static inline struct blk_flush_queue *blk_get_flush_queue( + struct request_queue *q) +{ + return q->fq; +} + static inline void __blk_get_queue(struct request_queue *q) { kobject_get(&q->kobj); @@ -89,6 +106,7 @@ void blk_insert_flush(struct request *rq); static inline struct request *__elv_next_request(struct request_queue *q) { struct request *rq; + struct blk_flush_queue *fq = blk_get_flush_queue(q); while (1) { if (!list_empty(&q->queue_head)) { @@ -111,9 +129,9 @@ static inline struct request *__elv_next_request(struct request_queue *q) * should be restarted later. Please see flush_end_io() for * details. */ - if (q->flush_pending_idx != q->flush_running_idx && + if (fq->flush_pending_idx != fq->flush_running_idx && !queue_flush_queueable(q)) { - q->flush_queue_delayed = 1; + fq->flush_queue_delayed = 1; return NULL; } if (unlikely(blk_queue_bypass(q)) || -- cgit v1.2.3 From ba483388e3058b3e412632a84e6bf1f134beaf3d Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Thu, 25 Sep 2014 23:23:44 +0800 Subject: block: remove blk_init_flush() and its pair Now mission of the two helpers is over, and just call blk_alloc_flush_queue() and blk_free_flush_queue() directly. Signed-off-by: Ming Lei Signed-off-by: Jens Axboe --- block/blk-core.c | 5 +++-- block/blk-flush.c | 19 ++----------------- block/blk-mq.c | 3 ++- block/blk-sysfs.c | 2 +- block/blk.h | 4 ++-- 5 files changed, 10 insertions(+), 23 deletions(-) (limited to 'block') diff --git a/block/blk-core.c b/block/blk-core.c index cfaca8ca6cc4..dba0a8350807 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -704,7 +704,8 @@ blk_init_allocated_queue(struct request_queue *q, request_fn_proc *rfn, if (!q) return NULL; - if (blk_init_flush(q)) + q->fq = blk_alloc_flush_queue(q); + if (!q->fq) return NULL; if (blk_init_rl(&q->root_rl, q, GFP_KERNEL)) @@ -740,7 +741,7 @@ blk_init_allocated_queue(struct request_queue *q, request_fn_proc *rfn, return q; fail: - blk_exit_flush(q); + blk_free_flush_queue(q->fq); return NULL; } EXPORT_SYMBOL(blk_init_allocated_queue); diff --git a/block/blk-flush.c b/block/blk-flush.c index b01a86d6bf86..d66cbf2b2bc8 100644 --- a/block/blk-flush.c +++ b/block/blk-flush.c @@ -479,8 +479,7 @@ int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask, } EXPORT_SYMBOL(blkdev_issue_flush); -static struct blk_flush_queue *blk_alloc_flush_queue( - struct request_queue *q) +struct blk_flush_queue *blk_alloc_flush_queue(struct request_queue *q) { struct blk_flush_queue *fq; int rq_sz = sizeof(struct request); @@ -511,7 +510,7 @@ static struct blk_flush_queue *blk_alloc_flush_queue( return NULL; } -static void blk_free_flush_queue(struct blk_flush_queue *fq) +void blk_free_flush_queue(struct blk_flush_queue *fq) { /* bio based request queue hasn't flush queue */ if (!fq) @@ -520,17 +519,3 @@ static void blk_free_flush_queue(struct blk_flush_queue *fq) kfree(fq->flush_rq); kfree(fq); } - -int blk_init_flush(struct request_queue *q) -{ - q->fq = blk_alloc_flush_queue(q); - if (!q->fq) - return -ENOMEM; - - return 0; -} - -void blk_exit_flush(struct request_queue *q) -{ - blk_free_flush_queue(q->fq); -} diff --git a/block/blk-mq.c b/block/blk-mq.c index d39e8a5eaeaa..59ca79634cb9 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -1861,7 +1861,8 @@ struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set) blk_mq_add_queue_tag_set(set, q); - if (blk_init_flush(q)) + q->fq = blk_alloc_flush_queue(q); + if (!q->fq) goto err_hw_queues; blk_mq_map_swqueue(q); diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c index 949075952119..718cffc4c678 100644 --- a/block/blk-sysfs.c +++ b/block/blk-sysfs.c @@ -517,7 +517,7 @@ static void blk_release_queue(struct kobject *kobj) if (q->queue_tags) __blk_queue_free_tags(q); - blk_exit_flush(q); + blk_free_flush_queue(q->fq); if (q->mq_ops) blk_mq_free_queue(q); diff --git a/block/blk.h b/block/blk.h index 833c4ac6c4eb..9eaa6e91b13f 100644 --- a/block/blk.h +++ b/block/blk.h @@ -39,8 +39,8 @@ static inline void __blk_get_queue(struct request_queue *q) kobject_get(&q->kobj); } -int blk_init_flush(struct request_queue *q); -void blk_exit_flush(struct request_queue *q); +struct blk_flush_queue *blk_alloc_flush_queue(struct request_queue *q); +void blk_free_flush_queue(struct blk_flush_queue *fq); int blk_init_rl(struct request_list *rl, struct request_queue *q, gfp_t gfp_mask); -- cgit v1.2.3 From 0bae352da54a95435f721705d3670a6eaefdcf87 Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Thu, 25 Sep 2014 23:23:45 +0800 Subject: block: flush: avoid to figure out flush queue unnecessarily Just figuring out flush queue at the entry of kicking off flush machinery and request's completion handler, then pass it through. Reviewed-by: Christoph Hellwig Signed-off-by: Ming Lei Signed-off-by: Jens Axboe --- block/blk-flush.c | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) (limited to 'block') diff --git a/block/blk-flush.c b/block/blk-flush.c index d66cbf2b2bc8..9bc5b4f35c23 100644 --- a/block/blk-flush.c +++ b/block/blk-flush.c @@ -91,7 +91,8 @@ enum { FLUSH_PENDING_TIMEOUT = 5 * HZ, }; -static bool blk_kick_flush(struct request_queue *q); +static bool blk_kick_flush(struct request_queue *q, + struct blk_flush_queue *fq); static unsigned int blk_flush_policy(unsigned int fflags, struct request *rq) { @@ -148,6 +149,7 @@ static bool blk_flush_queue_rq(struct request *rq, bool add_front) /** * blk_flush_complete_seq - complete flush sequence * @rq: FLUSH/FUA request being sequenced + * @fq: flush queue * @seq: sequences to complete (mask of %REQ_FSEQ_*, can be zero) * @error: whether an error occurred * @@ -160,11 +162,11 @@ static bool blk_flush_queue_rq(struct request *rq, bool add_front) * RETURNS: * %true if requests were added to the dispatch queue, %false otherwise. */ -static bool blk_flush_complete_seq(struct request *rq, unsigned int seq, - int error) +static bool blk_flush_complete_seq(struct request *rq, + struct blk_flush_queue *fq, + unsigned int seq, int error) { struct request_queue *q = rq->q; - struct blk_flush_queue *fq = blk_get_flush_queue(q); struct list_head *pending = &fq->flush_queue[fq->flush_pending_idx]; bool queued = false, kicked; @@ -210,7 +212,7 @@ static bool blk_flush_complete_seq(struct request *rq, unsigned int seq, BUG(); } - kicked = blk_kick_flush(q); + kicked = blk_kick_flush(q, fq); return kicked | queued; } @@ -242,7 +244,7 @@ static void flush_end_io(struct request *flush_rq, int error) unsigned int seq = blk_flush_cur_seq(rq); BUG_ON(seq != REQ_FSEQ_PREFLUSH && seq != REQ_FSEQ_POSTFLUSH); - queued |= blk_flush_complete_seq(rq, seq, error); + queued |= blk_flush_complete_seq(rq, fq, seq, error); } /* @@ -268,6 +270,7 @@ static void flush_end_io(struct request *flush_rq, int error) /** * blk_kick_flush - consider issuing flush request * @q: request_queue being kicked + * @fq: flush queue * * Flush related states of @q have changed, consider issuing flush request. * Please read the comment at the top of this file for more info. @@ -278,9 +281,8 @@ static void flush_end_io(struct request *flush_rq, int error) * RETURNS: * %true if flush was issued, %false otherwise. */ -static bool blk_kick_flush(struct request_queue *q) +static bool blk_kick_flush(struct request_queue *q, struct blk_flush_queue *fq) { - struct blk_flush_queue *fq = blk_get_flush_queue(q); struct list_head *pending = &fq->flush_queue[fq->flush_pending_idx]; struct request *first_rq = list_first_entry(pending, struct request, flush.list); @@ -317,12 +319,13 @@ static bool blk_kick_flush(struct request_queue *q) static void flush_data_end_io(struct request *rq, int error) { struct request_queue *q = rq->q; + struct blk_flush_queue *fq = blk_get_flush_queue(q); /* * After populating an empty queue, kick it to avoid stall. Read * the comment in flush_end_io(). */ - if (blk_flush_complete_seq(rq, REQ_FSEQ_DATA, error)) + if (blk_flush_complete_seq(rq, fq, REQ_FSEQ_DATA, error)) blk_run_queue_async(q); } @@ -342,7 +345,7 @@ static void mq_flush_data_end_io(struct request *rq, int error) * the comment in flush_end_io(). */ spin_lock_irqsave(&fq->mq_flush_lock, flags); - if (blk_flush_complete_seq(rq, REQ_FSEQ_DATA, error)) + if (blk_flush_complete_seq(rq, fq, REQ_FSEQ_DATA, error)) blk_mq_run_hw_queue(hctx, true); spin_unlock_irqrestore(&fq->mq_flush_lock, flags); } @@ -364,6 +367,7 @@ void blk_insert_flush(struct request *rq) struct request_queue *q = rq->q; unsigned int fflags = q->flush_flags; /* may change, cache */ unsigned int policy = blk_flush_policy(fflags, rq); + struct blk_flush_queue *fq = blk_get_flush_queue(q); /* * @policy now records what operations need to be done. Adjust @@ -412,18 +416,16 @@ void blk_insert_flush(struct request *rq) rq->cmd_flags |= REQ_FLUSH_SEQ; rq->flush.saved_end_io = rq->end_io; /* Usually NULL */ if (q->mq_ops) { - struct blk_flush_queue *fq = blk_get_flush_queue(q); - rq->end_io = mq_flush_data_end_io; spin_lock_irq(&fq->mq_flush_lock); - blk_flush_complete_seq(rq, REQ_FSEQ_ACTIONS & ~policy, 0); + blk_flush_complete_seq(rq, fq, REQ_FSEQ_ACTIONS & ~policy, 0); spin_unlock_irq(&fq->mq_flush_lock); return; } rq->end_io = flush_data_end_io; - blk_flush_complete_seq(rq, REQ_FSEQ_ACTIONS & ~policy, 0); + blk_flush_complete_seq(rq, fq, REQ_FSEQ_ACTIONS & ~policy, 0); } /** -- cgit v1.2.3 From e97c293cdf77263abdc021de280516e0017afc84 Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Thu, 25 Sep 2014 23:23:46 +0800 Subject: block: introduce 'blk_mq_ctx' parameter to blk_get_flush_queue This patch adds 'blk_mq_ctx' parameter to blk_get_flush_queue(), so that this function can find the corresponding blk_flush_queue bound with current mq context since the flush queue will become per hw-queue. For legacy queue, the parameter can be simply 'NULL'. For multiqueue case, the parameter should be set as the context from which the related request is originated. With this context info, the hw queue and related flush queue can be found easily. Signed-off-by: Ming Lei Signed-off-by: Jens Axboe --- block/blk-core.c | 2 +- block/blk-flush.c | 11 +++++------ block/blk-mq.c | 3 ++- block/blk.h | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) (limited to 'block') diff --git a/block/blk-core.c b/block/blk-core.c index dba0a8350807..b1dd4e086740 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -390,7 +390,7 @@ static void __blk_drain_queue(struct request_queue *q, bool drain_all) * be drained. Check all the queues and counters. */ if (drain_all) { - struct blk_flush_queue *fq = blk_get_flush_queue(q); + struct blk_flush_queue *fq = blk_get_flush_queue(q, NULL); drain |= !list_empty(&q->queue_head); for (i = 0; i < 2; i++) { drain |= q->nr_rqs[i]; diff --git a/block/blk-flush.c b/block/blk-flush.c index 9bc5b4f35c23..004d95e4098e 100644 --- a/block/blk-flush.c +++ b/block/blk-flush.c @@ -223,7 +223,7 @@ static void flush_end_io(struct request *flush_rq, int error) bool queued = false; struct request *rq, *n; unsigned long flags = 0; - struct blk_flush_queue *fq = blk_get_flush_queue(q); + struct blk_flush_queue *fq = blk_get_flush_queue(q, flush_rq->mq_ctx); if (q->mq_ops) { spin_lock_irqsave(&fq->mq_flush_lock, flags); @@ -319,7 +319,7 @@ static bool blk_kick_flush(struct request_queue *q, struct blk_flush_queue *fq) static void flush_data_end_io(struct request *rq, int error) { struct request_queue *q = rq->q; - struct blk_flush_queue *fq = blk_get_flush_queue(q); + struct blk_flush_queue *fq = blk_get_flush_queue(q, NULL); /* * After populating an empty queue, kick it to avoid stall. Read @@ -333,11 +333,10 @@ static void mq_flush_data_end_io(struct request *rq, int error) { struct request_queue *q = rq->q; struct blk_mq_hw_ctx *hctx; - struct blk_mq_ctx *ctx; + struct blk_mq_ctx *ctx = rq->mq_ctx; unsigned long flags; - struct blk_flush_queue *fq = blk_get_flush_queue(q); + struct blk_flush_queue *fq = blk_get_flush_queue(q, ctx); - ctx = rq->mq_ctx; hctx = q->mq_ops->map_queue(q, ctx->cpu); /* @@ -367,7 +366,7 @@ void blk_insert_flush(struct request *rq) struct request_queue *q = rq->q; unsigned int fflags = q->flush_flags; /* may change, cache */ unsigned int policy = blk_flush_policy(fflags, rq); - struct blk_flush_queue *fq = blk_get_flush_queue(q); + struct blk_flush_queue *fq = blk_get_flush_queue(q, rq->mq_ctx); /* * @policy now records what operations need to be done. Adjust diff --git a/block/blk-mq.c b/block/blk-mq.c index 59ca79634cb9..53b6def12fc4 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -518,7 +518,8 @@ static inline bool is_flush_request(struct request *rq, struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag) { struct request *rq = tags->rqs[tag]; - struct blk_flush_queue *fq = blk_get_flush_queue(rq->q); + /* mq_ctx of flush rq is always cloned from the corresponding req */ + struct blk_flush_queue *fq = blk_get_flush_queue(rq->q, rq->mq_ctx); if (!is_flush_request(rq, fq, tag)) return rq; diff --git a/block/blk.h b/block/blk.h index 9eaa6e91b13f..7ecdd8517e69 100644 --- a/block/blk.h +++ b/block/blk.h @@ -29,7 +29,7 @@ extern struct kobj_type blk_queue_ktype; extern struct ida blk_queue_ida; static inline struct blk_flush_queue *blk_get_flush_queue( - struct request_queue *q) + struct request_queue *q, struct blk_mq_ctx *ctx) { return q->fq; } @@ -106,7 +106,7 @@ void blk_insert_flush(struct request *rq); static inline struct request *__elv_next_request(struct request_queue *q) { struct request *rq; - struct blk_flush_queue *fq = blk_get_flush_queue(q); + struct blk_flush_queue *fq = blk_get_flush_queue(q, NULL); while (1) { if (!list_empty(&q->queue_head)) { -- cgit v1.2.3 From f70ced09170761acb69840cafaace4abc72cba4b Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Thu, 25 Sep 2014 23:23:47 +0800 Subject: blk-mq: support per-distpatch_queue flush machinery This patch supports to run one single flush machinery for each blk-mq dispatch queue, so that: - current init_request and exit_request callbacks can cover flush request too, then the buggy copying way of initializing flush request's pdu can be fixed - flushing performance gets improved in case of multi hw-queue In fio sync write test over virtio-blk(4 hw queues, ioengine=sync, iodepth=64, numjobs=4, bs=4K), it is observed that througput gets increased a lot over my test environment: - throughput: +70% in case of virtio-blk over null_blk - throughput: +30% in case of virtio-blk over SSD image The multi virtqueue feature isn't merged to QEMU yet, and patches for the feature can be found in below tree: git://kernel.ubuntu.com/ming/qemu.git v2.1.0-mq.4 And simply passing 'num_queues=4 vectors=5' should be enough to enable multi queue(quad queue) feature for QEMU virtio-blk. Suggested-by: Christoph Hellwig Signed-off-by: Ming Lei Signed-off-by: Jens Axboe --- block/blk-core.c | 2 +- block/blk-flush.c | 21 ++++++++++++++------- block/blk-mq.c | 50 ++++++++++++++++++++++++-------------------------- block/blk-sysfs.c | 4 ++-- block/blk.h | 16 +++++++++++++--- 5 files changed, 54 insertions(+), 39 deletions(-) (limited to 'block') diff --git a/block/blk-core.c b/block/blk-core.c index b1dd4e086740..e1c2775c7597 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -704,7 +704,7 @@ blk_init_allocated_queue(struct request_queue *q, request_fn_proc *rfn, if (!q) return NULL; - q->fq = blk_alloc_flush_queue(q); + q->fq = blk_alloc_flush_queue(q, NUMA_NO_NODE, 0); if (!q->fq) return NULL; diff --git a/block/blk-flush.c b/block/blk-flush.c index 004d95e4098e..20badd7b9d1b 100644 --- a/block/blk-flush.c +++ b/block/blk-flush.c @@ -305,8 +305,15 @@ static bool blk_kick_flush(struct request_queue *q, struct blk_flush_queue *fq) fq->flush_pending_idx ^= 1; blk_rq_init(q, flush_rq); - if (q->mq_ops) - blk_mq_clone_flush_request(flush_rq, first_rq); + + /* + * Borrow tag from the first request since they can't + * be in flight at the same time. + */ + if (q->mq_ops) { + flush_rq->mq_ctx = first_rq->mq_ctx; + flush_rq->tag = first_rq->tag; + } flush_rq->cmd_type = REQ_TYPE_FS; flush_rq->cmd_flags = WRITE_FLUSH | REQ_FLUSH_SEQ; @@ -480,22 +487,22 @@ int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask, } EXPORT_SYMBOL(blkdev_issue_flush); -struct blk_flush_queue *blk_alloc_flush_queue(struct request_queue *q) +struct blk_flush_queue *blk_alloc_flush_queue(struct request_queue *q, + int node, int cmd_size) { struct blk_flush_queue *fq; int rq_sz = sizeof(struct request); - fq = kzalloc(sizeof(*fq), GFP_KERNEL); + fq = kzalloc_node(sizeof(*fq), GFP_KERNEL, node); if (!fq) goto fail; if (q->mq_ops) { spin_lock_init(&fq->mq_flush_lock); - rq_sz = round_up(rq_sz + q->tag_set->cmd_size, - cache_line_size()); + rq_sz = round_up(rq_sz + cmd_size, cache_line_size()); } - fq->flush_rq = kzalloc(rq_sz, GFP_KERNEL); + fq->flush_rq = kzalloc_node(rq_sz, GFP_KERNEL, node); if (!fq->flush_rq) goto fail_rq; diff --git a/block/blk-mq.c b/block/blk-mq.c index 53b6def12fc4..4e7a31466139 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -281,26 +281,6 @@ void blk_mq_free_request(struct request *rq) __blk_mq_free_request(hctx, ctx, rq); } -/* - * Clone all relevant state from a request that has been put on hold in - * the flush state machine into the preallocated flush request that hangs - * off the request queue. - * - * For a driver the flush request should be invisible, that's why we are - * impersonating the original request here. - */ -void blk_mq_clone_flush_request(struct request *flush_rq, - struct request *orig_rq) -{ - struct blk_mq_hw_ctx *hctx = - orig_rq->q->mq_ops->map_queue(orig_rq->q, orig_rq->mq_ctx->cpu); - - flush_rq->mq_ctx = orig_rq->mq_ctx; - flush_rq->tag = orig_rq->tag; - memcpy(blk_mq_rq_to_pdu(flush_rq), blk_mq_rq_to_pdu(orig_rq), - hctx->cmd_size); -} - inline void __blk_mq_end_request(struct request *rq, int error) { blk_account_io_done(rq); @@ -1516,12 +1496,20 @@ static void blk_mq_exit_hctx(struct request_queue *q, struct blk_mq_tag_set *set, struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx) { + unsigned flush_start_tag = set->queue_depth; + blk_mq_tag_idle(hctx); + if (set->ops->exit_request) + set->ops->exit_request(set->driver_data, + hctx->fq->flush_rq, hctx_idx, + flush_start_tag + hctx_idx); + if (set->ops->exit_hctx) set->ops->exit_hctx(hctx, hctx_idx); blk_mq_unregister_cpu_notifier(&hctx->cpu_notifier); + blk_free_flush_queue(hctx->fq); kfree(hctx->ctxs); blk_mq_free_bitmap(&hctx->ctx_map); } @@ -1556,6 +1544,7 @@ static int blk_mq_init_hctx(struct request_queue *q, struct blk_mq_hw_ctx *hctx, unsigned hctx_idx) { int node; + unsigned flush_start_tag = set->queue_depth; node = hctx->numa_node; if (node == NUMA_NO_NODE) @@ -1594,8 +1583,23 @@ static int blk_mq_init_hctx(struct request_queue *q, set->ops->init_hctx(hctx, set->driver_data, hctx_idx)) goto free_bitmap; + hctx->fq = blk_alloc_flush_queue(q, hctx->numa_node, set->cmd_size); + if (!hctx->fq) + goto exit_hctx; + + if (set->ops->init_request && + set->ops->init_request(set->driver_data, + hctx->fq->flush_rq, hctx_idx, + flush_start_tag + hctx_idx, node)) + goto free_fq; + return 0; + free_fq: + kfree(hctx->fq); + exit_hctx: + if (set->ops->exit_hctx) + set->ops->exit_hctx(hctx, hctx_idx); free_bitmap: blk_mq_free_bitmap(&hctx->ctx_map); free_ctxs: @@ -1862,16 +1866,10 @@ struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set) blk_mq_add_queue_tag_set(set, q); - q->fq = blk_alloc_flush_queue(q); - if (!q->fq) - goto err_hw_queues; - blk_mq_map_swqueue(q); return q; -err_hw_queues: - blk_mq_exit_hw_queues(q, set, set->nr_hw_queues); err_hw: blk_cleanup_queue(q); err_hctxs: diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c index 718cffc4c678..e8f38a36c625 100644 --- a/block/blk-sysfs.c +++ b/block/blk-sysfs.c @@ -517,10 +517,10 @@ static void blk_release_queue(struct kobject *kobj) if (q->queue_tags) __blk_queue_free_tags(q); - blk_free_flush_queue(q->fq); - if (q->mq_ops) blk_mq_free_queue(q); + else + blk_free_flush_queue(q->fq); blk_trace_shutdown(q); diff --git a/block/blk.h b/block/blk.h index 7ecdd8517e69..43b036185712 100644 --- a/block/blk.h +++ b/block/blk.h @@ -2,6 +2,8 @@ #define BLK_INTERNAL_H #include +#include +#include "blk-mq.h" /* Amount of time in which a process may batch requests */ #define BLK_BATCH_TIME (HZ/50UL) @@ -31,7 +33,14 @@ extern struct ida blk_queue_ida; static inline struct blk_flush_queue *blk_get_flush_queue( struct request_queue *q, struct blk_mq_ctx *ctx) { - return q->fq; + struct blk_mq_hw_ctx *hctx; + + if (!q->mq_ops) + return q->fq; + + hctx = q->mq_ops->map_queue(q, ctx->cpu); + + return hctx->fq; } static inline void __blk_get_queue(struct request_queue *q) @@ -39,8 +48,9 @@ static inline void __blk_get_queue(struct request_queue *q) kobject_get(&q->kobj); } -struct blk_flush_queue *blk_alloc_flush_queue(struct request_queue *q); -void blk_free_flush_queue(struct blk_flush_queue *fq); +struct blk_flush_queue *blk_alloc_flush_queue(struct request_queue *q, + int node, int cmd_size); +void blk_free_flush_queue(struct blk_flush_queue *q); int blk_init_rl(struct request_list *rl, struct request_queue *q, gfp_t gfp_mask); -- cgit v1.2.3 From e7258c1a269e0967856c81d182c286a78f5ecf15 Mon Sep 17 00:00:00 2001 From: "Martin K. Petersen" Date: Fri, 26 Sep 2014 19:19:55 -0400 Subject: block: Get rid of bdev_integrity_enabled() bdev_integrity_enabled() is only used by bio_integrity_enabled(). Combine these two functions. Signed-off-by: Martin K. Petersen Reviewed-by: Christoph Hellwig Reviewed-by: Sagi Grimberg Signed-off-by: Jens Axboe --- block/bio-integrity.c | 39 +++++++++++++++++---------------------- 1 file changed, 17 insertions(+), 22 deletions(-) (limited to 'block') diff --git a/block/bio-integrity.c b/block/bio-integrity.c index f14b4abbebd8..36b788552c3e 100644 --- a/block/bio-integrity.c +++ b/block/bio-integrity.c @@ -147,24 +147,6 @@ int bio_integrity_add_page(struct bio *bio, struct page *page, } EXPORT_SYMBOL(bio_integrity_add_page); -static int bdev_integrity_enabled(struct block_device *bdev, int rw) -{ - struct blk_integrity *bi = bdev_get_integrity(bdev); - - if (bi == NULL) - return 0; - - if (rw == READ && bi->verify_fn != NULL && - (bi->flags & INTEGRITY_FLAG_READ)) - return 1; - - if (rw == WRITE && bi->generate_fn != NULL && - (bi->flags & INTEGRITY_FLAG_WRITE)) - return 1; - - return 0; -} - /** * bio_integrity_enabled - Check whether integrity can be passed * @bio: bio to check @@ -174,16 +156,29 @@ static int bdev_integrity_enabled(struct block_device *bdev, int rw) * set prior to calling. The functions honors the write_generate and * read_verify flags in sysfs. */ -int bio_integrity_enabled(struct bio *bio) +bool bio_integrity_enabled(struct bio *bio) { + struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev); + if (!bio_is_rw(bio)) - return 0; + return false; /* Already protected? */ if (bio_integrity(bio)) - return 0; + return false; + + if (bi == NULL) + return false; + + if (bio_data_dir(bio) == READ && bi->verify_fn != NULL && + (bi->flags & INTEGRITY_FLAG_READ)) + return true; + + if (bio_data_dir(bio) == WRITE && bi->generate_fn != NULL && + (bi->flags & INTEGRITY_FLAG_WRITE)) + return true; - return bdev_integrity_enabled(bio->bi_bdev, bio_data_dir(bio)); + return false; } EXPORT_SYMBOL(bio_integrity_enabled); -- cgit v1.2.3 From 180b2f95dd331010a9930a65c8a18d6d81b94dc1 Mon Sep 17 00:00:00 2001 From: "Martin K. Petersen" Date: Fri, 26 Sep 2014 19:19:56 -0400 Subject: block: Replace bi_integrity with bi_special For commands like REQ_COPY we need a way to pass extra information along with each bio. Like integrity metadata this information must be available at the bottom of the stack so bi_private does not suffice. Rename the existing bi_integrity field to bi_special and make it a union so we can have different bio extensions for each class of command. We previously used bi_integrity != NULL as a way to identify whether a bio had integrity metadata or not. Introduce a REQ_INTEGRITY to be the indicator now that bi_special can contain different things. In addition, bio_integrity(bio) will now return a pointer to the integrity payload (when applicable). Signed-off-by: Martin K. Petersen Reviewed-by: Christoph Hellwig Reviewed-by: Sagi Grimberg Signed-off-by: Jens Axboe --- block/bio-integrity.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) (limited to 'block') diff --git a/block/bio-integrity.c b/block/bio-integrity.c index 36b788552c3e..bd3125c3c124 100644 --- a/block/bio-integrity.c +++ b/block/bio-integrity.c @@ -79,6 +79,7 @@ struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio, bip->bip_slab = idx; bip->bip_bio = bio; bio->bi_integrity = bip; + bio->bi_rw |= REQ_INTEGRITY; return bip; err: @@ -96,7 +97,7 @@ EXPORT_SYMBOL(bio_integrity_alloc); */ void bio_integrity_free(struct bio *bio) { - struct bio_integrity_payload *bip = bio->bi_integrity; + struct bio_integrity_payload *bip = bio_integrity(bio); struct bio_set *bs = bio->bi_pool; if (bip->bip_owns_buf) @@ -128,7 +129,7 @@ EXPORT_SYMBOL(bio_integrity_free); int bio_integrity_add_page(struct bio *bio, struct page *page, unsigned int len, unsigned int offset) { - struct bio_integrity_payload *bip = bio->bi_integrity; + struct bio_integrity_payload *bip = bio_integrity(bio); struct bio_vec *iv; if (bip->bip_vcnt >= bip->bip_max_vcnt) { @@ -229,7 +230,7 @@ EXPORT_SYMBOL(bio_integrity_tag_size); static int bio_integrity_tag(struct bio *bio, void *tag_buf, unsigned int len, int set) { - struct bio_integrity_payload *bip = bio->bi_integrity; + struct bio_integrity_payload *bip = bio_integrity(bio); struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev); unsigned int nr_sectors; @@ -304,12 +305,12 @@ static int bio_integrity_generate_verify(struct bio *bio, int operate) struct bio_vec *bv; sector_t sector; unsigned int sectors, ret = 0, i; - void *prot_buf = bio->bi_integrity->bip_buf; + void *prot_buf = bio_integrity(bio)->bip_buf; if (operate) sector = bio->bi_iter.bi_sector; else - sector = bio->bi_integrity->bip_iter.bi_sector; + sector = bio_integrity(bio)->bip_iter.bi_sector; bix.disk_name = bio->bi_bdev->bd_disk->disk_name; bix.sector_size = bi->sector_size; @@ -505,7 +506,7 @@ static void bio_integrity_verify_fn(struct work_struct *work) */ void bio_integrity_endio(struct bio *bio, int error) { - struct bio_integrity_payload *bip = bio->bi_integrity; + struct bio_integrity_payload *bip = bio_integrity(bio); BUG_ON(bip->bip_bio != bio); @@ -536,7 +537,7 @@ EXPORT_SYMBOL(bio_integrity_endio); */ void bio_integrity_advance(struct bio *bio, unsigned int bytes_done) { - struct bio_integrity_payload *bip = bio->bi_integrity; + struct bio_integrity_payload *bip = bio_integrity(bio); struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev); unsigned bytes = bio_integrity_bytes(bi, bytes_done >> 9); @@ -558,7 +559,7 @@ EXPORT_SYMBOL(bio_integrity_advance); void bio_integrity_trim(struct bio *bio, unsigned int offset, unsigned int sectors) { - struct bio_integrity_payload *bip = bio->bi_integrity; + struct bio_integrity_payload *bip = bio_integrity(bio); struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev); bio_integrity_advance(bio, offset << 9); @@ -577,7 +578,7 @@ EXPORT_SYMBOL(bio_integrity_trim); int bio_integrity_clone(struct bio *bio, struct bio *bio_src, gfp_t gfp_mask) { - struct bio_integrity_payload *bip_src = bio_src->bi_integrity; + struct bio_integrity_payload *bip_src = bio_integrity(bio_src); struct bio_integrity_payload *bip; BUG_ON(bip_src == NULL); -- cgit v1.2.3 From 8492b68bc4025e7bce1d57761bd7c047efda2f81 Mon Sep 17 00:00:00 2001 From: "Martin K. Petersen" Date: Fri, 26 Sep 2014 19:19:57 -0400 Subject: block: Remove integrity tagging functions None of the filesystems appear interested in using the integrity tagging feature. Potentially because very few storage devices actually permit using the application tag space. Remove the tagging functions. Signed-off-by: Martin K. Petersen Reviewed-by: Christoph Hellwig Reviewed-by: Sagi Grimberg Signed-off-by: Jens Axboe --- block/bio-integrity.c | 94 +-------------------------------------------------- block/blk-integrity.c | 2 -- 2 files changed, 1 insertion(+), 95 deletions(-) (limited to 'block') diff --git a/block/bio-integrity.c b/block/bio-integrity.c index bd3125c3c124..367bb24bb9f1 100644 --- a/block/bio-integrity.c +++ b/block/bio-integrity.c @@ -209,90 +209,6 @@ static inline unsigned int bio_integrity_bytes(struct blk_integrity *bi, return bio_integrity_hw_sectors(bi, sectors) * bi->tuple_size; } -/** - * bio_integrity_tag_size - Retrieve integrity tag space - * @bio: bio to inspect - * - * Description: Returns the maximum number of tag bytes that can be - * attached to this bio. Filesystems can use this to determine how - * much metadata to attach to an I/O. - */ -unsigned int bio_integrity_tag_size(struct bio *bio) -{ - struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev); - - BUG_ON(bio->bi_iter.bi_size == 0); - - return bi->tag_size * (bio->bi_iter.bi_size / bi->sector_size); -} -EXPORT_SYMBOL(bio_integrity_tag_size); - -static int bio_integrity_tag(struct bio *bio, void *tag_buf, unsigned int len, - int set) -{ - struct bio_integrity_payload *bip = bio_integrity(bio); - struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev); - unsigned int nr_sectors; - - BUG_ON(bip->bip_buf == NULL); - - if (bi->tag_size == 0) - return -1; - - nr_sectors = bio_integrity_hw_sectors(bi, - DIV_ROUND_UP(len, bi->tag_size)); - - if (nr_sectors * bi->tuple_size > bip->bip_iter.bi_size) { - printk(KERN_ERR "%s: tag too big for bio: %u > %u\n", __func__, - nr_sectors * bi->tuple_size, bip->bip_iter.bi_size); - return -1; - } - - if (set) - bi->set_tag_fn(bip->bip_buf, tag_buf, nr_sectors); - else - bi->get_tag_fn(bip->bip_buf, tag_buf, nr_sectors); - - return 0; -} - -/** - * bio_integrity_set_tag - Attach a tag buffer to a bio - * @bio: bio to attach buffer to - * @tag_buf: Pointer to a buffer containing tag data - * @len: Length of the included buffer - * - * Description: Use this function to tag a bio by leveraging the extra - * space provided by devices formatted with integrity protection. The - * size of the integrity buffer must be <= to the size reported by - * bio_integrity_tag_size(). - */ -int bio_integrity_set_tag(struct bio *bio, void *tag_buf, unsigned int len) -{ - BUG_ON(bio_data_dir(bio) != WRITE); - - return bio_integrity_tag(bio, tag_buf, len, 1); -} -EXPORT_SYMBOL(bio_integrity_set_tag); - -/** - * bio_integrity_get_tag - Retrieve a tag buffer from a bio - * @bio: bio to retrieve buffer from - * @tag_buf: Pointer to a buffer for the tag data - * @len: Length of the target buffer - * - * Description: Use this function to retrieve the tag buffer from a - * completed I/O. The size of the integrity buffer must be <= to the - * size reported by bio_integrity_tag_size(). - */ -int bio_integrity_get_tag(struct bio *bio, void *tag_buf, unsigned int len) -{ - BUG_ON(bio_data_dir(bio) != READ); - - return bio_integrity_tag(bio, tag_buf, len, 0); -} -EXPORT_SYMBOL(bio_integrity_get_tag); - /** * bio_integrity_generate_verify - Generate/verify integrity metadata for a bio * @bio: bio to generate/verify integrity metadata for @@ -355,14 +271,6 @@ static void bio_integrity_generate(struct bio *bio) bio_integrity_generate_verify(bio, 1); } -static inline unsigned short blk_integrity_tuple_size(struct blk_integrity *bi) -{ - if (bi) - return bi->tuple_size; - - return 0; -} - /** * bio_integrity_prep - Prepare bio for integrity I/O * @bio: bio to prepare @@ -393,7 +301,7 @@ int bio_integrity_prep(struct bio *bio) sectors = bio_integrity_hw_sectors(bi, bio_sectors(bio)); /* Allocate kernel buffer for protection data */ - len = sectors * blk_integrity_tuple_size(bi); + len = sectors * bi->tuple_size; buf = kmalloc(len, GFP_NOIO | q->bounce_gfp); if (unlikely(buf == NULL)) { printk(KERN_ERR "could not allocate integrity buffer\n"); diff --git a/block/blk-integrity.c b/block/blk-integrity.c index 7fbab84399e6..7ac17160ab69 100644 --- a/block/blk-integrity.c +++ b/block/blk-integrity.c @@ -418,8 +418,6 @@ int blk_integrity_register(struct gendisk *disk, struct blk_integrity *template) bi->generate_fn = template->generate_fn; bi->verify_fn = template->verify_fn; bi->tuple_size = template->tuple_size; - bi->set_tag_fn = template->set_tag_fn; - bi->get_tag_fn = template->get_tag_fn; bi->tag_size = template->tag_size; } else bi->name = bi_unsupported_name; -- cgit v1.2.3 From 5f9378fa9ca214977b5bfc12197c67eea450fc40 Mon Sep 17 00:00:00 2001 From: "Martin K. Petersen" Date: Fri, 26 Sep 2014 19:19:58 -0400 Subject: block: Remove bip_buf bip_buf is not really needed so we can remove it. Signed-off-by: Martin K. Petersen Reviewed-by: Christoph Hellwig Reviewed-by: Sagi Grimberg Signed-off-by: Jens Axboe --- block/bio-integrity.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'block') diff --git a/block/bio-integrity.c b/block/bio-integrity.c index 367bb24bb9f1..e84f7fb8694b 100644 --- a/block/bio-integrity.c +++ b/block/bio-integrity.c @@ -101,7 +101,8 @@ void bio_integrity_free(struct bio *bio) struct bio_set *bs = bio->bi_pool; if (bip->bip_owns_buf) - kfree(bip->bip_buf); + kfree(page_address(bip->bip_vec->bv_page) + + bip->bip_vec->bv_offset); if (bs) { if (bip->bip_slab != BIO_POOL_NONE) @@ -219,14 +220,16 @@ static int bio_integrity_generate_verify(struct bio *bio, int operate) struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev); struct blk_integrity_exchg bix; struct bio_vec *bv; + struct bio_integrity_payload *bip = bio_integrity(bio); sector_t sector; unsigned int sectors, ret = 0, i; - void *prot_buf = bio_integrity(bio)->bip_buf; + void *prot_buf = page_address(bip->bip_vec->bv_page) + + bip->bip_vec->bv_offset; if (operate) sector = bio->bi_iter.bi_sector; else - sector = bio_integrity(bio)->bip_iter.bi_sector; + sector = bip->bip_iter.bi_sector; bix.disk_name = bio->bi_bdev->bd_disk->disk_name; bix.sector_size = bi->sector_size; @@ -321,7 +324,6 @@ int bio_integrity_prep(struct bio *bio) } bip->bip_owns_buf = 1; - bip->bip_buf = buf; bip->bip_iter.bi_size = len; bip->bip_iter.bi_sector = bio->bi_iter.bi_sector; -- cgit v1.2.3 From 3be91c4a3d090bd700bd6ee5bf457c1bbf189a4f Mon Sep 17 00:00:00 2001 From: "Martin K. Petersen" Date: Fri, 26 Sep 2014 19:19:59 -0400 Subject: block: Deprecate the use of the term sector in the context of block integrity The protection interval is not necessarily tied to the logical block size of a block device. Stop using the terms "sector" and "sectors". Going forward we will use the term "seed" to describe the initial reference tag value for a given I/O. "Interval" will be used to describe the portion of the data buffer that a given piece of protection information is associated with. Signed-off-by: Martin K. Petersen Reviewed-by: Christoph Hellwig Reviewed-by: Sagi Grimberg Signed-off-by: Jens Axboe --- block/bio-integrity.c | 42 +++++++++++++++++++++--------------------- block/blk-integrity.c | 10 +++++----- 2 files changed, 26 insertions(+), 26 deletions(-) (limited to 'block') diff --git a/block/bio-integrity.c b/block/bio-integrity.c index e84f7fb8694b..6a3aacf57b19 100644 --- a/block/bio-integrity.c +++ b/block/bio-integrity.c @@ -185,20 +185,20 @@ bool bio_integrity_enabled(struct bio *bio) EXPORT_SYMBOL(bio_integrity_enabled); /** - * bio_integrity_hw_sectors - Convert 512b sectors to hardware ditto + * bio_integrity_intervals - Return number of integrity intervals for a bio * @bi: blk_integrity profile for device - * @sectors: Number of 512 sectors to convert + * @sectors: Size of the bio in 512-byte sectors * * Description: The block layer calculates everything in 512 byte - * sectors but integrity metadata is done in terms of the hardware - * sector size of the storage device. Convert the block layer sectors - * to physical sectors. + * sectors but integrity metadata is done in terms of the data integrity + * interval size of the storage device. Convert the block layer sectors + * to the appropriate number of integrity intervals. */ -static inline unsigned int bio_integrity_hw_sectors(struct blk_integrity *bi, - unsigned int sectors) +static inline unsigned int bio_integrity_intervals(struct blk_integrity *bi, + unsigned int sectors) { /* At this point there are only 512b or 4096b DIF/EPP devices */ - if (bi->sector_size == 4096) + if (bi->interval == 4096) return sectors >>= 3; return sectors; @@ -207,7 +207,7 @@ static inline unsigned int bio_integrity_hw_sectors(struct blk_integrity *bi, static inline unsigned int bio_integrity_bytes(struct blk_integrity *bi, unsigned int sectors) { - return bio_integrity_hw_sectors(bi, sectors) * bi->tuple_size; + return bio_integrity_intervals(bi, sectors) * bi->tuple_size; } /** @@ -221,25 +221,25 @@ static int bio_integrity_generate_verify(struct bio *bio, int operate) struct blk_integrity_exchg bix; struct bio_vec *bv; struct bio_integrity_payload *bip = bio_integrity(bio); - sector_t sector; - unsigned int sectors, ret = 0, i; + sector_t seed; + unsigned int intervals, ret = 0, i; void *prot_buf = page_address(bip->bip_vec->bv_page) + bip->bip_vec->bv_offset; if (operate) - sector = bio->bi_iter.bi_sector; + seed = bio->bi_iter.bi_sector; else - sector = bip->bip_iter.bi_sector; + seed = bip->bip_iter.bi_sector; bix.disk_name = bio->bi_bdev->bd_disk->disk_name; - bix.sector_size = bi->sector_size; + bix.interval = bi->interval; bio_for_each_segment_all(bv, bio, i) { void *kaddr = kmap_atomic(bv->bv_page); bix.data_buf = kaddr + bv->bv_offset; bix.data_size = bv->bv_len; bix.prot_buf = prot_buf; - bix.sector = sector; + bix.seed = seed; if (operate) bi->generate_fn(&bix); @@ -251,9 +251,9 @@ static int bio_integrity_generate_verify(struct bio *bio, int operate) } } - sectors = bv->bv_len / bi->sector_size; - sector += sectors; - prot_buf += sectors * bi->tuple_size; + intervals = bv->bv_len / bi->interval; + seed += intervals; + prot_buf += intervals * bi->tuple_size; kunmap_atomic(kaddr); } @@ -294,17 +294,17 @@ int bio_integrity_prep(struct bio *bio) unsigned long start, end; unsigned int len, nr_pages; unsigned int bytes, offset, i; - unsigned int sectors; + unsigned int intervals; bi = bdev_get_integrity(bio->bi_bdev); q = bdev_get_queue(bio->bi_bdev); BUG_ON(bi == NULL); BUG_ON(bio_integrity(bio)); - sectors = bio_integrity_hw_sectors(bi, bio_sectors(bio)); + intervals = bio_integrity_intervals(bi, bio_sectors(bio)); /* Allocate kernel buffer for protection data */ - len = sectors * bi->tuple_size; + len = intervals * bi->tuple_size; buf = kmalloc(len, GFP_NOIO | q->bounce_gfp); if (unlikely(buf == NULL)) { printk(KERN_ERR "could not allocate integrity buffer\n"); diff --git a/block/blk-integrity.c b/block/blk-integrity.c index 7ac17160ab69..3a83a7d08177 100644 --- a/block/blk-integrity.c +++ b/block/blk-integrity.c @@ -154,10 +154,10 @@ int blk_integrity_compare(struct gendisk *gd1, struct gendisk *gd2) if (!b1 || !b2) return -1; - if (b1->sector_size != b2->sector_size) { - printk(KERN_ERR "%s: %s/%s sector sz %u != %u\n", __func__, - gd1->disk_name, gd2->disk_name, - b1->sector_size, b2->sector_size); + if (b1->interval != b2->interval) { + pr_err("%s: %s/%s protection interval %u != %u\n", + __func__, gd1->disk_name, gd2->disk_name, + b1->interval, b2->interval); return -1; } @@ -407,7 +407,7 @@ int blk_integrity_register(struct gendisk *disk, struct blk_integrity *template) kobject_uevent(&bi->kobj, KOBJ_ADD); bi->flags |= INTEGRITY_FLAG_READ | INTEGRITY_FLAG_WRITE; - bi->sector_size = queue_logical_block_size(disk->queue); + bi->interval = queue_logical_block_size(disk->queue); disk->integrity = bi; } else bi = disk->integrity; -- cgit v1.2.3 From 5a2aa873059fec8b6385071d9c0802893a9b2f41 Mon Sep 17 00:00:00 2001 From: "Martin K. Petersen" Date: Fri, 26 Sep 2014 19:20:00 -0400 Subject: block: Make protection interval calculation generic Now that the protection interval has been detached from the sector size we need to be able to handle sizes that are different from 4K and 512. Make the interval calculation generic. Signed-off-by: Martin K. Petersen Reviewed-by: Christoph Hellwig Reviewed-by: Sagi Grimberg Signed-off-by: Jens Axboe --- block/bio-integrity.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'block') diff --git a/block/bio-integrity.c b/block/bio-integrity.c index 6a3aacf57b19..cf40837e7710 100644 --- a/block/bio-integrity.c +++ b/block/bio-integrity.c @@ -197,11 +197,7 @@ EXPORT_SYMBOL(bio_integrity_enabled); static inline unsigned int bio_integrity_intervals(struct blk_integrity *bi, unsigned int sectors) { - /* At this point there are only 512b or 4096b DIF/EPP devices */ - if (bi->interval == 4096) - return sectors >>= 3; - - return sectors; + return sectors >> (ilog2(bi->interval) - 9); } static inline unsigned int bio_integrity_bytes(struct blk_integrity *bi, -- cgit v1.2.3 From 1859308853b19c4daf4afaab910d3d52ac1ec2ff Mon Sep 17 00:00:00 2001 From: "Martin K. Petersen" Date: Fri, 26 Sep 2014 19:20:01 -0400 Subject: block: Clean up the code used to generate and verify integrity metadata Instead of the "operate" parameter we pass in a seed value and a pointer to a function that can be used to process the integrity metadata. The generation function is changed to have a return value to fit into this scheme. Signed-off-by: Martin K. Petersen Reviewed-by: Sagi Grimberg Signed-off-by: Jens Axboe --- block/bio-integrity.c | 82 ++++++++++++++------------------------------------- 1 file changed, 22 insertions(+), 60 deletions(-) (limited to 'block') diff --git a/block/bio-integrity.c b/block/bio-integrity.c index cf40837e7710..fe4de033b34c 100644 --- a/block/bio-integrity.c +++ b/block/bio-integrity.c @@ -207,69 +207,43 @@ static inline unsigned int bio_integrity_bytes(struct blk_integrity *bi, } /** - * bio_integrity_generate_verify - Generate/verify integrity metadata for a bio + * bio_integrity_process - Process integrity metadata for a bio * @bio: bio to generate/verify integrity metadata for - * @operate: operate number, 1 for generate, 0 for verify + * @proc_fn: Pointer to the relevant processing function */ -static int bio_integrity_generate_verify(struct bio *bio, int operate) +static int bio_integrity_process(struct bio *bio, + integrity_processing_fn *proc_fn) { struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev); - struct blk_integrity_exchg bix; + struct blk_integrity_iter iter; struct bio_vec *bv; struct bio_integrity_payload *bip = bio_integrity(bio); - sector_t seed; - unsigned int intervals, ret = 0, i; + unsigned int i, ret = 0; void *prot_buf = page_address(bip->bip_vec->bv_page) + bip->bip_vec->bv_offset; - if (operate) - seed = bio->bi_iter.bi_sector; - else - seed = bip->bip_iter.bi_sector; - - bix.disk_name = bio->bi_bdev->bd_disk->disk_name; - bix.interval = bi->interval; + iter.disk_name = bio->bi_bdev->bd_disk->disk_name; + iter.interval = bi->interval; + iter.seed = bip_get_seed(bip); + iter.prot_buf = prot_buf; bio_for_each_segment_all(bv, bio, i) { void *kaddr = kmap_atomic(bv->bv_page); - bix.data_buf = kaddr + bv->bv_offset; - bix.data_size = bv->bv_len; - bix.prot_buf = prot_buf; - bix.seed = seed; - - if (operate) - bi->generate_fn(&bix); - else { - ret = bi->verify_fn(&bix); - if (ret) { - kunmap_atomic(kaddr); - return ret; - } - } - intervals = bv->bv_len / bi->interval; - seed += intervals; - prot_buf += intervals * bi->tuple_size; + iter.data_buf = kaddr + bv->bv_offset; + iter.data_size = bv->bv_len; + + ret = proc_fn(&iter); + if (ret) { + kunmap_atomic(kaddr); + return ret; + } kunmap_atomic(kaddr); } return ret; } -/** - * bio_integrity_generate - Generate integrity metadata for a bio - * @bio: bio to generate integrity metadata for - * - * Description: Generates integrity metadata for a bio by calling the - * block device's generation callback function. The bio must have a - * bip attached with enough room to accommodate the generated - * integrity metadata. - */ -static void bio_integrity_generate(struct bio *bio) -{ - bio_integrity_generate_verify(bio, 1); -} - /** * bio_integrity_prep - Prepare bio for integrity I/O * @bio: bio to prepare @@ -321,7 +295,7 @@ int bio_integrity_prep(struct bio *bio) bip->bip_owns_buf = 1; bip->bip_iter.bi_size = len; - bip->bip_iter.bi_sector = bio->bi_iter.bi_sector; + bip_set_seed(bip, bio->bi_iter.bi_sector); /* Map it */ offset = offset_in_page(buf); @@ -357,25 +331,12 @@ int bio_integrity_prep(struct bio *bio) /* Auto-generate integrity metadata if this is a write */ if (bio_data_dir(bio) == WRITE) - bio_integrity_generate(bio); + bio_integrity_process(bio, bi->generate_fn); return 0; } EXPORT_SYMBOL(bio_integrity_prep); -/** - * bio_integrity_verify - Verify integrity metadata for a bio - * @bio: bio to verify - * - * Description: This function is called to verify the integrity of a - * bio. The data in the bio io_vec is compared to the integrity - * metadata returned by the HBA. - */ -static int bio_integrity_verify(struct bio *bio) -{ - return bio_integrity_generate_verify(bio, 0); -} - /** * bio_integrity_verify_fn - Integrity I/O completion worker * @work: Work struct stored in bio to be verified @@ -389,9 +350,10 @@ static void bio_integrity_verify_fn(struct work_struct *work) struct bio_integrity_payload *bip = container_of(work, struct bio_integrity_payload, bip_work); struct bio *bio = bip->bip_bio; + struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev); int error; - error = bio_integrity_verify(bio); + error = bio_integrity_process(bio, bi->verify_fn); /* Restore original bio completion handler */ bio->bi_end_io = bip->bip_end_io; -- cgit v1.2.3 From 8288f496eb1b1905c425e92eaf1abbb29119217b Mon Sep 17 00:00:00 2001 From: "Martin K. Petersen" Date: Fri, 26 Sep 2014 19:20:02 -0400 Subject: block: Add prefix to block integrity profile flags Add a BLK_ prefix to the integrity profile flags. Also rename the flags to be more consistent with the generate/verify terminology in the rest of the integrity code. Signed-off-by: Martin K. Petersen Reviewed-by: Christoph Hellwig Reviewed-by: Sagi Grimberg Signed-off-by: Jens Axboe --- block/bio-integrity.c | 4 ++-- block/blk-integrity.c | 43 ++++++++++++++++++++++--------------------- 2 files changed, 24 insertions(+), 23 deletions(-) (limited to 'block') diff --git a/block/bio-integrity.c b/block/bio-integrity.c index fe4de033b34c..e64733bb29b6 100644 --- a/block/bio-integrity.c +++ b/block/bio-integrity.c @@ -173,11 +173,11 @@ bool bio_integrity_enabled(struct bio *bio) return false; if (bio_data_dir(bio) == READ && bi->verify_fn != NULL && - (bi->flags & INTEGRITY_FLAG_READ)) + (bi->flags & BLK_INTEGRITY_VERIFY)) return true; if (bio_data_dir(bio) == WRITE && bi->generate_fn != NULL && - (bi->flags & INTEGRITY_FLAG_WRITE)) + (bi->flags & BLK_INTEGRITY_GENERATE)) return true; return false; diff --git a/block/blk-integrity.c b/block/blk-integrity.c index 3a83a7d08177..a7436ccc936b 100644 --- a/block/blk-integrity.c +++ b/block/blk-integrity.c @@ -269,42 +269,42 @@ static ssize_t integrity_tag_size_show(struct blk_integrity *bi, char *page) return sprintf(page, "0\n"); } -static ssize_t integrity_read_store(struct blk_integrity *bi, - const char *page, size_t count) +static ssize_t integrity_verify_store(struct blk_integrity *bi, + const char *page, size_t count) { char *p = (char *) page; unsigned long val = simple_strtoul(p, &p, 10); if (val) - bi->flags |= INTEGRITY_FLAG_READ; + bi->flags |= BLK_INTEGRITY_VERIFY; else - bi->flags &= ~INTEGRITY_FLAG_READ; + bi->flags &= ~BLK_INTEGRITY_VERIFY; return count; } -static ssize_t integrity_read_show(struct blk_integrity *bi, char *page) +static ssize_t integrity_verify_show(struct blk_integrity *bi, char *page) { - return sprintf(page, "%d\n", (bi->flags & INTEGRITY_FLAG_READ) != 0); + return sprintf(page, "%d\n", (bi->flags & BLK_INTEGRITY_VERIFY) != 0); } -static ssize_t integrity_write_store(struct blk_integrity *bi, - const char *page, size_t count) +static ssize_t integrity_generate_store(struct blk_integrity *bi, + const char *page, size_t count) { char *p = (char *) page; unsigned long val = simple_strtoul(p, &p, 10); if (val) - bi->flags |= INTEGRITY_FLAG_WRITE; + bi->flags |= BLK_INTEGRITY_GENERATE; else - bi->flags &= ~INTEGRITY_FLAG_WRITE; + bi->flags &= ~BLK_INTEGRITY_GENERATE; return count; } -static ssize_t integrity_write_show(struct blk_integrity *bi, char *page) +static ssize_t integrity_generate_show(struct blk_integrity *bi, char *page) { - return sprintf(page, "%d\n", (bi->flags & INTEGRITY_FLAG_WRITE) != 0); + return sprintf(page, "%d\n", (bi->flags & BLK_INTEGRITY_GENERATE) != 0); } static struct integrity_sysfs_entry integrity_format_entry = { @@ -317,23 +317,23 @@ static struct integrity_sysfs_entry integrity_tag_size_entry = { .show = integrity_tag_size_show, }; -static struct integrity_sysfs_entry integrity_read_entry = { +static struct integrity_sysfs_entry integrity_verify_entry = { .attr = { .name = "read_verify", .mode = S_IRUGO | S_IWUSR }, - .show = integrity_read_show, - .store = integrity_read_store, + .show = integrity_verify_show, + .store = integrity_verify_store, }; -static struct integrity_sysfs_entry integrity_write_entry = { +static struct integrity_sysfs_entry integrity_generate_entry = { .attr = { .name = "write_generate", .mode = S_IRUGO | S_IWUSR }, - .show = integrity_write_show, - .store = integrity_write_store, + .show = integrity_generate_show, + .store = integrity_generate_store, }; static struct attribute *integrity_attrs[] = { &integrity_format_entry.attr, &integrity_tag_size_entry.attr, - &integrity_read_entry.attr, - &integrity_write_entry.attr, + &integrity_verify_entry.attr, + &integrity_generate_entry.attr, NULL, }; @@ -406,7 +406,7 @@ int blk_integrity_register(struct gendisk *disk, struct blk_integrity *template) kobject_uevent(&bi->kobj, KOBJ_ADD); - bi->flags |= INTEGRITY_FLAG_READ | INTEGRITY_FLAG_WRITE; + bi->flags |= BLK_INTEGRITY_VERIFY | BLK_INTEGRITY_GENERATE; bi->interval = queue_logical_block_size(disk->queue); disk->integrity = bi; } else @@ -419,6 +419,7 @@ int blk_integrity_register(struct gendisk *disk, struct blk_integrity *template) bi->verify_fn = template->verify_fn; bi->tuple_size = template->tuple_size; bi->tag_size = template->tag_size; + bi->flags |= template->flags; } else bi->name = bi_unsupported_name; -- cgit v1.2.3 From 3aec2f41a8baeb70aa77556a4e4dcec7d9d70b4d Mon Sep 17 00:00:00 2001 From: "Martin K. Petersen" Date: Fri, 26 Sep 2014 19:20:03 -0400 Subject: block: Add a disk flag to block integrity profile So far we have relied on the app tag size to determine whether a disk has been formatted with T10 protection information or not. However, not all target devices provide application tag storage. Add a flag to the block integrity profile that indicates whether the disk has been formatted with protection information. Signed-off-by: Martin K. Petersen Reviewed-by: Sagi Grimberg Signed-off-by: Jens Axboe --- block/blk-integrity.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'block') diff --git a/block/blk-integrity.c b/block/blk-integrity.c index a7436ccc936b..1c6ba442cd91 100644 --- a/block/blk-integrity.c +++ b/block/blk-integrity.c @@ -307,6 +307,12 @@ static ssize_t integrity_generate_show(struct blk_integrity *bi, char *page) return sprintf(page, "%d\n", (bi->flags & BLK_INTEGRITY_GENERATE) != 0); } +static ssize_t integrity_device_show(struct blk_integrity *bi, char *page) +{ + return sprintf(page, "%u\n", + (bi->flags & BLK_INTEGRITY_DEVICE_CAPABLE) != 0); +} + static struct integrity_sysfs_entry integrity_format_entry = { .attr = { .name = "format", .mode = S_IRUGO }, .show = integrity_format_show, @@ -329,11 +335,17 @@ static struct integrity_sysfs_entry integrity_generate_entry = { .store = integrity_generate_store, }; +static struct integrity_sysfs_entry integrity_device_entry = { + .attr = { .name = "device_is_integrity_capable", .mode = S_IRUGO }, + .show = integrity_device_show, +}; + static struct attribute *integrity_attrs[] = { &integrity_format_entry.attr, &integrity_tag_size_entry.attr, &integrity_verify_entry.attr, &integrity_generate_entry.attr, + &integrity_device_entry.attr, NULL, }; -- cgit v1.2.3 From b1f01388574c9329922f760fc2a7335c2d14b08b Mon Sep 17 00:00:00 2001 From: "Martin K. Petersen" Date: Fri, 26 Sep 2014 19:20:04 -0400 Subject: block: Relocate bio integrity flags Move flags affecting the integrity code out of the bio bi_flags and into the block integrity payload. Signed-off-by: Martin K. Petersen Reviewed-by: Sagi Grimberg Signed-off-by: Jens Axboe --- block/bio-integrity.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'block') diff --git a/block/bio-integrity.c b/block/bio-integrity.c index e64733bb29b6..26aa901b961f 100644 --- a/block/bio-integrity.c +++ b/block/bio-integrity.c @@ -100,7 +100,7 @@ void bio_integrity_free(struct bio *bio) struct bio_integrity_payload *bip = bio_integrity(bio); struct bio_set *bs = bio->bi_pool; - if (bip->bip_owns_buf) + if (bip->bip_flags & BIP_BLOCK_INTEGRITY) kfree(page_address(bip->bip_vec->bv_page) + bip->bip_vec->bv_offset); @@ -293,7 +293,7 @@ int bio_integrity_prep(struct bio *bio) return -EIO; } - bip->bip_owns_buf = 1; + bip->bip_flags |= BIP_BLOCK_INTEGRITY; bip->bip_iter.bi_size = len; bip_set_seed(bip, bio->bi_iter.bi_sector); -- cgit v1.2.3 From aae7df50190a640e51bfe11c93f94741ac82ff0b Mon Sep 17 00:00:00 2001 From: "Martin K. Petersen" Date: Fri, 26 Sep 2014 19:20:05 -0400 Subject: block: Integrity checksum flag Make the choice of checksum a per-I/O property by introducing a flag that can be inspected by the SCSI layer. There are several reasons for this: 1. It allows us to switch choice of checksum without unloading and reloading the HBA driver. 2. During error recovery we need to be able to tell the HBA that checksums read from disk should not be verified and converted to IP checksums. 3. For error injection purposes we need to be able to write a bad guard tag to storage. Since the storage device only supports T10 CRC we need to be able to disable IP checksum conversion on the HBA. Signed-off-by: Martin K. Petersen Reviewed-by: Sagi Grimberg Signed-off-by: Jens Axboe --- block/bio-integrity.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'block') diff --git a/block/bio-integrity.c b/block/bio-integrity.c index 26aa901b961f..8e0548484dd3 100644 --- a/block/bio-integrity.c +++ b/block/bio-integrity.c @@ -297,6 +297,9 @@ int bio_integrity_prep(struct bio *bio) bip->bip_iter.bi_size = len; bip_set_seed(bip, bio->bi_iter.bi_sector); + if (bi->flags & BLK_INTEGRITY_IP_CHECKSUM) + bip->bip_flags |= BIP_IP_CHECKSUM; + /* Map it */ offset = offset_in_page(buf); for (i = 0 ; i < nr_pages ; i++) { -- cgit v1.2.3 From 4eaf99beadcefbf126fa05e66fb40fca999e09fd Mon Sep 17 00:00:00 2001 From: "Martin K. Petersen" Date: Fri, 26 Sep 2014 19:20:06 -0400 Subject: block: Don't merge requests if integrity flags differ We'd occasionally merge requests with conflicting integrity flags. Introduce a merge helper which checks that the requests have compatible integrity payloads. Signed-off-by: Martin K. Petersen Reviewed-by: Christoph Hellwig Reviewed-by: Sagi Grimberg Signed-off-by: Jens Axboe --- block/blk-integrity.c | 36 ++++++++++++++++++++++++++---------- block/blk-merge.c | 6 +++--- 2 files changed, 29 insertions(+), 13 deletions(-) (limited to 'block') diff --git a/block/blk-integrity.c b/block/blk-integrity.c index 1c6ba442cd91..79ffb4855af0 100644 --- a/block/blk-integrity.c +++ b/block/blk-integrity.c @@ -186,37 +186,53 @@ int blk_integrity_compare(struct gendisk *gd1, struct gendisk *gd2) } EXPORT_SYMBOL(blk_integrity_compare); -int blk_integrity_merge_rq(struct request_queue *q, struct request *req, - struct request *next) +bool blk_integrity_merge_rq(struct request_queue *q, struct request *req, + struct request *next) { - if (blk_integrity_rq(req) != blk_integrity_rq(next)) - return -1; + if (blk_integrity_rq(req) == 0 && blk_integrity_rq(next) == 0) + return true; + + if (blk_integrity_rq(req) == 0 || blk_integrity_rq(next) == 0) + return false; + + if (bio_integrity(req->bio)->bip_flags != + bio_integrity(next->bio)->bip_flags) + return false; if (req->nr_integrity_segments + next->nr_integrity_segments > q->limits.max_integrity_segments) - return -1; + return false; - return 0; + return true; } EXPORT_SYMBOL(blk_integrity_merge_rq); -int blk_integrity_merge_bio(struct request_queue *q, struct request *req, - struct bio *bio) +bool blk_integrity_merge_bio(struct request_queue *q, struct request *req, + struct bio *bio) { int nr_integrity_segs; struct bio *next = bio->bi_next; + if (blk_integrity_rq(req) == 0 && bio_integrity(bio) == NULL) + return true; + + if (blk_integrity_rq(req) == 0 || bio_integrity(bio) == NULL) + return false; + + if (bio_integrity(req->bio)->bip_flags != bio_integrity(bio)->bip_flags) + return false; + bio->bi_next = NULL; nr_integrity_segs = blk_rq_count_integrity_sg(q, bio); bio->bi_next = next; if (req->nr_integrity_segments + nr_integrity_segs > q->limits.max_integrity_segments) - return -1; + return false; req->nr_integrity_segments += nr_integrity_segs; - return 0; + return true; } EXPORT_SYMBOL(blk_integrity_merge_bio); diff --git a/block/blk-merge.c b/block/blk-merge.c index 77881798f793..f71bad35b4cc 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -313,7 +313,7 @@ static inline int ll_new_hw_segment(struct request_queue *q, if (req->nr_phys_segments + nr_phys_segs > queue_max_segments(q)) goto no_merge; - if (bio_integrity(bio) && blk_integrity_merge_bio(q, req, bio)) + if (blk_integrity_merge_bio(q, req, bio) == false) goto no_merge; /* @@ -410,7 +410,7 @@ static int ll_merge_requests_fn(struct request_queue *q, struct request *req, if (total_phys_segments > queue_max_segments(q)) return 0; - if (blk_integrity_rq(req) && blk_integrity_merge_rq(q, req, next)) + if (blk_integrity_merge_rq(q, req, next) == false) return 0; /* Merge is OK... */ @@ -590,7 +590,7 @@ bool blk_rq_merge_ok(struct request *rq, struct bio *bio) return false; /* only merge integrity protected bio into ditto rq */ - if (bio_integrity(bio) != blk_integrity_rq(rq)) + if (blk_integrity_merge_bio(rq->q, rq, bio) == false) return false; /* must be using the same buffer */ -- cgit v1.2.3 From 2341c2f8c33196d02cf5a721746eea4e3c06674a Mon Sep 17 00:00:00 2001 From: "Martin K. Petersen" Date: Fri, 26 Sep 2014 19:20:07 -0400 Subject: block: Add T10 Protection Information functions The T10 Protection Information format is also used by some devices that do not go through the SCSI layer (virtual block devices, NVMe). Relocate the relevant functions to a block layer library that can be used without involving SCSI. Signed-off-by: Martin K. Petersen Reviewed-by: Christoph Hellwig Signed-off-by: Jens Axboe --- block/Kconfig | 1 + block/Makefile | 4 +- block/t10-pi.c | 197 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 200 insertions(+), 2 deletions(-) create mode 100644 block/t10-pi.c (limited to 'block') diff --git a/block/Kconfig b/block/Kconfig index 2429515c05c2..161491d0a879 100644 --- a/block/Kconfig +++ b/block/Kconfig @@ -77,6 +77,7 @@ config BLK_DEV_BSGLIB config BLK_DEV_INTEGRITY bool "Block layer data integrity support" + select CRC_T10DIF if BLK_DEV_INTEGRITY ---help--- Some storage devices allow extra information to be stored/retrieved to help protect the data. The block layer diff --git a/block/Makefile b/block/Makefile index a2ce6ac935ec..00ecc97629db 100644 --- a/block/Makefile +++ b/block/Makefile @@ -20,6 +20,6 @@ obj-$(CONFIG_IOSCHED_DEADLINE) += deadline-iosched.o obj-$(CONFIG_IOSCHED_CFQ) += cfq-iosched.o obj-$(CONFIG_BLOCK_COMPAT) += compat_ioctl.o -obj-$(CONFIG_BLK_DEV_INTEGRITY) += blk-integrity.o obj-$(CONFIG_BLK_CMDLINE_PARSER) += cmdline-parser.o -obj-$(CONFIG_BLK_DEV_INTEGRITY) += bio-integrity.o +obj-$(CONFIG_BLK_DEV_INTEGRITY) += bio-integrity.o blk-integrity.o t10-pi.o + diff --git a/block/t10-pi.c b/block/t10-pi.c new file mode 100644 index 000000000000..24d6e9715318 --- /dev/null +++ b/block/t10-pi.c @@ -0,0 +1,197 @@ +/* + * t10_pi.c - Functions for generating and verifying T10 Protection + * Information. + * + * Copyright (C) 2007, 2008, 2014 Oracle Corporation + * Written by: Martin K. Petersen + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License version + * 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; see the file COPYING. If not, write to + * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, + * USA. + * + */ + +#include +#include +#include +#include + +typedef __be16 (csum_fn) (void *, unsigned int); + +static const __be16 APP_ESCAPE = (__force __be16) 0xffff; +static const __be32 REF_ESCAPE = (__force __be32) 0xffffffff; + +static __be16 t10_pi_crc_fn(void *data, unsigned int len) +{ + return cpu_to_be16(crc_t10dif(data, len)); +} + +static __be16 t10_pi_ip_fn(void *data, unsigned int len) +{ + return (__force __be16)ip_compute_csum(data, len); +} + +/* + * Type 1 and Type 2 protection use the same format: 16 bit guard tag, + * 16 bit app tag, 32 bit reference tag. Type 3 does not define the ref + * tag. + */ +static int t10_pi_generate(struct blk_integrity_iter *iter, csum_fn *fn, + unsigned int type) +{ + unsigned int i; + + for (i = 0 ; i < iter->data_size ; i += iter->interval) { + struct t10_pi_tuple *pi = iter->prot_buf; + + pi->guard_tag = fn(iter->data_buf, iter->interval); + pi->app_tag = 0; + + if (type == 1) + pi->ref_tag = cpu_to_be32(lower_32_bits(iter->seed)); + else + pi->ref_tag = 0; + + iter->data_buf += iter->interval; + iter->prot_buf += sizeof(struct t10_pi_tuple); + iter->seed++; + } + + return 0; +} + +static int t10_pi_verify(struct blk_integrity_iter *iter, csum_fn *fn, + unsigned int type) +{ + unsigned int i; + + for (i = 0 ; i < iter->data_size ; i += iter->interval) { + struct t10_pi_tuple *pi = iter->prot_buf; + __be16 csum; + + switch (type) { + case 1: + case 2: + if (pi->app_tag == APP_ESCAPE) + goto next; + + if (be32_to_cpu(pi->ref_tag) != + lower_32_bits(iter->seed)) { + pr_err("%s: ref tag error at location %llu " \ + "(rcvd %u)\n", iter->disk_name, + (unsigned long long) + iter->seed, be32_to_cpu(pi->ref_tag)); + return -EILSEQ; + } + break; + case 3: + if (pi->app_tag == APP_ESCAPE && + pi->ref_tag == REF_ESCAPE) + goto next; + break; + } + + csum = fn(iter->data_buf, iter->interval); + + if (pi->guard_tag != csum) { + pr_err("%s: guard tag error at sector %llu " \ + "(rcvd %04x, want %04x)\n", iter->disk_name, + (unsigned long long)iter->seed, + be16_to_cpu(pi->guard_tag), be16_to_cpu(csum)); + return -EILSEQ; + } + +next: + iter->data_buf += iter->interval; + iter->prot_buf += sizeof(struct t10_pi_tuple); + iter->seed++; + } + + return 0; +} + +static int t10_pi_type1_generate_crc(struct blk_integrity_iter *iter) +{ + return t10_pi_generate(iter, t10_pi_crc_fn, 1); +} + +static int t10_pi_type1_generate_ip(struct blk_integrity_iter *iter) +{ + return t10_pi_generate(iter, t10_pi_ip_fn, 1); +} + +static int t10_pi_type1_verify_crc(struct blk_integrity_iter *iter) +{ + return t10_pi_verify(iter, t10_pi_crc_fn, 1); +} + +static int t10_pi_type1_verify_ip(struct blk_integrity_iter *iter) +{ + return t10_pi_verify(iter, t10_pi_ip_fn, 1); +} + +static int t10_pi_type3_generate_crc(struct blk_integrity_iter *iter) +{ + return t10_pi_generate(iter, t10_pi_crc_fn, 3); +} + +static int t10_pi_type3_generate_ip(struct blk_integrity_iter *iter) +{ + return t10_pi_generate(iter, t10_pi_ip_fn, 3); +} + +static int t10_pi_type3_verify_crc(struct blk_integrity_iter *iter) +{ + return t10_pi_verify(iter, t10_pi_crc_fn, 3); +} + +static int t10_pi_type3_verify_ip(struct blk_integrity_iter *iter) +{ + return t10_pi_verify(iter, t10_pi_ip_fn, 3); +} + +struct blk_integrity t10_pi_type1_crc = { + .name = "T10-DIF-TYPE1-CRC", + .generate_fn = t10_pi_type1_generate_crc, + .verify_fn = t10_pi_type1_verify_crc, + .tuple_size = sizeof(struct t10_pi_tuple), + .tag_size = 0, +}; +EXPORT_SYMBOL(t10_pi_type1_crc); + +struct blk_integrity t10_pi_type1_ip = { + .name = "T10-DIF-TYPE1-IP", + .generate_fn = t10_pi_type1_generate_ip, + .verify_fn = t10_pi_type1_verify_ip, + .tuple_size = sizeof(struct t10_pi_tuple), + .tag_size = 0, +}; +EXPORT_SYMBOL(t10_pi_type1_ip); + +struct blk_integrity t10_pi_type3_crc = { + .name = "T10-DIF-TYPE3-CRC", + .generate_fn = t10_pi_type3_generate_crc, + .verify_fn = t10_pi_type3_verify_crc, + .tuple_size = sizeof(struct t10_pi_tuple), + .tag_size = 0, +}; +EXPORT_SYMBOL(t10_pi_type3_crc); + +struct blk_integrity t10_pi_type3_ip = { + .name = "T10-DIF-TYPE3-IP", + .generate_fn = t10_pi_type3_generate_ip, + .verify_fn = t10_pi_type3_verify_ip, + .tuple_size = sizeof(struct t10_pi_tuple), + .tag_size = 0, +}; +EXPORT_SYMBOL(t10_pi_type3_ip); -- cgit v1.2.3 From 582940508b5d589229d0232e0eeee8fef0d54809 Mon Sep 17 00:00:00 2001 From: Rasmus Villemoes Date: Tue, 16 Sep 2014 22:51:16 +0200 Subject: block: Replace strnicmp with strncasecmp The kernel used to contain two functions for length-delimited, case-insensitive string comparison, strnicmp with correct semantics and a slightly buggy strncasecmp. The latter is the POSIX name, so strnicmp was renamed to strncasecmp, and strnicmp made into a wrapper for the new strncasecmp to avoid breaking existing users. To allow the compat wrapper strnicmp to be removed at some point in the future, and to avoid the extra indirection cost, do s/strnicmp/strncasecmp/g. Cc: Jens Axboe Signed-off-by: Rasmus Villemoes Signed-off-by: Jens Axboe --- block/partitions/mac.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'block') diff --git a/block/partitions/mac.c b/block/partitions/mac.c index 76d8ba6379a9..c2c48ec64b27 100644 --- a/block/partitions/mac.c +++ b/block/partitions/mac.c @@ -81,7 +81,7 @@ int mac_partition(struct parsed_partitions *state) be32_to_cpu(part->start_block) * (secsize/512), be32_to_cpu(part->block_count) * (secsize/512)); - if (!strnicmp(part->type, "Linux_RAID", 10)) + if (!strncasecmp(part->type, "Linux_RAID", 10)) state->parts[slot].flags = ADDPART_FLAG_RAID; #ifdef CONFIG_PPC_PMAC /* @@ -100,7 +100,7 @@ int mac_partition(struct parsed_partitions *state) goodness++; if (strcasecmp(part->type, "Apple_UNIX_SVR2") == 0 - || (strnicmp(part->type, "Linux", 5) == 0 + || (strncasecmp(part->type, "Linux", 5) == 0 && strcasecmp(part->type, "Linux_swap") != 0)) { int i, l; @@ -109,13 +109,13 @@ int mac_partition(struct parsed_partitions *state) if (strcmp(part->name, "/") == 0) goodness++; for (i = 0; i <= l - 4; ++i) { - if (strnicmp(part->name + i, "root", + if (strncasecmp(part->name + i, "root", 4) == 0) { goodness += 2; break; } } - if (strnicmp(part->name, "swap", 4) == 0) + if (strncasecmp(part->name, "swap", 4) == 0) goodness--; } -- cgit v1.2.3 From 4a0efdc933680d908de11712a774a2c9492c3d5a Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Wed, 1 Oct 2014 14:32:31 +0200 Subject: block: misplaced rq_complete tracepoint The rq_complete tracepoint was never issued for empty requests, causing the resulting blktrace information to never show any completion for those request. Signed-off-by: Hannes Reinecke Acked-by: Tejun Heo Signed-off-by: Jens Axboe --- block/blk-core.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'block') diff --git a/block/blk-core.c b/block/blk-core.c index e1c2775c7597..4aa9ccd02a50 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -2400,11 +2400,11 @@ bool blk_update_request(struct request *req, int error, unsigned int nr_bytes) { int total_bytes; + trace_block_rq_complete(req->q, req, nr_bytes); + if (!req->bio) return false; - trace_block_rq_complete(req->q, req, nr_bytes); - /* * For fs requests, rq is just carrier of independent bio's * and each partial completion should be handled separately. -- cgit v1.2.3 From 11dfce509eaa35e8fc81cb50d0910c0e235fd7e2 Mon Sep 17 00:00:00 2001 From: Junichi Nomura Date: Fri, 3 Oct 2014 17:27:11 -0400 Subject: block: use bio_clone_fast() in blk_rq_prep_clone() Request cloning clones bios in the request to track the completion of each bio. For that purpose, we can use bio_clone_fast() instead of bio_clone() to avoid unnecessary allocation and copy of bvecs. This patch reduces memory footprint of request-based device-mapper (about 1-4KB for each request) and is a preparation for further reduction of memory usage by removing unused bvec mempool. Signed-off-by: Jun'ichi Nomura Signed-off-by: Mike Snitzer Signed-off-by: Jens Axboe --- block/blk-core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'block') diff --git a/block/blk-core.c b/block/blk-core.c index 4aa9ccd02a50..ffcb47af35f3 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -2926,7 +2926,7 @@ int blk_rq_prep_clone(struct request *rq, struct request *rq_src, blk_rq_init(NULL, rq); __rq_for_each_bio(bio_src, rq_src) { - bio = bio_clone_bioset(bio_src, gfp_mask, bs); + bio = bio_clone_fast(bio_src, gfp_mask, bs); if (!bio) goto free_and_out; -- cgit v1.2.3 From d8f429e1669b9709f5b669aac9d734dbe0640891 Mon Sep 17 00:00:00 2001 From: Junichi Nomura Date: Fri, 3 Oct 2014 17:27:12 -0400 Subject: block: add bioset_create_nobvec() Users of bio_clone_fast() do not want bios with their own bvecs. Allocating a bvec mempool as part of the bioset intended for such users is a waste of memory. bioset_create_nobvec() creates a bioset that doesn't have the bvec mempool. Signed-off-by: Jun'ichi Nomura Signed-off-by: Mike Snitzer Signed-off-by: Jens Axboe --- block/bio.c | 61 ++++++++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 44 insertions(+), 17 deletions(-) (limited to 'block') diff --git a/block/bio.c b/block/bio.c index 3e6331d25d90..3e6e1986a5b2 100644 --- a/block/bio.c +++ b/block/bio.c @@ -428,6 +428,9 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, int nr_iovecs, struct bio_set *bs) front_pad = 0; inline_vecs = nr_iovecs; } else { + /* should not use nobvec bioset for nr_iovecs > 0 */ + if (WARN_ON_ONCE(!bs->bvec_pool && nr_iovecs > 0)) + return NULL; /* * generic_make_request() converts recursion to iteration; this * means if we're running beneath it, any bios we allocate and @@ -1900,20 +1903,9 @@ void bioset_free(struct bio_set *bs) } EXPORT_SYMBOL(bioset_free); -/** - * bioset_create - Create a bio_set - * @pool_size: Number of bio and bio_vecs to cache in the mempool - * @front_pad: Number of bytes to allocate in front of the returned bio - * - * Description: - * Set up a bio_set to be used with @bio_alloc_bioset. Allows the caller - * to ask for a number of bytes to be allocated in front of the bio. - * Front pad allocation is useful for embedding the bio inside - * another structure, to avoid allocating extra data to go with the bio. - * Note that the bio must be embedded at the END of that structure always, - * or things will break badly. - */ -struct bio_set *bioset_create(unsigned int pool_size, unsigned int front_pad) +static struct bio_set *__bioset_create(unsigned int pool_size, + unsigned int front_pad, + bool create_bvec_pool) { unsigned int back_pad = BIO_INLINE_VECS * sizeof(struct bio_vec); struct bio_set *bs; @@ -1938,9 +1930,11 @@ struct bio_set *bioset_create(unsigned int pool_size, unsigned int front_pad) if (!bs->bio_pool) goto bad; - bs->bvec_pool = biovec_create_pool(pool_size); - if (!bs->bvec_pool) - goto bad; + if (create_bvec_pool) { + bs->bvec_pool = biovec_create_pool(pool_size); + if (!bs->bvec_pool) + goto bad; + } bs->rescue_workqueue = alloc_workqueue("bioset", WQ_MEM_RECLAIM, 0); if (!bs->rescue_workqueue) @@ -1951,8 +1945,41 @@ bad: bioset_free(bs); return NULL; } + +/** + * bioset_create - Create a bio_set + * @pool_size: Number of bio and bio_vecs to cache in the mempool + * @front_pad: Number of bytes to allocate in front of the returned bio + * + * Description: + * Set up a bio_set to be used with @bio_alloc_bioset. Allows the caller + * to ask for a number of bytes to be allocated in front of the bio. + * Front pad allocation is useful for embedding the bio inside + * another structure, to avoid allocating extra data to go with the bio. + * Note that the bio must be embedded at the END of that structure always, + * or things will break badly. + */ +struct bio_set *bioset_create(unsigned int pool_size, unsigned int front_pad) +{ + return __bioset_create(pool_size, front_pad, true); +} EXPORT_SYMBOL(bioset_create); +/** + * bioset_create_nobvec - Create a bio_set without bio_vec mempool + * @pool_size: Number of bio to cache in the mempool + * @front_pad: Number of bytes to allocate in front of the returned bio + * + * Description: + * Same functionality as bioset_create() except that mempool is not + * created for bio_vecs. Saving some memory for bio_clone_fast() users. + */ +struct bio_set *bioset_create_nobvec(unsigned int pool_size, unsigned int front_pad) +{ + return __bioset_create(pool_size, front_pad, false); +} +EXPORT_SYMBOL(bioset_create_nobvec); + #ifdef CONFIG_BLK_CGROUP /** * bio_associate_current - associate a bio with %current -- cgit v1.2.3 From abab13b5c4fd1fec4f9a61622548012d93dc2831 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 7 Oct 2014 08:39:20 -0600 Subject: blk-mq: fix potential hang if rolling wakeup depth is too high We currently divide the queue depth by 4 as our batch wakeup count, but we split the wakeups over BT_WAIT_QUEUES number of wait queues. This defaults to 8. If the product of the resulting batch wake count and BT_WAIT_QUEUES is higher than the device queue depth, we can get into a situation where a task goes to sleep waiting for a request, but never gets woken up. Reported-by: Bart Van Assche Fixes: 4bb659b156996 Cc: stable@kernel.org Signed-off-by: Jens Axboe --- block/blk-mq-tag.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'block') diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c index b08788086414..146fd02659ec 100644 --- a/block/blk-mq-tag.c +++ b/block/blk-mq-tag.c @@ -455,8 +455,8 @@ static void bt_update_count(struct blk_mq_bitmap_tags *bt, } bt->wake_cnt = BT_WAIT_BATCH; - if (bt->wake_cnt > depth / 4) - bt->wake_cnt = max(1U, depth / 4); + if (bt->wake_cnt > depth / BT_WAIT_QUEUES) + bt->wake_cnt = max(1U, depth / BT_WAIT_QUEUES); bt->depth = depth; } -- cgit v1.2.3 From 9d8f0bcca6ffa024a822ce4ab1008ab663f06672 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Tue, 7 Oct 2014 08:45:21 -0600 Subject: blk-mq: Make bt_clear_tag() easier to read Eliminate a backwards goto statement from bt_clear_tag(). Signed-off-by: Bart Van Assche Signed-off-by: Jens Axboe --- block/blk-mq-tag.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'block') diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c index 146fd02659ec..8317175a3009 100644 --- a/block/blk-mq-tag.c +++ b/block/blk-mq-tag.c @@ -351,15 +351,12 @@ static void bt_clear_tag(struct blk_mq_bitmap_tags *bt, unsigned int tag) return; wait_cnt = atomic_dec_return(&bs->wait_cnt); + if (unlikely(wait_cnt < 0)) + wait_cnt = atomic_inc_return(&bs->wait_cnt); if (wait_cnt == 0) { -wake: atomic_add(bt->wake_cnt, &bs->wait_cnt); bt_index_atomic_inc(&bt->wake_index); wake_up(&bs->wait); - } else if (wait_cnt < 0) { - wait_cnt = atomic_inc_return(&bs->wait_cnt); - if (!wait_cnt) - goto wake; } } -- cgit v1.2.3 From b8839b8c55f3fdd60dc36abcda7e0266aff7985c Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Wed, 8 Oct 2014 18:26:13 -0400 Subject: block: fix alignment_offset math that assumes io_min is a power-of-2 The math in both blk_stack_limits() and queue_limit_alignment_offset() assume that a block device's io_min (aka minimum_io_size) is always a power-of-2. Fix the math such that it works for non-power-of-2 io_min. This issue (of alignment_offset != 0) became apparent when testing dm-thinp with a thinp blocksize that matches a RAID6 stripesize of 1280K. Commit fdfb4c8c1 ("dm thin: set minimum_io_size to pool's data block size") unlocked the potential for alignment_offset != 0 due to the dm-thin-pool's io_min possibly being a non-power-of-2. Signed-off-by: Mike Snitzer Cc: stable@vger.kernel.org Acked-by: Martin K. Petersen Signed-off-by: Jens Axboe --- block/blk-settings.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'block') diff --git a/block/blk-settings.c b/block/blk-settings.c index f1a1795a5683..aa02247d227e 100644 --- a/block/blk-settings.c +++ b/block/blk-settings.c @@ -574,7 +574,7 @@ int blk_stack_limits(struct queue_limits *t, struct queue_limits *b, bottom = max(b->physical_block_size, b->io_min) + alignment; /* Verify that top and bottom intervals line up */ - if (max(top, bottom) & (min(top, bottom) - 1)) { + if (max(top, bottom) % min(top, bottom)) { t->misaligned = 1; ret = -1; } @@ -619,7 +619,7 @@ int blk_stack_limits(struct queue_limits *t, struct queue_limits *b, /* Find lowest common alignment_offset */ t->alignment_offset = lcm(t->alignment_offset, alignment) - & (max(t->physical_block_size, t->io_min) - 1); + % max(t->physical_block_size, t->io_min); /* Verify that new alignment_offset is on a logical block boundary */ if (t->alignment_offset & (t->logical_block_size - 1)) { -- cgit v1.2.3 From 764f612c6c3c231b9c12cfae7c328ccc9c453258 Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Thu, 9 Oct 2014 23:17:35 +0800 Subject: blk-merge: don't compute bi_phys_segments from bi_vcnt for cloned bio It isn't correct to figure out req->bi_phys_segments from bio->bi_vcnt if the bio is cloned. Signed-off-by: Ming Lei Tested-by: Jeff Mahoney Signed-off-by: Jens Axboe --- block/blk-merge.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'block') diff --git a/block/blk-merge.c b/block/blk-merge.c index f71bad35b4cc..ba99351c0f58 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -97,14 +97,18 @@ void blk_recalc_rq_segments(struct request *rq) void blk_recount_segments(struct request_queue *q, struct bio *bio) { - if (test_bit(QUEUE_FLAG_NO_SG_MERGE, &q->queue_flags) && + bool no_sg_merge = !!test_bit(QUEUE_FLAG_NO_SG_MERGE, + &q->queue_flags); + + if (no_sg_merge && !bio_flagged(bio, BIO_CLONED) && bio->bi_vcnt < queue_max_segments(q)) bio->bi_phys_segments = bio->bi_vcnt; else { struct bio *nxt = bio->bi_next; bio->bi_next = NULL; - bio->bi_phys_segments = __blk_recalc_rq_segments(q, bio, false); + bio->bi_phys_segments = __blk_recalc_rq_segments(q, bio, + no_sg_merge); bio->bi_next = nxt; } -- cgit v1.2.3 From ef3ecb66bcd6b2076dc8782e1315cf2807b73c0c Mon Sep 17 00:00:00 2001 From: Robert Elliott Date: Wed, 27 Aug 2014 10:50:31 -0500 Subject: block: make blk_update_request print prefix match ratelimited prefix In blk_update_request, change the printk_ratelimited prefix from end_request to blk_update_request so it matches the name printed if rate limiting occurs. Old: [10234.933106] blk_update_request: 174 callbacks suppressed [10234.934940] end_request: critical target error, dev sdr, sector 16 [10234.949788] end_request: critical target error, dev sdr, sector 16 New: [16863.445173] blk_update_request: 398 callbacks suppressed [16863.447029] blk_update_request: critical target error, dev sdr, sector 1442066176 [16863.449383] blk_update_request: critical target error, dev sdr, sector 802802888 [16863.451680] blk_update_request: critical target error, dev sdr, sector 1609535456 Signed-off-by: Robert Elliott Reviewed-by: Webb Scales Signed-off-by: Jens Axboe --- block/blk-core.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'block') diff --git a/block/blk-core.c b/block/blk-core.c index ffcb47af35f3..ecc124ec53bb 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -2444,8 +2444,8 @@ bool blk_update_request(struct request *req, int error, unsigned int nr_bytes) error_type = "I/O"; break; } - printk_ratelimited(KERN_ERR "end_request: %s error, dev %s, sector %llu\n", - error_type, req->rq_disk ? + printk_ratelimited(KERN_ERR "%s: %s error, dev %s, sector %llu\n", + __func__, error_type, req->rq_disk ? req->rq_disk->disk_name : "?", (unsigned long long)blk_rq_pos(req)); -- cgit v1.2.3 From 7b2b10e0e2c65ebc11314e1af9924d0824ec1562 Mon Sep 17 00:00:00 2001 From: Robert Elliott Date: Wed, 27 Aug 2014 10:50:36 -0500 Subject: block: include func name in __get_request prints In __get_request calls to printk_ratelimited, include the function name so the callbacks suppressed message matches the messages that are printed, and add "dev" before the device name so it matches other block layer messages. Signed-off-by: Robert Elliott Reviewed-by: Webb Scales Signed-off-by: Jens Axboe --- block/blk-core.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'block') diff --git a/block/blk-core.c b/block/blk-core.c index ecc124ec53bb..d6ec7db9a9f4 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -1060,8 +1060,8 @@ fail_elvpriv: * shouldn't stall IO. Treat this request as !elvpriv. This will * disturb iosched and blkcg but weird is bettern than dead. */ - printk_ratelimited(KERN_WARNING "%s: request aux data allocation failed, iosched may be disturbed\n", - dev_name(q->backing_dev_info.dev)); + printk_ratelimited(KERN_WARNING "%s: dev %s: request aux data allocation failed, iosched may be disturbed\n", + __func__, dev_name(q->backing_dev_info.dev)); rq->cmd_flags &= ~REQ_ELVPRIV; rq->elv.icq = NULL; -- cgit v1.2.3 From b65c7491cb865577e83e6b7fae2aa2f4ea457c38 Mon Sep 17 00:00:00 2001 From: Gu Zheng Date: Mon, 13 Oct 2014 14:07:27 -0700 Subject: bio-integrity: remove the needless fail handle of bip_slab creating bip_slab is created with SLAB_PANIC, so the fail handler is unneeded. Signed-off-by: Gu Zheng Signed-off-by: Andrew Morton Signed-off-by: Jens Axboe --- block/bio-integrity.c | 2 -- 1 file changed, 2 deletions(-) (limited to 'block') diff --git a/block/bio-integrity.c b/block/bio-integrity.c index 8e0548484dd3..0984232e429f 100644 --- a/block/bio-integrity.c +++ b/block/bio-integrity.c @@ -513,6 +513,4 @@ void __init bio_integrity_init(void) sizeof(struct bio_integrity_payload) + sizeof(struct bio_vec) * BIP_INLINE_VECS, 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL); - if (!bip_slab) - panic("Failed to create slab\n"); } -- cgit v1.2.3 From a86073e48ae85c9b50127facb0cc45bbd35972a1 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Mon, 13 Oct 2014 15:41:54 -0600 Subject: blk-mq: allocate cpumask on the home node All other allocs are done on the specific node, somehow the cpumask for hw queue runs was missed. Fix that by using zalloc_cpumask_var_node() in blk_mq_init_queue(). Signed-off-by: Jens Axboe --- block/blk-mq.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'block') diff --git a/block/blk-mq.c b/block/blk-mq.c index 4e7a31466139..79aa11b3efa5 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -1802,7 +1802,8 @@ struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set) if (!hctxs[i]) goto err_hctxs; - if (!zalloc_cpumask_var(&hctxs[i]->cpumask, GFP_KERNEL)) + if (!zalloc_cpumask_var_node(&hctxs[i]->cpumask, GFP_KERNEL, + node)) goto err_hctxs; atomic_set(&hctxs[i]->nr_active, 0); -- cgit v1.2.3