blk-mq: rework I/O completions
Rework I/O completions to work more like the old code path. blk_mq_end_io now stays out of the business of deferring completions to others CPUs and calling blk_mark_rq_complete. The latter is very important to allow completing requests that have timed out and thus are already marked completed, the former allows using the IPI callout even for driver specific completions instead of having to reimplement them. Signed-off-by: Christoph Hellwig <hch@lst.de> Signed-off-by: Jens Axboe <axboe@fb.com>
This commit is contained in:
parent
c4540a7d8c
commit
30a91cb4ef
4 changed files with 37 additions and 24 deletions
|
@ -326,7 +326,7 @@ static void blk_mq_bio_endio(struct request *rq, struct bio *bio, int error)
|
||||||
bio_endio(bio, error);
|
bio_endio(bio, error);
|
||||||
}
|
}
|
||||||
|
|
||||||
void blk_mq_complete_request(struct request *rq, int error)
|
void blk_mq_end_io(struct request *rq, int error)
|
||||||
{
|
{
|
||||||
struct bio *bio = rq->bio;
|
struct bio *bio = rq->bio;
|
||||||
unsigned int bytes = 0;
|
unsigned int bytes = 0;
|
||||||
|
@ -351,46 +351,53 @@ void blk_mq_complete_request(struct request *rq, int error)
|
||||||
else
|
else
|
||||||
blk_mq_free_request(rq);
|
blk_mq_free_request(rq);
|
||||||
}
|
}
|
||||||
|
EXPORT_SYMBOL(blk_mq_end_io);
|
||||||
|
|
||||||
void __blk_mq_end_io(struct request *rq, int error)
|
static void __blk_mq_complete_request_remote(void *data)
|
||||||
{
|
|
||||||
if (!blk_mark_rq_complete(rq))
|
|
||||||
blk_mq_complete_request(rq, error);
|
|
||||||
}
|
|
||||||
|
|
||||||
static void blk_mq_end_io_remote(void *data)
|
|
||||||
{
|
{
|
||||||
struct request *rq = data;
|
struct request *rq = data;
|
||||||
|
|
||||||
__blk_mq_end_io(rq, rq->errors);
|
rq->q->softirq_done_fn(rq);
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
void __blk_mq_complete_request(struct request *rq)
|
||||||
* End IO on this request on a multiqueue enabled driver. We'll either do
|
|
||||||
* it directly inline, or punt to a local IPI handler on the matching
|
|
||||||
* remote CPU.
|
|
||||||
*/
|
|
||||||
void blk_mq_end_io(struct request *rq, int error)
|
|
||||||
{
|
{
|
||||||
struct blk_mq_ctx *ctx = rq->mq_ctx;
|
struct blk_mq_ctx *ctx = rq->mq_ctx;
|
||||||
int cpu;
|
int cpu;
|
||||||
|
|
||||||
if (!ctx->ipi_redirect)
|
if (!ctx->ipi_redirect) {
|
||||||
return __blk_mq_end_io(rq, error);
|
rq->q->softirq_done_fn(rq);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
cpu = get_cpu();
|
cpu = get_cpu();
|
||||||
if (cpu != ctx->cpu && cpu_online(ctx->cpu)) {
|
if (cpu != ctx->cpu && cpu_online(ctx->cpu)) {
|
||||||
rq->errors = error;
|
rq->csd.func = __blk_mq_complete_request_remote;
|
||||||
rq->csd.func = blk_mq_end_io_remote;
|
|
||||||
rq->csd.info = rq;
|
rq->csd.info = rq;
|
||||||
rq->csd.flags = 0;
|
rq->csd.flags = 0;
|
||||||
__smp_call_function_single(ctx->cpu, &rq->csd, 0);
|
__smp_call_function_single(ctx->cpu, &rq->csd, 0);
|
||||||
} else {
|
} else {
|
||||||
__blk_mq_end_io(rq, error);
|
rq->q->softirq_done_fn(rq);
|
||||||
}
|
}
|
||||||
put_cpu();
|
put_cpu();
|
||||||
}
|
}
|
||||||
EXPORT_SYMBOL(blk_mq_end_io);
|
|
||||||
|
/**
|
||||||
|
* blk_mq_complete_request - end I/O on a request
|
||||||
|
* @rq: the request being processed
|
||||||
|
*
|
||||||
|
* Description:
|
||||||
|
* Ends all I/O on a request. It does not handle partial completions.
|
||||||
|
* The actual completion happens out-of-order, through a IPI handler.
|
||||||
|
**/
|
||||||
|
void blk_mq_complete_request(struct request *rq)
|
||||||
|
{
|
||||||
|
if (unlikely(blk_should_fake_timeout(rq->q)))
|
||||||
|
return;
|
||||||
|
if (!blk_mark_rq_complete(rq))
|
||||||
|
__blk_mq_complete_request(rq);
|
||||||
|
}
|
||||||
|
EXPORT_SYMBOL(blk_mq_complete_request);
|
||||||
|
|
||||||
static void blk_mq_start_request(struct request *rq)
|
static void blk_mq_start_request(struct request *rq)
|
||||||
{
|
{
|
||||||
|
@ -1399,6 +1406,9 @@ struct request_queue *blk_mq_init_queue(struct blk_mq_reg *reg,
|
||||||
if (reg->timeout)
|
if (reg->timeout)
|
||||||
blk_queue_rq_timeout(q, reg->timeout);
|
blk_queue_rq_timeout(q, reg->timeout);
|
||||||
|
|
||||||
|
if (reg->ops->complete)
|
||||||
|
blk_queue_softirq_done(q, reg->ops->complete);
|
||||||
|
|
||||||
blk_mq_init_flush(q);
|
blk_mq_init_flush(q);
|
||||||
blk_mq_init_cpu_queues(q, reg->nr_hw_queues);
|
blk_mq_init_cpu_queues(q, reg->nr_hw_queues);
|
||||||
|
|
||||||
|
|
|
@ -22,8 +22,7 @@ struct blk_mq_ctx {
|
||||||
struct kobject kobj;
|
struct kobject kobj;
|
||||||
};
|
};
|
||||||
|
|
||||||
void __blk_mq_end_io(struct request *rq, int error);
|
void __blk_mq_complete_request(struct request *rq);
|
||||||
void blk_mq_complete_request(struct request *rq, int error);
|
|
||||||
void blk_mq_run_request(struct request *rq, bool run_queue, bool async);
|
void blk_mq_run_request(struct request *rq, bool run_queue, bool async);
|
||||||
void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async);
|
void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async);
|
||||||
void blk_mq_init_flush(struct request_queue *q);
|
void blk_mq_init_flush(struct request_queue *q);
|
||||||
|
|
|
@ -91,7 +91,7 @@ static void blk_rq_timed_out(struct request *req)
|
||||||
case BLK_EH_HANDLED:
|
case BLK_EH_HANDLED:
|
||||||
/* Can we use req->errors here? */
|
/* Can we use req->errors here? */
|
||||||
if (q->mq_ops)
|
if (q->mq_ops)
|
||||||
blk_mq_complete_request(req, req->errors);
|
__blk_mq_complete_request(req);
|
||||||
else
|
else
|
||||||
__blk_complete_request(req);
|
__blk_complete_request(req);
|
||||||
break;
|
break;
|
||||||
|
|
|
@ -86,6 +86,8 @@ struct blk_mq_ops {
|
||||||
*/
|
*/
|
||||||
rq_timed_out_fn *timeout;
|
rq_timed_out_fn *timeout;
|
||||||
|
|
||||||
|
softirq_done_fn *complete;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Override for hctx allocations (should probably go)
|
* Override for hctx allocations (should probably go)
|
||||||
*/
|
*/
|
||||||
|
@ -137,6 +139,8 @@ void blk_mq_free_single_hw_queue(struct blk_mq_hw_ctx *, unsigned int);
|
||||||
|
|
||||||
void blk_mq_end_io(struct request *rq, int error);
|
void blk_mq_end_io(struct request *rq, int error);
|
||||||
|
|
||||||
|
void blk_mq_complete_request(struct request *rq);
|
||||||
|
|
||||||
void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx);
|
void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx);
|
||||||
void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx);
|
void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx);
|
||||||
void blk_mq_stop_hw_queues(struct request_queue *q);
|
void blk_mq_stop_hw_queues(struct request_queue *q);
|
||||||
|
|
Loading…
Reference in a new issue