8e8320c931
If we have shared tags enabled, then every IO completion will trigger
a full loop of every queue belonging to a tag set, and every hardware
queue for each of those queues, even if nothing needs to be done.
This causes a massive performance regression if you have a lot of
shared devices.
Instead of doing this huge full scan on every IO, add an atomic
counter to the main queue that tracks how many hardware queues have
been marked as needing a restart. With that, we can avoid looking for
restartable queues, if we don't have to.
Max reports that this restores performance. Before this patch, 4K
IOPS was limited to 22-23K IOPS. With the patch, we are running at
950-970K IOPS.
Fixes: 6d8c6c0f97 ("blk-mq: Restart a single queue if tag sets are shared")
Reported-by: Max Gurtovoy <maxg@mellanox.com>
Tested-by: Max Gurtovoy <maxg@mellanox.com>
Reviewed-by: Bart Van Assche <bart.vanassche@sandisk.com>
Tested-by: Bart Van Assche <bart.vanassche@wdc.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
123 lines
3.4 KiB
C
123 lines
3.4 KiB
C
#ifndef BLK_MQ_SCHED_H
|
|
#define BLK_MQ_SCHED_H
|
|
|
|
#include "blk-mq.h"
|
|
#include "blk-mq-tag.h"
|
|
|
|
void blk_mq_sched_free_hctx_data(struct request_queue *q,
|
|
void (*exit)(struct blk_mq_hw_ctx *));
|
|
|
|
struct request *blk_mq_sched_get_request(struct request_queue *q, struct bio *bio, unsigned int op, struct blk_mq_alloc_data *data);
|
|
void blk_mq_sched_put_request(struct request *rq);
|
|
|
|
void blk_mq_sched_request_inserted(struct request *rq);
|
|
bool blk_mq_sched_try_merge(struct request_queue *q, struct bio *bio,
|
|
struct request **merged_request);
|
|
bool __blk_mq_sched_bio_merge(struct request_queue *q, struct bio *bio);
|
|
bool blk_mq_sched_try_insert_merge(struct request_queue *q, struct request *rq);
|
|
void blk_mq_sched_restart(struct blk_mq_hw_ctx *hctx);
|
|
|
|
void blk_mq_sched_insert_request(struct request *rq, bool at_head,
|
|
bool run_queue, bool async, bool can_block);
|
|
void blk_mq_sched_insert_requests(struct request_queue *q,
|
|
struct blk_mq_ctx *ctx,
|
|
struct list_head *list, bool run_queue_async);
|
|
|
|
void blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx);
|
|
|
|
int blk_mq_init_sched(struct request_queue *q, struct elevator_type *e);
|
|
void blk_mq_exit_sched(struct request_queue *q, struct elevator_queue *e);
|
|
|
|
int blk_mq_sched_init_hctx(struct request_queue *q, struct blk_mq_hw_ctx *hctx,
|
|
unsigned int hctx_idx);
|
|
void blk_mq_sched_exit_hctx(struct request_queue *q, struct blk_mq_hw_ctx *hctx,
|
|
unsigned int hctx_idx);
|
|
|
|
int blk_mq_sched_init(struct request_queue *q);
|
|
|
|
static inline bool
|
|
blk_mq_sched_bio_merge(struct request_queue *q, struct bio *bio)
|
|
{
|
|
struct elevator_queue *e = q->elevator;
|
|
|
|
if (!e || blk_queue_nomerges(q) || !bio_mergeable(bio))
|
|
return false;
|
|
|
|
return __blk_mq_sched_bio_merge(q, bio);
|
|
}
|
|
|
|
static inline int blk_mq_sched_get_rq_priv(struct request_queue *q,
|
|
struct request *rq,
|
|
struct bio *bio)
|
|
{
|
|
struct elevator_queue *e = q->elevator;
|
|
|
|
if (e && e->type->ops.mq.get_rq_priv)
|
|
return e->type->ops.mq.get_rq_priv(q, rq, bio);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static inline void blk_mq_sched_put_rq_priv(struct request_queue *q,
|
|
struct request *rq)
|
|
{
|
|
struct elevator_queue *e = q->elevator;
|
|
|
|
if (e && e->type->ops.mq.put_rq_priv)
|
|
e->type->ops.mq.put_rq_priv(q, rq);
|
|
}
|
|
|
|
static inline bool
|
|
blk_mq_sched_allow_merge(struct request_queue *q, struct request *rq,
|
|
struct bio *bio)
|
|
{
|
|
struct elevator_queue *e = q->elevator;
|
|
|
|
if (e && e->type->ops.mq.allow_merge)
|
|
return e->type->ops.mq.allow_merge(q, rq, bio);
|
|
|
|
return true;
|
|
}
|
|
|
|
static inline void blk_mq_sched_completed_request(struct request *rq)
|
|
{
|
|
struct elevator_queue *e = rq->q->elevator;
|
|
|
|
if (e && e->type->ops.mq.completed_request)
|
|
e->type->ops.mq.completed_request(rq);
|
|
}
|
|
|
|
static inline void blk_mq_sched_started_request(struct request *rq)
|
|
{
|
|
struct request_queue *q = rq->q;
|
|
struct elevator_queue *e = q->elevator;
|
|
|
|
if (e && e->type->ops.mq.started_request)
|
|
e->type->ops.mq.started_request(rq);
|
|
}
|
|
|
|
static inline void blk_mq_sched_requeue_request(struct request *rq)
|
|
{
|
|
struct request_queue *q = rq->q;
|
|
struct elevator_queue *e = q->elevator;
|
|
|
|
if (e && e->type->ops.mq.requeue_request)
|
|
e->type->ops.mq.requeue_request(rq);
|
|
}
|
|
|
|
static inline bool blk_mq_sched_has_work(struct blk_mq_hw_ctx *hctx)
|
|
{
|
|
struct elevator_queue *e = hctx->queue->elevator;
|
|
|
|
if (e && e->type->ops.mq.has_work)
|
|
return e->type->ops.mq.has_work(hctx);
|
|
|
|
return false;
|
|
}
|
|
|
|
static inline bool blk_mq_sched_needs_restart(struct blk_mq_hw_ctx *hctx)
|
|
{
|
|
return test_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state);
|
|
}
|
|
|
|
#endif
|