Commit 2f8f1336 authored by Ming Lei's avatar Ming Lei Committed by Jens Axboe
Browse files

blk-mq: always free hctx after request queue is freed

In normal queue cleanup path, hctx is released after request queue
is freed, see blk_mq_release().

However, in __blk_mq_update_nr_hw_queues(), hctx may be freed because
of hw queues shrinking. This way is easy to cause use-after-free,
because: one implicit rule is that it is safe to call almost all block
layer APIs if the request queue is alive; and one hctx may be retrieved
by one API, then the hctx can be freed by blk_mq_update_nr_hw_queues();
finally use-after-free is triggered.

Fixes this issue by always freeing hctx after releasing request queue.
If some hctxs are removed in blk_mq_update_nr_hw_queues(), introduce
a per-queue list to hold them, then try to resuse these hctxs if numa
node is matched.

Cc: Dongli Zhang <>
Cc: James Smart <>
Cc: Bart Van Assche <>
Cc: Martin K . Petersen <>,
Cc: Christoph Hellwig <>,
Cc: James E . J . Bottomley <>,
Reviewed-by: default avatarHannes Reinecke <>
Tested-by: default avatarJames Smart <>
Signed-off-by: default avatarMing Lei <>
Signed-off-by: default avatarJens Axboe <>
parent 7c6c5b7c
......@@ -2269,6 +2269,10 @@ static void blk_mq_exit_hctx(struct request_queue *q,
set->ops->exit_hctx(hctx, hctx_idx);
list_add(&hctx->hctx_list, &q->unused_hctx_list);
static void blk_mq_exit_hw_queues(struct request_queue *q,
......@@ -2351,6 +2355,8 @@ blk_mq_alloc_hctx(struct request_queue *q, struct blk_mq_tag_set *set,
hctx->queue = q;
hctx->flags = set->flags & ~BLK_MQ_F_TAG_SHARED;
* Allocate space for all possible cpus to avoid allocation at
* runtime
......@@ -2664,15 +2670,17 @@ static int blk_mq_alloc_ctxs(struct request_queue *q)
void blk_mq_release(struct request_queue *q)
struct blk_mq_hw_ctx *hctx;
unsigned int i;
struct blk_mq_hw_ctx *hctx, *next;
int i;
/* hctx kobj stays in hctx */
queue_for_each_hw_ctx(q, hctx, i) {
if (!hctx)
queue_for_each_hw_ctx(q, hctx, i)
WARN_ON_ONCE(hctx && list_empty(&hctx->hctx_list));
/* all hctx are in .unused_hctx_list now */
list_for_each_entry_safe(hctx, next, &q->unused_hctx_list, hctx_list) {
......@@ -2739,9 +2747,22 @@ static struct blk_mq_hw_ctx *blk_mq_alloc_and_init_hctx(
struct blk_mq_tag_set *set, struct request_queue *q,
int hctx_idx, int node)
struct blk_mq_hw_ctx *hctx;
struct blk_mq_hw_ctx *hctx = NULL, *tmp;
hctx = blk_mq_alloc_hctx(q, set, node);
/* reuse dead hctx first */
list_for_each_entry(tmp, &q->unused_hctx_list, hctx_list) {
if (tmp->numa_node == node) {
hctx = tmp;
if (hctx)
if (!hctx)
hctx = blk_mq_alloc_hctx(q, set, node);
if (!hctx)
goto fail;
......@@ -2779,10 +2800,8 @@ static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set,
hctx = blk_mq_alloc_and_init_hctx(set, q, i, node);
if (hctx) {
if (hctxs[i]) {
if (hctxs[i])
blk_mq_exit_hctx(q, set, hctxs[i], i);
hctxs[i] = hctx;
} else {
if (hctxs[i])
......@@ -2813,9 +2832,7 @@ static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set,
if (hctx->tags)
blk_mq_free_map_and_requests(set, j);
blk_mq_exit_hctx(q, set, hctx, j);
hctxs[j] = NULL;
......@@ -2858,6 +2875,9 @@ struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
if (!q->queue_hw_ctx)
goto err_sys_init;
blk_mq_realloc_hw_ctxs(set, q);
if (!q->nr_hw_queues)
goto err_hctxs;
......@@ -70,6 +70,8 @@ struct blk_mq_hw_ctx {
struct dentry *sched_debugfs_dir;
struct list_head hctx_list;
/* Must be the last member - see also blk_mq_hw_ctx_size(). */
struct srcu_struct srcu[0];
......@@ -535,6 +535,13 @@ struct request_queue {
struct mutex sysfs_lock;
* for reusing dead hctx instance in case of updating
* nr_hw_queues
struct list_head unused_hctx_list;
spinlock_t unused_hctx_lock;
atomic_t mq_freeze_depth;
#if defined(CONFIG_BLK_DEV_BSG)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment