block/blk-mq.c

   1 #include <linux/kernel.h>
   2 #include <linux/module.h>
   3 #include <linux/backing-dev.h>
   4 #include <linux/bio.h>
   5 #include <linux/blkdev.h>
   6 #include <linux/mm.h>
   7 #include <linux/init.h>
   8 #include <linux/slab.h>
   9 #include <linux/workqueue.h>
  10 #include <linux/smp.h>
  11 #include <linux/llist.h>
  12 #include <linux/list_sort.h>
  13 #include <linux/cpu.h>
  14 #include <linux/cache.h>
  15 #include <linux/sched/sysctl.h>
  16 #include <linux/delay.h>
  17
  18 #include <trace/events/block.h>
  19
  20 #include <linux/blk-mq.h>
  21 #include "blk.h"
  22 #include "blk-mq.h"
  23 #include "blk-mq-tag.h"
  24
  25 static DEFINE_MUTEX(all_q_mutex);
  26 static LIST_HEAD(all_q_list);
  27
  28 static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx);
  29
  30 static struct blk_mq_ctx *__blk_mq_get_ctx(struct request_queue *q,
  31                                            unsigned int cpu)
  32 {
  33         return per_cpu_ptr(q->queue_ctx, cpu);
  34 }
  35
  36 /*
  37  * This assumes per-cpu software queueing queues. They could be per-node
  38  * as well, for instance. For now this is hardcoded as-is. Note that we don't
  39  * care about preemption, since we know the ctx's are persistent. This does
  40  * mean that we can't rely on ctx always matching the currently running CPU.
  41  */
  42 static struct blk_mq_ctx *blk_mq_get_ctx(struct request_queue *q)
  43 {
  44         return __blk_mq_get_ctx(q, get_cpu());
  45 }
  46
  47 static void blk_mq_put_ctx(struct blk_mq_ctx *ctx)
  48 {
  49         put_cpu();
  50 }
  51
  52 /*
  53  * Check if any of the ctx's have pending work in this hardware queue
  54  */
  55 static bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx)
  56 {
  57         unsigned int i;
  58
  59         for (i = 0; i < hctx->ctx_map.map_size; i++)
  60                 if (hctx->ctx_map.map[i].word)
  61                         return true;
  62
  63         return false;
  64 }
  65
  66 static inline struct blk_align_bitmap *get_bm(struct blk_mq_hw_ctx *hctx,
  67                                               struct blk_mq_ctx *ctx)
  68 {
  69         return &hctx->ctx_map.map[ctx->index_hw / hctx->ctx_map.bits_per_word];
  70 }
  71
  72 #define CTX_TO_BIT(hctx, ctx)   \
  73         ((ctx)->index_hw & ((hctx)->ctx_map.bits_per_word - 1))
  74
  75 /*
  76  * Mark this ctx as having pending work in this hardware queue
  77  */
  78 static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx,
  79                                      struct blk_mq_ctx *ctx)
  80 {
  81         struct blk_align_bitmap *bm = get_bm(hctx, ctx);
  82
  83         if (!test_bit(CTX_TO_BIT(hctx, ctx), &bm->word))
  84                 set_bit(CTX_TO_BIT(hctx, ctx), &bm->word);
  85 }
  86
  87 static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx,
  88                                       struct blk_mq_ctx *ctx)
  89 {
  90         struct blk_align_bitmap *bm = get_bm(hctx, ctx);
  91
  92         clear_bit(CTX_TO_BIT(hctx, ctx), &bm->word);
  93 }
  94
  95 static struct request *__blk_mq_alloc_request(struct blk_mq_hw_ctx *hctx,
  96                                               struct blk_mq_ctx *ctx,
  97                                               gfp_t gfp, bool reserved)
  98 {
  99         struct request *rq;
 100         unsigned int tag;
 101
 102         tag = blk_mq_get_tag(hctx, &ctx->last_tag, gfp, reserved);
 103         if (tag != BLK_MQ_TAG_FAIL) {
 104                 rq = hctx->tags->rqs[tag];
 105
 106                 rq->cmd_flags = 0;
 107                 if (blk_mq_tag_busy(hctx)) {
 108                         rq->cmd_flags = REQ_MQ_INFLIGHT;
 109                         atomic_inc(&hctx->nr_active);
 110                 }
 111
 112                 rq->tag = tag;
 113                 return rq;
 114         }
 115
 116         return NULL;
 117 }
 118
 119 static int blk_mq_queue_enter(struct request_queue *q)
 120 {
 121         int ret;
 122
 123         __percpu_counter_add(&q->mq_usage_counter, 1, 1000000);
 124         smp_wmb();
 125         /* we have problems to freeze the queue if it's initializing */
 126         if (!blk_queue_bypass(q) || !blk_queue_init_done(q))
 127                 return 0;
 128
 129         __percpu_counter_add(&q->mq_usage_counter, -1, 1000000);
 130
 131         spin_lock_irq(q->queue_lock);
 132         ret = wait_event_interruptible_lock_irq(q->mq_freeze_wq,
 133                 !blk_queue_bypass(q) || blk_queue_dying(q),
 134                 *q->queue_lock);
 135         /* inc usage with lock hold to avoid freeze_queue runs here */
 136         if (!ret && !blk_queue_dying(q))
 137                 __percpu_counter_add(&q->mq_usage_counter, 1, 1000000);
 138         else if (blk_queue_dying(q))
 139                 ret = -ENODEV;
 140         spin_unlock_irq(q->queue_lock);
 141
 142         return ret;
 143 }
 144
 145 static void blk_mq_queue_exit(struct request_queue *q)
 146 {
 147         __percpu_counter_add(&q->mq_usage_counter, -1, 1000000);
 148 }
 149
 150 static void __blk_mq_drain_queue(struct request_queue *q)
 151 {
 152         while (true) {
 153                 s64 count;
 154
 155                 spin_lock_irq(q->queue_lock);
 156                 count = percpu_counter_sum(&q->mq_usage_counter);
 157                 spin_unlock_irq(q->queue_lock);
 158
 159                 if (count == 0)
 160                         break;
 161                 blk_mq_run_queues(q, false);
 162                 msleep(10);
 163         }
 164 }
 165
 166 /*
 167  * Guarantee no request is in use, so we can change any data structure of
 168  * the queue afterward.
 169  */
 170 static void blk_mq_freeze_queue(struct request_queue *q)
 171 {
 172         bool drain;
 173
 174         spin_lock_irq(q->queue_lock);
 175         drain = !q->bypass_depth++;
 176         queue_flag_set(QUEUE_FLAG_BYPASS, q);
 177         spin_unlock_irq(q->queue_lock);
 178
 179         if (drain)
 180                 __blk_mq_drain_queue(q);
 181 }
 182
 183 void blk_mq_drain_queue(struct request_queue *q)
 184 {
 185         __blk_mq_drain_queue(q);
 186 }
 187
 188 static void blk_mq_unfreeze_queue(struct request_queue *q)
 189 {
 190         bool wake = false;
 191
 192         spin_lock_irq(q->queue_lock);
 193         if (!--q->bypass_depth) {
 194                 queue_flag_clear(QUEUE_FLAG_BYPASS, q);
 195                 wake = true;
 196         }
 197         WARN_ON_ONCE(q->bypass_depth < 0);
 198         spin_unlock_irq(q->queue_lock);
 199         if (wake)
 200                 wake_up_all(&q->mq_freeze_wq);
 201 }
 202
 203 bool blk_mq_can_queue(struct blk_mq_hw_ctx *hctx)
 204 {
 205         return blk_mq_has_free_tags(hctx->tags);
 206 }
 207 EXPORT_SYMBOL(blk_mq_can_queue);
 208
 209 static void blk_mq_rq_ctx_init(struct request_queue *q, struct blk_mq_ctx *ctx,
 210                                struct request *rq, unsigned int rw_flags)
 211 {
 212         if (blk_queue_io_stat(q))
 213                 rw_flags |= REQ_IO_STAT;
 214
 215         INIT_LIST_HEAD(&rq->queuelist);
 216         /* csd/requeue_work/fifo_time is initialized before use */
 217         rq->q = q;
 218         rq->mq_ctx = ctx;
 219         rq->cmd_flags |= rw_flags;
 220         rq->cmd_type = 0;
 221         /* do not touch atomic flags, it needs atomic ops against the timer */
 222         rq->cpu = -1;
 223         rq->__data_len = 0;
 224         rq->__sector = (sector_t) -1;
 225         rq->bio = NULL;
 226         rq->biotail = NULL;
 227         INIT_HLIST_NODE(&rq->hash);
 228         RB_CLEAR_NODE(&rq->rb_node);
 229         memset(&rq->flush, 0, max(sizeof(rq->flush), sizeof(rq->elv)));
 230         rq->rq_disk = NULL;
 231         rq->part = NULL;
 232         rq->start_time = jiffies;
 233 #ifdef CONFIG_BLK_CGROUP
 234         rq->rl = NULL;
 235         set_start_time_ns(rq);
 236         rq->io_start_time_ns = 0;
 237 #endif
 238         rq->nr_phys_segments = 0;
 239 #if defined(CONFIG_BLK_DEV_INTEGRITY)
 240         rq->nr_integrity_segments = 0;
 241 #endif
 242         rq->ioprio = 0;
 243         rq->special = NULL;
 244         /* tag was already set */
 245         rq->errors = 0;
 246         memset(rq->__cmd, 0, sizeof(rq->__cmd));
 247         rq->cmd = rq->__cmd;
 248         rq->cmd_len = BLK_MAX_CDB;
 249
 250         rq->extra_len = 0;
 251         rq->sense_len = 0;
 252         rq->resid_len = 0;
 253         rq->sense = NULL;
 254
 255         rq->deadline = 0;
 256         INIT_LIST_HEAD(&rq->timeout_list);
 257         rq->timeout = 0;
 258         rq->retries = 0;
 259         rq->end_io = NULL;
 260         rq->end_io_data = NULL;
 261         rq->next_rq = NULL;
 262
 263         ctx->rq_dispatched[rw_is_sync(rw_flags)]++;
 264 }
 265
 266 static struct request *blk_mq_alloc_request_pinned(struct request_queue *q,
 267                                                    int rw, gfp_t gfp,
 268                                                    bool reserved)
 269 {
 270         struct request *rq;
 271
 272         do {
 273                 struct blk_mq_ctx *ctx = blk_mq_get_ctx(q);
 274                 struct blk_mq_hw_ctx *hctx = q->mq_ops->map_queue(q, ctx->cpu);
 275
 276                 rq = __blk_mq_alloc_request(hctx, ctx, gfp & ~__GFP_WAIT,
 277                                                 reserved);
 278                 if (rq) {
 279                         blk_mq_rq_ctx_init(q, ctx, rq, rw);
 280                         break;
 281                 }
 282
 283                 if (gfp & __GFP_WAIT) {
 284                         __blk_mq_run_hw_queue(hctx);
 285                         blk_mq_put_ctx(ctx);
 286                 } else {
 287                         blk_mq_put_ctx(ctx);
 288                         break;
 289                 }
 290
 291                 blk_mq_wait_for_tags(hctx, reserved);
 292         } while (1);
 293
 294         return rq;
 295 }
 296
 297 struct request *blk_mq_alloc_request(struct request_queue *q, int rw, gfp_t gfp,
 298                 bool reserved)
 299 {
 300         struct request *rq;
 301
 302         if (blk_mq_queue_enter(q))
 303                 return NULL;
 304
 305         rq = blk_mq_alloc_request_pinned(q, rw, gfp, reserved);
 306         if (rq)
 307                 blk_mq_put_ctx(rq->mq_ctx);
 308         return rq;
 309 }
 310 EXPORT_SYMBOL(blk_mq_alloc_request);
 311
 312 static void __blk_mq_free_request(struct blk_mq_hw_ctx *hctx,
 313                                   struct blk_mq_ctx *ctx, struct request *rq)
 314 {
 315         const int tag = rq->tag;
 316         struct request_queue *q = rq->q;
 317
 318         if (rq->cmd_flags & REQ_MQ_INFLIGHT)
 319                 atomic_dec(&hctx->nr_active);
 320
 321         clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
 322         blk_mq_put_tag(hctx, tag, &ctx->last_tag);
 323         blk_mq_queue_exit(q);
 324 }
 325
 326 void blk_mq_free_request(struct request *rq)
 327 {
 328         struct blk_mq_ctx *ctx = rq->mq_ctx;
 329         struct blk_mq_hw_ctx *hctx;
 330         struct request_queue *q = rq->q;
 331
 332         ctx->rq_completed[rq_is_sync(rq)]++;
 333
 334         hctx = q->mq_ops->map_queue(q, ctx->cpu);
 335         __blk_mq_free_request(hctx, ctx, rq);
 336 }
 337
 338 /*
 339  * Clone all relevant state from a request that has been put on hold in
 340  * the flush state machine into the preallocated flush request that hangs
 341  * off the request queue.
 342  *
 343  * For a driver the flush request should be invisible, that's why we are
 344  * impersonating the original request here.
 345  */
 346 void blk_mq_clone_flush_request(struct request *flush_rq,
 347                 struct request *orig_rq)
 348 {
 349         struct blk_mq_hw_ctx *hctx =
 350                 orig_rq->q->mq_ops->map_queue(orig_rq->q, orig_rq->mq_ctx->cpu);
 351
 352         flush_rq->mq_ctx = orig_rq->mq_ctx;
 353         flush_rq->tag = orig_rq->tag;
 354         memcpy(blk_mq_rq_to_pdu(flush_rq), blk_mq_rq_to_pdu(orig_rq),
 355                 hctx->cmd_size);
 356 }
 357
 358 inline void __blk_mq_end_io(struct request *rq, int error)
 359 {
 360         blk_account_io_done(rq);
 361
 362         if (rq->end_io) {
 363                 rq->end_io(rq, error);
 364         } else {
 365                 if (unlikely(blk_bidi_rq(rq)))
 366                         blk_mq_free_request(rq->next_rq);
 367                 blk_mq_free_request(rq);
 368         }
 369 }
 370 EXPORT_SYMBOL(__blk_mq_end_io);
 371
 372 void blk_mq_end_io(struct request *rq, int error)
 373 {
 374         if (blk_update_request(rq, error, blk_rq_bytes(rq)))
 375                 BUG();
 376         __blk_mq_end_io(rq, error);
 377 }
 378 EXPORT_SYMBOL(blk_mq_end_io);
 379
 380 static void __blk_mq_complete_request_remote(void *data)
 381 {
 382         struct request *rq = data;
 383
 384         rq->q->softirq_done_fn(rq);
 385 }
 386
 387 void __blk_mq_complete_request(struct request *rq)
 388 {
 389         struct blk_mq_ctx *ctx = rq->mq_ctx;
 390         bool shared = false;
 391         int cpu;
 392
 393         if (!test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags)) {
 394                 rq->q->softirq_done_fn(rq);
 395                 return;
 396         }
 397
 398         cpu = get_cpu();
 399         if (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags))
 400                 shared = cpus_share_cache(cpu, ctx->cpu);
 401
 402         if (cpu != ctx->cpu && !shared && cpu_online(ctx->cpu)) {
 403                 rq->csd.func = __blk_mq_complete_request_remote;
 404                 rq->csd.info = rq;
 405                 rq->csd.flags = 0;
 406                 smp_call_function_single_async(ctx->cpu, &rq->csd);
 407         } else {
 408                 rq->q->softirq_done_fn(rq);
 409         }
 410         put_cpu();
 411 }
 412
 413 /**
 414  * blk_mq_complete_request - end I/O on a request
 415  * @rq:         the request being processed
 416  *
 417  * Description:
 418  *      Ends all I/O on a request. It does not handle partial completions.
 419  *      The actual completion happens out-of-order, through a IPI handler.
 420  **/
 421 void blk_mq_complete_request(struct request *rq)
 422 {
 423         struct request_queue *q = rq->q;
 424
 425         if (unlikely(blk_should_fake_timeout(q)))
 426                 return;
 427         if (!blk_mark_rq_complete(rq)) {
 428                 if (q->softirq_done_fn)
 429                         __blk_mq_complete_request(rq);
 430                 else
 431                         blk_mq_end_io(rq, rq->errors);
 432         }
 433 }
 434 EXPORT_SYMBOL(blk_mq_complete_request);
 435
 436 static void blk_mq_start_request(struct request *rq, bool last)
 437 {
 438         struct request_queue *q = rq->q;
 439
 440         trace_block_rq_issue(q, rq);
 441
 442         rq->resid_len = blk_rq_bytes(rq);
 443         if (unlikely(blk_bidi_rq(rq)))
 444                 rq->next_rq->resid_len = blk_rq_bytes(rq->next_rq);
 445
 446         /*
 447          * Just mark start time and set the started bit. Due to memory
 448          * ordering, we know we'll see the correct deadline as long as
 449          * REQ_ATOMIC_STARTED is seen. Use the default queue timeout,
 450          * unless one has been set in the request.
 451          */
 452         if (!rq->timeout)
 453                 rq->deadline = jiffies + q->rq_timeout;
 454         else
 455                 rq->deadline = jiffies + rq->timeout;
 456
 457         /*
 458          * Mark us as started and clear complete. Complete might have been
 459          * set if requeue raced with timeout, which then marked it as
 460          * complete. So be sure to clear complete again when we start
 461          * the request, otherwise we'll ignore the completion event.
 462          */
 463         set_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
 464         clear_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags);
 465
 466         if (q->dma_drain_size && blk_rq_bytes(rq)) {
 467                 /*
 468                  * Make sure space for the drain appears.  We know we can do
 469                  * this because max_hw_segments has been adjusted to be one
 470                  * fewer than the device can handle.
 471                  */
 472                 rq->nr_phys_segments++;
 473         }
 474
 475         /*
 476          * Flag the last request in the series so that drivers know when IO
 477          * should be kicked off, if they don't do it on a per-request basis.
 478          *
 479          * Note: the flag isn't the only condition drivers should do kick off.
 480          * If drive is busy, the last request might not have the bit set.
 481          */
 482         if (last)
 483                 rq->cmd_flags |= REQ_END;
 484 }
 485
 486 static void __blk_mq_requeue_request(struct request *rq)
 487 {
 488         struct request_queue *q = rq->q;
 489
 490         trace_block_rq_requeue(q, rq);
 491         clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
 492
 493         rq->cmd_flags &= ~REQ_END;
 494
 495         if (q->dma_drain_size && blk_rq_bytes(rq))
 496                 rq->nr_phys_segments--;
 497 }
 498
 499 void blk_mq_requeue_request(struct request *rq)
 500 {
 501         __blk_mq_requeue_request(rq);
 502         blk_clear_rq_complete(rq);
 503
 504         BUG_ON(blk_queued_rq(rq));
 505         blk_mq_add_to_requeue_list(rq, true);
 506 }
 507 EXPORT_SYMBOL(blk_mq_requeue_request);
 508
 509 static void blk_mq_requeue_work(struct work_struct *work)
 510 {
 511         struct request_queue *q =
 512                 container_of(work, struct request_queue, requeue_work);
 513         LIST_HEAD(rq_list);
 514         struct request *rq, *next;
 515         unsigned long flags;
 516
 517         spin_lock_irqsave(&q->requeue_lock, flags);
 518         list_splice_init(&q->requeue_list, &rq_list);
 519         spin_unlock_irqrestore(&q->requeue_lock, flags);
 520
 521         list_for_each_entry_safe(rq, next, &rq_list, queuelist) {
 522                 if (!(rq->cmd_flags & REQ_SOFTBARRIER))
 523                         continue;
 524
 525                 rq->cmd_flags &= ~REQ_SOFTBARRIER;
 526                 list_del_init(&rq->queuelist);
 527                 blk_mq_insert_request(rq, true, false, false);
 528         }
 529
 530         while (!list_empty(&rq_list)) {
 531                 rq = list_entry(rq_list.next, struct request, queuelist);
 532                 list_del_init(&rq->queuelist);
 533                 blk_mq_insert_request(rq, false, false, false);
 534         }
 535
 536         blk_mq_run_queues(q, false);
 537 }
 538
 539 void blk_mq_add_to_requeue_list(struct request *rq, bool at_head)
 540 {
 541         struct request_queue *q = rq->q;
 542         unsigned long flags;
 543
 544         /*
 545          * We abuse this flag that is otherwise used by the I/O scheduler to
 546          * request head insertation from the workqueue.
 547          */
 548         BUG_ON(rq->cmd_flags & REQ_SOFTBARRIER);
 549
 550         spin_lock_irqsave(&q->requeue_lock, flags);
 551         if (at_head) {
 552                 rq->cmd_flags |= REQ_SOFTBARRIER;
 553                 list_add(&rq->queuelist, &q->requeue_list);
 554         } else {
 555                 list_add_tail(&rq->queuelist, &q->requeue_list);
 556         }
 557         spin_unlock_irqrestore(&q->requeue_lock, flags);
 558 }
 559 EXPORT_SYMBOL(blk_mq_add_to_requeue_list);
 560
 561 void blk_mq_kick_requeue_list(struct request_queue *q)
 562 {
 563         kblockd_schedule_work(&q->requeue_work);
 564 }
 565 EXPORT_SYMBOL(blk_mq_kick_requeue_list);
 566
 567 struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag)
 568 {
 569         return tags->rqs[tag];
 570 }
 571 EXPORT_SYMBOL(blk_mq_tag_to_rq);
 572
 573 struct blk_mq_timeout_data {
 574         struct blk_mq_hw_ctx *hctx;
 575         unsigned long *next;
 576         unsigned int *next_set;
 577 };
 578
 579 static void blk_mq_timeout_check(void *__data, unsigned long *free_tags)
 580 {
 581         struct blk_mq_timeout_data *data = __data;
 582         struct blk_mq_hw_ctx *hctx = data->hctx;
 583         unsigned int tag;
 584
 585          /* It may not be in flight yet (this is where
 586          * the REQ_ATOMIC_STARTED flag comes in). The requests are
 587          * statically allocated, so we know it's always safe to access the
 588          * memory associated with a bit offset into ->rqs[].
 589          */
 590         tag = 0;
 591         do {
 592                 struct request *rq;
 593
 594                 tag = find_next_zero_bit(free_tags, hctx->tags->nr_tags, tag);
 595                 if (tag >= hctx->tags->nr_tags)
 596                         break;
 597
 598                 rq = blk_mq_tag_to_rq(hctx->tags, tag++);
 599                 if (rq->q != hctx->queue)
 600                         continue;
 601                 if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags))
 602                         continue;
 603
 604                 blk_rq_check_expired(rq, data->next, data->next_set);
 605         } while (1);
 606 }
 607
 608 static void blk_mq_hw_ctx_check_timeout(struct blk_mq_hw_ctx *hctx,
 609                                         unsigned long *next,
 610                                         unsigned int *next_set)
 611 {
 612         struct blk_mq_timeout_data data = {
 613                 .hctx           = hctx,
 614                 .next           = next,
 615                 .next_set       = next_set,
 616         };
 617
 618         /*
 619          * Ask the tagging code to iterate busy requests, so we can
 620          * check them for timeout.
 621          */
 622         blk_mq_tag_busy_iter(hctx->tags, blk_mq_timeout_check, &data);
 623 }
 624
 625 static enum blk_eh_timer_return blk_mq_rq_timed_out(struct request *rq)
 626 {
 627         struct request_queue *q = rq->q;
 628
 629         /*
 630          * We know that complete is set at this point. If STARTED isn't set
 631          * anymore, then the request isn't active and the "timeout" should
 632          * just be ignored. This can happen due to the bitflag ordering.
 633          * Timeout first checks if STARTED is set, and if it is, assumes
 634          * the request is active. But if we race with completion, then
 635          * we both flags will get cleared. So check here again, and ignore
 636          * a timeout event with a request that isn't active.
 637          */
 638         if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags))
 639                 return BLK_EH_NOT_HANDLED;
 640
 641         if (!q->mq_ops->timeout)
 642                 return BLK_EH_RESET_TIMER;
 643
 644         return q->mq_ops->timeout(rq);
 645 }
 646
 647 static void blk_mq_rq_timer(unsigned long data)
 648 {
 649         struct request_queue *q = (struct request_queue *) data;
 650         struct blk_mq_hw_ctx *hctx;
 651         unsigned long next = 0;
 652         int i, next_set = 0;
 653
 654         queue_for_each_hw_ctx(q, hctx, i) {
 655                 /*
 656                  * If not software queues are currently mapped to this
 657                  * hardware queue, there's nothing to check
 658                  */
 659                 if (!hctx->nr_ctx || !hctx->tags)
 660                         continue;
 661
 662                 blk_mq_hw_ctx_check_timeout(hctx, &next, &next_set);
 663         }
 664
 665         if (next_set) {
 666                 next = blk_rq_timeout(round_jiffies_up(next));
 667                 mod_timer(&q->timeout, next);
 668         } else {
 669                 queue_for_each_hw_ctx(q, hctx, i)
 670                         blk_mq_tag_idle(hctx);
 671         }
 672 }
 673
 674 /*
 675  * Reverse check our software queue for entries that we could potentially
 676  * merge with. Currently includes a hand-wavy stop count of 8, to not spend
 677  * too much time checking for merges.
 678  */
 679 static bool blk_mq_attempt_merge(struct request_queue *q,
 680                                  struct blk_mq_ctx *ctx, struct bio *bio)
 681 {
 682         struct request *rq;
 683         int checked = 8;
 684
 685         list_for_each_entry_reverse(rq, &ctx->rq_list, queuelist) {
 686                 int el_ret;
 687
 688                 if (!checked--)
 689                         break;
 690
 691                 if (!blk_rq_merge_ok(rq, bio))
 692                         continue;
 693
 694                 el_ret = blk_try_merge(rq, bio);
 695                 if (el_ret == ELEVATOR_BACK_MERGE) {
 696                         if (bio_attempt_back_merge(q, rq, bio)) {
 697                                 ctx->rq_merged++;
 698                                 return true;
 699                         }
 700                         break;
 701                 } else if (el_ret == ELEVATOR_FRONT_MERGE) {
 702                         if (bio_attempt_front_merge(q, rq, bio)) {
 703                                 ctx->rq_merged++;
 704                                 return true;
 705                         }
 706                         break;
 707                 }
 708         }
 709
 710         return false;
 711 }
 712
 713 /*
 714  * Process software queues that have been marked busy, splicing them
 715  * to the for-dispatch
 716  */
 717 static void flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list)
 718 {
 719         struct blk_mq_ctx *ctx;
 720         int i;
 721
 722         for (i = 0; i < hctx->ctx_map.map_size; i++) {
 723                 struct blk_align_bitmap *bm = &hctx->ctx_map.map[i];
 724                 unsigned int off, bit;
 725
 726                 if (!bm->word)
 727                         continue;
 728
 729                 bit = 0;
 730                 off = i * hctx->ctx_map.bits_per_word;
 731                 do {
 732                         bit = find_next_bit(&bm->word, bm->depth, bit);
 733                         if (bit >= bm->depth)
 734                                 break;
 735
 736                         ctx = hctx->ctxs[bit + off];
 737                         clear_bit(bit, &bm->word);
 738                         spin_lock(&ctx->lock);
 739                         list_splice_tail_init(&ctx->rq_list, list);
 740                         spin_unlock(&ctx->lock);
 741
 742                         bit++;
 743                 } while (1);
 744         }
 745 }
 746
 747 /*
 748  * Run this hardware queue, pulling any software queues mapped to it in.
 749  * Note that this function currently has various problems around ordering
 750  * of IO. In particular, we'd like FIFO behaviour on handling existing
 751  * items on the hctx->dispatch list. Ignore that for now.
 752  */
 753 static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
 754 {
 755         struct request_queue *q = hctx->queue;
 756         struct request *rq;
 757         LIST_HEAD(rq_list);
 758         int queued;
 759
 760         WARN_ON(!cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask));
 761
 762         if (unlikely(test_bit(BLK_MQ_S_STOPPED, &hctx->state)))
 763                 return;
 764
 765         hctx->run++;
 766
 767         /*
 768          * Touch any software queue that has pending entries.
 769          */
 770         flush_busy_ctxs(hctx, &rq_list);
 771
 772         /*
 773          * If we have previous entries on our dispatch list, grab them
 774          * and stuff them at the front for more fair dispatch.
 775          */
 776         if (!list_empty_careful(&hctx->dispatch)) {
 777                 spin_lock(&hctx->lock);
 778                 if (!list_empty(&hctx->dispatch))
 779                         list_splice_init(&hctx->dispatch, &rq_list);
 780                 spin_unlock(&hctx->lock);
 781         }
 782
 783         /*
 784          * Now process all the entries, sending them to the driver.
 785          */
 786         queued = 0;
 787         while (!list_empty(&rq_list)) {
 788                 int ret;
 789
 790                 rq = list_first_entry(&rq_list, struct request, queuelist);
 791                 list_del_init(&rq->queuelist);
 792
 793                 blk_mq_start_request(rq, list_empty(&rq_list));
 794
 795                 ret = q->mq_ops->queue_rq(hctx, rq);
 796                 switch (ret) {
 797                 case BLK_MQ_RQ_QUEUE_OK:
 798                         queued++;
 799                         continue;
 800                 case BLK_MQ_RQ_QUEUE_BUSY:
 801                         list_add(&rq->queuelist, &rq_list);
 802                         __blk_mq_requeue_request(rq);
 803                         break;
 804                 default:
 805                         pr_err("blk-mq: bad return on queue: %d\n", ret);
 806                 case BLK_MQ_RQ_QUEUE_ERROR:
 807                         rq->errors = -EIO;
 808                         blk_mq_end_io(rq, rq->errors);
 809                         break;
 810                 }
 811
 812                 if (ret == BLK_MQ_RQ_QUEUE_BUSY)
 813                         break;
 814         }
 815
 816         if (!queued)
 817                 hctx->dispatched[0]++;
 818         else if (queued < (1 << (BLK_MQ_MAX_DISPATCH_ORDER - 1)))
 819                 hctx->dispatched[ilog2(queued) + 1]++;
 820
 821         /*
 822          * Any items that need requeuing? Stuff them into hctx->dispatch,
 823          * that is where we will continue on next queue run.
 824          */
 825         if (!list_empty(&rq_list)) {
 826                 spin_lock(&hctx->lock);
 827                 list_splice(&rq_list, &hctx->dispatch);
 828                 spin_unlock(&hctx->lock);
 829         }
 830 }
 831
 832 /*
 833  * It'd be great if the workqueue API had a way to pass
 834  * in a mask and had some smarts for more clever placement.
 835  * For now we just round-robin here, switching for every
 836  * BLK_MQ_CPU_WORK_BATCH queued items.
 837  */
 838 static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx)
 839 {
 840         int cpu = hctx->next_cpu;
 841
 842         if (--hctx->next_cpu_batch <= 0) {
 843                 int next_cpu;
 844
 845                 next_cpu = cpumask_next(hctx->next_cpu, hctx->cpumask);
 846                 if (next_cpu >= nr_cpu_ids)
 847                         next_cpu = cpumask_first(hctx->cpumask);
 848
 849                 hctx->next_cpu = next_cpu;
 850                 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
 851         }
 852
 853         return cpu;
 854 }
 855
 856 void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
 857 {
 858         if (unlikely(test_bit(BLK_MQ_S_STOPPED, &hctx->state)))
 859                 return;
 860
 861         if (!async && cpumask_test_cpu(smp_processor_id(), hctx->cpumask))
 862                 __blk_mq_run_hw_queue(hctx);
 863         else if (hctx->queue->nr_hw_queues == 1)
 864                 kblockd_schedule_delayed_work(&hctx->run_work, 0);
 865         else {
 866                 unsigned int cpu;
 867
 868                 cpu = blk_mq_hctx_next_cpu(hctx);
 869                 kblockd_schedule_delayed_work_on(cpu, &hctx->run_work, 0);
 870         }
 871 }
 872
 873 void blk_mq_run_queues(struct request_queue *q, bool async)
 874 {
 875         struct blk_mq_hw_ctx *hctx;
 876         int i;
 877
 878         queue_for_each_hw_ctx(q, hctx, i) {
 879                 if ((!blk_mq_hctx_has_pending(hctx) &&
 880                     list_empty_careful(&hctx->dispatch)) ||
 881                     test_bit(BLK_MQ_S_STOPPED, &hctx->state))
 882                         continue;
 883
 884                 preempt_disable();
 885                 blk_mq_run_hw_queue(hctx, async);
 886                 preempt_enable();
 887         }
 888 }
 889 EXPORT_SYMBOL(blk_mq_run_queues);
 890
 891 void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx)
 892 {
 893         cancel_delayed_work(&hctx->run_work);
 894         cancel_delayed_work(&hctx->delay_work);
 895         set_bit(BLK_MQ_S_STOPPED, &hctx->state);
 896 }
 897 EXPORT_SYMBOL(blk_mq_stop_hw_queue);
 898
 899 void blk_mq_stop_hw_queues(struct request_queue *q)
 900 {
 901         struct blk_mq_hw_ctx *hctx;
 902         int i;
 903
 904         queue_for_each_hw_ctx(q, hctx, i)
 905                 blk_mq_stop_hw_queue(hctx);
 906 }
 907 EXPORT_SYMBOL(blk_mq_stop_hw_queues);
 908
 909 void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx)
 910 {
 911         clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
 912
 913         preempt_disable();
 914         __blk_mq_run_hw_queue(hctx);
 915         preempt_enable();
 916 }
 917 EXPORT_SYMBOL(blk_mq_start_hw_queue);
 918
 919 void blk_mq_start_hw_queues(struct request_queue *q)
 920 {
 921         struct blk_mq_hw_ctx *hctx;
 922         int i;
 923
 924         queue_for_each_hw_ctx(q, hctx, i)
 925                 blk_mq_start_hw_queue(hctx);
 926 }
 927 EXPORT_SYMBOL(blk_mq_start_hw_queues);
 928
 929
 930 void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async)
 931 {
 932         struct blk_mq_hw_ctx *hctx;
 933         int i;
 934
 935         queue_for_each_hw_ctx(q, hctx, i) {
 936                 if (!test_bit(BLK_MQ_S_STOPPED, &hctx->state))
 937                         continue;
 938
 939                 clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
 940                 preempt_disable();
 941                 blk_mq_run_hw_queue(hctx, async);
 942                 preempt_enable();
 943         }
 944 }
 945 EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues);
 946
 947 static void blk_mq_run_work_fn(struct work_struct *work)
 948 {
 949         struct blk_mq_hw_ctx *hctx;
 950
 951         hctx = container_of(work, struct blk_mq_hw_ctx, run_work.work);
 952
 953         __blk_mq_run_hw_queue(hctx);
 954 }
 955
 956 static void blk_mq_delay_work_fn(struct work_struct *work)
 957 {
 958         struct blk_mq_hw_ctx *hctx;
 959
 960         hctx = container_of(work, struct blk_mq_hw_ctx, delay_work.work);
 961
 962         if (test_and_clear_bit(BLK_MQ_S_STOPPED, &hctx->state))
 963                 __blk_mq_run_hw_queue(hctx);
 964 }
 965
 966 void blk_mq_delay_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs)
 967 {
 968         unsigned long tmo = msecs_to_jiffies(msecs);
 969
 970         if (hctx->queue->nr_hw_queues == 1)
 971                 kblockd_schedule_delayed_work(&hctx->delay_work, tmo);
 972         else {
 973                 unsigned int cpu;
 974
 975                 cpu = blk_mq_hctx_next_cpu(hctx);
 976                 kblockd_schedule_delayed_work_on(cpu, &hctx->delay_work, tmo);
 977         }
 978 }
 979 EXPORT_SYMBOL(blk_mq_delay_queue);
 980
 981 static void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx,
 982                                     struct request *rq, bool at_head)
 983 {
 984         struct blk_mq_ctx *ctx = rq->mq_ctx;
 985
 986         trace_block_rq_insert(hctx->queue, rq);
 987
 988         if (at_head)
 989                 list_add(&rq->queuelist, &ctx->rq_list);
 990         else
 991                 list_add_tail(&rq->queuelist, &ctx->rq_list);
 992
 993         blk_mq_hctx_mark_pending(hctx, ctx);
 994
 995         /*
 996          * We do this early, to ensure we are on the right CPU.
 997          */
 998         blk_add_timer(rq);
 999 }
1000
1001 void blk_mq_insert_request(struct request *rq, bool at_head, bool run_queue,
1002                 bool async)
1003 {
1004         struct request_queue *q = rq->q;
1005         struct blk_mq_hw_ctx *hctx;
1006         struct blk_mq_ctx *ctx = rq->mq_ctx, *current_ctx;
1007
1008         current_ctx = blk_mq_get_ctx(q);
1009         if (!cpu_online(ctx->cpu))
1010                 rq->mq_ctx = ctx = current_ctx;
1011
1012         hctx = q->mq_ops->map_queue(q, ctx->cpu);
1013
1014         if (rq->cmd_flags & (REQ_FLUSH | REQ_FUA) &&
1015             !(rq->cmd_flags & (REQ_FLUSH_SEQ))) {
1016                 blk_insert_flush(rq);
1017         } else {
1018                 spin_lock(&ctx->lock);
1019                 __blk_mq_insert_request(hctx, rq, at_head);
1020                 spin_unlock(&ctx->lock);
1021         }
1022
1023         if (run_queue)
1024                 blk_mq_run_hw_queue(hctx, async);
1025
1026         blk_mq_put_ctx(current_ctx);
1027 }
1028
1029 static void blk_mq_insert_requests(struct request_queue *q,
1030                                      struct blk_mq_ctx *ctx,
1031                                      struct list_head *list,
1032                                      int depth,
1033                                      bool from_schedule)
1034
1035 {
1036         struct blk_mq_hw_ctx *hctx;
1037         struct blk_mq_ctx *current_ctx;
1038
1039         trace_block_unplug(q, depth, !from_schedule);
1040
1041         current_ctx = blk_mq_get_ctx(q);
1042
1043         if (!cpu_online(ctx->cpu))
1044                 ctx = current_ctx;
1045         hctx = q->mq_ops->map_queue(q, ctx->cpu);
1046
1047         /*
1048          * preemption doesn't flush plug list, so it's possible ctx->cpu is
1049          * offline now
1050          */
1051         spin_lock(&ctx->lock);
1052         while (!list_empty(list)) {
1053                 struct request *rq;
1054
1055                 rq = list_first_entry(list, struct request, queuelist);
1056                 list_del_init(&rq->queuelist);
1057                 rq->mq_ctx = ctx;
1058                 __blk_mq_insert_request(hctx, rq, false);
1059         }
1060         spin_unlock(&ctx->lock);
1061
1062         blk_mq_run_hw_queue(hctx, from_schedule);
1063         blk_mq_put_ctx(current_ctx);
1064 }
1065
1066 static int plug_ctx_cmp(void *priv, struct list_head *a, struct list_head *b)
1067 {
1068         struct request *rqa = container_of(a, struct request, queuelist);
1069         struct request *rqb = container_of(b, struct request, queuelist);
1070
1071         return !(rqa->mq_ctx < rqb->mq_ctx ||
1072                  (rqa->mq_ctx == rqb->mq_ctx &&
1073                   blk_rq_pos(rqa) < blk_rq_pos(rqb)));
1074 }
1075
1076 void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
1077 {
1078         struct blk_mq_ctx *this_ctx;
1079         struct request_queue *this_q;
1080         struct request *rq;
1081         LIST_HEAD(list);
1082         LIST_HEAD(ctx_list);
1083         unsigned int depth;
1084
1085         list_splice_init(&plug->mq_list, &list);
1086
1087         list_sort(NULL, &list, plug_ctx_cmp);
1088
1089         this_q = NULL;
1090         this_ctx = NULL;
1091         depth = 0;
1092
1093         while (!list_empty(&list)) {
1094                 rq = list_entry_rq(list.next);
1095                 list_del_init(&rq->queuelist);
1096                 BUG_ON(!rq->q);
1097                 if (rq->mq_ctx != this_ctx) {
1098                         if (this_ctx) {
1099                                 blk_mq_insert_requests(this_q, this_ctx,
1100                                                         &ctx_list, depth,
1101                                                         from_schedule);
1102                         }
1103
1104                         this_ctx = rq->mq_ctx;
1105                         this_q = rq->q;
1106                         depth = 0;
1107                 }
1108
1109                 depth++;
1110                 list_add_tail(&rq->queuelist, &ctx_list);
1111         }
1112
1113         /*
1114          * If 'this_ctx' is set, we know we have entries to complete
1115          * on 'ctx_list'. Do those.
1116          */
1117         if (this_ctx) {
1118                 blk_mq_insert_requests(this_q, this_ctx, &ctx_list, depth,
1119                                        from_schedule);
1120         }
1121 }
1122
1123 static void blk_mq_bio_to_request(struct request *rq, struct bio *bio)
1124 {
1125         init_request_from_bio(rq, bio);
1126         blk_account_io_start(rq, 1);
1127 }
1128
1129 static inline bool blk_mq_merge_queue_io(struct blk_mq_hw_ctx *hctx,
1130                                          struct blk_mq_ctx *ctx,
1131                                          struct request *rq, struct bio *bio)
1132 {
1133         struct request_queue *q = hctx->queue;
1134
1135         if (!(hctx->flags & BLK_MQ_F_SHOULD_MERGE)) {
1136                 blk_mq_bio_to_request(rq, bio);
1137                 spin_lock(&ctx->lock);
1138 insert_rq:
1139                 __blk_mq_insert_request(hctx, rq, false);
1140                 spin_unlock(&ctx->lock);
1141                 return false;
1142         } else {
1143                 spin_lock(&ctx->lock);
1144                 if (!blk_mq_attempt_merge(q, ctx, bio)) {
1145                         blk_mq_bio_to_request(rq, bio);
1146                         goto insert_rq;
1147                 }
1148
1149                 spin_unlock(&ctx->lock);
1150                 __blk_mq_free_request(hctx, ctx, rq);
1151                 return true;
1152         }
1153 }
1154
1155 struct blk_map_ctx {
1156         struct blk_mq_hw_ctx *hctx;
1157         struct blk_mq_ctx *ctx;
1158 };
1159
1160 static struct request *blk_mq_map_request(struct request_queue *q,
1161                                           struct bio *bio,
1162                                           struct blk_map_ctx *data)
1163 {
1164         struct blk_mq_hw_ctx *hctx;
1165         struct blk_mq_ctx *ctx;
1166         struct request *rq;
1167         int rw = bio_data_dir(bio);
1168
1169         if (unlikely(blk_mq_queue_enter(q))) {
1170                 bio_endio(bio, -EIO);
1171                 return NULL;
1172         }
1173
1174         ctx = blk_mq_get_ctx(q);
1175         hctx = q->mq_ops->map_queue(q, ctx->cpu);
1176
1177         if (rw_is_sync(bio->bi_rw))
1178                 rw |= REQ_SYNC;
1179
1180         trace_block_getrq(q, bio, rw);
1181         rq = __blk_mq_alloc_request(hctx, ctx, GFP_ATOMIC, false);
1182         if (likely(rq))
1183                 blk_mq_rq_ctx_init(q, ctx, rq, rw);
1184         else {
1185                 blk_mq_put_ctx(ctx);
1186                 trace_block_sleeprq(q, bio, rw);
1187                 rq = blk_mq_alloc_request_pinned(q, rw, __GFP_WAIT|GFP_ATOMIC,
1188                                                         false);
1189                 ctx = rq->mq_ctx;
1190                 hctx = q->mq_ops->map_queue(q, ctx->cpu);
1191         }
1192
1193         hctx->queued++;
1194         data->hctx = hctx;
1195         data->ctx = ctx;
1196         return rq;
1197 }
1198
1199 /*
1200  * Multiple hardware queue variant. This will not use per-process plugs,
1201  * but will attempt to bypass the hctx queueing if we can go straight to
1202  * hardware for SYNC IO.
1203  */
1204 static void blk_mq_make_request(struct request_queue *q, struct bio *bio)
1205 {
1206         const int is_sync = rw_is_sync(bio->bi_rw);
1207         const int is_flush_fua = bio->bi_rw & (REQ_FLUSH | REQ_FUA);
1208         struct blk_map_ctx data;
1209         struct request *rq;
1210
1211         blk_queue_bounce(q, &bio);
1212
1213         if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
1214                 bio_endio(bio, -EIO);
1215                 return;
1216         }
1217
1218         rq = blk_mq_map_request(q, bio, &data);
1219         if (unlikely(!rq))
1220                 return;
1221
1222         if (unlikely(is_flush_fua)) {
1223                 blk_mq_bio_to_request(rq, bio);
1224                 blk_insert_flush(rq);
1225                 goto run_queue;
1226         }
1227
1228         if (is_sync) {
1229                 int ret;
1230
1231                 blk_mq_bio_to_request(rq, bio);
1232                 blk_mq_start_request(rq, true);
1233
1234                 /*
1235                  * For OK queue, we are done. For error, kill it. Any other
1236                  * error (busy), just add it to our list as we previously
1237                  * would have done
1238                  */
1239                 ret = q->mq_ops->queue_rq(data.hctx, rq);
1240                 if (ret == BLK_MQ_RQ_QUEUE_OK)
1241                         goto done;
1242                 else {
1243                         __blk_mq_requeue_request(rq);
1244
1245                         if (ret == BLK_MQ_RQ_QUEUE_ERROR) {
1246                                 rq->errors = -EIO;
1247                                 blk_mq_end_io(rq, rq->errors);
1248                                 goto done;
1249                         }
1250                 }
1251         }
1252
1253         if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {
1254                 /*
1255                  * For a SYNC request, send it to the hardware immediately. For
1256                  * an ASYNC request, just ensure that we run it later on. The
1257                  * latter allows for merging opportunities and more efficient
1258                  * dispatching.
1259                  */
1260 run_queue:
1261                 blk_mq_run_hw_queue(data.hctx, !is_sync || is_flush_fua);
1262         }
1263 done:
1264         blk_mq_put_ctx(data.ctx);
1265 }
1266
1267 /*
1268  * Single hardware queue variant. This will attempt to use any per-process
1269  * plug for merging and IO deferral.
1270  */
1271 static void blk_sq_make_request(struct request_queue *q, struct bio *bio)
1272 {
1273         const int is_sync = rw_is_sync(bio->bi_rw);
1274         const int is_flush_fua = bio->bi_rw & (REQ_FLUSH | REQ_FUA);
1275         unsigned int use_plug, request_count = 0;
1276         struct blk_map_ctx data;
1277         struct request *rq;
1278
1279         /*
1280          * If we have multiple hardware queues, just go directly to
1281          * one of those for sync IO.
1282          */
1283         use_plug = !is_flush_fua && !is_sync;
1284
1285         blk_queue_bounce(q, &bio);
1286
1287         if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
1288                 bio_endio(bio, -EIO);
1289                 return;
1290         }
1291
1292         if (use_plug && !blk_queue_nomerges(q) &&
1293             blk_attempt_plug_merge(q, bio, &request_count))
1294                 return;
1295
1296         rq = blk_mq_map_request(q, bio, &data);
1297
1298         if (unlikely(is_flush_fua)) {
1299                 blk_mq_bio_to_request(rq, bio);
1300                 blk_insert_flush(rq);
1301                 goto run_queue;
1302         }
1303
1304         /*
1305          * A task plug currently exists. Since this is completely lockless,
1306          * utilize that to temporarily store requests until the task is
1307          * either done or scheduled away.
1308          */
1309         if (use_plug) {
1310                 struct blk_plug *plug = current->plug;
1311
1312                 if (plug) {
1313                         blk_mq_bio_to_request(rq, bio);
1314                         if (list_empty(&plug->mq_list))
1315                                 trace_block_plug(q);
1316                         else if (request_count >= BLK_MAX_REQUEST_COUNT) {
1317                                 blk_flush_plug_list(plug, false);
1318                                 trace_block_plug(q);
1319                         }
1320                         list_add_tail(&rq->queuelist, &plug->mq_list);
1321                         blk_mq_put_ctx(data.ctx);
1322                         return;
1323                 }
1324         }
1325
1326         if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {
1327                 /*
1328                  * For a SYNC request, send it to the hardware immediately. For
1329                  * an ASYNC request, just ensure that we run it later on. The
1330                  * latter allows for merging opportunities and more efficient
1331                  * dispatching.
1332                  */
1333 run_queue:
1334                 blk_mq_run_hw_queue(data.hctx, !is_sync || is_flush_fua);
1335         }
1336
1337         blk_mq_put_ctx(data.ctx);
1338 }
1339
1340 /*
1341  * Default mapping to a software queue, since we use one per CPU.
1342  */
1343 struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, const int cpu)
1344 {
1345         return q->queue_hw_ctx[q->mq_map[cpu]];
1346 }
1347 EXPORT_SYMBOL(blk_mq_map_queue);
1348
1349 struct blk_mq_hw_ctx *blk_mq_alloc_single_hw_queue(struct blk_mq_tag_set *set,
1350                                                    unsigned int hctx_index,
1351                                                    int node)
1352 {
1353         return kzalloc_node(sizeof(struct blk_mq_hw_ctx), GFP_KERNEL, node);
1354 }
1355 EXPORT_SYMBOL(blk_mq_alloc_single_hw_queue);
1356
1357 void blk_mq_free_single_hw_queue(struct blk_mq_hw_ctx *hctx,
1358                                  unsigned int hctx_index)
1359 {
1360         kfree(hctx);
1361 }
1362 EXPORT_SYMBOL(blk_mq_free_single_hw_queue);
1363
1364 static void blk_mq_free_rq_map(struct blk_mq_tag_set *set,
1365                 struct blk_mq_tags *tags, unsigned int hctx_idx)
1366 {
1367         struct page *page;
1368
1369         if (tags->rqs && set->ops->exit_request) {
1370                 int i;
1371
1372                 for (i = 0; i < tags->nr_tags; i++) {
1373                         if (!tags->rqs[i])
1374                                 continue;
1375                         set->ops->exit_request(set->driver_data, tags->rqs[i],
1376                                                 hctx_idx, i);
1377                 }
1378         }
1379
1380         while (!list_empty(&tags->page_list)) {
1381                 page = list_first_entry(&tags->page_list, struct page, lru);
1382                 list_del_init(&page->lru);
1383                 __free_pages(page, page->private);
1384         }
1385
1386         kfree(tags->rqs);
1387
1388         blk_mq_free_tags(tags);
1389 }
1390
1391 static size_t order_to_size(unsigned int order)
1392 {
1393         return (size_t)PAGE_SIZE << order;
1394 }
1395
1396 static struct blk_mq_tags *blk_mq_init_rq_map(struct blk_mq_tag_set *set,
1397                 unsigned int hctx_idx)
1398 {
1399         struct blk_mq_tags *tags;
1400         unsigned int i, j, entries_per_page, max_order = 4;
1401         size_t rq_size, left;
1402
1403         tags = blk_mq_init_tags(set->queue_depth, set->reserved_tags,
1404                                 set->numa_node);
1405         if (!tags)
1406                 return NULL;
1407
1408         INIT_LIST_HEAD(&tags->page_list);
1409
1410         tags->rqs = kmalloc_node(set->queue_depth * sizeof(struct request *),
1411                                         GFP_KERNEL, set->numa_node);
1412         if (!tags->rqs) {
1413                 blk_mq_free_tags(tags);
1414                 return NULL;
1415         }
1416
1417         /*
1418          * rq_size is the size of the request plus driver payload, rounded
1419          * to the cacheline size
1420          */
1421         rq_size = round_up(sizeof(struct request) + set->cmd_size,
1422                                 cache_line_size());
1423         left = rq_size * set->queue_depth;
1424
1425         for (i = 0; i < set->queue_depth; ) {
1426                 int this_order = max_order;
1427                 struct page *page;
1428                 int to_do;
1429                 void *p;
1430
1431                 while (left < order_to_size(this_order - 1) && this_order)
1432                         this_order--;
1433
1434                 do {
1435                         page = alloc_pages_node(set->numa_node, GFP_KERNEL,
1436                                                 this_order);
1437                         if (page)
1438                                 break;
1439                         if (!this_order--)
1440                                 break;
1441                         if (order_to_size(this_order) < rq_size)
1442                                 break;
1443                 } while (1);
1444
1445                 if (!page)
1446                         goto fail;
1447
1448                 page->private = this_order;
1449                 list_add_tail(&page->lru, &tags->page_list);
1450
1451                 p = page_address(page);
1452                 entries_per_page = order_to_size(this_order) / rq_size;
1453                 to_do = min(entries_per_page, set->queue_depth - i);
1454                 left -= to_do * rq_size;
1455                 for (j = 0; j < to_do; j++) {
1456                         tags->rqs[i] = p;
1457                         if (set->ops->init_request) {
1458                                 if (set->ops->init_request(set->driver_data,
1459                                                 tags->rqs[i], hctx_idx, i,
1460                                                 set->numa_node))
1461                                         goto fail;
1462                         }
1463
1464                         p += rq_size;
1465                         i++;
1466                 }
1467         }
1468
1469         return tags;
1470
1471 fail:
1472         pr_warn("%s: failed to allocate requests\n", __func__);
1473         blk_mq_free_rq_map(set, tags, hctx_idx);
1474         return NULL;
1475 }
1476
1477 static void blk_mq_free_bitmap(struct blk_mq_ctxmap *bitmap)
1478 {
1479         kfree(bitmap->map);
1480 }
1481
1482 static int blk_mq_alloc_bitmap(struct blk_mq_ctxmap *bitmap, int node)
1483 {
1484         unsigned int bpw = 8, total, num_maps, i;
1485
1486         bitmap->bits_per_word = bpw;
1487
1488         num_maps = ALIGN(nr_cpu_ids, bpw) / bpw;
1489         bitmap->map = kzalloc_node(num_maps * sizeof(struct blk_align_bitmap),
1490                                         GFP_KERNEL, node);
1491         if (!bitmap->map)
1492                 return -ENOMEM;
1493
1494         bitmap->map_size = num_maps;
1495
1496         total = nr_cpu_ids;
1497         for (i = 0; i < num_maps; i++) {
1498                 bitmap->map[i].depth = min(total, bitmap->bits_per_word);
1499                 total -= bitmap->map[i].depth;
1500         }
1501
1502         return 0;
1503 }
1504
1505 static int blk_mq_hctx_cpu_offline(struct blk_mq_hw_ctx *hctx, int cpu)
1506 {
1507         struct request_queue *q = hctx->queue;
1508         struct blk_mq_ctx *ctx;
1509         LIST_HEAD(tmp);
1510
1511         /*
1512          * Move ctx entries to new CPU, if this one is going away.
1513          */
1514         ctx = __blk_mq_get_ctx(q, cpu);
1515
1516         spin_lock(&ctx->lock);
1517         if (!list_empty(&ctx->rq_list)) {
1518                 list_splice_init(&ctx->rq_list, &tmp);
1519                 blk_mq_hctx_clear_pending(hctx, ctx);
1520         }
1521         spin_unlock(&ctx->lock);
1522
1523         if (list_empty(&tmp))
1524                 return NOTIFY_OK;
1525
1526         ctx = blk_mq_get_ctx(q);
1527         spin_lock(&ctx->lock);
1528
1529         while (!list_empty(&tmp)) {
1530                 struct request *rq;
1531
1532                 rq = list_first_entry(&tmp, struct request, queuelist);
1533                 rq->mq_ctx = ctx;
1534                 list_move_tail(&rq->queuelist, &ctx->rq_list);
1535         }
1536
1537         hctx = q->mq_ops->map_queue(q, ctx->cpu);
1538         blk_mq_hctx_mark_pending(hctx, ctx);
1539
1540         spin_unlock(&ctx->lock);
1541
1542         blk_mq_run_hw_queue(hctx, true);
1543         blk_mq_put_ctx(ctx);
1544         return NOTIFY_OK;
1545 }
1546
1547 static int blk_mq_hctx_cpu_online(struct blk_mq_hw_ctx *hctx, int cpu)
1548 {
1549         struct request_queue *q = hctx->queue;
1550         struct blk_mq_tag_set *set = q->tag_set;
1551
1552         if (set->tags[hctx->queue_num])
1553                 return NOTIFY_OK;
1554
1555         set->tags[hctx->queue_num] = blk_mq_init_rq_map(set, hctx->queue_num);
1556         if (!set->tags[hctx->queue_num])
1557                 return NOTIFY_STOP;
1558
1559         hctx->tags = set->tags[hctx->queue_num];
1560         return NOTIFY_OK;
1561 }
1562
1563 static int blk_mq_hctx_notify(void *data, unsigned long action,
1564                               unsigned int cpu)
1565 {
1566         struct blk_mq_hw_ctx *hctx = data;
1567
1568         if (action == CPU_DEAD || action == CPU_DEAD_FROZEN)
1569                 return blk_mq_hctx_cpu_offline(hctx, cpu);
1570         else if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN)
1571                 return blk_mq_hctx_cpu_online(hctx, cpu);
1572
1573         return NOTIFY_OK;
1574 }
1575
1576 static void blk_mq_exit_hw_queues(struct request_queue *q,
1577                 struct blk_mq_tag_set *set, int nr_queue)
1578 {
1579         struct blk_mq_hw_ctx *hctx;
1580         unsigned int i;
1581
1582         queue_for_each_hw_ctx(q, hctx, i) {
1583                 if (i == nr_queue)
1584                         break;
1585
1586                 if (set->ops->exit_hctx)
1587                         set->ops->exit_hctx(hctx, i);
1588
1589                 blk_mq_unregister_cpu_notifier(&hctx->cpu_notifier);
1590                 kfree(hctx->ctxs);
1591                 blk_mq_free_bitmap(&hctx->ctx_map);
1592         }
1593
1594 }
1595
1596 static void blk_mq_free_hw_queues(struct request_queue *q,
1597                 struct blk_mq_tag_set *set)
1598 {
1599         struct blk_mq_hw_ctx *hctx;
1600         unsigned int i;
1601
1602         queue_for_each_hw_ctx(q, hctx, i) {
1603                 free_cpumask_var(hctx->cpumask);
1604                 set->ops->free_hctx(hctx, i);
1605         }
1606 }
1607
1608 static int blk_mq_init_hw_queues(struct request_queue *q,
1609                 struct blk_mq_tag_set *set)
1610 {
1611         struct blk_mq_hw_ctx *hctx;
1612         unsigned int i;
1613
1614         /*
1615          * Initialize hardware queues
1616          */
1617         queue_for_each_hw_ctx(q, hctx, i) {
1618                 int node;
1619
1620                 node = hctx->numa_node;
1621                 if (node == NUMA_NO_NODE)
1622                         node = hctx->numa_node = set->numa_node;
1623
1624                 INIT_DELAYED_WORK(&hctx->run_work, blk_mq_run_work_fn);
1625                 INIT_DELAYED_WORK(&hctx->delay_work, blk_mq_delay_work_fn);
1626                 spin_lock_init(&hctx->lock);
1627                 INIT_LIST_HEAD(&hctx->dispatch);
1628                 hctx->queue = q;
1629                 hctx->queue_num = i;
1630                 hctx->flags = set->flags;
1631                 hctx->cmd_size = set->cmd_size;
1632
1633                 blk_mq_init_cpu_notifier(&hctx->cpu_notifier,
1634                                                 blk_mq_hctx_notify, hctx);
1635                 blk_mq_register_cpu_notifier(&hctx->cpu_notifier);
1636
1637                 hctx->tags = set->tags[i];
1638
1639                 /*
1640                  * Allocate space for all possible cpus to avoid allocation in
1641                  * runtime
1642                  */
1643                 hctx->ctxs = kmalloc_node(nr_cpu_ids * sizeof(void *),
1644                                                 GFP_KERNEL, node);
1645                 if (!hctx->ctxs)
1646                         break;
1647
1648                 if (blk_mq_alloc_bitmap(&hctx->ctx_map, node))
1649                         break;
1650
1651                 hctx->nr_ctx = 0;
1652
1653                 if (set->ops->init_hctx &&
1654                     set->ops->init_hctx(hctx, set->driver_data, i))
1655                         break;
1656         }
1657
1658         if (i == q->nr_hw_queues)
1659                 return 0;
1660
1661         /*
1662          * Init failed
1663          */
1664         blk_mq_exit_hw_queues(q, set, i);
1665
1666         return 1;
1667 }
1668
1669 static void blk_mq_init_cpu_queues(struct request_queue *q,
1670                                    unsigned int nr_hw_queues)
1671 {
1672         unsigned int i;
1673
1674         for_each_possible_cpu(i) {
1675                 struct blk_mq_ctx *__ctx = per_cpu_ptr(q->queue_ctx, i);
1676                 struct blk_mq_hw_ctx *hctx;
1677
1678                 memset(__ctx, 0, sizeof(*__ctx));
1679                 __ctx->cpu = i;
1680                 spin_lock_init(&__ctx->lock);
1681                 INIT_LIST_HEAD(&__ctx->rq_list);
1682                 __ctx->queue = q;
1683
1684                 /* If the cpu isn't online, the cpu is mapped to first hctx */
1685                 if (!cpu_online(i))
1686                         continue;
1687
1688                 hctx = q->mq_ops->map_queue(q, i);
1689                 cpumask_set_cpu(i, hctx->cpumask);
1690                 hctx->nr_ctx++;
1691
1692                 /*
1693                  * Set local node, IFF we have more than one hw queue. If
1694                  * not, we remain on the home node of the device
1695                  */
1696                 if (nr_hw_queues > 1 && hctx->numa_node == NUMA_NO_NODE)
1697                         hctx->numa_node = cpu_to_node(i);
1698         }
1699 }
1700
1701 static void blk_mq_map_swqueue(struct request_queue *q)
1702 {
1703         unsigned int i;
1704         struct blk_mq_hw_ctx *hctx;
1705         struct blk_mq_ctx *ctx;
1706
1707         queue_for_each_hw_ctx(q, hctx, i) {
1708                 cpumask_clear(hctx->cpumask);
1709                 hctx->nr_ctx = 0;
1710         }
1711
1712         /*
1713          * Map software to hardware queues
1714          */
1715         queue_for_each_ctx(q, ctx, i) {
1716                 /* If the cpu isn't online, the cpu is mapped to first hctx */
1717                 if (!cpu_online(i))
1718                         continue;
1719
1720                 hctx = q->mq_ops->map_queue(q, i);
1721                 cpumask_set_cpu(i, hctx->cpumask);
1722                 ctx->index_hw = hctx->nr_ctx;
1723                 hctx->ctxs[hctx->nr_ctx++] = ctx;
1724         }
1725
1726         queue_for_each_hw_ctx(q, hctx, i) {
1727                 /*
1728                  * If not software queues are mapped to this hardware queue,
1729                  * disable it and free the request entries
1730                  */
1731                 if (!hctx->nr_ctx) {
1732                         struct blk_mq_tag_set *set = q->tag_set;
1733
1734                         if (set->tags[i]) {
1735                                 blk_mq_free_rq_map(set, set->tags[i], i);
1736                                 set->tags[i] = NULL;
1737                                 hctx->tags = NULL;
1738                         }
1739                         continue;
1740                 }
1741
1742                 /*
1743                  * Initialize batch roundrobin counts
1744                  */
1745                 hctx->next_cpu = cpumask_first(hctx->cpumask);
1746                 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
1747         }
1748 }
1749
1750 static void blk_mq_update_tag_set_depth(struct blk_mq_tag_set *set)
1751 {
1752         struct blk_mq_hw_ctx *hctx;
1753         struct request_queue *q;
1754         bool shared;
1755         int i;
1756
1757         if (set->tag_list.next == set->tag_list.prev)
1758                 shared = false;
1759         else
1760                 shared = true;
1761
1762         list_for_each_entry(q, &set->tag_list, tag_set_list) {
1763                 blk_mq_freeze_queue(q);
1764
1765                 queue_for_each_hw_ctx(q, hctx, i) {
1766                         if (shared)
1767                                 hctx->flags |= BLK_MQ_F_TAG_SHARED;
1768                         else
1769                                 hctx->flags &= ~BLK_MQ_F_TAG_SHARED;
1770                 }
1771                 blk_mq_unfreeze_queue(q);
1772         }
1773 }
1774
1775 static void blk_mq_del_queue_tag_set(struct request_queue *q)
1776 {
1777         struct blk_mq_tag_set *set = q->tag_set;
1778
1779         blk_mq_freeze_queue(q);
1780
1781         mutex_lock(&set->tag_list_lock);
1782         list_del_init(&q->tag_set_list);
1783         blk_mq_update_tag_set_depth(set);
1784         mutex_unlock(&set->tag_list_lock);
1785
1786         blk_mq_unfreeze_queue(q);
1787 }
1788
1789 static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,
1790                                      struct request_queue *q)
1791 {
1792         q->tag_set = set;
1793
1794         mutex_lock(&set->tag_list_lock);
1795         list_add_tail(&q->tag_set_list, &set->tag_list);
1796         blk_mq_update_tag_set_depth(set);
1797         mutex_unlock(&set->tag_list_lock);
1798 }
1799
1800 struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set)
1801 {
1802         struct blk_mq_hw_ctx **hctxs;
1803         struct blk_mq_ctx *ctx;
1804         struct request_queue *q;
1805         unsigned int *map;
1806         int i;
1807
1808         ctx = alloc_percpu(struct blk_mq_ctx);
1809         if (!ctx)
1810                 return ERR_PTR(-ENOMEM);
1811
1812         hctxs = kmalloc_node(set->nr_hw_queues * sizeof(*hctxs), GFP_KERNEL,
1813                         set->numa_node);
1814
1815         if (!hctxs)
1816                 goto err_percpu;
1817
1818         map = blk_mq_make_queue_map(set);
1819         if (!map)
1820                 goto err_map;
1821
1822         for (i = 0; i < set->nr_hw_queues; i++) {
1823                 int node = blk_mq_hw_queue_to_node(map, i);
1824
1825                 hctxs[i] = set->ops->alloc_hctx(set, i, node);
1826                 if (!hctxs[i])
1827                         goto err_hctxs;
1828
1829                 if (!zalloc_cpumask_var(&hctxs[i]->cpumask, GFP_KERNEL))
1830                         goto err_hctxs;
1831
1832                 atomic_set(&hctxs[i]->nr_active, 0);
1833                 hctxs[i]->numa_node = node;
1834                 hctxs[i]->queue_num = i;
1835         }
1836
1837         q = blk_alloc_queue_node(GFP_KERNEL, set->numa_node);
1838         if (!q)
1839                 goto err_hctxs;
1840
1841         if (percpu_counter_init(&q->mq_usage_counter, 0))
1842                 goto err_map;
1843
1844         setup_timer(&q->timeout, blk_mq_rq_timer, (unsigned long) q);
1845         blk_queue_rq_timeout(q, 30000);
1846
1847         q->nr_queues = nr_cpu_ids;
1848         q->nr_hw_queues = set->nr_hw_queues;
1849         q->mq_map = map;
1850
1851         q->queue_ctx = ctx;
1852         q->queue_hw_ctx = hctxs;
1853
1854         q->mq_ops = set->ops;
1855         q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT;
1856
1857         q->sg_reserved_size = INT_MAX;
1858
1859         INIT_WORK(&q->requeue_work, blk_mq_requeue_work);
1860         INIT_LIST_HEAD(&q->requeue_list);
1861         spin_lock_init(&q->requeue_lock);
1862
1863         if (q->nr_hw_queues > 1)
1864                 blk_queue_make_request(q, blk_mq_make_request);
1865         else
1866                 blk_queue_make_request(q, blk_sq_make_request);
1867
1868         blk_queue_rq_timed_out(q, blk_mq_rq_timed_out);
1869         if (set->timeout)
1870                 blk_queue_rq_timeout(q, set->timeout);
1871
1872         /*
1873          * Do this after blk_queue_make_request() overrides it...
1874          */
1875         q->nr_requests = set->queue_depth;
1876
1877         if (set->ops->complete)
1878                 blk_queue_softirq_done(q, set->ops->complete);
1879
1880         blk_mq_init_flush(q);
1881         blk_mq_init_cpu_queues(q, set->nr_hw_queues);
1882
1883         q->flush_rq = kzalloc(round_up(sizeof(struct request) +
1884                                 set->cmd_size, cache_line_size()),
1885                                 GFP_KERNEL);
1886         if (!q->flush_rq)
1887                 goto err_hw;
1888
1889         if (blk_mq_init_hw_queues(q, set))
1890                 goto err_flush_rq;
1891
1892         mutex_lock(&all_q_mutex);
1893         list_add_tail(&q->all_q_node, &all_q_list);
1894         mutex_unlock(&all_q_mutex);
1895
1896         blk_mq_add_queue_tag_set(set, q);
1897
1898         blk_mq_map_swqueue(q);
1899
1900         return q;
1901
1902 err_flush_rq:
1903         kfree(q->flush_rq);
1904 err_hw:
1905         blk_cleanup_queue(q);
1906 err_hctxs:
1907         kfree(map);
1908         for (i = 0; i < set->nr_hw_queues; i++) {
1909                 if (!hctxs[i])
1910                         break;
1911                 free_cpumask_var(hctxs[i]->cpumask);
1912                 set->ops->free_hctx(hctxs[i], i);
1913         }
1914 err_map:
1915         kfree(hctxs);
1916 err_percpu:
1917         free_percpu(ctx);
1918         return ERR_PTR(-ENOMEM);
1919 }
1920 EXPORT_SYMBOL(blk_mq_init_queue);
1921
1922 void blk_mq_free_queue(struct request_queue *q)
1923 {
1924         struct blk_mq_tag_set   *set = q->tag_set;
1925
1926         blk_mq_del_queue_tag_set(q);
1927
1928         blk_mq_exit_hw_queues(q, set, set->nr_hw_queues);
1929         blk_mq_free_hw_queues(q, set);
1930
1931         percpu_counter_destroy(&q->mq_usage_counter);
1932
1933         free_percpu(q->queue_ctx);
1934         kfree(q->queue_hw_ctx);
1935         kfree(q->mq_map);
1936
1937         q->queue_ctx = NULL;
1938         q->queue_hw_ctx = NULL;
1939         q->mq_map = NULL;
1940
1941         mutex_lock(&all_q_mutex);
1942         list_del_init(&q->all_q_node);
1943         mutex_unlock(&all_q_mutex);
1944 }
1945
1946 /* Basically redo blk_mq_init_queue with queue frozen */
1947 static void blk_mq_queue_reinit(struct request_queue *q)
1948 {
1949         blk_mq_freeze_queue(q);
1950
1951         blk_mq_update_queue_map(q->mq_map, q->nr_hw_queues);
1952
1953         /*
1954          * redo blk_mq_init_cpu_queues and blk_mq_init_hw_queues. FIXME: maybe
1955          * we should change hctx numa_node according to new topology (this
1956          * involves free and re-allocate memory, worthy doing?)
1957          */
1958
1959         blk_mq_map_swqueue(q);
1960
1961         blk_mq_unfreeze_queue(q);
1962 }
1963
1964 static int blk_mq_queue_reinit_notify(struct notifier_block *nb,
1965                                       unsigned long action, void *hcpu)
1966 {
1967         struct request_queue *q;
1968
1969         /*
1970          * Before new mappings are established, hotadded cpu might already
1971          * start handling requests. This doesn't break anything as we map
1972          * offline CPUs to first hardware queue. We will re-init the queue
1973          * below to get optimal settings.
1974          */
1975         if (action != CPU_DEAD && action != CPU_DEAD_FROZEN &&
1976             action != CPU_ONLINE && action != CPU_ONLINE_FROZEN)
1977                 return NOTIFY_OK;
1978
1979         mutex_lock(&all_q_mutex);
1980         list_for_each_entry(q, &all_q_list, all_q_node)
1981                 blk_mq_queue_reinit(q);
1982         mutex_unlock(&all_q_mutex);
1983         return NOTIFY_OK;
1984 }
1985
1986 int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set)
1987 {
1988         int i;
1989
1990         if (!set->nr_hw_queues)
1991                 return -EINVAL;
1992         if (!set->queue_depth || set->queue_depth > BLK_MQ_MAX_DEPTH)
1993                 return -EINVAL;
1994         if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN)
1995                 return -EINVAL;
1996
1997         if (!set->nr_hw_queues ||
1998             !set->ops->queue_rq || !set->ops->map_queue ||
1999             !set->ops->alloc_hctx || !set->ops->free_hctx)
2000                 return -EINVAL;
2001
2002
2003         set->tags = kmalloc_node(set->nr_hw_queues *
2004                                  sizeof(struct blk_mq_tags *),
2005                                  GFP_KERNEL, set->numa_node);
2006         if (!set->tags)
2007                 goto out;
2008
2009         for (i = 0; i < set->nr_hw_queues; i++) {
2010                 set->tags[i] = blk_mq_init_rq_map(set, i);
2011                 if (!set->tags[i])
2012                         goto out_unwind;
2013         }
2014
2015         mutex_init(&set->tag_list_lock);
2016         INIT_LIST_HEAD(&set->tag_list);
2017
2018         return 0;
2019
2020 out_unwind:
2021         while (--i >= 0)
2022                 blk_mq_free_rq_map(set, set->tags[i], i);
2023 out:
2024         return -ENOMEM;
2025 }
2026 EXPORT_SYMBOL(blk_mq_alloc_tag_set);
2027
2028 void blk_mq_free_tag_set(struct blk_mq_tag_set *set)
2029 {
2030         int i;
2031
2032         for (i = 0; i < set->nr_hw_queues; i++) {
2033                 if (set->tags[i])
2034                         blk_mq_free_rq_map(set, set->tags[i], i);
2035         }
2036
2037         kfree(set->tags);
2038 }
2039 EXPORT_SYMBOL(blk_mq_free_tag_set);
2040
2041 int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr)
2042 {
2043         struct blk_mq_tag_set *set = q->tag_set;
2044         struct blk_mq_hw_ctx *hctx;
2045         int i, ret;
2046
2047         if (!set || nr > set->queue_depth)
2048                 return -EINVAL;
2049
2050         ret = 0;
2051         queue_for_each_hw_ctx(q, hctx, i) {
2052                 ret = blk_mq_tag_update_depth(hctx->tags, nr);
2053                 if (ret)
2054                         break;
2055         }
2056
2057         if (!ret)
2058                 q->nr_requests = nr;
2059
2060         return ret;
2061 }
2062
2063 void blk_mq_disable_hotplug(void)
2064 {
2065         mutex_lock(&all_q_mutex);
2066 }
2067
2068 void blk_mq_enable_hotplug(void)
2069 {
2070         mutex_unlock(&all_q_mutex);
2071 }
2072
2073 static int __init blk_mq_init(void)
2074 {
2075         blk_mq_cpu_init();
2076
2077         /* Must be called after percpu_counter_hotcpu_callback() */
2078         hotcpu_notifier(blk_mq_queue_reinit_notify, -10);
2079
2080         return 0;
2081 }
2082 subsys_initcall(blk_mq_init);