block/blk-mq.c

   1 /*
   2  * Block multiqueue core code
   3  *
   4  * Copyright (C) 2013-2014 Jens Axboe
   5  * Copyright (C) 2013-2014 Christoph Hellwig
   6  */
   7 #include <linux/kernel.h>
   8 #include <linux/module.h>
   9 #include <linux/backing-dev.h>
  10 #include <linux/bio.h>
  11 #include <linux/blkdev.h>
  12 #include <linux/mm.h>
  13 #include <linux/init.h>
  14 #include <linux/slab.h>
  15 #include <linux/workqueue.h>
  16 #include <linux/smp.h>
  17 #include <linux/llist.h>
  18 #include <linux/list_sort.h>
  19 #include <linux/cpu.h>
  20 #include <linux/cache.h>
  21 #include <linux/sched/sysctl.h>
  22 #include <linux/delay.h>
  23
  24 #include <trace/events/block.h>
  25
  26 #include <linux/blk-mq.h>
  27 #include "blk.h"
  28 #include "blk-mq.h"
  29 #include "blk-mq-tag.h"
  30
  31 static DEFINE_MUTEX(all_q_mutex);
  32 static LIST_HEAD(all_q_list);
  33
  34 static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx);
  35
  36 /*
  37  * Check if any of the ctx's have pending work in this hardware queue
  38  */
  39 static bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx)
  40 {
  41         unsigned int i;
  42
  43         for (i = 0; i < hctx->ctx_map.map_size; i++)
  44                 if (hctx->ctx_map.map[i].word)
  45                         return true;
  46
  47         return false;
  48 }
  49
  50 static inline struct blk_align_bitmap *get_bm(struct blk_mq_hw_ctx *hctx,
  51                                               struct blk_mq_ctx *ctx)
  52 {
  53         return &hctx->ctx_map.map[ctx->index_hw / hctx->ctx_map.bits_per_word];
  54 }
  55
  56 #define CTX_TO_BIT(hctx, ctx)   \
  57         ((ctx)->index_hw & ((hctx)->ctx_map.bits_per_word - 1))
  58
  59 /*
  60  * Mark this ctx as having pending work in this hardware queue
  61  */
  62 static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx,
  63                                      struct blk_mq_ctx *ctx)
  64 {
  65         struct blk_align_bitmap *bm = get_bm(hctx, ctx);
  66
  67         if (!test_bit(CTX_TO_BIT(hctx, ctx), &bm->word))
  68                 set_bit(CTX_TO_BIT(hctx, ctx), &bm->word);
  69 }
  70
  71 static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx,
  72                                       struct blk_mq_ctx *ctx)
  73 {
  74         struct blk_align_bitmap *bm = get_bm(hctx, ctx);
  75
  76         clear_bit(CTX_TO_BIT(hctx, ctx), &bm->word);
  77 }
  78
  79 static int blk_mq_queue_enter(struct request_queue *q)
  80 {
  81         int ret;
  82
  83         __percpu_counter_add(&q->mq_usage_counter, 1, 1000000);
  84         smp_wmb();
  85         /* we have problems to freeze the queue if it's initializing */
  86         if (!blk_queue_bypass(q) || !blk_queue_init_done(q))
  87                 return 0;
  88
  89         __percpu_counter_add(&q->mq_usage_counter, -1, 1000000);
  90
  91         spin_lock_irq(q->queue_lock);
  92         ret = wait_event_interruptible_lock_irq(q->mq_freeze_wq,
  93                 !blk_queue_bypass(q) || blk_queue_dying(q),
  94                 *q->queue_lock);
  95         /* inc usage with lock hold to avoid freeze_queue runs here */
  96         if (!ret && !blk_queue_dying(q))
  97                 __percpu_counter_add(&q->mq_usage_counter, 1, 1000000);
  98         else if (blk_queue_dying(q))
  99                 ret = -ENODEV;
 100         spin_unlock_irq(q->queue_lock);
 101
 102         return ret;
 103 }
 104
 105 static void blk_mq_queue_exit(struct request_queue *q)
 106 {
 107         __percpu_counter_add(&q->mq_usage_counter, -1, 1000000);
 108 }
 109
 110 static void __blk_mq_drain_queue(struct request_queue *q)
 111 {
 112         while (true) {
 113                 s64 count;
 114
 115                 spin_lock_irq(q->queue_lock);
 116                 count = percpu_counter_sum(&q->mq_usage_counter);
 117                 spin_unlock_irq(q->queue_lock);
 118
 119                 if (count == 0)
 120                         break;
 121                 blk_mq_run_queues(q, false);
 122                 msleep(10);
 123         }
 124 }
 125
 126 /*
 127  * Guarantee no request is in use, so we can change any data structure of
 128  * the queue afterward.
 129  */
 130 static void blk_mq_freeze_queue(struct request_queue *q)
 131 {
 132         bool drain;
 133
 134         spin_lock_irq(q->queue_lock);
 135         drain = !q->bypass_depth++;
 136         queue_flag_set(QUEUE_FLAG_BYPASS, q);
 137         spin_unlock_irq(q->queue_lock);
 138
 139         if (drain)
 140                 __blk_mq_drain_queue(q);
 141 }
 142
 143 void blk_mq_drain_queue(struct request_queue *q)
 144 {
 145         __blk_mq_drain_queue(q);
 146 }
 147
 148 static void blk_mq_unfreeze_queue(struct request_queue *q)
 149 {
 150         bool wake = false;
 151
 152         spin_lock_irq(q->queue_lock);
 153         if (!--q->bypass_depth) {
 154                 queue_flag_clear(QUEUE_FLAG_BYPASS, q);
 155                 wake = true;
 156         }
 157         WARN_ON_ONCE(q->bypass_depth < 0);
 158         spin_unlock_irq(q->queue_lock);
 159         if (wake)
 160                 wake_up_all(&q->mq_freeze_wq);
 161 }
 162
 163 bool blk_mq_can_queue(struct blk_mq_hw_ctx *hctx)
 164 {
 165         return blk_mq_has_free_tags(hctx->tags);
 166 }
 167 EXPORT_SYMBOL(blk_mq_can_queue);
 168
 169 static void blk_mq_rq_ctx_init(struct request_queue *q, struct blk_mq_ctx *ctx,
 170                                struct request *rq, unsigned int rw_flags)
 171 {
 172         if (blk_queue_io_stat(q))
 173                 rw_flags |= REQ_IO_STAT;
 174
 175         INIT_LIST_HEAD(&rq->queuelist);
 176         /* csd/requeue_work/fifo_time is initialized before use */
 177         rq->q = q;
 178         rq->mq_ctx = ctx;
 179         rq->cmd_flags |= rw_flags;
 180         /* do not touch atomic flags, it needs atomic ops against the timer */
 181         rq->cpu = -1;
 182         INIT_HLIST_NODE(&rq->hash);
 183         RB_CLEAR_NODE(&rq->rb_node);
 184         rq->rq_disk = NULL;
 185         rq->part = NULL;
 186 #ifdef CONFIG_BLK_CGROUP
 187         rq->rl = NULL;
 188         set_start_time_ns(rq);
 189         rq->io_start_time_ns = 0;
 190 #endif
 191         rq->nr_phys_segments = 0;
 192 #if defined(CONFIG_BLK_DEV_INTEGRITY)
 193         rq->nr_integrity_segments = 0;
 194 #endif
 195         rq->special = NULL;
 196         /* tag was already set */
 197         rq->errors = 0;
 198
 199         rq->extra_len = 0;
 200         rq->sense_len = 0;
 201         rq->resid_len = 0;
 202         rq->sense = NULL;
 203
 204         INIT_LIST_HEAD(&rq->timeout_list);
 205         rq->end_io = NULL;
 206         rq->end_io_data = NULL;
 207         rq->next_rq = NULL;
 208
 209         ctx->rq_dispatched[rw_is_sync(rw_flags)]++;
 210 }
 211
 212 static struct request *
 213 __blk_mq_alloc_request(struct blk_mq_alloc_data *data, int rw)
 214 {
 215         struct request *rq;
 216         unsigned int tag;
 217
 218         tag = blk_mq_get_tag(data);
 219         if (tag != BLK_MQ_TAG_FAIL) {
 220                 rq = data->hctx->tags->rqs[tag];
 221
 222                 rq->cmd_flags = 0;
 223                 if (blk_mq_tag_busy(data->hctx)) {
 224                         rq->cmd_flags = REQ_MQ_INFLIGHT;
 225                         atomic_inc(&data->hctx->nr_active);
 226                 }
 227
 228                 rq->tag = tag;
 229                 blk_mq_rq_ctx_init(data->q, data->ctx, rq, rw);
 230                 return rq;
 231         }
 232
 233         return NULL;
 234 }
 235
 236 struct request *blk_mq_alloc_request(struct request_queue *q, int rw, gfp_t gfp,
 237                 bool reserved)
 238 {
 239         struct blk_mq_ctx *ctx;
 240         struct blk_mq_hw_ctx *hctx;
 241         struct request *rq;
 242         struct blk_mq_alloc_data alloc_data;
 243
 244         if (blk_mq_queue_enter(q))
 245                 return NULL;
 246
 247         ctx = blk_mq_get_ctx(q);
 248         hctx = q->mq_ops->map_queue(q, ctx->cpu);
 249         blk_mq_set_alloc_data(&alloc_data, q, gfp & ~__GFP_WAIT,
 250                         reserved, ctx, hctx);
 251
 252         rq = __blk_mq_alloc_request(&alloc_data, rw);
 253         if (!rq && (gfp & __GFP_WAIT)) {
 254                 __blk_mq_run_hw_queue(hctx);
 255                 blk_mq_put_ctx(ctx);
 256
 257                 ctx = blk_mq_get_ctx(q);
 258                 hctx = q->mq_ops->map_queue(q, ctx->cpu);
 259                 blk_mq_set_alloc_data(&alloc_data, q, gfp, reserved, ctx,
 260                                 hctx);
 261                 rq =  __blk_mq_alloc_request(&alloc_data, rw);
 262                 ctx = alloc_data.ctx;
 263         }
 264         blk_mq_put_ctx(ctx);
 265         return rq;
 266 }
 267 EXPORT_SYMBOL(blk_mq_alloc_request);
 268
 269 static void __blk_mq_free_request(struct blk_mq_hw_ctx *hctx,
 270                                   struct blk_mq_ctx *ctx, struct request *rq)
 271 {
 272         const int tag = rq->tag;
 273         struct request_queue *q = rq->q;
 274
 275         if (rq->cmd_flags & REQ_MQ_INFLIGHT)
 276                 atomic_dec(&hctx->nr_active);
 277
 278         clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
 279         blk_mq_put_tag(hctx, tag, &ctx->last_tag);
 280         blk_mq_queue_exit(q);
 281 }
 282
 283 void blk_mq_free_request(struct request *rq)
 284 {
 285         struct blk_mq_ctx *ctx = rq->mq_ctx;
 286         struct blk_mq_hw_ctx *hctx;
 287         struct request_queue *q = rq->q;
 288
 289         ctx->rq_completed[rq_is_sync(rq)]++;
 290
 291         hctx = q->mq_ops->map_queue(q, ctx->cpu);
 292         __blk_mq_free_request(hctx, ctx, rq);
 293 }
 294
 295 /*
 296  * Clone all relevant state from a request that has been put on hold in
 297  * the flush state machine into the preallocated flush request that hangs
 298  * off the request queue.
 299  *
 300  * For a driver the flush request should be invisible, that's why we are
 301  * impersonating the original request here.
 302  */
 303 void blk_mq_clone_flush_request(struct request *flush_rq,
 304                 struct request *orig_rq)
 305 {
 306         struct blk_mq_hw_ctx *hctx =
 307                 orig_rq->q->mq_ops->map_queue(orig_rq->q, orig_rq->mq_ctx->cpu);
 308
 309         flush_rq->mq_ctx = orig_rq->mq_ctx;
 310         flush_rq->tag = orig_rq->tag;
 311         memcpy(blk_mq_rq_to_pdu(flush_rq), blk_mq_rq_to_pdu(orig_rq),
 312                 hctx->cmd_size);
 313 }
 314
 315 inline void __blk_mq_end_io(struct request *rq, int error)
 316 {
 317         blk_account_io_done(rq);
 318
 319         if (rq->end_io) {
 320                 rq->end_io(rq, error);
 321         } else {
 322                 if (unlikely(blk_bidi_rq(rq)))
 323                         blk_mq_free_request(rq->next_rq);
 324                 blk_mq_free_request(rq);
 325         }
 326 }
 327 EXPORT_SYMBOL(__blk_mq_end_io);
 328
 329 void blk_mq_end_io(struct request *rq, int error)
 330 {
 331         if (blk_update_request(rq, error, blk_rq_bytes(rq)))
 332                 BUG();
 333         __blk_mq_end_io(rq, error);
 334 }
 335 EXPORT_SYMBOL(blk_mq_end_io);
 336
 337 static void __blk_mq_complete_request_remote(void *data)
 338 {
 339         struct request *rq = data;
 340
 341         rq->q->softirq_done_fn(rq);
 342 }
 343
 344 static void blk_mq_ipi_complete_request(struct request *rq)
 345 {
 346         struct blk_mq_ctx *ctx = rq->mq_ctx;
 347         bool shared = false;
 348         int cpu;
 349
 350         if (!test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags)) {
 351                 rq->q->softirq_done_fn(rq);
 352                 return;
 353         }
 354
 355         cpu = get_cpu();
 356         if (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags))
 357                 shared = cpus_share_cache(cpu, ctx->cpu);
 358
 359         if (cpu != ctx->cpu && !shared && cpu_online(ctx->cpu)) {
 360                 rq->csd.func = __blk_mq_complete_request_remote;
 361                 rq->csd.info = rq;
 362                 rq->csd.flags = 0;
 363                 smp_call_function_single_async(ctx->cpu, &rq->csd);
 364         } else {
 365                 rq->q->softirq_done_fn(rq);
 366         }
 367         put_cpu();
 368 }
 369
 370 void __blk_mq_complete_request(struct request *rq)
 371 {
 372         struct request_queue *q = rq->q;
 373
 374         if (!q->softirq_done_fn)
 375                 blk_mq_end_io(rq, rq->errors);
 376         else
 377                 blk_mq_ipi_complete_request(rq);
 378 }
 379
 380 /**
 381  * blk_mq_complete_request - end I/O on a request
 382  * @rq:         the request being processed
 383  *
 384  * Description:
 385  *      Ends all I/O on a request. It does not handle partial completions.
 386  *      The actual completion happens out-of-order, through a IPI handler.
 387  **/
 388 void blk_mq_complete_request(struct request *rq)
 389 {
 390         struct request_queue *q = rq->q;
 391
 392         if (unlikely(blk_should_fake_timeout(q)))
 393                 return;
 394         if (!blk_mark_rq_complete(rq))
 395                 __blk_mq_complete_request(rq);
 396 }
 397 EXPORT_SYMBOL(blk_mq_complete_request);
 398
 399 static void blk_mq_start_request(struct request *rq, bool last)
 400 {
 401         struct request_queue *q = rq->q;
 402
 403         trace_block_rq_issue(q, rq);
 404
 405         rq->resid_len = blk_rq_bytes(rq);
 406         if (unlikely(blk_bidi_rq(rq)))
 407                 rq->next_rq->resid_len = blk_rq_bytes(rq->next_rq);
 408
 409         /*
 410          * Just mark start time and set the started bit. Due to memory
 411          * ordering, we know we'll see the correct deadline as long as
 412          * REQ_ATOMIC_STARTED is seen. Use the default queue timeout,
 413          * unless one has been set in the request.
 414          */
 415         if (!rq->timeout)
 416                 rq->deadline = jiffies + q->rq_timeout;
 417         else
 418                 rq->deadline = jiffies + rq->timeout;
 419
 420         /*
 421          * Mark us as started and clear complete. Complete might have been
 422          * set if requeue raced with timeout, which then marked it as
 423          * complete. So be sure to clear complete again when we start
 424          * the request, otherwise we'll ignore the completion event.
 425          */
 426         if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags))
 427                 set_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
 428         if (test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags))
 429                 clear_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags);
 430
 431         if (q->dma_drain_size && blk_rq_bytes(rq)) {
 432                 /*
 433                  * Make sure space for the drain appears.  We know we can do
 434                  * this because max_hw_segments has been adjusted to be one
 435                  * fewer than the device can handle.
 436                  */
 437                 rq->nr_phys_segments++;
 438         }
 439
 440         /*
 441          * Flag the last request in the series so that drivers know when IO
 442          * should be kicked off, if they don't do it on a per-request basis.
 443          *
 444          * Note: the flag isn't the only condition drivers should do kick off.
 445          * If drive is busy, the last request might not have the bit set.
 446          */
 447         if (last)
 448                 rq->cmd_flags |= REQ_END;
 449 }
 450
 451 static void __blk_mq_requeue_request(struct request *rq)
 452 {
 453         struct request_queue *q = rq->q;
 454
 455         trace_block_rq_requeue(q, rq);
 456         clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
 457
 458         rq->cmd_flags &= ~REQ_END;
 459
 460         if (q->dma_drain_size && blk_rq_bytes(rq))
 461                 rq->nr_phys_segments--;
 462 }
 463
 464 void blk_mq_requeue_request(struct request *rq)
 465 {
 466         __blk_mq_requeue_request(rq);
 467         blk_clear_rq_complete(rq);
 468
 469         BUG_ON(blk_queued_rq(rq));
 470         blk_mq_add_to_requeue_list(rq, true);
 471 }
 472 EXPORT_SYMBOL(blk_mq_requeue_request);
 473
 474 static void blk_mq_requeue_work(struct work_struct *work)
 475 {
 476         struct request_queue *q =
 477                 container_of(work, struct request_queue, requeue_work);
 478         LIST_HEAD(rq_list);
 479         struct request *rq, *next;
 480         unsigned long flags;
 481
 482         spin_lock_irqsave(&q->requeue_lock, flags);
 483         list_splice_init(&q->requeue_list, &rq_list);
 484         spin_unlock_irqrestore(&q->requeue_lock, flags);
 485
 486         list_for_each_entry_safe(rq, next, &rq_list, queuelist) {
 487                 if (!(rq->cmd_flags & REQ_SOFTBARRIER))
 488                         continue;
 489
 490                 rq->cmd_flags &= ~REQ_SOFTBARRIER;
 491                 list_del_init(&rq->queuelist);
 492                 blk_mq_insert_request(rq, true, false, false);
 493         }
 494
 495         while (!list_empty(&rq_list)) {
 496                 rq = list_entry(rq_list.next, struct request, queuelist);
 497                 list_del_init(&rq->queuelist);
 498                 blk_mq_insert_request(rq, false, false, false);
 499         }
 500
 501         blk_mq_run_queues(q, false);
 502 }
 503
 504 void blk_mq_add_to_requeue_list(struct request *rq, bool at_head)
 505 {
 506         struct request_queue *q = rq->q;
 507         unsigned long flags;
 508
 509         /*
 510          * We abuse this flag that is otherwise used by the I/O scheduler to
 511          * request head insertation from the workqueue.
 512          */
 513         BUG_ON(rq->cmd_flags & REQ_SOFTBARRIER);
 514
 515         spin_lock_irqsave(&q->requeue_lock, flags);
 516         if (at_head) {
 517                 rq->cmd_flags |= REQ_SOFTBARRIER;
 518                 list_add(&rq->queuelist, &q->requeue_list);
 519         } else {
 520                 list_add_tail(&rq->queuelist, &q->requeue_list);
 521         }
 522         spin_unlock_irqrestore(&q->requeue_lock, flags);
 523 }
 524 EXPORT_SYMBOL(blk_mq_add_to_requeue_list);
 525
 526 void blk_mq_kick_requeue_list(struct request_queue *q)
 527 {
 528         kblockd_schedule_work(&q->requeue_work);
 529 }
 530 EXPORT_SYMBOL(blk_mq_kick_requeue_list);
 531
 532 struct request *blk_mq_tag_to_rq(struct blk_mq_hw_ctx *hctx, unsigned int tag)
 533 {
 534         struct request_queue *q = hctx->queue;
 535
 536         if ((q->flush_rq->cmd_flags & REQ_FLUSH_SEQ) &&
 537             q->flush_rq->tag == tag)
 538                 return q->flush_rq;
 539
 540         return hctx->tags->rqs[tag];
 541 }
 542 EXPORT_SYMBOL(blk_mq_tag_to_rq);
 543
 544 struct blk_mq_timeout_data {
 545         struct blk_mq_hw_ctx *hctx;
 546         unsigned long *next;
 547         unsigned int *next_set;
 548 };
 549
 550 static void blk_mq_timeout_check(void *__data, unsigned long *free_tags)
 551 {
 552         struct blk_mq_timeout_data *data = __data;
 553         struct blk_mq_hw_ctx *hctx = data->hctx;
 554         unsigned int tag;
 555
 556          /* It may not be in flight yet (this is where
 557          * the REQ_ATOMIC_STARTED flag comes in). The requests are
 558          * statically allocated, so we know it's always safe to access the
 559          * memory associated with a bit offset into ->rqs[].
 560          */
 561         tag = 0;
 562         do {
 563                 struct request *rq;
 564
 565                 tag = find_next_zero_bit(free_tags, hctx->tags->nr_tags, tag);
 566                 if (tag >= hctx->tags->nr_tags)
 567                         break;
 568
 569                 rq = blk_mq_tag_to_rq(hctx, tag++);
 570                 if (rq->q != hctx->queue)
 571                         continue;
 572                 if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags))
 573                         continue;
 574
 575                 blk_rq_check_expired(rq, data->next, data->next_set);
 576         } while (1);
 577 }
 578
 579 static void blk_mq_hw_ctx_check_timeout(struct blk_mq_hw_ctx *hctx,
 580                                         unsigned long *next,
 581                                         unsigned int *next_set)
 582 {
 583         struct blk_mq_timeout_data data = {
 584                 .hctx           = hctx,
 585                 .next           = next,
 586                 .next_set       = next_set,
 587         };
 588
 589         /*
 590          * Ask the tagging code to iterate busy requests, so we can
 591          * check them for timeout.
 592          */
 593         blk_mq_tag_busy_iter(hctx->tags, blk_mq_timeout_check, &data);
 594 }
 595
 596 static enum blk_eh_timer_return blk_mq_rq_timed_out(struct request *rq)
 597 {
 598         struct request_queue *q = rq->q;
 599
 600         /*
 601          * We know that complete is set at this point. If STARTED isn't set
 602          * anymore, then the request isn't active and the "timeout" should
 603          * just be ignored. This can happen due to the bitflag ordering.
 604          * Timeout first checks if STARTED is set, and if it is, assumes
 605          * the request is active. But if we race with completion, then
 606          * we both flags will get cleared. So check here again, and ignore
 607          * a timeout event with a request that isn't active.
 608          */
 609         if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags))
 610                 return BLK_EH_NOT_HANDLED;
 611
 612         if (!q->mq_ops->timeout)
 613                 return BLK_EH_RESET_TIMER;
 614
 615         return q->mq_ops->timeout(rq);
 616 }
 617
 618 static void blk_mq_rq_timer(unsigned long data)
 619 {
 620         struct request_queue *q = (struct request_queue *) data;
 621         struct blk_mq_hw_ctx *hctx;
 622         unsigned long next = 0;
 623         int i, next_set = 0;
 624
 625         queue_for_each_hw_ctx(q, hctx, i) {
 626                 /*
 627                  * If not software queues are currently mapped to this
 628                  * hardware queue, there's nothing to check
 629                  */
 630                 if (!hctx->nr_ctx || !hctx->tags)
 631                         continue;
 632
 633                 blk_mq_hw_ctx_check_timeout(hctx, &next, &next_set);
 634         }
 635
 636         if (next_set) {
 637                 next = blk_rq_timeout(round_jiffies_up(next));
 638                 mod_timer(&q->timeout, next);
 639         } else {
 640                 queue_for_each_hw_ctx(q, hctx, i)
 641                         blk_mq_tag_idle(hctx);
 642         }
 643 }
 644
 645 /*
 646  * Reverse check our software queue for entries that we could potentially
 647  * merge with. Currently includes a hand-wavy stop count of 8, to not spend
 648  * too much time checking for merges.
 649  */
 650 static bool blk_mq_attempt_merge(struct request_queue *q,
 651                                  struct blk_mq_ctx *ctx, struct bio *bio)
 652 {
 653         struct request *rq;
 654         int checked = 8;
 655
 656         list_for_each_entry_reverse(rq, &ctx->rq_list, queuelist) {
 657                 int el_ret;
 658
 659                 if (!checked--)
 660                         break;
 661
 662                 if (!blk_rq_merge_ok(rq, bio))
 663                         continue;
 664
 665                 el_ret = blk_try_merge(rq, bio);
 666                 if (el_ret == ELEVATOR_BACK_MERGE) {
 667                         if (bio_attempt_back_merge(q, rq, bio)) {
 668                                 ctx->rq_merged++;
 669                                 return true;
 670                         }
 671                         break;
 672                 } else if (el_ret == ELEVATOR_FRONT_MERGE) {
 673                         if (bio_attempt_front_merge(q, rq, bio)) {
 674                                 ctx->rq_merged++;
 675                                 return true;
 676                         }
 677                         break;
 678                 }
 679         }
 680
 681         return false;
 682 }
 683
 684 /*
 685  * Process software queues that have been marked busy, splicing them
 686  * to the for-dispatch
 687  */
 688 static void flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list)
 689 {
 690         struct blk_mq_ctx *ctx;
 691         int i;
 692
 693         for (i = 0; i < hctx->ctx_map.map_size; i++) {
 694                 struct blk_align_bitmap *bm = &hctx->ctx_map.map[i];
 695                 unsigned int off, bit;
 696
 697                 if (!bm->word)
 698                         continue;
 699
 700                 bit = 0;
 701                 off = i * hctx->ctx_map.bits_per_word;
 702                 do {
 703                         bit = find_next_bit(&bm->word, bm->depth, bit);
 704                         if (bit >= bm->depth)
 705                                 break;
 706
 707                         ctx = hctx->ctxs[bit + off];
 708                         clear_bit(bit, &bm->word);
 709                         spin_lock(&ctx->lock);
 710                         list_splice_tail_init(&ctx->rq_list, list);
 711                         spin_unlock(&ctx->lock);
 712
 713                         bit++;
 714                 } while (1);
 715         }
 716 }
 717
 718 /*
 719  * Run this hardware queue, pulling any software queues mapped to it in.
 720  * Note that this function currently has various problems around ordering
 721  * of IO. In particular, we'd like FIFO behaviour on handling existing
 722  * items on the hctx->dispatch list. Ignore that for now.
 723  */
 724 static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
 725 {
 726         struct request_queue *q = hctx->queue;
 727         struct request *rq;
 728         LIST_HEAD(rq_list);
 729         int queued;
 730
 731         WARN_ON(!cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask));
 732
 733         if (unlikely(test_bit(BLK_MQ_S_STOPPED, &hctx->state)))
 734                 return;
 735
 736         hctx->run++;
 737
 738         /*
 739          * Touch any software queue that has pending entries.
 740          */
 741         flush_busy_ctxs(hctx, &rq_list);
 742
 743         /*
 744          * If we have previous entries on our dispatch list, grab them
 745          * and stuff them at the front for more fair dispatch.
 746          */
 747         if (!list_empty_careful(&hctx->dispatch)) {
 748                 spin_lock(&hctx->lock);
 749                 if (!list_empty(&hctx->dispatch))
 750                         list_splice_init(&hctx->dispatch, &rq_list);
 751                 spin_unlock(&hctx->lock);
 752         }
 753
 754         /*
 755          * Now process all the entries, sending them to the driver.
 756          */
 757         queued = 0;
 758         while (!list_empty(&rq_list)) {
 759                 int ret;
 760
 761                 rq = list_first_entry(&rq_list, struct request, queuelist);
 762                 list_del_init(&rq->queuelist);
 763
 764                 blk_mq_start_request(rq, list_empty(&rq_list));
 765
 766                 ret = q->mq_ops->queue_rq(hctx, rq);
 767                 switch (ret) {
 768                 case BLK_MQ_RQ_QUEUE_OK:
 769                         queued++;
 770                         continue;
 771                 case BLK_MQ_RQ_QUEUE_BUSY:
 772                         list_add(&rq->queuelist, &rq_list);
 773                         __blk_mq_requeue_request(rq);
 774                         break;
 775                 default:
 776                         pr_err("blk-mq: bad return on queue: %d\n", ret);
 777                 case BLK_MQ_RQ_QUEUE_ERROR:
 778                         rq->errors = -EIO;
 779                         blk_mq_end_io(rq, rq->errors);
 780                         break;
 781                 }
 782
 783                 if (ret == BLK_MQ_RQ_QUEUE_BUSY)
 784                         break;
 785         }
 786
 787         if (!queued)
 788                 hctx->dispatched[0]++;
 789         else if (queued < (1 << (BLK_MQ_MAX_DISPATCH_ORDER - 1)))
 790                 hctx->dispatched[ilog2(queued) + 1]++;
 791
 792         /*
 793          * Any items that need requeuing? Stuff them into hctx->dispatch,
 794          * that is where we will continue on next queue run.
 795          */
 796         if (!list_empty(&rq_list)) {
 797                 spin_lock(&hctx->lock);
 798                 list_splice(&rq_list, &hctx->dispatch);
 799                 spin_unlock(&hctx->lock);
 800         }
 801 }
 802
 803 /*
 804  * It'd be great if the workqueue API had a way to pass
 805  * in a mask and had some smarts for more clever placement.
 806  * For now we just round-robin here, switching for every
 807  * BLK_MQ_CPU_WORK_BATCH queued items.
 808  */
 809 static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx)
 810 {
 811         int cpu = hctx->next_cpu;
 812
 813         if (--hctx->next_cpu_batch <= 0) {
 814                 int next_cpu;
 815
 816                 next_cpu = cpumask_next(hctx->next_cpu, hctx->cpumask);
 817                 if (next_cpu >= nr_cpu_ids)
 818                         next_cpu = cpumask_first(hctx->cpumask);
 819
 820                 hctx->next_cpu = next_cpu;
 821                 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
 822         }
 823
 824         return cpu;
 825 }
 826
 827 void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
 828 {
 829         if (unlikely(test_bit(BLK_MQ_S_STOPPED, &hctx->state)))
 830                 return;
 831
 832         if (!async && cpumask_test_cpu(smp_processor_id(), hctx->cpumask))
 833                 __blk_mq_run_hw_queue(hctx);
 834         else if (hctx->queue->nr_hw_queues == 1)
 835                 kblockd_schedule_delayed_work(&hctx->run_work, 0);
 836         else {
 837                 unsigned int cpu;
 838
 839                 cpu = blk_mq_hctx_next_cpu(hctx);
 840                 kblockd_schedule_delayed_work_on(cpu, &hctx->run_work, 0);
 841         }
 842 }
 843
 844 void blk_mq_run_queues(struct request_queue *q, bool async)
 845 {
 846         struct blk_mq_hw_ctx *hctx;
 847         int i;
 848
 849         queue_for_each_hw_ctx(q, hctx, i) {
 850                 if ((!blk_mq_hctx_has_pending(hctx) &&
 851                     list_empty_careful(&hctx->dispatch)) ||
 852                     test_bit(BLK_MQ_S_STOPPED, &hctx->state))
 853                         continue;
 854
 855                 preempt_disable();
 856                 blk_mq_run_hw_queue(hctx, async);
 857                 preempt_enable();
 858         }
 859 }
 860 EXPORT_SYMBOL(blk_mq_run_queues);
 861
 862 void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx)
 863 {
 864         cancel_delayed_work(&hctx->run_work);
 865         cancel_delayed_work(&hctx->delay_work);
 866         set_bit(BLK_MQ_S_STOPPED, &hctx->state);
 867 }
 868 EXPORT_SYMBOL(blk_mq_stop_hw_queue);
 869
 870 void blk_mq_stop_hw_queues(struct request_queue *q)
 871 {
 872         struct blk_mq_hw_ctx *hctx;
 873         int i;
 874
 875         queue_for_each_hw_ctx(q, hctx, i)
 876                 blk_mq_stop_hw_queue(hctx);
 877 }
 878 EXPORT_SYMBOL(blk_mq_stop_hw_queues);
 879
 880 void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx)
 881 {
 882         clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
 883
 884         preempt_disable();
 885         __blk_mq_run_hw_queue(hctx);
 886         preempt_enable();
 887 }
 888 EXPORT_SYMBOL(blk_mq_start_hw_queue);
 889
 890 void blk_mq_start_hw_queues(struct request_queue *q)
 891 {
 892         struct blk_mq_hw_ctx *hctx;
 893         int i;
 894
 895         queue_for_each_hw_ctx(q, hctx, i)
 896                 blk_mq_start_hw_queue(hctx);
 897 }
 898 EXPORT_SYMBOL(blk_mq_start_hw_queues);
 899
 900
 901 void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async)
 902 {
 903         struct blk_mq_hw_ctx *hctx;
 904         int i;
 905
 906         queue_for_each_hw_ctx(q, hctx, i) {
 907                 if (!test_bit(BLK_MQ_S_STOPPED, &hctx->state))
 908                         continue;
 909
 910                 clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
 911                 preempt_disable();
 912                 blk_mq_run_hw_queue(hctx, async);
 913                 preempt_enable();
 914         }
 915 }
 916 EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues);
 917
 918 static void blk_mq_run_work_fn(struct work_struct *work)
 919 {
 920         struct blk_mq_hw_ctx *hctx;
 921
 922         hctx = container_of(work, struct blk_mq_hw_ctx, run_work.work);
 923
 924         __blk_mq_run_hw_queue(hctx);
 925 }
 926
 927 static void blk_mq_delay_work_fn(struct work_struct *work)
 928 {
 929         struct blk_mq_hw_ctx *hctx;
 930
 931         hctx = container_of(work, struct blk_mq_hw_ctx, delay_work.work);
 932
 933         if (test_and_clear_bit(BLK_MQ_S_STOPPED, &hctx->state))
 934                 __blk_mq_run_hw_queue(hctx);
 935 }
 936
 937 void blk_mq_delay_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs)
 938 {
 939         unsigned long tmo = msecs_to_jiffies(msecs);
 940
 941         if (hctx->queue->nr_hw_queues == 1)
 942                 kblockd_schedule_delayed_work(&hctx->delay_work, tmo);
 943         else {
 944                 unsigned int cpu;
 945
 946                 cpu = blk_mq_hctx_next_cpu(hctx);
 947                 kblockd_schedule_delayed_work_on(cpu, &hctx->delay_work, tmo);
 948         }
 949 }
 950 EXPORT_SYMBOL(blk_mq_delay_queue);
 951
 952 static void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx,
 953                                     struct request *rq, bool at_head)
 954 {
 955         struct blk_mq_ctx *ctx = rq->mq_ctx;
 956
 957         trace_block_rq_insert(hctx->queue, rq);
 958
 959         if (at_head)
 960                 list_add(&rq->queuelist, &ctx->rq_list);
 961         else
 962                 list_add_tail(&rq->queuelist, &ctx->rq_list);
 963
 964         blk_mq_hctx_mark_pending(hctx, ctx);
 965
 966         /*
 967          * We do this early, to ensure we are on the right CPU.
 968          */
 969         blk_add_timer(rq);
 970 }
 971
 972 void blk_mq_insert_request(struct request *rq, bool at_head, bool run_queue,
 973                 bool async)
 974 {
 975         struct request_queue *q = rq->q;
 976         struct blk_mq_hw_ctx *hctx;
 977         struct blk_mq_ctx *ctx = rq->mq_ctx, *current_ctx;
 978
 979         current_ctx = blk_mq_get_ctx(q);
 980         if (!cpu_online(ctx->cpu))
 981                 rq->mq_ctx = ctx = current_ctx;
 982
 983         hctx = q->mq_ops->map_queue(q, ctx->cpu);
 984
 985         if (rq->cmd_flags & (REQ_FLUSH | REQ_FUA) &&
 986             !(rq->cmd_flags & (REQ_FLUSH_SEQ))) {
 987                 blk_insert_flush(rq);
 988         } else {
 989                 spin_lock(&ctx->lock);
 990                 __blk_mq_insert_request(hctx, rq, at_head);
 991                 spin_unlock(&ctx->lock);
 992         }
 993
 994         if (run_queue)
 995                 blk_mq_run_hw_queue(hctx, async);
 996
 997         blk_mq_put_ctx(current_ctx);
 998 }
 999
1000 static void blk_mq_insert_requests(struct request_queue *q,
1001                                      struct blk_mq_ctx *ctx,
1002                                      struct list_head *list,
1003                                      int depth,
1004                                      bool from_schedule)
1005
1006 {
1007         struct blk_mq_hw_ctx *hctx;
1008         struct blk_mq_ctx *current_ctx;
1009
1010         trace_block_unplug(q, depth, !from_schedule);
1011
1012         current_ctx = blk_mq_get_ctx(q);
1013
1014         if (!cpu_online(ctx->cpu))
1015                 ctx = current_ctx;
1016         hctx = q->mq_ops->map_queue(q, ctx->cpu);
1017
1018         /*
1019          * preemption doesn't flush plug list, so it's possible ctx->cpu is
1020          * offline now
1021          */
1022         spin_lock(&ctx->lock);
1023         while (!list_empty(list)) {
1024                 struct request *rq;
1025
1026                 rq = list_first_entry(list, struct request, queuelist);
1027                 list_del_init(&rq->queuelist);
1028                 rq->mq_ctx = ctx;
1029                 __blk_mq_insert_request(hctx, rq, false);
1030         }
1031         spin_unlock(&ctx->lock);
1032
1033         blk_mq_run_hw_queue(hctx, from_schedule);
1034         blk_mq_put_ctx(current_ctx);
1035 }
1036
1037 static int plug_ctx_cmp(void *priv, struct list_head *a, struct list_head *b)
1038 {
1039         struct request *rqa = container_of(a, struct request, queuelist);
1040         struct request *rqb = container_of(b, struct request, queuelist);
1041
1042         return !(rqa->mq_ctx < rqb->mq_ctx ||
1043                  (rqa->mq_ctx == rqb->mq_ctx &&
1044                   blk_rq_pos(rqa) < blk_rq_pos(rqb)));
1045 }
1046
1047 void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
1048 {
1049         struct blk_mq_ctx *this_ctx;
1050         struct request_queue *this_q;
1051         struct request *rq;
1052         LIST_HEAD(list);
1053         LIST_HEAD(ctx_list);
1054         unsigned int depth;
1055
1056         list_splice_init(&plug->mq_list, &list);
1057
1058         list_sort(NULL, &list, plug_ctx_cmp);
1059
1060         this_q = NULL;
1061         this_ctx = NULL;
1062         depth = 0;
1063
1064         while (!list_empty(&list)) {
1065                 rq = list_entry_rq(list.next);
1066                 list_del_init(&rq->queuelist);
1067                 BUG_ON(!rq->q);
1068                 if (rq->mq_ctx != this_ctx) {
1069                         if (this_ctx) {
1070                                 blk_mq_insert_requests(this_q, this_ctx,
1071                                                         &ctx_list, depth,
1072                                                         from_schedule);
1073                         }
1074
1075                         this_ctx = rq->mq_ctx;
1076                         this_q = rq->q;
1077                         depth = 0;
1078                 }
1079
1080                 depth++;
1081                 list_add_tail(&rq->queuelist, &ctx_list);
1082         }
1083
1084         /*
1085          * If 'this_ctx' is set, we know we have entries to complete
1086          * on 'ctx_list'. Do those.
1087          */
1088         if (this_ctx) {
1089                 blk_mq_insert_requests(this_q, this_ctx, &ctx_list, depth,
1090                                        from_schedule);
1091         }
1092 }
1093
1094 static void blk_mq_bio_to_request(struct request *rq, struct bio *bio)
1095 {
1096         init_request_from_bio(rq, bio);
1097
1098         if (blk_do_io_stat(rq)) {
1099                 rq->start_time = jiffies;
1100                 blk_account_io_start(rq, 1);
1101         }
1102 }
1103
1104 static inline bool blk_mq_merge_queue_io(struct blk_mq_hw_ctx *hctx,
1105                                          struct blk_mq_ctx *ctx,
1106                                          struct request *rq, struct bio *bio)
1107 {
1108         struct request_queue *q = hctx->queue;
1109
1110         if (!(hctx->flags & BLK_MQ_F_SHOULD_MERGE)) {
1111                 blk_mq_bio_to_request(rq, bio);
1112                 spin_lock(&ctx->lock);
1113 insert_rq:
1114                 __blk_mq_insert_request(hctx, rq, false);
1115                 spin_unlock(&ctx->lock);
1116                 return false;
1117         } else {
1118                 spin_lock(&ctx->lock);
1119                 if (!blk_mq_attempt_merge(q, ctx, bio)) {
1120                         blk_mq_bio_to_request(rq, bio);
1121                         goto insert_rq;
1122                 }
1123
1124                 spin_unlock(&ctx->lock);
1125                 __blk_mq_free_request(hctx, ctx, rq);
1126                 return true;
1127         }
1128 }
1129
1130 struct blk_map_ctx {
1131         struct blk_mq_hw_ctx *hctx;
1132         struct blk_mq_ctx *ctx;
1133 };
1134
1135 static struct request *blk_mq_map_request(struct request_queue *q,
1136                                           struct bio *bio,
1137                                           struct blk_map_ctx *data)
1138 {
1139         struct blk_mq_hw_ctx *hctx;
1140         struct blk_mq_ctx *ctx;
1141         struct request *rq;
1142         int rw = bio_data_dir(bio);
1143         struct blk_mq_alloc_data alloc_data;
1144
1145         if (unlikely(blk_mq_queue_enter(q))) {
1146                 bio_endio(bio, -EIO);
1147                 return NULL;
1148         }
1149
1150         ctx = blk_mq_get_ctx(q);
1151         hctx = q->mq_ops->map_queue(q, ctx->cpu);
1152
1153         if (rw_is_sync(bio->bi_rw))
1154                 rw |= REQ_SYNC;
1155
1156         trace_block_getrq(q, bio, rw);
1157         blk_mq_set_alloc_data(&alloc_data, q, GFP_ATOMIC, false, ctx,
1158                         hctx);
1159         rq = __blk_mq_alloc_request(&alloc_data, rw);
1160         if (unlikely(!rq)) {
1161                 __blk_mq_run_hw_queue(hctx);
1162                 blk_mq_put_ctx(ctx);
1163                 trace_block_sleeprq(q, bio, rw);
1164
1165                 ctx = blk_mq_get_ctx(q);
1166                 hctx = q->mq_ops->map_queue(q, ctx->cpu);
1167                 blk_mq_set_alloc_data(&alloc_data, q,
1168                                 __GFP_WAIT|GFP_ATOMIC, false, ctx, hctx);
1169                 rq = __blk_mq_alloc_request(&alloc_data, rw);
1170                 ctx = alloc_data.ctx;
1171                 hctx = alloc_data.hctx;
1172         }
1173
1174         hctx->queued++;
1175         data->hctx = hctx;
1176         data->ctx = ctx;
1177         return rq;
1178 }
1179
1180 /*
1181  * Multiple hardware queue variant. This will not use per-process plugs,
1182  * but will attempt to bypass the hctx queueing if we can go straight to
1183  * hardware for SYNC IO.
1184  */
1185 static void blk_mq_make_request(struct request_queue *q, struct bio *bio)
1186 {
1187         const int is_sync = rw_is_sync(bio->bi_rw);
1188         const int is_flush_fua = bio->bi_rw & (REQ_FLUSH | REQ_FUA);
1189         struct blk_map_ctx data;
1190         struct request *rq;
1191
1192         blk_queue_bounce(q, &bio);
1193
1194         if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
1195                 bio_endio(bio, -EIO);
1196                 return;
1197         }
1198
1199         rq = blk_mq_map_request(q, bio, &data);
1200         if (unlikely(!rq))
1201                 return;
1202
1203         if (unlikely(is_flush_fua)) {
1204                 blk_mq_bio_to_request(rq, bio);
1205                 blk_insert_flush(rq);
1206                 goto run_queue;
1207         }
1208
1209         if (is_sync) {
1210                 int ret;
1211
1212                 blk_mq_bio_to_request(rq, bio);
1213                 blk_mq_start_request(rq, true);
1214                 blk_add_timer(rq);
1215
1216                 /*
1217                  * For OK queue, we are done. For error, kill it. Any other
1218                  * error (busy), just add it to our list as we previously
1219                  * would have done
1220                  */
1221                 ret = q->mq_ops->queue_rq(data.hctx, rq);
1222                 if (ret == BLK_MQ_RQ_QUEUE_OK)
1223                         goto done;
1224                 else {
1225                         __blk_mq_requeue_request(rq);
1226
1227                         if (ret == BLK_MQ_RQ_QUEUE_ERROR) {
1228                                 rq->errors = -EIO;
1229                                 blk_mq_end_io(rq, rq->errors);
1230                                 goto done;
1231                         }
1232                 }
1233         }
1234
1235         if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {
1236                 /*
1237                  * For a SYNC request, send it to the hardware immediately. For
1238                  * an ASYNC request, just ensure that we run it later on. The
1239                  * latter allows for merging opportunities and more efficient
1240                  * dispatching.
1241                  */
1242 run_queue:
1243                 blk_mq_run_hw_queue(data.hctx, !is_sync || is_flush_fua);
1244         }
1245 done:
1246         blk_mq_put_ctx(data.ctx);
1247 }
1248
1249 /*
1250  * Single hardware queue variant. This will attempt to use any per-process
1251  * plug for merging and IO deferral.
1252  */
1253 static void blk_sq_make_request(struct request_queue *q, struct bio *bio)
1254 {
1255         const int is_sync = rw_is_sync(bio->bi_rw);
1256         const int is_flush_fua = bio->bi_rw & (REQ_FLUSH | REQ_FUA);
1257         unsigned int use_plug, request_count = 0;
1258         struct blk_map_ctx data;
1259         struct request *rq;
1260
1261         /*
1262          * If we have multiple hardware queues, just go directly to
1263          * one of those for sync IO.
1264          */
1265         use_plug = !is_flush_fua && !is_sync;
1266
1267         blk_queue_bounce(q, &bio);
1268
1269         if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
1270                 bio_endio(bio, -EIO);
1271                 return;
1272         }
1273
1274         if (use_plug && !blk_queue_nomerges(q) &&
1275             blk_attempt_plug_merge(q, bio, &request_count))
1276                 return;
1277
1278         rq = blk_mq_map_request(q, bio, &data);
1279
1280         if (unlikely(is_flush_fua)) {
1281                 blk_mq_bio_to_request(rq, bio);
1282                 blk_insert_flush(rq);
1283                 goto run_queue;
1284         }
1285
1286         /*
1287          * A task plug currently exists. Since this is completely lockless,
1288          * utilize that to temporarily store requests until the task is
1289          * either done or scheduled away.
1290          */
1291         if (use_plug) {
1292                 struct blk_plug *plug = current->plug;
1293
1294                 if (plug) {
1295                         blk_mq_bio_to_request(rq, bio);
1296                         if (list_empty(&plug->mq_list))
1297                                 trace_block_plug(q);
1298                         else if (request_count >= BLK_MAX_REQUEST_COUNT) {
1299                                 blk_flush_plug_list(plug, false);
1300                                 trace_block_plug(q);
1301                         }
1302                         list_add_tail(&rq->queuelist, &plug->mq_list);
1303                         blk_mq_put_ctx(data.ctx);
1304                         return;
1305                 }
1306         }
1307
1308         if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {
1309                 /*
1310                  * For a SYNC request, send it to the hardware immediately. For
1311                  * an ASYNC request, just ensure that we run it later on. The
1312                  * latter allows for merging opportunities and more efficient
1313                  * dispatching.
1314                  */
1315 run_queue:
1316                 blk_mq_run_hw_queue(data.hctx, !is_sync || is_flush_fua);
1317         }
1318
1319         blk_mq_put_ctx(data.ctx);
1320 }
1321
1322 /*
1323  * Default mapping to a software queue, since we use one per CPU.
1324  */
1325 struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, const int cpu)
1326 {
1327         return q->queue_hw_ctx[q->mq_map[cpu]];
1328 }
1329 EXPORT_SYMBOL(blk_mq_map_queue);
1330
1331 static void blk_mq_free_rq_map(struct blk_mq_tag_set *set,
1332                 struct blk_mq_tags *tags, unsigned int hctx_idx)
1333 {
1334         struct page *page;
1335
1336         if (tags->rqs && set->ops->exit_request) {
1337                 int i;
1338
1339                 for (i = 0; i < tags->nr_tags; i++) {
1340                         if (!tags->rqs[i])
1341                                 continue;
1342                         set->ops->exit_request(set->driver_data, tags->rqs[i],
1343                                                 hctx_idx, i);
1344                 }
1345         }
1346
1347         while (!list_empty(&tags->page_list)) {
1348                 page = list_first_entry(&tags->page_list, struct page, lru);
1349                 list_del_init(&page->lru);
1350                 __free_pages(page, page->private);
1351         }
1352
1353         kfree(tags->rqs);
1354
1355         blk_mq_free_tags(tags);
1356 }
1357
1358 static size_t order_to_size(unsigned int order)
1359 {
1360         return (size_t)PAGE_SIZE << order;
1361 }
1362
1363 static struct blk_mq_tags *blk_mq_init_rq_map(struct blk_mq_tag_set *set,
1364                 unsigned int hctx_idx)
1365 {
1366         struct blk_mq_tags *tags;
1367         unsigned int i, j, entries_per_page, max_order = 4;
1368         size_t rq_size, left;
1369
1370         tags = blk_mq_init_tags(set->queue_depth, set->reserved_tags,
1371                                 set->numa_node);
1372         if (!tags)
1373                 return NULL;
1374
1375         INIT_LIST_HEAD(&tags->page_list);
1376
1377         tags->rqs = kmalloc_node(set->queue_depth * sizeof(struct request *),
1378                                         GFP_KERNEL, set->numa_node);
1379         if (!tags->rqs) {
1380                 blk_mq_free_tags(tags);
1381                 return NULL;
1382         }
1383
1384         /*
1385          * rq_size is the size of the request plus driver payload, rounded
1386          * to the cacheline size
1387          */
1388         rq_size = round_up(sizeof(struct request) + set->cmd_size,
1389                                 cache_line_size());
1390         left = rq_size * set->queue_depth;
1391
1392         for (i = 0; i < set->queue_depth; ) {
1393                 int this_order = max_order;
1394                 struct page *page;
1395                 int to_do;
1396                 void *p;
1397
1398                 while (left < order_to_size(this_order - 1) && this_order)
1399                         this_order--;
1400
1401                 do {
1402                         page = alloc_pages_node(set->numa_node, GFP_KERNEL,
1403                                                 this_order);
1404                         if (page)
1405                                 break;
1406                         if (!this_order--)
1407                                 break;
1408                         if (order_to_size(this_order) < rq_size)
1409                                 break;
1410                 } while (1);
1411
1412                 if (!page)
1413                         goto fail;
1414
1415                 page->private = this_order;
1416                 list_add_tail(&page->lru, &tags->page_list);
1417
1418                 p = page_address(page);
1419                 entries_per_page = order_to_size(this_order) / rq_size;
1420                 to_do = min(entries_per_page, set->queue_depth - i);
1421                 left -= to_do * rq_size;
1422                 for (j = 0; j < to_do; j++) {
1423                         tags->rqs[i] = p;
1424                         if (set->ops->init_request) {
1425                                 if (set->ops->init_request(set->driver_data,
1426                                                 tags->rqs[i], hctx_idx, i,
1427                                                 set->numa_node))
1428                                         goto fail;
1429                         }
1430
1431                         p += rq_size;
1432                         i++;
1433                 }
1434         }
1435
1436         return tags;
1437
1438 fail:
1439         pr_warn("%s: failed to allocate requests\n", __func__);
1440         blk_mq_free_rq_map(set, tags, hctx_idx);
1441         return NULL;
1442 }
1443
1444 static void blk_mq_free_bitmap(struct blk_mq_ctxmap *bitmap)
1445 {
1446         kfree(bitmap->map);
1447 }
1448
1449 static int blk_mq_alloc_bitmap(struct blk_mq_ctxmap *bitmap, int node)
1450 {
1451         unsigned int bpw = 8, total, num_maps, i;
1452
1453         bitmap->bits_per_word = bpw;
1454
1455         num_maps = ALIGN(nr_cpu_ids, bpw) / bpw;
1456         bitmap->map = kzalloc_node(num_maps * sizeof(struct blk_align_bitmap),
1457                                         GFP_KERNEL, node);
1458         if (!bitmap->map)
1459                 return -ENOMEM;
1460
1461         bitmap->map_size = num_maps;
1462
1463         total = nr_cpu_ids;
1464         for (i = 0; i < num_maps; i++) {
1465                 bitmap->map[i].depth = min(total, bitmap->bits_per_word);
1466                 total -= bitmap->map[i].depth;
1467         }
1468
1469         return 0;
1470 }
1471
1472 static int blk_mq_hctx_cpu_offline(struct blk_mq_hw_ctx *hctx, int cpu)
1473 {
1474         struct request_queue *q = hctx->queue;
1475         struct blk_mq_ctx *ctx;
1476         LIST_HEAD(tmp);
1477
1478         /*
1479          * Move ctx entries to new CPU, if this one is going away.
1480          */
1481         ctx = __blk_mq_get_ctx(q, cpu);
1482
1483         spin_lock(&ctx->lock);
1484         if (!list_empty(&ctx->rq_list)) {
1485                 list_splice_init(&ctx->rq_list, &tmp);
1486                 blk_mq_hctx_clear_pending(hctx, ctx);
1487         }
1488         spin_unlock(&ctx->lock);
1489
1490         if (list_empty(&tmp))
1491                 return NOTIFY_OK;
1492
1493         ctx = blk_mq_get_ctx(q);
1494         spin_lock(&ctx->lock);
1495
1496         while (!list_empty(&tmp)) {
1497                 struct request *rq;
1498
1499                 rq = list_first_entry(&tmp, struct request, queuelist);
1500                 rq->mq_ctx = ctx;
1501                 list_move_tail(&rq->queuelist, &ctx->rq_list);
1502         }
1503
1504         hctx = q->mq_ops->map_queue(q, ctx->cpu);
1505         blk_mq_hctx_mark_pending(hctx, ctx);
1506
1507         spin_unlock(&ctx->lock);
1508
1509         blk_mq_run_hw_queue(hctx, true);
1510         blk_mq_put_ctx(ctx);
1511         return NOTIFY_OK;
1512 }
1513
1514 static int blk_mq_hctx_cpu_online(struct blk_mq_hw_ctx *hctx, int cpu)
1515 {
1516         struct request_queue *q = hctx->queue;
1517         struct blk_mq_tag_set *set = q->tag_set;
1518
1519         if (set->tags[hctx->queue_num])
1520                 return NOTIFY_OK;
1521
1522         set->tags[hctx->queue_num] = blk_mq_init_rq_map(set, hctx->queue_num);
1523         if (!set->tags[hctx->queue_num])
1524                 return NOTIFY_STOP;
1525
1526         hctx->tags = set->tags[hctx->queue_num];
1527         return NOTIFY_OK;
1528 }
1529
1530 static int blk_mq_hctx_notify(void *data, unsigned long action,
1531                               unsigned int cpu)
1532 {
1533         struct blk_mq_hw_ctx *hctx = data;
1534
1535         if (action == CPU_DEAD || action == CPU_DEAD_FROZEN)
1536                 return blk_mq_hctx_cpu_offline(hctx, cpu);
1537         else if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN)
1538                 return blk_mq_hctx_cpu_online(hctx, cpu);
1539
1540         return NOTIFY_OK;
1541 }
1542
1543 static void blk_mq_exit_hw_queues(struct request_queue *q,
1544                 struct blk_mq_tag_set *set, int nr_queue)
1545 {
1546         struct blk_mq_hw_ctx *hctx;
1547         unsigned int i;
1548
1549         queue_for_each_hw_ctx(q, hctx, i) {
1550                 if (i == nr_queue)
1551                         break;
1552
1553                 if (set->ops->exit_hctx)
1554                         set->ops->exit_hctx(hctx, i);
1555
1556                 blk_mq_unregister_cpu_notifier(&hctx->cpu_notifier);
1557                 kfree(hctx->ctxs);
1558                 blk_mq_free_bitmap(&hctx->ctx_map);
1559         }
1560
1561 }
1562
1563 static void blk_mq_free_hw_queues(struct request_queue *q,
1564                 struct blk_mq_tag_set *set)
1565 {
1566         struct blk_mq_hw_ctx *hctx;
1567         unsigned int i;
1568
1569         queue_for_each_hw_ctx(q, hctx, i) {
1570                 free_cpumask_var(hctx->cpumask);
1571                 kfree(hctx);
1572         }
1573 }
1574
1575 static int blk_mq_init_hw_queues(struct request_queue *q,
1576                 struct blk_mq_tag_set *set)
1577 {
1578         struct blk_mq_hw_ctx *hctx;
1579         unsigned int i;
1580
1581         /*
1582          * Initialize hardware queues
1583          */
1584         queue_for_each_hw_ctx(q, hctx, i) {
1585                 int node;
1586
1587                 node = hctx->numa_node;
1588                 if (node == NUMA_NO_NODE)
1589                         node = hctx->numa_node = set->numa_node;
1590
1591                 INIT_DELAYED_WORK(&hctx->run_work, blk_mq_run_work_fn);
1592                 INIT_DELAYED_WORK(&hctx->delay_work, blk_mq_delay_work_fn);
1593                 spin_lock_init(&hctx->lock);
1594                 INIT_LIST_HEAD(&hctx->dispatch);
1595                 hctx->queue = q;
1596                 hctx->queue_num = i;
1597                 hctx->flags = set->flags;
1598                 hctx->cmd_size = set->cmd_size;
1599
1600                 blk_mq_init_cpu_notifier(&hctx->cpu_notifier,
1601                                                 blk_mq_hctx_notify, hctx);
1602                 blk_mq_register_cpu_notifier(&hctx->cpu_notifier);
1603
1604                 hctx->tags = set->tags[i];
1605
1606                 /*
1607                  * Allocate space for all possible cpus to avoid allocation in
1608                  * runtime
1609                  */
1610                 hctx->ctxs = kmalloc_node(nr_cpu_ids * sizeof(void *),
1611                                                 GFP_KERNEL, node);
1612                 if (!hctx->ctxs)
1613                         break;
1614
1615                 if (blk_mq_alloc_bitmap(&hctx->ctx_map, node))
1616                         break;
1617
1618                 hctx->nr_ctx = 0;
1619
1620                 if (set->ops->init_hctx &&
1621                     set->ops->init_hctx(hctx, set->driver_data, i))
1622                         break;
1623         }
1624
1625         if (i == q->nr_hw_queues)
1626                 return 0;
1627
1628         /*
1629          * Init failed
1630          */
1631         blk_mq_exit_hw_queues(q, set, i);
1632
1633         return 1;
1634 }
1635
1636 static void blk_mq_init_cpu_queues(struct request_queue *q,
1637                                    unsigned int nr_hw_queues)
1638 {
1639         unsigned int i;
1640
1641         for_each_possible_cpu(i) {
1642                 struct blk_mq_ctx *__ctx = per_cpu_ptr(q->queue_ctx, i);
1643                 struct blk_mq_hw_ctx *hctx;
1644
1645                 memset(__ctx, 0, sizeof(*__ctx));
1646                 __ctx->cpu = i;
1647                 spin_lock_init(&__ctx->lock);
1648                 INIT_LIST_HEAD(&__ctx->rq_list);
1649                 __ctx->queue = q;
1650
1651                 /* If the cpu isn't online, the cpu is mapped to first hctx */
1652                 if (!cpu_online(i))
1653                         continue;
1654
1655                 hctx = q->mq_ops->map_queue(q, i);
1656                 cpumask_set_cpu(i, hctx->cpumask);
1657                 hctx->nr_ctx++;
1658
1659                 /*
1660                  * Set local node, IFF we have more than one hw queue. If
1661                  * not, we remain on the home node of the device
1662                  */
1663                 if (nr_hw_queues > 1 && hctx->numa_node == NUMA_NO_NODE)
1664                         hctx->numa_node = cpu_to_node(i);
1665         }
1666 }
1667
1668 static void blk_mq_map_swqueue(struct request_queue *q)
1669 {
1670         unsigned int i;
1671         struct blk_mq_hw_ctx *hctx;
1672         struct blk_mq_ctx *ctx;
1673
1674         queue_for_each_hw_ctx(q, hctx, i) {
1675                 cpumask_clear(hctx->cpumask);
1676                 hctx->nr_ctx = 0;
1677         }
1678
1679         /*
1680          * Map software to hardware queues
1681          */
1682         queue_for_each_ctx(q, ctx, i) {
1683                 /* If the cpu isn't online, the cpu is mapped to first hctx */
1684                 if (!cpu_online(i))
1685                         continue;
1686
1687                 hctx = q->mq_ops->map_queue(q, i);
1688                 cpumask_set_cpu(i, hctx->cpumask);
1689                 ctx->index_hw = hctx->nr_ctx;
1690                 hctx->ctxs[hctx->nr_ctx++] = ctx;
1691         }
1692
1693         queue_for_each_hw_ctx(q, hctx, i) {
1694                 /*
1695                  * If not software queues are mapped to this hardware queue,
1696                  * disable it and free the request entries
1697                  */
1698                 if (!hctx->nr_ctx) {
1699                         struct blk_mq_tag_set *set = q->tag_set;
1700
1701                         if (set->tags[i]) {
1702                                 blk_mq_free_rq_map(set, set->tags[i], i);
1703                                 set->tags[i] = NULL;
1704                                 hctx->tags = NULL;
1705                         }
1706                         continue;
1707                 }
1708
1709                 /*
1710                  * Initialize batch roundrobin counts
1711                  */
1712                 hctx->next_cpu = cpumask_first(hctx->cpumask);
1713                 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
1714         }
1715 }
1716
1717 static void blk_mq_update_tag_set_depth(struct blk_mq_tag_set *set)
1718 {
1719         struct blk_mq_hw_ctx *hctx;
1720         struct request_queue *q;
1721         bool shared;
1722         int i;
1723
1724         if (set->tag_list.next == set->tag_list.prev)
1725                 shared = false;
1726         else
1727                 shared = true;
1728
1729         list_for_each_entry(q, &set->tag_list, tag_set_list) {
1730                 blk_mq_freeze_queue(q);
1731
1732                 queue_for_each_hw_ctx(q, hctx, i) {
1733                         if (shared)
1734                                 hctx->flags |= BLK_MQ_F_TAG_SHARED;
1735                         else
1736                                 hctx->flags &= ~BLK_MQ_F_TAG_SHARED;
1737                 }
1738                 blk_mq_unfreeze_queue(q);
1739         }
1740 }
1741
1742 static void blk_mq_del_queue_tag_set(struct request_queue *q)
1743 {
1744         struct blk_mq_tag_set *set = q->tag_set;
1745
1746         blk_mq_freeze_queue(q);
1747
1748         mutex_lock(&set->tag_list_lock);
1749         list_del_init(&q->tag_set_list);
1750         blk_mq_update_tag_set_depth(set);
1751         mutex_unlock(&set->tag_list_lock);
1752
1753         blk_mq_unfreeze_queue(q);
1754 }
1755
1756 static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,
1757                                      struct request_queue *q)
1758 {
1759         q->tag_set = set;
1760
1761         mutex_lock(&set->tag_list_lock);
1762         list_add_tail(&q->tag_set_list, &set->tag_list);
1763         blk_mq_update_tag_set_depth(set);
1764         mutex_unlock(&set->tag_list_lock);
1765 }
1766
1767 struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set)
1768 {
1769         struct blk_mq_hw_ctx **hctxs;
1770         struct blk_mq_ctx *ctx;
1771         struct request_queue *q;
1772         unsigned int *map;
1773         int i;
1774
1775         ctx = alloc_percpu(struct blk_mq_ctx);
1776         if (!ctx)
1777                 return ERR_PTR(-ENOMEM);
1778
1779         hctxs = kmalloc_node(set->nr_hw_queues * sizeof(*hctxs), GFP_KERNEL,
1780                         set->numa_node);
1781
1782         if (!hctxs)
1783                 goto err_percpu;
1784
1785         map = blk_mq_make_queue_map(set);
1786         if (!map)
1787                 goto err_map;
1788
1789         for (i = 0; i < set->nr_hw_queues; i++) {
1790                 int node = blk_mq_hw_queue_to_node(map, i);
1791
1792                 hctxs[i] = kzalloc_node(sizeof(struct blk_mq_hw_ctx),
1793                                         GFP_KERNEL, node);
1794                 if (!hctxs[i])
1795                         goto err_hctxs;
1796
1797                 if (!zalloc_cpumask_var(&hctxs[i]->cpumask, GFP_KERNEL))
1798                         goto err_hctxs;
1799
1800                 atomic_set(&hctxs[i]->nr_active, 0);
1801                 hctxs[i]->numa_node = node;
1802                 hctxs[i]->queue_num = i;
1803         }
1804
1805         q = blk_alloc_queue_node(GFP_KERNEL, set->numa_node);
1806         if (!q)
1807                 goto err_hctxs;
1808
1809         if (percpu_counter_init(&q->mq_usage_counter, 0))
1810                 goto err_map;
1811
1812         setup_timer(&q->timeout, blk_mq_rq_timer, (unsigned long) q);
1813         blk_queue_rq_timeout(q, 30000);
1814
1815         q->nr_queues = nr_cpu_ids;
1816         q->nr_hw_queues = set->nr_hw_queues;
1817         q->mq_map = map;
1818
1819         q->queue_ctx = ctx;
1820         q->queue_hw_ctx = hctxs;
1821
1822         q->mq_ops = set->ops;
1823         q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT;
1824
1825         if (!(set->flags & BLK_MQ_F_SG_MERGE))
1826                 q->queue_flags |= 1 << QUEUE_FLAG_NO_SG_MERGE;
1827
1828         q->sg_reserved_size = INT_MAX;
1829
1830         INIT_WORK(&q->requeue_work, blk_mq_requeue_work);
1831         INIT_LIST_HEAD(&q->requeue_list);
1832         spin_lock_init(&q->requeue_lock);
1833
1834         if (q->nr_hw_queues > 1)
1835                 blk_queue_make_request(q, blk_mq_make_request);
1836         else
1837                 blk_queue_make_request(q, blk_sq_make_request);
1838
1839         blk_queue_rq_timed_out(q, blk_mq_rq_timed_out);
1840         if (set->timeout)
1841                 blk_queue_rq_timeout(q, set->timeout);
1842
1843         /*
1844          * Do this after blk_queue_make_request() overrides it...
1845          */
1846         q->nr_requests = set->queue_depth;
1847
1848         if (set->ops->complete)
1849                 blk_queue_softirq_done(q, set->ops->complete);
1850
1851         blk_mq_init_flush(q);
1852         blk_mq_init_cpu_queues(q, set->nr_hw_queues);
1853
1854         q->flush_rq = kzalloc(round_up(sizeof(struct request) +
1855                                 set->cmd_size, cache_line_size()),
1856                                 GFP_KERNEL);
1857         if (!q->flush_rq)
1858                 goto err_hw;
1859
1860         if (blk_mq_init_hw_queues(q, set))
1861                 goto err_flush_rq;
1862
1863         mutex_lock(&all_q_mutex);
1864         list_add_tail(&q->all_q_node, &all_q_list);
1865         mutex_unlock(&all_q_mutex);
1866
1867         blk_mq_add_queue_tag_set(set, q);
1868
1869         blk_mq_map_swqueue(q);
1870
1871         return q;
1872
1873 err_flush_rq:
1874         kfree(q->flush_rq);
1875 err_hw:
1876         blk_cleanup_queue(q);
1877 err_hctxs:
1878         kfree(map);
1879         for (i = 0; i < set->nr_hw_queues; i++) {
1880                 if (!hctxs[i])
1881                         break;
1882                 free_cpumask_var(hctxs[i]->cpumask);
1883                 kfree(hctxs[i]);
1884         }
1885 err_map:
1886         kfree(hctxs);
1887 err_percpu:
1888         free_percpu(ctx);
1889         return ERR_PTR(-ENOMEM);
1890 }
1891 EXPORT_SYMBOL(blk_mq_init_queue);
1892
1893 void blk_mq_free_queue(struct request_queue *q)
1894 {
1895         struct blk_mq_tag_set   *set = q->tag_set;
1896
1897         blk_mq_del_queue_tag_set(q);
1898
1899         blk_mq_exit_hw_queues(q, set, set->nr_hw_queues);
1900         blk_mq_free_hw_queues(q, set);
1901
1902         percpu_counter_destroy(&q->mq_usage_counter);
1903
1904         free_percpu(q->queue_ctx);
1905         kfree(q->queue_hw_ctx);
1906         kfree(q->mq_map);
1907
1908         q->queue_ctx = NULL;
1909         q->queue_hw_ctx = NULL;
1910         q->mq_map = NULL;
1911
1912         mutex_lock(&all_q_mutex);
1913         list_del_init(&q->all_q_node);
1914         mutex_unlock(&all_q_mutex);
1915 }
1916
1917 /* Basically redo blk_mq_init_queue with queue frozen */
1918 static void blk_mq_queue_reinit(struct request_queue *q)
1919 {
1920         blk_mq_freeze_queue(q);
1921
1922         blk_mq_sysfs_unregister(q);
1923
1924         blk_mq_update_queue_map(q->mq_map, q->nr_hw_queues);
1925
1926         /*
1927          * redo blk_mq_init_cpu_queues and blk_mq_init_hw_queues. FIXME: maybe
1928          * we should change hctx numa_node according to new topology (this
1929          * involves free and re-allocate memory, worthy doing?)
1930          */
1931
1932         blk_mq_map_swqueue(q);
1933
1934         blk_mq_sysfs_register(q);
1935
1936         blk_mq_unfreeze_queue(q);
1937 }
1938
1939 static int blk_mq_queue_reinit_notify(struct notifier_block *nb,
1940                                       unsigned long action, void *hcpu)
1941 {
1942         struct request_queue *q;
1943
1944         /*
1945          * Before new mappings are established, hotadded cpu might already
1946          * start handling requests. This doesn't break anything as we map
1947          * offline CPUs to first hardware queue. We will re-init the queue
1948          * below to get optimal settings.
1949          */
1950         if (action != CPU_DEAD && action != CPU_DEAD_FROZEN &&
1951             action != CPU_ONLINE && action != CPU_ONLINE_FROZEN)
1952                 return NOTIFY_OK;
1953
1954         mutex_lock(&all_q_mutex);
1955         list_for_each_entry(q, &all_q_list, all_q_node)
1956                 blk_mq_queue_reinit(q);
1957         mutex_unlock(&all_q_mutex);
1958         return NOTIFY_OK;
1959 }
1960
1961 int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set)
1962 {
1963         int i;
1964
1965         if (!set->nr_hw_queues)
1966                 return -EINVAL;
1967         if (!set->queue_depth || set->queue_depth > BLK_MQ_MAX_DEPTH)
1968                 return -EINVAL;
1969         if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN)
1970                 return -EINVAL;
1971
1972         if (!set->nr_hw_queues || !set->ops->queue_rq || !set->ops->map_queue)
1973                 return -EINVAL;
1974
1975
1976         set->tags = kmalloc_node(set->nr_hw_queues *
1977                                  sizeof(struct blk_mq_tags *),
1978                                  GFP_KERNEL, set->numa_node);
1979         if (!set->tags)
1980                 goto out;
1981
1982         for (i = 0; i < set->nr_hw_queues; i++) {
1983                 set->tags[i] = blk_mq_init_rq_map(set, i);
1984                 if (!set->tags[i])
1985                         goto out_unwind;
1986         }
1987
1988         mutex_init(&set->tag_list_lock);
1989         INIT_LIST_HEAD(&set->tag_list);
1990
1991         return 0;
1992
1993 out_unwind:
1994         while (--i >= 0)
1995                 blk_mq_free_rq_map(set, set->tags[i], i);
1996 out:
1997         return -ENOMEM;
1998 }
1999 EXPORT_SYMBOL(blk_mq_alloc_tag_set);
2000
2001 void blk_mq_free_tag_set(struct blk_mq_tag_set *set)
2002 {
2003         int i;
2004
2005         for (i = 0; i < set->nr_hw_queues; i++) {
2006                 if (set->tags[i])
2007                         blk_mq_free_rq_map(set, set->tags[i], i);
2008         }
2009
2010         kfree(set->tags);
2011 }
2012 EXPORT_SYMBOL(blk_mq_free_tag_set);
2013
2014 int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr)
2015 {
2016         struct blk_mq_tag_set *set = q->tag_set;
2017         struct blk_mq_hw_ctx *hctx;
2018         int i, ret;
2019
2020         if (!set || nr > set->queue_depth)
2021                 return -EINVAL;
2022
2023         ret = 0;
2024         queue_for_each_hw_ctx(q, hctx, i) {
2025                 ret = blk_mq_tag_update_depth(hctx->tags, nr);
2026                 if (ret)
2027                         break;
2028         }
2029
2030         if (!ret)
2031                 q->nr_requests = nr;
2032
2033         return ret;
2034 }
2035
2036 void blk_mq_disable_hotplug(void)
2037 {
2038         mutex_lock(&all_q_mutex);
2039 }
2040
2041 void blk_mq_enable_hotplug(void)
2042 {
2043         mutex_unlock(&all_q_mutex);
2044 }
2045
2046 static int __init blk_mq_init(void)
2047 {
2048         blk_mq_cpu_init();
2049
2050         /* Must be called after percpu_counter_hotcpu_callback() */
2051         hotcpu_notifier(blk_mq_queue_reinit_notify, -10);
2052
2053         return 0;
2054 }
2055 subsys_initcall(blk_mq_init);