Merge tag 'gpio-v3.19-2' of git://git.kernel.org/pub/scm/linux/kernel/git/linusw...

[cascardo/linux.git] / block / blk-mq-tag.c

diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c
index eb55492..e3d4e40 100644 (file)
--- a/block/blk-mq-tag.c
+++ b/block/blk-mq-tag.c
@@ -137,6 +137,7 @@ static inline bool hctx_may_queue(struct blk_mq_hw_ctx *hctx,
 static int __bt_get_word(struct blk_align_bitmap *bm, unsigned int last_tag)
 {
        int tag, org_last_tag, end;
+       bool wrap = last_tag != 0;
 
        org_last_tag = last_tag;
        end = bm->depth;
@@ -148,15 +149,16 @@ restart:
                         * We started with an offset, start from 0 to
                         * exhaust the map.
                         */
-                       if (org_last_tag && last_tag) {
-                               end = last_tag;
+                       if (wrap) {
+                               wrap = false;
+                               end = org_last_tag;
                                last_tag = 0;
                                goto restart;
                        }
                        return -1;
                }
                last_tag = tag + 1;
-       } while (test_and_set_bit_lock(tag, &bm->word));
+       } while (test_and_set_bit(tag, &bm->word));
 
        return tag;
 }
@@ -246,8 +248,8 @@ static int bt_get(struct blk_mq_alloc_data *data,
        if (!(data->gfp & __GFP_WAIT))
                return -1;
 
-       bs = bt_wait_ptr(bt, hctx);
        do {
+               bs = bt_wait_ptr(bt, hctx);
                prepare_to_wait(&bs->wait, &wait, TASK_UNINTERRUPTIBLE);
 
                tag = __bt_get(hctx, bt, last_tag);
@@ -261,6 +263,14 @@ static int bt_get(struct blk_mq_alloc_data *data,
                 */
                blk_mq_run_hw_queue(hctx, false);
 
+               /*
+                * Retry tag allocation after running the hardware queue,
+                * as running the queue may also have found completions.
+                */
+               tag = __bt_get(hctx, bt, last_tag);
+               if (tag != -1)
+                       break;
+
                blk_mq_put_ctx(data->ctx);
 
                io_schedule();
@@ -275,8 +285,6 @@ static int bt_get(struct blk_mq_alloc_data *data,
                        hctx = data->hctx;
                        bt = &hctx->tags->bitmap_tags;
                }
-               finish_wait(&bs->wait, &wait);
-               bs = bt_wait_ptr(bt, hctx);
        } while (1);
 
        finish_wait(&bs->wait, &wait);
@@ -347,11 +355,10 @@ static void bt_clear_tag(struct blk_mq_bitmap_tags *bt, unsigned int tag)
        struct bt_wait_state *bs;
        int wait_cnt;
 
-       /*
-        * The unlock memory barrier need to order access to req in free
-        * path and clearing tag bit
-        */
-       clear_bit_unlock(TAG_TO_BIT(bt, tag), &bt->map[index].word);
+       clear_bit(TAG_TO_BIT(bt, tag), &bt->map[index].word);
+
+       /* Ensure that the wait list checks occur after clear_bit(). */
+       smp_mb();
 
        bs = bt_wake_ptr(bt);
        if (!bs)
@@ -579,6 +586,34 @@ int blk_mq_tag_update_depth(struct blk_mq_tags *tags, unsigned int tdepth)
        return 0;
 }
 
+/**
+ * blk_mq_unique_tag() - return a tag that is unique queue-wide
+ * @rq: request for which to compute a unique tag
+ *
+ * The tag field in struct request is unique per hardware queue but not over
+ * all hardware queues. Hence this function that returns a tag with the
+ * hardware context index in the upper bits and the per hardware queue tag in
+ * the lower bits.
+ *
+ * Note: When called for a request that is queued on a non-multiqueue request
+ * queue, the hardware context index is set to zero.
+ */
+u32 blk_mq_unique_tag(struct request *rq)
+{
+       struct request_queue *q = rq->q;
+       struct blk_mq_hw_ctx *hctx;
+       int hwq = 0;
+
+       if (q->mq_ops) {
+               hctx = q->mq_ops->map_queue(q, rq->mq_ctx->cpu);
+               hwq = hctx->queue_num;
+       }
+
+       return (hwq << BLK_MQ_UNIQUE_TAG_BITS) |
+               (rq->tag & BLK_MQ_UNIQUE_TAG_MASK);
+}
+EXPORT_SYMBOL(blk_mq_unique_tag);
+
 ssize_t blk_mq_tag_sysfs_show(struct blk_mq_tags *tags, char *page)
 {
        char *orig_page = page;