Merge tag 'iwlwifi-next-for-kalle-2014-12-30' of https://git.kernel.org/pub/scm/linux...

[cascardo/linux.git] / block / blk-mq-tag.c

diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c
index 728b9a4..32e8dbb 100644 (file)
--- a/block/blk-mq-tag.c
+++ b/block/blk-mq-tag.c
@@ -137,6 +137,7 @@ static inline bool hctx_may_queue(struct blk_mq_hw_ctx *hctx,
 static int __bt_get_word(struct blk_align_bitmap *bm, unsigned int last_tag)
 {
        int tag, org_last_tag, end;
+       bool wrap = last_tag != 0;
 
        org_last_tag = last_tag;
        end = bm->depth;
@@ -148,15 +149,16 @@ restart:
                         * We started with an offset, start from 0 to
                         * exhaust the map.
                         */
-                       if (org_last_tag && last_tag) {
-                               end = last_tag;
+                       if (wrap) {
+                               wrap = false;
+                               end = org_last_tag;
                                last_tag = 0;
                                goto restart;
                        }
                        return -1;
                }
                last_tag = tag + 1;
-       } while (test_and_set_bit_lock(tag, &bm->word));
+       } while (test_and_set_bit(tag, &bm->word));
 
        return tag;
 }
@@ -254,6 +256,21 @@ static int bt_get(struct blk_mq_alloc_data *data,
                if (tag != -1)
                        break;
 
+               /*
+                * We're out of tags on this hardware queue, kick any
+                * pending IO submits before going to sleep waiting for
+                * some to complete.
+                */
+               blk_mq_run_hw_queue(hctx, false);
+
+               /*
+                * Retry tag allocation after running the hardware queue,
+                * as running the queue may also have found completions.
+                */
+               tag = __bt_get(hctx, bt, last_tag);
+               if (tag != -1)
+                       break;
+
                blk_mq_put_ctx(data->ctx);
 
                io_schedule();
@@ -340,11 +357,10 @@ static void bt_clear_tag(struct blk_mq_bitmap_tags *bt, unsigned int tag)
        struct bt_wait_state *bs;
        int wait_cnt;
 
-       /*
-        * The unlock memory barrier need to order access to req in free
-        * path and clearing tag bit
-        */
-       clear_bit_unlock(TAG_TO_BIT(bt, tag), &bt->map[index].word);
+       clear_bit(TAG_TO_BIT(bt, tag), &bt->map[index].word);
+
+       /* Ensure that the wait list checks occur after clear_bit(). */
+       smp_mb();
 
        bs = bt_wake_ptr(bt);
        if (!bs)
@@ -360,21 +376,6 @@ static void bt_clear_tag(struct blk_mq_bitmap_tags *bt, unsigned int tag)
        }
 }
 
-static void __blk_mq_put_tag(struct blk_mq_tags *tags, unsigned int tag)
-{
-       BUG_ON(tag >= tags->nr_tags);
-
-       bt_clear_tag(&tags->bitmap_tags, tag);
-}
-
-static void __blk_mq_put_reserved_tag(struct blk_mq_tags *tags,
-                                     unsigned int tag)
-{
-       BUG_ON(tag >= tags->nr_reserved_tags);
-
-       bt_clear_tag(&tags->breserved_tags, tag);
-}
-
 void blk_mq_put_tag(struct blk_mq_hw_ctx *hctx, unsigned int tag,
                    unsigned int *last_tag)
 {
@@ -383,10 +384,13 @@ void blk_mq_put_tag(struct blk_mq_hw_ctx *hctx, unsigned int tag,
        if (tag >= tags->nr_reserved_tags) {
                const int real_tag = tag - tags->nr_reserved_tags;
 
-               __blk_mq_put_tag(tags, real_tag);
+               BUG_ON(real_tag >= tags->nr_tags);
+               bt_clear_tag(&tags->bitmap_tags, real_tag);
                *last_tag = real_tag;
-       } else
-               __blk_mq_put_reserved_tag(tags, tag);
+       } else {
+               BUG_ON(tag >= tags->nr_reserved_tags);
+               bt_clear_tag(&tags->breserved_tags, tag);
+       }
 }
 
 static void bt_for_each(struct blk_mq_hw_ctx *hctx,