sched/wait: Avoid abort_exclusive_wait() in ___wait_event()

[cascardo/linux.git] / kernel / sched / wait.c

diff --git a/kernel/sched/wait.c b/kernel/sched/wait.c
index f15d6b6..2612393 100644 (file)
--- a/kernel/sched/wait.c
+++ b/kernel/sched/wait.c
@@ -199,24 +199,39 @@ EXPORT_SYMBOL(prepare_to_wait_exclusive);
 long prepare_to_wait_event(wait_queue_head_t *q, wait_queue_t *wait, int state)
 {
        unsigned long flags;
-
-       if (signal_pending_state(state, current))
-               return -ERESTARTSYS;
+       long ret = 0;
 
        wait->private = current;
        wait->func = autoremove_wake_function;
 
        spin_lock_irqsave(&q->lock, flags);
-       if (list_empty(&wait->task_list)) {
-               if (wait->flags & WQ_FLAG_EXCLUSIVE)
-                       __add_wait_queue_tail(q, wait);
-               else
-                       __add_wait_queue(q, wait);
+       if (unlikely(signal_pending_state(state, current))) {
+               /*
+                * Exclusive waiter must not fail if it was selected by wakeup,
+                * it should "consume" the condition we were waiting for.
+                *
+                * The caller will recheck the condition and return success if
+                * we were already woken up, we can not miss the event because
+                * wakeup locks/unlocks the same q->lock.
+                *
+                * But we need to ensure that set-condition + wakeup after that
+                * can't see us, it should wake up another exclusive waiter if
+                * we fail.
+                */
+               list_del_init(&wait->task_list);
+               ret = -ERESTARTSYS;
+       } else {
+               if (list_empty(&wait->task_list)) {
+                       if (wait->flags & WQ_FLAG_EXCLUSIVE)
+                               __add_wait_queue_tail(q, wait);
+                       else
+                               __add_wait_queue(q, wait);
+               }
+               set_current_state(state);
        }
-       set_current_state(state);
        spin_unlock_irqrestore(&q->lock, flags);
 
-       return 0;
+       return ret;
 }
 EXPORT_SYMBOL(prepare_to_wait_event);
 
@@ -259,7 +274,6 @@ EXPORT_SYMBOL(finish_wait);
  * abort_exclusive_wait - abort exclusive waiting in a queue
  * @q: waitqueue waited on
  * @wait: wait descriptor
- * @mode: runstate of the waiter to be woken
  * @key: key to identify a wait bit queue or %NULL
  *
  * Sets current thread back to running state and removes
@@ -273,8 +287,7 @@ EXPORT_SYMBOL(finish_wait);
  * aborts and is woken up concurrently and no one wakes up
  * the next waiter.
  */
-void abort_exclusive_wait(wait_queue_head_t *q, wait_queue_t *wait,
-                       unsigned int mode, void *key)
+void abort_exclusive_wait(wait_queue_head_t *q, wait_queue_t *wait, void *key)
 {
        unsigned long flags;
 
@@ -283,7 +296,7 @@ void abort_exclusive_wait(wait_queue_head_t *q, wait_queue_t *wait,
        if (!list_empty(&wait->task_list))
                list_del_init(&wait->task_list);
        else if (waitqueue_active(q))
-               __wake_up_locked_key(q, mode, key);
+               __wake_up_locked_key(q, TASK_NORMAL, key);
        spin_unlock_irqrestore(&q->lock, flags);
 }
 EXPORT_SYMBOL(abort_exclusive_wait);
@@ -434,7 +447,7 @@ __wait_on_bit_lock(wait_queue_head_t *wq, struct wait_bit_queue *q,
                ret = action(&q->key, mode);
                if (!ret)
                        continue;
-               abort_exclusive_wait(wq, &q->wait, mode, &q->key);
+               abort_exclusive_wait(wq, &q->wait, &q->key);
                return ret;
        } while (test_and_set_bit(q->key.bit_nr, q->key.flags));
        finish_wait(wq, &q->wait);