Merge tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/borntraeger...

[cascardo/linux.git] / arch / x86 / include / asm / spinlock.h

diff --git a/arch/x86/include/asm/spinlock.h b/arch/x86/include/asm/spinlock.h
index 9295016..625660f 100644 (file)
--- a/arch/x86/include/asm/spinlock.h
+++ b/arch/x86/include/asm/spinlock.h
@@ -92,7 +92,7 @@ static __always_inline void arch_spin_lock(arch_spinlock_t *lock)
                unsigned count = SPIN_THRESHOLD;
 
                do {
-                       if (ACCESS_ONCE(lock->tickets.head) == inc.tail)
+                       if (READ_ONCE(lock->tickets.head) == inc.tail)
                                goto out;
                        cpu_relax();
                } while (--count);
@@ -105,7 +105,7 @@ static __always_inline int arch_spin_trylock(arch_spinlock_t *lock)
 {
        arch_spinlock_t old, new;
 
-       old.tickets = ACCESS_ONCE(lock->tickets);
+       old.tickets = READ_ONCE(lock->tickets);
        if (old.tickets.head != (old.tickets.tail & ~TICKET_SLOWPATH_FLAG))
                return 0;
 
@@ -162,14 +162,14 @@ static __always_inline void arch_spin_unlock(arch_spinlock_t *lock)
 
 static inline int arch_spin_is_locked(arch_spinlock_t *lock)
 {
-       struct __raw_tickets tmp = ACCESS_ONCE(lock->tickets);
+       struct __raw_tickets tmp = READ_ONCE(lock->tickets);
 
        return tmp.tail != tmp.head;
 }
 
 static inline int arch_spin_is_contended(arch_spinlock_t *lock)
 {
-       struct __raw_tickets tmp = ACCESS_ONCE(lock->tickets);
+       struct __raw_tickets tmp = READ_ONCE(lock->tickets);
 
        return (__ticket_t)(tmp.tail - tmp.head) > TICKET_LOCK_INC;
 }
@@ -183,8 +183,20 @@ static __always_inline void arch_spin_lock_flags(arch_spinlock_t *lock,
 
 static inline void arch_spin_unlock_wait(arch_spinlock_t *lock)
 {
-       while (arch_spin_is_locked(lock))
+       __ticket_t head = ACCESS_ONCE(lock->tickets.head);
+
+       for (;;) {
+               struct __raw_tickets tmp = ACCESS_ONCE(lock->tickets);
+               /*
+                * We need to check "unlocked" in a loop, tmp.head == head
+                * can be false positive because of overflow.
+                */
+               if (tmp.head == (tmp.tail & ~TICKET_SLOWPATH_FLAG) ||
+                   tmp.head != head)
+                       break;
+
                cpu_relax();
+       }
 }
 
 /*