s390/rwlock: use directed yield for write-locked rwlocks

Add an owner field to the arch_rwlock_t to be able to pass the timeslice
of a virtual CPU with diagnose 0x9c to the lock owner in case the rwlock
is write-locked. The undirected yield in case the rwlock is acquired
writable but the lock is read-locked is removed.

Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>

diff --git a/arch/s390/include/asm/smp.h b/arch/s390/include/asm/smp.h
index 4f13079..762d4f8 100644 (file)
--- a/arch/s390/include/asm/smp.h
+++ b/arch/s390/include/asm/smp.h
@@ -29,7 +29,6 @@ extern int smp_find_processor_id(u16 address);
 extern int smp_store_status(int cpu);
 extern int smp_vcpu_scheduled(int cpu);
 extern void smp_yield_cpu(int cpu);
-extern void smp_yield(void);
 extern void smp_cpu_set_polarization(int cpu, int val);
 extern int smp_cpu_get_polarization(int cpu);
 extern void smp_fill_possible_mask(void);
@@ -50,7 +49,6 @@ static inline int smp_find_processor_id(u16 address) { return 0; }
 static inline int smp_store_status(int cpu) { return 0; }
 static inline int smp_vcpu_scheduled(int cpu) { return 1; }
 static inline void smp_yield_cpu(int cpu) { }
-static inline void smp_yield(void) { }
 static inline void smp_fill_possible_mask(void) { }
 
 #endif /* CONFIG_SMP */

diff --git a/arch/s390/include/asm/spinlock.h b/arch/s390/include/asm/spinlock.h
index d26ad2a..e986541 100644 (file)
--- a/arch/s390/include/asm/spinlock.h
+++ b/arch/s390/include/asm/spinlock.h
@@ -37,11 +37,17 @@ _raw_compare_and_swap(unsigned int *lock, unsigned int old, unsigned int new)
  * (the type definitions are in asm/spinlock_types.h)
  */
 
+void arch_lock_relax(unsigned int cpu);
+
 void arch_spin_lock_wait(arch_spinlock_t *);
 int arch_spin_trylock_retry(arch_spinlock_t *);
-void arch_spin_relax(arch_spinlock_t *);
 void arch_spin_lock_wait_flags(arch_spinlock_t *, unsigned long flags);
 
+static inline void arch_spin_relax(arch_spinlock_t *lock)
+{
+       arch_lock_relax(lock->lock);
+}
+
 static inline u32 arch_spin_lockval(int cpu)
 {
        return ~cpu;
@@ -170,17 +176,21 @@ static inline void arch_write_lock(arch_rwlock_t *rw)
 {
        if (!arch_write_trylock_once(rw))
                _raw_write_lock_wait(rw);
+       rw->owner = SPINLOCK_LOCKVAL;
 }
 
 static inline void arch_write_lock_flags(arch_rwlock_t *rw, unsigned long flags)
 {
        if (!arch_write_trylock_once(rw))
                _raw_write_lock_wait_flags(rw, flags);
+       rw->owner = SPINLOCK_LOCKVAL;
 }
 
 static inline void arch_write_unlock(arch_rwlock_t *rw)
 {
        typecheck(unsigned int, rw->lock);
+
+       rw->owner = 0;
        asm volatile(
                __ASM_BARRIER
                "st     %1,%0\n"
@@ -198,12 +208,20 @@ static inline int arch_read_trylock(arch_rwlock_t *rw)
 
 static inline int arch_write_trylock(arch_rwlock_t *rw)
 {
-       if (!arch_write_trylock_once(rw))
-               return _raw_write_trylock_retry(rw);
+       if (!arch_write_trylock_once(rw) && !_raw_write_trylock_retry(rw))
+               return 0;
+       rw->owner = SPINLOCK_LOCKVAL;
        return 1;
 }
 
-#define arch_read_relax(lock)  cpu_relax()
-#define arch_write_relax(lock) cpu_relax()
+static inline void arch_read_relax(arch_rwlock_t *rw)
+{
+       arch_lock_relax(rw->owner);
+}
+
+static inline void arch_write_relax(arch_rwlock_t *rw)
+{
+       arch_lock_relax(rw->owner);
+}
 
 #endif /* __ASM_SPINLOCK_H */

diff --git a/arch/s390/include/asm/spinlock_types.h b/arch/s390/include/asm/spinlock_types.h
index b2cd6ff..d84b693 100644 (file)
--- a/arch/s390/include/asm/spinlock_types.h
+++ b/arch/s390/include/asm/spinlock_types.h
@@ -13,6 +13,7 @@ typedef struct {
 
 typedef struct {
        unsigned int lock;
+       unsigned int owner;
 } arch_rwlock_t;
 
 #define __ARCH_RW_LOCK_UNLOCKED                { 0 }

diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c
index 243c7e5..abec97b 100644 (file)
--- a/arch/s390/kernel/smp.c
+++ b/arch/s390/kernel/smp.c
@@ -333,12 +333,6 @@ int smp_vcpu_scheduled(int cpu)
        return pcpu_running(pcpu_devices + cpu);
 }
 
-void smp_yield(void)
-{
-       if (MACHINE_HAS_DIAG44)
-               asm volatile("diag 0,0,0x44");
-}
-
 void smp_yield_cpu(int cpu)
 {
        if (MACHINE_HAS_DIAG9C)

diff --git a/arch/s390/lib/spinlock.c b/arch/s390/lib/spinlock.c
index 5b0e445..5f63ac5 100644 (file)
--- a/arch/s390/lib/spinlock.c
+++ b/arch/s390/lib/spinlock.c
@@ -98,17 +98,6 @@ void arch_spin_lock_wait_flags(arch_spinlock_t *lp, unsigned long flags)
 }
 EXPORT_SYMBOL(arch_spin_lock_wait_flags);
 
-void arch_spin_relax(arch_spinlock_t *lp)
-{
-       unsigned int cpu = lp->lock;
-       if (cpu != 0) {
-               if (MACHINE_IS_VM || MACHINE_IS_KVM ||
-                   !smp_vcpu_scheduled(~cpu))
-                       smp_yield_cpu(~cpu);
-       }
-}
-EXPORT_SYMBOL(arch_spin_relax);
-
 int arch_spin_trylock_retry(arch_spinlock_t *lp)
 {
        int count;
@@ -122,15 +111,18 @@ EXPORT_SYMBOL(arch_spin_trylock_retry);
 
 void _raw_read_lock_wait(arch_rwlock_t *rw)
 {
-       unsigned int old;
+       unsigned int owner, old;
        int count = spin_retry;
 
+       owner = 0;
        while (1) {
                if (count-- <= 0) {
-                       smp_yield();
+                       if (owner && !smp_vcpu_scheduled(~owner))
+                               smp_yield_cpu(~owner);
                        count = spin_retry;
                }
                old = ACCESS_ONCE(rw->lock);
+               owner = ACCESS_ONCE(rw->owner);
                if ((int) old < 0)
                        continue;
                if (_raw_compare_and_swap(&rw->lock, old, old + 1))
@@ -141,16 +133,19 @@ EXPORT_SYMBOL(_raw_read_lock_wait);
 
 void _raw_read_lock_wait_flags(arch_rwlock_t *rw, unsigned long flags)
 {
-       unsigned int old;
+       unsigned int owner, old;
        int count = spin_retry;
 
        local_irq_restore(flags);
+       owner = 0;
        while (1) {
                if (count-- <= 0) {
-                       smp_yield();
+                       if (owner && !smp_vcpu_scheduled(~owner))
+                               smp_yield_cpu(~owner);
                        count = spin_retry;
                }
                old = ACCESS_ONCE(rw->lock);
+               owner = ACCESS_ONCE(rw->owner);
                if ((int) old < 0)
                        continue;
                local_irq_disable();
@@ -179,15 +174,18 @@ EXPORT_SYMBOL(_raw_read_trylock_retry);
 
 void _raw_write_lock_wait(arch_rwlock_t *rw)
 {
-       unsigned int old;
+       unsigned int owner, old;
        int count = spin_retry;
 
+       owner = 0;
        while (1) {
                if (count-- <= 0) {
-                       smp_yield();
+                       if (owner && !smp_vcpu_scheduled(~owner))
+                               smp_yield_cpu(~owner);
                        count = spin_retry;
                }
                old = ACCESS_ONCE(rw->lock);
+               owner = ACCESS_ONCE(rw->owner);
                if (old)
                        continue;
                if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000))
@@ -198,16 +196,19 @@ EXPORT_SYMBOL(_raw_write_lock_wait);
 
 void _raw_write_lock_wait_flags(arch_rwlock_t *rw, unsigned long flags)
 {
-       unsigned int old;
+       unsigned int owner, old;
        int count = spin_retry;
 
        local_irq_restore(flags);
+       owner = 0;
        while (1) {
                if (count-- <= 0) {
-                       smp_yield();
+                       if (owner && !smp_vcpu_scheduled(~owner))
+                               smp_yield_cpu(~owner);
                        count = spin_retry;
                }
                old = ACCESS_ONCE(rw->lock);
+               owner = ACCESS_ONCE(rw->owner);
                if (old)
                        continue;
                local_irq_disable();
@@ -233,3 +234,13 @@ int _raw_write_trylock_retry(arch_rwlock_t *rw)
        return 0;
 }
 EXPORT_SYMBOL(_raw_write_trylock_retry);
+
+void arch_lock_relax(unsigned int cpu)
+{
+       if (!cpu)
+               return;
+       if (MACHINE_IS_LPAR && smp_vcpu_scheduled(~cpu))
+               return;
+       smp_yield_cpu(~cpu);
+}
+EXPORT_SYMBOL(arch_lock_relax);