arch/s390/lib/spinlock.c

   1 /*
   2  *    Out of line spinlock code.
   3  *
   4  *    Copyright IBM Corp. 2004, 2006
   5  *    Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com)
   6  */
   7
   8 #include <linux/types.h>
   9 #include <linux/module.h>
  10 #include <linux/spinlock.h>
  11 #include <linux/init.h>
  12 #include <linux/smp.h>
  13 #include <asm/io.h>
  14
  15 int spin_retry = 1000;
  16
  17 /**
  18  * spin_retry= parameter
  19  */
  20 static int __init spin_retry_setup(char *str)
  21 {
  22         spin_retry = simple_strtoul(str, &str, 0);
  23         return 1;
  24 }
  25 __setup("spin_retry=", spin_retry_setup);
  26
  27 void arch_spin_lock_wait(arch_spinlock_t *lp)
  28 {
  29         unsigned int cpu = SPINLOCK_LOCKVAL;
  30         unsigned int owner;
  31         int count;
  32
  33         while (1) {
  34                 owner = ACCESS_ONCE(lp->lock);
  35                 /* Try to get the lock if it is free. */
  36                 if (!owner) {
  37                         if (_raw_compare_and_swap(&lp->lock, 0, cpu))
  38                                 return;
  39                         continue;
  40                 }
  41                 /* Check if the lock owner is running. */
  42                 if (!smp_vcpu_scheduled(~owner)) {
  43                         smp_yield_cpu(~owner);
  44                         continue;
  45                 }
  46                 /* Loop for a while on the lock value. */
  47                 count = spin_retry;
  48                 do {
  49                         owner = ACCESS_ONCE(lp->lock);
  50                 } while (owner && count-- > 0);
  51                 if (!owner)
  52                         continue;
  53                 /*
  54                  * For multiple layers of hypervisors, e.g. z/VM + LPAR
  55                  * yield the CPU if the lock is still unavailable.
  56                  */
  57                 if (!MACHINE_IS_LPAR)
  58                         smp_yield_cpu(~owner);
  59         }
  60 }
  61 EXPORT_SYMBOL(arch_spin_lock_wait);
  62
  63 void arch_spin_lock_wait_flags(arch_spinlock_t *lp, unsigned long flags)
  64 {
  65         unsigned int cpu = SPINLOCK_LOCKVAL;
  66         unsigned int owner;
  67         int count;
  68
  69         local_irq_restore(flags);
  70         while (1) {
  71                 owner = ACCESS_ONCE(lp->lock);
  72                 /* Try to get the lock if it is free. */
  73                 if (!owner) {
  74                         local_irq_disable();
  75                         if (_raw_compare_and_swap(&lp->lock, 0, cpu))
  76                                 return;
  77                         local_irq_restore(flags);
  78                 }
  79                 /* Check if the lock owner is running. */
  80                 if (!smp_vcpu_scheduled(~owner)) {
  81                         smp_yield_cpu(~owner);
  82                         continue;
  83                 }
  84                 /* Loop for a while on the lock value. */
  85                 count = spin_retry;
  86                 do {
  87                         owner = ACCESS_ONCE(lp->lock);
  88                 } while (owner && count-- > 0);
  89                 if (!owner)
  90                         continue;
  91                 /*
  92                  * For multiple layers of hypervisors, e.g. z/VM + LPAR
  93                  * yield the CPU if the lock is still unavailable.
  94                  */
  95                 if (!MACHINE_IS_LPAR)
  96                         smp_yield_cpu(~owner);
  97         }
  98 }
  99 EXPORT_SYMBOL(arch_spin_lock_wait_flags);
 100
 101 void arch_spin_relax(arch_spinlock_t *lp)
 102 {
 103         unsigned int cpu = lp->lock;
 104         if (cpu != 0) {
 105                 if (MACHINE_IS_VM || MACHINE_IS_KVM ||
 106                     !smp_vcpu_scheduled(~cpu))
 107                         smp_yield_cpu(~cpu);
 108         }
 109 }
 110 EXPORT_SYMBOL(arch_spin_relax);
 111
 112 int arch_spin_trylock_retry(arch_spinlock_t *lp)
 113 {
 114         int count;
 115
 116         for (count = spin_retry; count > 0; count--)
 117                 if (arch_spin_trylock_once(lp))
 118                         return 1;
 119         return 0;
 120 }
 121 EXPORT_SYMBOL(arch_spin_trylock_retry);
 122
 123 void _raw_read_lock_wait(arch_rwlock_t *rw)
 124 {
 125         unsigned int old;
 126         int count = spin_retry;
 127
 128         while (1) {
 129                 if (count-- <= 0) {
 130                         smp_yield();
 131                         count = spin_retry;
 132                 }
 133                 old = ACCESS_ONCE(rw->lock);
 134                 if ((int) old < 0)
 135                         continue;
 136                 if (_raw_compare_and_swap(&rw->lock, old, old + 1))
 137                         return;
 138         }
 139 }
 140 EXPORT_SYMBOL(_raw_read_lock_wait);
 141
 142 void _raw_read_lock_wait_flags(arch_rwlock_t *rw, unsigned long flags)
 143 {
 144         unsigned int old;
 145         int count = spin_retry;
 146
 147         local_irq_restore(flags);
 148         while (1) {
 149                 if (count-- <= 0) {
 150                         smp_yield();
 151                         count = spin_retry;
 152                 }
 153                 old = ACCESS_ONCE(rw->lock);
 154                 if ((int) old < 0)
 155                         continue;
 156                 local_irq_disable();
 157                 if (_raw_compare_and_swap(&rw->lock, old, old + 1))
 158                         return;
 159                 local_irq_restore(flags);
 160         }
 161 }
 162 EXPORT_SYMBOL(_raw_read_lock_wait_flags);
 163
 164 int _raw_read_trylock_retry(arch_rwlock_t *rw)
 165 {
 166         unsigned int old;
 167         int count = spin_retry;
 168
 169         while (count-- > 0) {
 170                 old = ACCESS_ONCE(rw->lock);
 171                 if ((int) old < 0)
 172                         continue;
 173                 if (_raw_compare_and_swap(&rw->lock, old, old + 1))
 174                         return 1;
 175         }
 176         return 0;
 177 }
 178 EXPORT_SYMBOL(_raw_read_trylock_retry);
 179
 180 void _raw_write_lock_wait(arch_rwlock_t *rw)
 181 {
 182         unsigned int old;
 183         int count = spin_retry;
 184
 185         while (1) {
 186                 if (count-- <= 0) {
 187                         smp_yield();
 188                         count = spin_retry;
 189                 }
 190                 old = ACCESS_ONCE(rw->lock);
 191                 if (old)
 192                         continue;
 193                 if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000))
 194                         return;
 195         }
 196 }
 197 EXPORT_SYMBOL(_raw_write_lock_wait);
 198
 199 void _raw_write_lock_wait_flags(arch_rwlock_t *rw, unsigned long flags)
 200 {
 201         unsigned int old;
 202         int count = spin_retry;
 203
 204         local_irq_restore(flags);
 205         while (1) {
 206                 if (count-- <= 0) {
 207                         smp_yield();
 208                         count = spin_retry;
 209                 }
 210                 old = ACCESS_ONCE(rw->lock);
 211                 if (old)
 212                         continue;
 213                 local_irq_disable();
 214                 if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000))
 215                         return;
 216                 local_irq_restore(flags);
 217         }
 218 }
 219 EXPORT_SYMBOL(_raw_write_lock_wait_flags);
 220
 221 int _raw_write_trylock_retry(arch_rwlock_t *rw)
 222 {
 223         unsigned int old;
 224         int count = spin_retry;
 225
 226         while (count-- > 0) {
 227                 old = ACCESS_ONCE(rw->lock);
 228                 if (old)
 229                         continue;
 230                 if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000))
 231                         return 1;
 232         }
 233         return 0;
 234 }
 235 EXPORT_SYMBOL(_raw_write_trylock_retry);