arch/mips/include/asm/spinlock.h

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * Copyright (C) 1999, 2000, 06 Ralf Baechle (ralf@linux-mips.org)
   7  * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
   8  */
   9 #ifndef _ASM_SPINLOCK_H
  10 #define _ASM_SPINLOCK_H
  11
  12 #include <linux/compiler.h>
  13
  14 #include <asm/barrier.h>
  15 #include <asm/compiler.h>
  16 #include <asm/war.h>
  17
  18 /*
  19  * Your basic SMP spinlocks, allowing only a single CPU anywhere
  20  *
  21  * Simple spin lock operations.  There are two variants, one clears IRQ's
  22  * on the local processor, one does not.
  23  *
  24  * These are fair FIFO ticket locks
  25  *
  26  * (the type definitions are in asm/spinlock_types.h)
  27  */
  28
  29
  30 /*
  31  * Ticket locks are conceptually two parts, one indicating the current head of
  32  * the queue, and the other indicating the current tail. The lock is acquired
  33  * by atomically noting the tail and incrementing it by one (thus adding
  34  * ourself to the queue and noting our position), then waiting until the head
  35  * becomes equal to the the initial value of the tail.
  36  */
  37
  38 static inline int arch_spin_is_locked(arch_spinlock_t *lock)
  39 {
  40         u32 counters = ACCESS_ONCE(lock->lock);
  41
  42         return ((counters >> 16) ^ counters) & 0xffff;
  43 }
  44
  45 #define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock)
  46 #define arch_spin_unlock_wait(x) \
  47         while (arch_spin_is_locked(x)) { cpu_relax(); }
  48
  49 static inline int arch_spin_is_contended(arch_spinlock_t *lock)
  50 {
  51         u32 counters = ACCESS_ONCE(lock->lock);
  52
  53         return (((counters >> 16) - counters) & 0xffff) > 1;
  54 }
  55 #define arch_spin_is_contended  arch_spin_is_contended
  56
  57 static inline void arch_spin_lock(arch_spinlock_t *lock)
  58 {
  59         int my_ticket;
  60         int tmp;
  61         int inc = 0x10000;
  62
  63         if (R10000_LLSC_WAR) {
  64                 __asm__ __volatile__ (
  65                 "       .set push               # arch_spin_lock        \n"
  66                 "       .set noreorder                                  \n"
  67                 "                                                       \n"
  68                 "1:     ll      %[ticket], %[ticket_ptr]                \n"
  69                 "       addu    %[my_ticket], %[ticket], %[inc]         \n"
  70                 "       sc      %[my_ticket], %[ticket_ptr]             \n"
  71                 "       beqzl   %[my_ticket], 1b                        \n"
  72                 "        nop                                            \n"
  73                 "       srl     %[my_ticket], %[ticket], 16             \n"
  74                 "       andi    %[ticket], %[ticket], 0xffff            \n"
  75                 "       bne     %[ticket], %[my_ticket], 4f             \n"
  76                 "        subu   %[ticket], %[my_ticket], %[ticket]      \n"
  77                 "2:                                                     \n"
  78                 "       .subsection 2                                   \n"
  79                 "4:     andi    %[ticket], %[ticket], 0xffff            \n"
  80                 "       sll     %[ticket], 5                            \n"
  81                 "                                                       \n"
  82                 "6:     bnez    %[ticket], 6b                           \n"
  83                 "        subu   %[ticket], 1                            \n"
  84                 "                                                       \n"
  85                 "       lhu     %[ticket], %[serving_now_ptr]           \n"
  86                 "       beq     %[ticket], %[my_ticket], 2b             \n"
  87                 "        subu   %[ticket], %[my_ticket], %[ticket]      \n"
  88                 "       b       4b                                      \n"
  89                 "        subu   %[ticket], %[ticket], 1                 \n"
  90                 "       .previous                                       \n"
  91                 "       .set pop                                        \n"
  92                 : [ticket_ptr] "+" GCC_OFF12_ASM() (lock->lock),
  93                   [serving_now_ptr] "+m" (lock->h.serving_now),
  94                   [ticket] "=&r" (tmp),
  95                   [my_ticket] "=&r" (my_ticket)
  96                 : [inc] "r" (inc));
  97         } else {
  98                 __asm__ __volatile__ (
  99                 "       .set push               # arch_spin_lock        \n"
 100                 "       .set noreorder                                  \n"
 101                 "                                                       \n"
 102                 "1:     ll      %[ticket], %[ticket_ptr]                \n"
 103                 "       addu    %[my_ticket], %[ticket], %[inc]         \n"
 104                 "       sc      %[my_ticket], %[ticket_ptr]             \n"
 105                 "       beqz    %[my_ticket], 1b                        \n"
 106                 "        srl    %[my_ticket], %[ticket], 16             \n"
 107                 "       andi    %[ticket], %[ticket], 0xffff            \n"
 108                 "       bne     %[ticket], %[my_ticket], 4f             \n"
 109                 "        subu   %[ticket], %[my_ticket], %[ticket]      \n"
 110                 "2:                                                     \n"
 111                 "       .subsection 2                                   \n"
 112                 "4:     andi    %[ticket], %[ticket], 0x1fff            \n"
 113                 "       sll     %[ticket], 5                            \n"
 114                 "                                                       \n"
 115                 "6:     bnez    %[ticket], 6b                           \n"
 116                 "        subu   %[ticket], 1                            \n"
 117                 "                                                       \n"
 118                 "       lhu     %[ticket], %[serving_now_ptr]           \n"
 119                 "       beq     %[ticket], %[my_ticket], 2b             \n"
 120                 "        subu   %[ticket], %[my_ticket], %[ticket]      \n"
 121                 "       b       4b                                      \n"
 122                 "        subu   %[ticket], %[ticket], 1                 \n"
 123                 "       .previous                                       \n"
 124                 "       .set pop                                        \n"
 125                 : [ticket_ptr] "+" GCC_OFF12_ASM() (lock->lock),
 126                   [serving_now_ptr] "+m" (lock->h.serving_now),
 127                   [ticket] "=&r" (tmp),
 128                   [my_ticket] "=&r" (my_ticket)
 129                 : [inc] "r" (inc));
 130         }
 131
 132         smp_llsc_mb();
 133 }
 134
 135 static inline void arch_spin_unlock(arch_spinlock_t *lock)
 136 {
 137         unsigned int serving_now = lock->h.serving_now + 1;
 138         wmb();
 139         lock->h.serving_now = (u16)serving_now;
 140         nudge_writes();
 141 }
 142
 143 static inline unsigned int arch_spin_trylock(arch_spinlock_t *lock)
 144 {
 145         int tmp, tmp2, tmp3;
 146         int inc = 0x10000;
 147
 148         if (R10000_LLSC_WAR) {
 149                 __asm__ __volatile__ (
 150                 "       .set push               # arch_spin_trylock     \n"
 151                 "       .set noreorder                                  \n"
 152                 "                                                       \n"
 153                 "1:     ll      %[ticket], %[ticket_ptr]                \n"
 154                 "       srl     %[my_ticket], %[ticket], 16             \n"
 155                 "       andi    %[now_serving], %[ticket], 0xffff       \n"
 156                 "       bne     %[my_ticket], %[now_serving], 3f        \n"
 157                 "        addu   %[ticket], %[ticket], %[inc]            \n"
 158                 "       sc      %[ticket], %[ticket_ptr]                \n"
 159                 "       beqzl   %[ticket], 1b                           \n"
 160                 "        li     %[ticket], 1                            \n"
 161                 "2:                                                     \n"
 162                 "       .subsection 2                                   \n"
 163                 "3:     b       2b                                      \n"
 164                 "        li     %[ticket], 0                            \n"
 165                 "       .previous                                       \n"
 166                 "       .set pop                                        \n"
 167                 : [ticket_ptr] "+" GCC_OFF12_ASM() (lock->lock),
 168                   [ticket] "=&r" (tmp),
 169                   [my_ticket] "=&r" (tmp2),
 170                   [now_serving] "=&r" (tmp3)
 171                 : [inc] "r" (inc));
 172         } else {
 173                 __asm__ __volatile__ (
 174                 "       .set push               # arch_spin_trylock     \n"
 175                 "       .set noreorder                                  \n"
 176                 "                                                       \n"
 177                 "1:     ll      %[ticket], %[ticket_ptr]                \n"
 178                 "       srl     %[my_ticket], %[ticket], 16             \n"
 179                 "       andi    %[now_serving], %[ticket], 0xffff       \n"
 180                 "       bne     %[my_ticket], %[now_serving], 3f        \n"
 181                 "        addu   %[ticket], %[ticket], %[inc]            \n"
 182                 "       sc      %[ticket], %[ticket_ptr]                \n"
 183                 "       beqz    %[ticket], 1b                           \n"
 184                 "        li     %[ticket], 1                            \n"
 185                 "2:                                                     \n"
 186                 "       .subsection 2                                   \n"
 187                 "3:     b       2b                                      \n"
 188                 "        li     %[ticket], 0                            \n"
 189                 "       .previous                                       \n"
 190                 "       .set pop                                        \n"
 191                 : [ticket_ptr] "+" GCC_OFF12_ASM() (lock->lock),
 192                   [ticket] "=&r" (tmp),
 193                   [my_ticket] "=&r" (tmp2),
 194                   [now_serving] "=&r" (tmp3)
 195                 : [inc] "r" (inc));
 196         }
 197
 198         smp_llsc_mb();
 199
 200         return tmp;
 201 }
 202
 203 /*
 204  * Read-write spinlocks, allowing multiple readers but only one writer.
 205  *
 206  * NOTE! it is quite common to have readers in interrupts but no interrupt
 207  * writers. For those circumstances we can "mix" irq-safe locks - any writer
 208  * needs to get a irq-safe write-lock, but readers can get non-irqsafe
 209  * read-locks.
 210  */
 211
 212 /*
 213  * read_can_lock - would read_trylock() succeed?
 214  * @lock: the rwlock in question.
 215  */
 216 #define arch_read_can_lock(rw)  ((rw)->lock >= 0)
 217
 218 /*
 219  * write_can_lock - would write_trylock() succeed?
 220  * @lock: the rwlock in question.
 221  */
 222 #define arch_write_can_lock(rw) (!(rw)->lock)
 223
 224 static inline void arch_read_lock(arch_rwlock_t *rw)
 225 {
 226         unsigned int tmp;
 227
 228         if (R10000_LLSC_WAR) {
 229                 __asm__ __volatile__(
 230                 "       .set    noreorder       # arch_read_lock        \n"
 231                 "1:     ll      %1, %2                                  \n"
 232                 "       bltz    %1, 1b                                  \n"
 233                 "        addu   %1, 1                                   \n"
 234                 "       sc      %1, %0                                  \n"
 235                 "       beqzl   %1, 1b                                  \n"
 236                 "        nop                                            \n"
 237                 "       .set    reorder                                 \n"
 238                 : "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp)
 239                 : GCC_OFF12_ASM() (rw->lock)
 240                 : "memory");
 241         } else {
 242                 do {
 243                         __asm__ __volatile__(
 244                         "1:     ll      %1, %2  # arch_read_lock        \n"
 245                         "       bltz    %1, 1b                          \n"
 246                         "        addu   %1, 1                           \n"
 247                         "2:     sc      %1, %0                          \n"
 248                         : "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp)
 249                         : GCC_OFF12_ASM() (rw->lock)
 250                         : "memory");
 251                 } while (unlikely(!tmp));
 252         }
 253
 254         smp_llsc_mb();
 255 }
 256
 257 /* Note the use of sub, not subu which will make the kernel die with an
 258    overflow exception if we ever try to unlock an rwlock that is already
 259    unlocked or is being held by a writer.  */
 260 static inline void arch_read_unlock(arch_rwlock_t *rw)
 261 {
 262         unsigned int tmp;
 263
 264         smp_mb__before_llsc();
 265
 266         if (R10000_LLSC_WAR) {
 267                 __asm__ __volatile__(
 268                 "1:     ll      %1, %2          # arch_read_unlock      \n"
 269                 "       sub     %1, 1                                   \n"
 270                 "       sc      %1, %0                                  \n"
 271                 "       beqzl   %1, 1b                                  \n"
 272                 : "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp)
 273                 : GCC_OFF12_ASM() (rw->lock)
 274                 : "memory");
 275         } else {
 276                 do {
 277                         __asm__ __volatile__(
 278                         "1:     ll      %1, %2  # arch_read_unlock      \n"
 279                         "       sub     %1, 1                           \n"
 280                         "       sc      %1, %0                          \n"
 281                         : "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp)
 282                         : GCC_OFF12_ASM() (rw->lock)
 283                         : "memory");
 284                 } while (unlikely(!tmp));
 285         }
 286 }
 287
 288 static inline void arch_write_lock(arch_rwlock_t *rw)
 289 {
 290         unsigned int tmp;
 291
 292         if (R10000_LLSC_WAR) {
 293                 __asm__ __volatile__(
 294                 "       .set    noreorder       # arch_write_lock       \n"
 295                 "1:     ll      %1, %2                                  \n"
 296                 "       bnez    %1, 1b                                  \n"
 297                 "        lui    %1, 0x8000                              \n"
 298                 "       sc      %1, %0                                  \n"
 299                 "       beqzl   %1, 1b                                  \n"
 300                 "        nop                                            \n"
 301                 "       .set    reorder                                 \n"
 302                 : "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp)
 303                 : GCC_OFF12_ASM() (rw->lock)
 304                 : "memory");
 305         } else {
 306                 do {
 307                         __asm__ __volatile__(
 308                         "1:     ll      %1, %2  # arch_write_lock       \n"
 309                         "       bnez    %1, 1b                          \n"
 310                         "        lui    %1, 0x8000                      \n"
 311                         "2:     sc      %1, %0                          \n"
 312                         : "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp)
 313                         : GCC_OFF12_ASM() (rw->lock)
 314                         : "memory");
 315                 } while (unlikely(!tmp));
 316         }
 317
 318         smp_llsc_mb();
 319 }
 320
 321 static inline void arch_write_unlock(arch_rwlock_t *rw)
 322 {
 323         smp_mb();
 324
 325         __asm__ __volatile__(
 326         "                               # arch_write_unlock     \n"
 327         "       sw      $0, %0                                  \n"
 328         : "=m" (rw->lock)
 329         : "m" (rw->lock)
 330         : "memory");
 331 }
 332
 333 static inline int arch_read_trylock(arch_rwlock_t *rw)
 334 {
 335         unsigned int tmp;
 336         int ret;
 337
 338         if (R10000_LLSC_WAR) {
 339                 __asm__ __volatile__(
 340                 "       .set    noreorder       # arch_read_trylock     \n"
 341                 "       li      %2, 0                                   \n"
 342                 "1:     ll      %1, %3                                  \n"
 343                 "       bltz    %1, 2f                                  \n"
 344                 "        addu   %1, 1                                   \n"
 345                 "       sc      %1, %0                                  \n"
 346                 "       .set    reorder                                 \n"
 347                 "       beqzl   %1, 1b                                  \n"
 348                 "        nop                                            \n"
 349                 __WEAK_LLSC_MB
 350                 "       li      %2, 1                                   \n"
 351                 "2:                                                     \n"
 352                 : "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp), "=&r" (ret)
 353                 : GCC_OFF12_ASM() (rw->lock)
 354                 : "memory");
 355         } else {
 356                 __asm__ __volatile__(
 357                 "       .set    noreorder       # arch_read_trylock     \n"
 358                 "       li      %2, 0                                   \n"
 359                 "1:     ll      %1, %3                                  \n"
 360                 "       bltz    %1, 2f                                  \n"
 361                 "        addu   %1, 1                                   \n"
 362                 "       sc      %1, %0                                  \n"
 363                 "       beqz    %1, 1b                                  \n"
 364                 "        nop                                            \n"
 365                 "       .set    reorder                                 \n"
 366                 __WEAK_LLSC_MB
 367                 "       li      %2, 1                                   \n"
 368                 "2:                                                     \n"
 369                 : "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp), "=&r" (ret)
 370                 : GCC_OFF12_ASM() (rw->lock)
 371                 : "memory");
 372         }
 373
 374         return ret;
 375 }
 376
 377 static inline int arch_write_trylock(arch_rwlock_t *rw)
 378 {
 379         unsigned int tmp;
 380         int ret;
 381
 382         if (R10000_LLSC_WAR) {
 383                 __asm__ __volatile__(
 384                 "       .set    noreorder       # arch_write_trylock    \n"
 385                 "       li      %2, 0                                   \n"
 386                 "1:     ll      %1, %3                                  \n"
 387                 "       bnez    %1, 2f                                  \n"
 388                 "        lui    %1, 0x8000                              \n"
 389                 "       sc      %1, %0                                  \n"
 390                 "       beqzl   %1, 1b                                  \n"
 391                 "        nop                                            \n"
 392                 __WEAK_LLSC_MB
 393                 "       li      %2, 1                                   \n"
 394                 "       .set    reorder                                 \n"
 395                 "2:                                                     \n"
 396                 : "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp), "=&r" (ret)
 397                 : GCC_OFF12_ASM() (rw->lock)
 398                 : "memory");
 399         } else {
 400                 do {
 401                         __asm__ __volatile__(
 402                         "       ll      %1, %3  # arch_write_trylock    \n"
 403                         "       li      %2, 0                           \n"
 404                         "       bnez    %1, 2f                          \n"
 405                         "       lui     %1, 0x8000                      \n"
 406                         "       sc      %1, %0                          \n"
 407                         "       li      %2, 1                           \n"
 408                         "2:                                             \n"
 409                         : "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp),
 410                           "=&r" (ret)
 411                         : GCC_OFF12_ASM() (rw->lock)
 412                         : "memory");
 413                 } while (unlikely(!tmp));
 414
 415                 smp_llsc_mb();
 416         }
 417
 418         return ret;
 419 }
 420
 421 #define arch_read_lock_flags(lock, flags) arch_read_lock(lock)
 422 #define arch_write_lock_flags(lock, flags) arch_write_lock(lock)
 423
 424 #define arch_spin_relax(lock)   cpu_relax()
 425 #define arch_read_relax(lock)   cpu_relax()
 426 #define arch_write_relax(lock)  cpu_relax()
 427
 428 #endif /* _ASM_SPINLOCK_H */