arch/mips/include/asm/atomic.h

   1 /*
   2  * Atomic operations that C can't guarantee us.  Useful for
   3  * resource counting etc..
   4  *
   5  * But use these as seldom as possible since they are much more slower
   6  * than regular operations.
   7  *
   8  * This file is subject to the terms and conditions of the GNU General Public
   9  * License.  See the file "COPYING" in the main directory of this archive
  10  * for more details.
  11  *
  12  * Copyright (C) 1996, 97, 99, 2000, 03, 04, 06 by Ralf Baechle
  13  */
  14 #ifndef _ASM_ATOMIC_H
  15 #define _ASM_ATOMIC_H
  16
  17 #include <linux/irqflags.h>
  18 #include <linux/types.h>
  19 #include <asm/barrier.h>
  20 #include <asm/compiler.h>
  21 #include <asm/cpu-features.h>
  22 #include <asm/cmpxchg.h>
  23 #include <asm/war.h>
  24
  25 #define ATOMIC_INIT(i)    { (i) }
  26
  27 /*
  28  * atomic_read - read atomic variable
  29  * @v: pointer of type atomic_t
  30  *
  31  * Atomically reads the value of @v.
  32  */
  33 #define atomic_read(v)          ACCESS_ONCE((v)->counter)
  34
  35 /*
  36  * atomic_set - set atomic variable
  37  * @v: pointer of type atomic_t
  38  * @i: required value
  39  *
  40  * Atomically sets the value of @v to @i.
  41  */
  42 #define atomic_set(v, i)                ((v)->counter = (i))
  43
  44 #define ATOMIC_OP(op, c_op, asm_op)                                           \
  45 static __inline__ void atomic_##op(int i, atomic_t * v)                       \
  46 {                                                                             \
  47         if (kernel_uses_llsc && R10000_LLSC_WAR) {                            \
  48                 int temp;                                                     \
  49                                                                               \
  50                 __asm__ __volatile__(                                         \
  51                 "       .set    arch=r4000                              \n"   \
  52                 "1:     ll      %0, %1          # atomic_" #op "        \n"   \
  53                 "       " #asm_op " %0, %2                              \n"   \
  54                 "       sc      %0, %1                                  \n"   \
  55                 "       beqzl   %0, 1b                                  \n"   \
  56                 "       .set    mips0                                   \n"   \
  57                 : "=&r" (temp), "+" GCC_OFF12_ASM() (v->counter)              \
  58                 : "Ir" (i));                                                  \
  59         } else if (kernel_uses_llsc) {                                        \
  60                 int temp;                                                     \
  61                                                                               \
  62                 do {                                                          \
  63                         __asm__ __volatile__(                                 \
  64                         "       .set    arch=r4000                      \n"   \
  65                         "       ll      %0, %1          # atomic_" #op "\n"   \
  66                         "       " #asm_op " %0, %2                      \n"   \
  67                         "       sc      %0, %1                          \n"   \
  68                         "       .set    mips0                           \n"   \
  69                         : "=&r" (temp), "+" GCC_OFF12_ASM() (v->counter)      \
  70                         : "Ir" (i));                                          \
  71                 } while (unlikely(!temp));                                    \
  72         } else {                                                              \
  73                 unsigned long flags;                                          \
  74                                                                               \
  75                 raw_local_irq_save(flags);                                    \
  76                 v->counter c_op i;                                            \
  77                 raw_local_irq_restore(flags);                                 \
  78         }                                                                     \
  79 }
  80
  81 #define ATOMIC_OP_RETURN(op, c_op, asm_op)                                    \
  82 static __inline__ int atomic_##op##_return(int i, atomic_t * v)               \
  83 {                                                                             \
  84         int result;                                                           \
  85                                                                               \
  86         smp_mb__before_llsc();                                                \
  87                                                                               \
  88         if (kernel_uses_llsc && R10000_LLSC_WAR) {                            \
  89                 int temp;                                                     \
  90                                                                               \
  91                 __asm__ __volatile__(                                         \
  92                 "       .set    arch=r4000                              \n"   \
  93                 "1:     ll      %1, %2          # atomic_" #op "_return \n"   \
  94                 "       " #asm_op " %0, %1, %3                          \n"   \
  95                 "       sc      %0, %2                                  \n"   \
  96                 "       beqzl   %0, 1b                                  \n"   \
  97                 "       " #asm_op " %0, %1, %3                          \n"   \
  98                 "       .set    mips0                                   \n"   \
  99                 : "=&r" (result), "=&r" (temp),                               \
 100                   "+" GCC_OFF12_ASM() (v->counter)                            \
 101                 : "Ir" (i));                                                  \
 102         } else if (kernel_uses_llsc) {                                        \
 103                 int temp;                                                     \
 104                                                                               \
 105                 do {                                                          \
 106                         __asm__ __volatile__(                                 \
 107                         "       .set    arch=r4000                      \n"   \
 108                         "       ll      %1, %2  # atomic_" #op "_return \n"   \
 109                         "       " #asm_op " %0, %1, %3                  \n"   \
 110                         "       sc      %0, %2                          \n"   \
 111                         "       .set    mips0                           \n"   \
 112                         : "=&r" (result), "=&r" (temp),                       \
 113                           "+" GCC_OFF12_ASM() (v->counter)                    \
 114                         : "Ir" (i));                                          \
 115                 } while (unlikely(!result));                                  \
 116                                                                               \
 117                 result = temp; result c_op i;                                 \
 118         } else {                                                              \
 119                 unsigned long flags;                                          \
 120                                                                               \
 121                 raw_local_irq_save(flags);                                    \
 122                 result = v->counter;                                          \
 123                 result c_op i;                                                \
 124                 v->counter = result;                                          \
 125                 raw_local_irq_restore(flags);                                 \
 126         }                                                                     \
 127                                                                               \
 128         smp_llsc_mb();                                                        \
 129                                                                               \
 130         return result;                                                        \
 131 }
 132
 133 #define ATOMIC_OPS(op, c_op, asm_op)                                          \
 134         ATOMIC_OP(op, c_op, asm_op)                                           \
 135         ATOMIC_OP_RETURN(op, c_op, asm_op)
 136
 137 ATOMIC_OPS(add, +=, addu)
 138 ATOMIC_OPS(sub, -=, subu)
 139
 140 #undef ATOMIC_OPS
 141 #undef ATOMIC_OP_RETURN
 142 #undef ATOMIC_OP
 143
 144 /*
 145  * atomic_sub_if_positive - conditionally subtract integer from atomic variable
 146  * @i: integer value to subtract
 147  * @v: pointer of type atomic_t
 148  *
 149  * Atomically test @v and subtract @i if @v is greater or equal than @i.
 150  * The function returns the old value of @v minus @i.
 151  */
 152 static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)
 153 {
 154         int result;
 155
 156         smp_mb__before_llsc();
 157
 158         if (kernel_uses_llsc && R10000_LLSC_WAR) {
 159                 int temp;
 160
 161                 __asm__ __volatile__(
 162                 "       .set    arch=r4000                              \n"
 163                 "1:     ll      %1, %2          # atomic_sub_if_positive\n"
 164                 "       subu    %0, %1, %3                              \n"
 165                 "       bltz    %0, 1f                                  \n"
 166                 "       sc      %0, %2                                  \n"
 167                 "       .set    noreorder                               \n"
 168                 "       beqzl   %0, 1b                                  \n"
 169                 "        subu   %0, %1, %3                              \n"
 170                 "       .set    reorder                                 \n"
 171                 "1:                                                     \n"
 172                 "       .set    mips0                                   \n"
 173                 : "=&r" (result), "=&r" (temp),
 174                   "+" GCC_OFF12_ASM() (v->counter)
 175                 : "Ir" (i), GCC_OFF12_ASM() (v->counter)
 176                 : "memory");
 177         } else if (kernel_uses_llsc) {
 178                 int temp;
 179
 180                 __asm__ __volatile__(
 181                 "       .set    arch=r4000                              \n"
 182                 "1:     ll      %1, %2          # atomic_sub_if_positive\n"
 183                 "       subu    %0, %1, %3                              \n"
 184                 "       bltz    %0, 1f                                  \n"
 185                 "       sc      %0, %2                                  \n"
 186                 "       .set    noreorder                               \n"
 187                 "       beqz    %0, 1b                                  \n"
 188                 "        subu   %0, %1, %3                              \n"
 189                 "       .set    reorder                                 \n"
 190                 "1:                                                     \n"
 191                 "       .set    mips0                                   \n"
 192                 : "=&r" (result), "=&r" (temp),
 193                   "+" GCC_OFF12_ASM() (v->counter)
 194                 : "Ir" (i));
 195         } else {
 196                 unsigned long flags;
 197
 198                 raw_local_irq_save(flags);
 199                 result = v->counter;
 200                 result -= i;
 201                 if (result >= 0)
 202                         v->counter = result;
 203                 raw_local_irq_restore(flags);
 204         }
 205
 206         smp_llsc_mb();
 207
 208         return result;
 209 }
 210
 211 #define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n)))
 212 #define atomic_xchg(v, new) (xchg(&((v)->counter), (new)))
 213
 214 /**
 215  * __atomic_add_unless - add unless the number is a given value
 216  * @v: pointer of type atomic_t
 217  * @a: the amount to add to v...
 218  * @u: ...unless v is equal to u.
 219  *
 220  * Atomically adds @a to @v, so long as it was not @u.
 221  * Returns the old value of @v.
 222  */
 223 static __inline__ int __atomic_add_unless(atomic_t *v, int a, int u)
 224 {
 225         int c, old;
 226         c = atomic_read(v);
 227         for (;;) {
 228                 if (unlikely(c == (u)))
 229                         break;
 230                 old = atomic_cmpxchg((v), c, c + (a));
 231                 if (likely(old == c))
 232                         break;
 233                 c = old;
 234         }
 235         return c;
 236 }
 237
 238 #define atomic_dec_return(v) atomic_sub_return(1, (v))
 239 #define atomic_inc_return(v) atomic_add_return(1, (v))
 240
 241 /*
 242  * atomic_sub_and_test - subtract value from variable and test result
 243  * @i: integer value to subtract
 244  * @v: pointer of type atomic_t
 245  *
 246  * Atomically subtracts @i from @v and returns
 247  * true if the result is zero, or false for all
 248  * other cases.
 249  */
 250 #define atomic_sub_and_test(i, v) (atomic_sub_return((i), (v)) == 0)
 251
 252 /*
 253  * atomic_inc_and_test - increment and test
 254  * @v: pointer of type atomic_t
 255  *
 256  * Atomically increments @v by 1
 257  * and returns true if the result is zero, or false for all
 258  * other cases.
 259  */
 260 #define atomic_inc_and_test(v) (atomic_inc_return(v) == 0)
 261
 262 /*
 263  * atomic_dec_and_test - decrement by 1 and test
 264  * @v: pointer of type atomic_t
 265  *
 266  * Atomically decrements @v by 1 and
 267  * returns true if the result is 0, or false for all other
 268  * cases.
 269  */
 270 #define atomic_dec_and_test(v) (atomic_sub_return(1, (v)) == 0)
 271
 272 /*
 273  * atomic_dec_if_positive - decrement by 1 if old value positive
 274  * @v: pointer of type atomic_t
 275  */
 276 #define atomic_dec_if_positive(v)       atomic_sub_if_positive(1, v)
 277
 278 /*
 279  * atomic_inc - increment atomic variable
 280  * @v: pointer of type atomic_t
 281  *
 282  * Atomically increments @v by 1.
 283  */
 284 #define atomic_inc(v) atomic_add(1, (v))
 285
 286 /*
 287  * atomic_dec - decrement and test
 288  * @v: pointer of type atomic_t
 289  *
 290  * Atomically decrements @v by 1.
 291  */
 292 #define atomic_dec(v) atomic_sub(1, (v))
 293
 294 /*
 295  * atomic_add_negative - add and test if negative
 296  * @v: pointer of type atomic_t
 297  * @i: integer value to add
 298  *
 299  * Atomically adds @i to @v and returns true
 300  * if the result is negative, or false when
 301  * result is greater than or equal to zero.
 302  */
 303 #define atomic_add_negative(i, v) (atomic_add_return(i, (v)) < 0)
 304
 305 #ifdef CONFIG_64BIT
 306
 307 #define ATOMIC64_INIT(i)    { (i) }
 308
 309 /*
 310  * atomic64_read - read atomic variable
 311  * @v: pointer of type atomic64_t
 312  *
 313  */
 314 #define atomic64_read(v)        ACCESS_ONCE((v)->counter)
 315
 316 /*
 317  * atomic64_set - set atomic variable
 318  * @v: pointer of type atomic64_t
 319  * @i: required value
 320  */
 321 #define atomic64_set(v, i)      ((v)->counter = (i))
 322
 323 #define ATOMIC64_OP(op, c_op, asm_op)                                         \
 324 static __inline__ void atomic64_##op(long i, atomic64_t * v)                  \
 325 {                                                                             \
 326         if (kernel_uses_llsc && R10000_LLSC_WAR) {                            \
 327                 long temp;                                                    \
 328                                                                               \
 329                 __asm__ __volatile__(                                         \
 330                 "       .set    arch=r4000                              \n"   \
 331                 "1:     lld     %0, %1          # atomic64_" #op "      \n"   \
 332                 "       " #asm_op " %0, %2                              \n"   \
 333                 "       scd     %0, %1                                  \n"   \
 334                 "       beqzl   %0, 1b                                  \n"   \
 335                 "       .set    mips0                                   \n"   \
 336                 : "=&r" (temp), "+" GCC_OFF12_ASM() (v->counter)              \
 337                 : "Ir" (i));                                                  \
 338         } else if (kernel_uses_llsc) {                                        \
 339                 long temp;                                                    \
 340                                                                               \
 341                 do {                                                          \
 342                         __asm__ __volatile__(                                 \
 343                         "       .set    arch=r4000                      \n"   \
 344                         "       lld     %0, %1          # atomic64_" #op "\n" \
 345                         "       " #asm_op " %0, %2                      \n"   \
 346                         "       scd     %0, %1                          \n"   \
 347                         "       .set    mips0                           \n"   \
 348                         : "=&r" (temp), "+" GCC_OFF12_ASM() (v->counter)      \
 349                         : "Ir" (i));                                          \
 350                 } while (unlikely(!temp));                                    \
 351         } else {                                                              \
 352                 unsigned long flags;                                          \
 353                                                                               \
 354                 raw_local_irq_save(flags);                                    \
 355                 v->counter c_op i;                                            \
 356                 raw_local_irq_restore(flags);                                 \
 357         }                                                                     \
 358 }
 359
 360 #define ATOMIC64_OP_RETURN(op, c_op, asm_op)                                  \
 361 static __inline__ long atomic64_##op##_return(long i, atomic64_t * v)         \
 362 {                                                                             \
 363         long result;                                                          \
 364                                                                               \
 365         smp_mb__before_llsc();                                                \
 366                                                                               \
 367         if (kernel_uses_llsc && R10000_LLSC_WAR) {                            \
 368                 long temp;                                                    \
 369                                                                               \
 370                 __asm__ __volatile__(                                         \
 371                 "       .set    arch=r4000                              \n"   \
 372                 "1:     lld     %1, %2          # atomic64_" #op "_return\n"  \
 373                 "       " #asm_op " %0, %1, %3                          \n"   \
 374                 "       scd     %0, %2                                  \n"   \
 375                 "       beqzl   %0, 1b                                  \n"   \
 376                 "       " #asm_op " %0, %1, %3                          \n"   \
 377                 "       .set    mips0                                   \n"   \
 378                 : "=&r" (result), "=&r" (temp),                               \
 379                   "+" GCC_OFF12_ASM() (v->counter)                            \
 380                 : "Ir" (i));                                                  \
 381         } else if (kernel_uses_llsc) {                                        \
 382                 long temp;                                                    \
 383                                                                               \
 384                 do {                                                          \
 385                         __asm__ __volatile__(                                 \
 386                         "       .set    arch=r4000                      \n"   \
 387                         "       lld     %1, %2  # atomic64_" #op "_return\n"  \
 388                         "       " #asm_op " %0, %1, %3                  \n"   \
 389                         "       scd     %0, %2                          \n"   \
 390                         "       .set    mips0                           \n"   \
 391                         : "=&r" (result), "=&r" (temp),                       \
 392                           "=" GCC_OFF12_ASM() (v->counter)                    \
 393                         : "Ir" (i), GCC_OFF12_ASM() (v->counter)              \
 394                         : "memory");                                          \
 395                 } while (unlikely(!result));                                  \
 396                                                                               \
 397                 result = temp; result c_op i;                                 \
 398         } else {                                                              \
 399                 unsigned long flags;                                          \
 400                                                                               \
 401                 raw_local_irq_save(flags);                                    \
 402                 result = v->counter;                                          \
 403                 result c_op i;                                                \
 404                 v->counter = result;                                          \
 405                 raw_local_irq_restore(flags);                                 \
 406         }                                                                     \
 407                                                                               \
 408         smp_llsc_mb();                                                        \
 409                                                                               \
 410         return result;                                                        \
 411 }
 412
 413 #define ATOMIC64_OPS(op, c_op, asm_op)                                        \
 414         ATOMIC64_OP(op, c_op, asm_op)                                         \
 415         ATOMIC64_OP_RETURN(op, c_op, asm_op)
 416
 417 ATOMIC64_OPS(add, +=, daddu)
 418 ATOMIC64_OPS(sub, -=, dsubu)
 419
 420 #undef ATOMIC64_OPS
 421 #undef ATOMIC64_OP_RETURN
 422 #undef ATOMIC64_OP
 423
 424 /*
 425  * atomic64_sub_if_positive - conditionally subtract integer from atomic
 426  *                            variable
 427  * @i: integer value to subtract
 428  * @v: pointer of type atomic64_t
 429  *
 430  * Atomically test @v and subtract @i if @v is greater or equal than @i.
 431  * The function returns the old value of @v minus @i.
 432  */
 433 static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v)
 434 {
 435         long result;
 436
 437         smp_mb__before_llsc();
 438
 439         if (kernel_uses_llsc && R10000_LLSC_WAR) {
 440                 long temp;
 441
 442                 __asm__ __volatile__(
 443                 "       .set    arch=r4000                              \n"
 444                 "1:     lld     %1, %2          # atomic64_sub_if_positive\n"
 445                 "       dsubu   %0, %1, %3                              \n"
 446                 "       bltz    %0, 1f                                  \n"
 447                 "       scd     %0, %2                                  \n"
 448                 "       .set    noreorder                               \n"
 449                 "       beqzl   %0, 1b                                  \n"
 450                 "        dsubu  %0, %1, %3                              \n"
 451                 "       .set    reorder                                 \n"
 452                 "1:                                                     \n"
 453                 "       .set    mips0                                   \n"
 454                 : "=&r" (result), "=&r" (temp),
 455                   "=" GCC_OFF12_ASM() (v->counter)
 456                 : "Ir" (i), GCC_OFF12_ASM() (v->counter)
 457                 : "memory");
 458         } else if (kernel_uses_llsc) {
 459                 long temp;
 460
 461                 __asm__ __volatile__(
 462                 "       .set    arch=r4000                              \n"
 463                 "1:     lld     %1, %2          # atomic64_sub_if_positive\n"
 464                 "       dsubu   %0, %1, %3                              \n"
 465                 "       bltz    %0, 1f                                  \n"
 466                 "       scd     %0, %2                                  \n"
 467                 "       .set    noreorder                               \n"
 468                 "       beqz    %0, 1b                                  \n"
 469                 "        dsubu  %0, %1, %3                              \n"
 470                 "       .set    reorder                                 \n"
 471                 "1:                                                     \n"
 472                 "       .set    mips0                                   \n"
 473                 : "=&r" (result), "=&r" (temp),
 474                   "+" GCC_OFF12_ASM() (v->counter)
 475                 : "Ir" (i));
 476         } else {
 477                 unsigned long flags;
 478
 479                 raw_local_irq_save(flags);
 480                 result = v->counter;
 481                 result -= i;
 482                 if (result >= 0)
 483                         v->counter = result;
 484                 raw_local_irq_restore(flags);
 485         }
 486
 487         smp_llsc_mb();
 488
 489         return result;
 490 }
 491
 492 #define atomic64_cmpxchg(v, o, n) \
 493         ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n)))
 494 #define atomic64_xchg(v, new) (xchg(&((v)->counter), (new)))
 495
 496 /**
 497  * atomic64_add_unless - add unless the number is a given value
 498  * @v: pointer of type atomic64_t
 499  * @a: the amount to add to v...
 500  * @u: ...unless v is equal to u.
 501  *
 502  * Atomically adds @a to @v, so long as it was not @u.
 503  * Returns the old value of @v.
 504  */
 505 static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u)
 506 {
 507         long c, old;
 508         c = atomic64_read(v);
 509         for (;;) {
 510                 if (unlikely(c == (u)))
 511                         break;
 512                 old = atomic64_cmpxchg((v), c, c + (a));
 513                 if (likely(old == c))
 514                         break;
 515                 c = old;
 516         }
 517         return c != (u);
 518 }
 519
 520 #define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0)
 521
 522 #define atomic64_dec_return(v) atomic64_sub_return(1, (v))
 523 #define atomic64_inc_return(v) atomic64_add_return(1, (v))
 524
 525 /*
 526  * atomic64_sub_and_test - subtract value from variable and test result
 527  * @i: integer value to subtract
 528  * @v: pointer of type atomic64_t
 529  *
 530  * Atomically subtracts @i from @v and returns
 531  * true if the result is zero, or false for all
 532  * other cases.
 533  */
 534 #define atomic64_sub_and_test(i, v) (atomic64_sub_return((i), (v)) == 0)
 535
 536 /*
 537  * atomic64_inc_and_test - increment and test
 538  * @v: pointer of type atomic64_t
 539  *
 540  * Atomically increments @v by 1
 541  * and returns true if the result is zero, or false for all
 542  * other cases.
 543  */
 544 #define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0)
 545
 546 /*
 547  * atomic64_dec_and_test - decrement by 1 and test
 548  * @v: pointer of type atomic64_t
 549  *
 550  * Atomically decrements @v by 1 and
 551  * returns true if the result is 0, or false for all other
 552  * cases.
 553  */
 554 #define atomic64_dec_and_test(v) (atomic64_sub_return(1, (v)) == 0)
 555
 556 /*
 557  * atomic64_dec_if_positive - decrement by 1 if old value positive
 558  * @v: pointer of type atomic64_t
 559  */
 560 #define atomic64_dec_if_positive(v)     atomic64_sub_if_positive(1, v)
 561
 562 /*
 563  * atomic64_inc - increment atomic variable
 564  * @v: pointer of type atomic64_t
 565  *
 566  * Atomically increments @v by 1.
 567  */
 568 #define atomic64_inc(v) atomic64_add(1, (v))
 569
 570 /*
 571  * atomic64_dec - decrement and test
 572  * @v: pointer of type atomic64_t
 573  *
 574  * Atomically decrements @v by 1.
 575  */
 576 #define atomic64_dec(v) atomic64_sub(1, (v))
 577
 578 /*
 579  * atomic64_add_negative - add and test if negative
 580  * @v: pointer of type atomic64_t
 581  * @i: integer value to add
 582  *
 583  * Atomically adds @i to @v and returns true
 584  * if the result is negative, or false when
 585  * result is greater than or equal to zero.
 586  */
 587 #define atomic64_add_negative(i, v) (atomic64_add_return(i, (v)) < 0)
 588
 589 #endif /* CONFIG_64BIT */
 590
 591 #endif /* _ASM_ATOMIC_H */