arch/arc/include/asm/atomic.h

   1 /*
   2  * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  */
   8
   9 #ifndef _ASM_ARC_ATOMIC_H
  10 #define _ASM_ARC_ATOMIC_H
  11
  12 #ifndef __ASSEMBLY__
  13
  14 #include <linux/types.h>
  15 #include <linux/compiler.h>
  16 #include <asm/cmpxchg.h>
  17 #include <asm/barrier.h>
  18 #include <asm/smp.h>
  19
  20 #ifndef CONFIG_ARC_PLAT_EZNPS
  21
  22 #define atomic_read(v)  READ_ONCE((v)->counter)
  23 #define ATOMIC_INIT(i)  { (i) }
  24
  25 #ifdef CONFIG_ARC_HAS_LLSC
  26
  27 #define atomic_set(v, i) WRITE_ONCE(((v)->counter), (i))
  28
  29 #define ATOMIC_OP(op, c_op, asm_op)                                     \
  30 static inline void atomic_##op(int i, atomic_t *v)                      \
  31 {                                                                       \
  32         unsigned int val;                                               \
  33                                                                         \
  34         __asm__ __volatile__(                                           \
  35         "1:     llock   %[val], [%[ctr]]                \n"             \
  36         "       " #asm_op " %[val], %[val], %[i]        \n"             \
  37         "       scond   %[val], [%[ctr]]                \n"             \
  38         "       bnz     1b                              \n"             \
  39         : [val] "=&r"   (val) /* Early clobber to prevent reg reuse */  \
  40         : [ctr] "r"     (&v->counter), /* Not "m": llock only supports reg direct addr mode */  \
  41           [i]   "ir"    (i)                                             \
  42         : "cc");                                                        \
  43 }                                                                       \
  44
  45 #define ATOMIC_OP_RETURN(op, c_op, asm_op)                              \
  46 static inline int atomic_##op##_return(int i, atomic_t *v)              \
  47 {                                                                       \
  48         unsigned int val;                                               \
  49                                                                         \
  50         /*                                                              \
  51          * Explicit full memory barrier needed before/after as          \
  52          * LLOCK/SCOND thmeselves don't provide any such semantics      \
  53          */                                                             \
  54         smp_mb();                                                       \
  55                                                                         \
  56         __asm__ __volatile__(                                           \
  57         "1:     llock   %[val], [%[ctr]]                \n"             \
  58         "       " #asm_op " %[val], %[val], %[i]        \n"             \
  59         "       scond   %[val], [%[ctr]]                \n"             \
  60         "       bnz     1b                              \n"             \
  61         : [val] "=&r"   (val)                                           \
  62         : [ctr] "r"     (&v->counter),                                  \
  63           [i]   "ir"    (i)                                             \
  64         : "cc");                                                        \
  65                                                                         \
  66         smp_mb();                                                       \
  67                                                                         \
  68         return val;                                                     \
  69 }
  70
  71 #define ATOMIC_FETCH_OP(op, c_op, asm_op)                               \
  72 static inline int atomic_fetch_##op(int i, atomic_t *v)                 \
  73 {                                                                       \
  74         unsigned int val, orig;                                         \
  75                                                                         \
  76         /*                                                              \
  77          * Explicit full memory barrier needed before/after as          \
  78          * LLOCK/SCOND thmeselves don't provide any such semantics      \
  79          */                                                             \
  80         smp_mb();                                                       \
  81                                                                         \
  82         __asm__ __volatile__(                                           \
  83         "1:     llock   %[orig], [%[ctr]]               \n"             \
  84         "       " #asm_op " %[val], %[orig], %[i]       \n"             \
  85         "       scond   %[val], [%[ctr]]                \n"             \
  86         "                                               \n"             \
  87         : [val] "=&r"   (val),                                          \
  88           [orig] "=&r" (orig)                                           \
  89         : [ctr] "r"     (&v->counter),                                  \
  90           [i]   "ir"    (i)                                             \
  91         : "cc");                                                        \
  92                                                                         \
  93         smp_mb();                                                       \
  94                                                                         \
  95         return orig;                                                    \
  96 }
  97
  98 #else   /* !CONFIG_ARC_HAS_LLSC */
  99
 100 #ifndef CONFIG_SMP
 101
 102  /* violating atomic_xxx API locking protocol in UP for optimization sake */
 103 #define atomic_set(v, i) WRITE_ONCE(((v)->counter), (i))
 104
 105 #else
 106
 107 static inline void atomic_set(atomic_t *v, int i)
 108 {
 109         /*
 110          * Independent of hardware support, all of the atomic_xxx() APIs need
 111          * to follow the same locking rules to make sure that a "hardware"
 112          * atomic insn (e.g. LD) doesn't clobber an "emulated" atomic insn
 113          * sequence
 114          *
 115          * Thus atomic_set() despite being 1 insn (and seemingly atomic)
 116          * requires the locking.
 117          */
 118         unsigned long flags;
 119
 120         atomic_ops_lock(flags);
 121         WRITE_ONCE(v->counter, i);
 122         atomic_ops_unlock(flags);
 123 }
 124
 125 #endif
 126
 127 /*
 128  * Non hardware assisted Atomic-R-M-W
 129  * Locking would change to irq-disabling only (UP) and spinlocks (SMP)
 130  */
 131
 132 #define ATOMIC_OP(op, c_op, asm_op)                                     \
 133 static inline void atomic_##op(int i, atomic_t *v)                      \
 134 {                                                                       \
 135         unsigned long flags;                                            \
 136                                                                         \
 137         atomic_ops_lock(flags);                                         \
 138         v->counter c_op i;                                              \
 139         atomic_ops_unlock(flags);                                       \
 140 }
 141
 142 #define ATOMIC_OP_RETURN(op, c_op, asm_op)                              \
 143 static inline int atomic_##op##_return(int i, atomic_t *v)              \
 144 {                                                                       \
 145         unsigned long flags;                                            \
 146         unsigned long temp;                                             \
 147                                                                         \
 148         /*                                                              \
 149          * spin lock/unlock provides the needed smp_mb() before/after   \
 150          */                                                             \
 151         atomic_ops_lock(flags);                                         \
 152         temp = v->counter;                                              \
 153         temp c_op i;                                                    \
 154         v->counter = temp;                                              \
 155         atomic_ops_unlock(flags);                                       \
 156                                                                         \
 157         return temp;                                                    \
 158 }
 159
 160 #define ATOMIC_FETCH_OP(op, c_op, asm_op)                               \
 161 static inline int atomic_fetch_##op(int i, atomic_t *v)                 \
 162 {                                                                       \
 163         unsigned long flags;                                            \
 164         unsigned long orig;                                             \
 165                                                                         \
 166         /*                                                              \
 167          * spin lock/unlock provides the needed smp_mb() before/after   \
 168          */                                                             \
 169         atomic_ops_lock(flags);                                         \
 170         orig = v->counter;                                              \
 171         v->counter c_op i;                                              \
 172         atomic_ops_unlock(flags);                                       \
 173                                                                         \
 174         return orig;                                                    \
 175 }
 176
 177 #endif /* !CONFIG_ARC_HAS_LLSC */
 178
 179 #define ATOMIC_OPS(op, c_op, asm_op)                                    \
 180         ATOMIC_OP(op, c_op, asm_op)                                     \
 181         ATOMIC_OP_RETURN(op, c_op, asm_op)                              \
 182         ATOMIC_FETCH_OP(op, c_op, asm_op)
 183
 184 ATOMIC_OPS(add, +=, add)
 185 ATOMIC_OPS(sub, -=, sub)
 186
 187 #define atomic_andnot atomic_andnot
 188
 189 #undef ATOMIC_OPS
 190 #define ATOMIC_OPS(op, c_op, asm_op)                                    \
 191         ATOMIC_OP(op, c_op, asm_op)                                     \
 192         ATOMIC_FETCH_OP(op, c_op, asm_op)
 193
 194 ATOMIC_OPS(and, &=, and)
 195 ATOMIC_OPS(andnot, &= ~, bic)
 196 ATOMIC_OPS(or, |=, or)
 197 ATOMIC_OPS(xor, ^=, xor)
 198
 199 #else /* CONFIG_ARC_PLAT_EZNPS */
 200
 201 static inline int atomic_read(const atomic_t *v)
 202 {
 203         int temp;
 204
 205         __asm__ __volatile__(
 206         "       ld.di %0, [%1]"
 207         : "=r"(temp)
 208         : "r"(&v->counter)
 209         : "memory");
 210         return temp;
 211 }
 212
 213 static inline void atomic_set(atomic_t *v, int i)
 214 {
 215         __asm__ __volatile__(
 216         "       st.di %0,[%1]"
 217         :
 218         : "r"(i), "r"(&v->counter)
 219         : "memory");
 220 }
 221
 222 #define ATOMIC_OP(op, c_op, asm_op)                                     \
 223 static inline void atomic_##op(int i, atomic_t *v)                      \
 224 {                                                                       \
 225         __asm__ __volatile__(                                           \
 226         "       mov r2, %0\n"                                           \
 227         "       mov r3, %1\n"                                           \
 228         "       .word %2\n"                                             \
 229         :                                                               \
 230         : "r"(i), "r"(&v->counter), "i"(asm_op)                         \
 231         : "r2", "r3", "memory");                                        \
 232 }                                                                       \
 233
 234 #define ATOMIC_OP_RETURN(op, c_op, asm_op)                              \
 235 static inline int atomic_##op##_return(int i, atomic_t *v)              \
 236 {                                                                       \
 237         unsigned int temp = i;                                          \
 238                                                                         \
 239         /* Explicit full memory barrier needed before/after */          \
 240         smp_mb();                                                       \
 241                                                                         \
 242         __asm__ __volatile__(                                           \
 243         "       mov r2, %0\n"                                           \
 244         "       mov r3, %1\n"                                           \
 245         "       .word %2\n"                                             \
 246         "       mov %0, r2"                                             \
 247         : "+r"(temp)                                                    \
 248         : "r"(&v->counter), "i"(asm_op)                                 \
 249         : "r2", "r3", "memory");                                        \
 250                                                                         \
 251         smp_mb();                                                       \
 252                                                                         \
 253         temp c_op i;                                                    \
 254                                                                         \
 255         return temp;                                                    \
 256 }
 257
 258 #define ATOMIC_FETCH_OP(op, c_op, asm_op)                               \
 259 static inline int atomic_fetch_##op(int i, atomic_t *v)                 \
 260 {                                                                       \
 261         unsigned int temp = i;                                          \
 262                                                                         \
 263         /* Explicit full memory barrier needed before/after */          \
 264         smp_mb();                                                       \
 265                                                                         \
 266         __asm__ __volatile__(                                           \
 267         "       mov r2, %0\n"                                           \
 268         "       mov r3, %1\n"                                           \
 269         "       .word %2\n"                                             \
 270         "       mov %0, r2"                                             \
 271         : "+r"(temp)                                                    \
 272         : "r"(&v->counter), "i"(asm_op)                                 \
 273         : "r2", "r3", "memory");                                        \
 274                                                                         \
 275         smp_mb();                                                       \
 276                                                                         \
 277         return temp;                                                    \
 278 }
 279
 280 #define ATOMIC_OPS(op, c_op, asm_op)                                    \
 281         ATOMIC_OP(op, c_op, asm_op)                                     \
 282         ATOMIC_OP_RETURN(op, c_op, asm_op)                              \
 283         ATOMIC_FETCH_OP(op, c_op, asm_op)
 284
 285 ATOMIC_OPS(add, +=, CTOP_INST_AADD_DI_R2_R2_R3)
 286 #define atomic_sub(i, v) atomic_add(-(i), (v))
 287 #define atomic_sub_return(i, v) atomic_add_return(-(i), (v))
 288
 289 #undef ATOMIC_OPS
 290 #define ATOMIC_OPS(op, c_op, asm_op)                                    \
 291         ATOMIC_OP(op, c_op, asm_op)                                     \
 292         ATOMIC_FETCH_OP(op, c_op, asm_op)
 293
 294 ATOMIC_OPS(and, &=, CTOP_INST_AAND_DI_R2_R2_R3)
 295 #define atomic_andnot(mask, v) atomic_and(~(mask), (v))
 296 ATOMIC_OPS(or, |=, CTOP_INST_AOR_DI_R2_R2_R3)
 297 ATOMIC_OPS(xor, ^=, CTOP_INST_AXOR_DI_R2_R2_R3)
 298
 299 #endif /* CONFIG_ARC_PLAT_EZNPS */
 300
 301 #undef ATOMIC_OPS
 302 #undef ATOMIC_FETCH_OP
 303 #undef ATOMIC_OP_RETURN
 304 #undef ATOMIC_OP
 305
 306 /**
 307  * __atomic_add_unless - add unless the number is a given value
 308  * @v: pointer of type atomic_t
 309  * @a: the amount to add to v...
 310  * @u: ...unless v is equal to u.
 311  *
 312  * Atomically adds @a to @v, so long as it was not @u.
 313  * Returns the old value of @v
 314  */
 315 #define __atomic_add_unless(v, a, u)                                    \
 316 ({                                                                      \
 317         int c, old;                                                     \
 318                                                                         \
 319         /*                                                              \
 320          * Explicit full memory barrier needed before/after as          \
 321          * LLOCK/SCOND thmeselves don't provide any such semantics      \
 322          */                                                             \
 323         smp_mb();                                                       \
 324                                                                         \
 325         c = atomic_read(v);                                             \
 326         while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c)\
 327                 c = old;                                                \
 328                                                                         \
 329         smp_mb();                                                       \
 330                                                                         \
 331         c;                                                              \
 332 })
 333
 334 #define atomic_inc_not_zero(v)          atomic_add_unless((v), 1, 0)
 335
 336 #define atomic_inc(v)                   atomic_add(1, v)
 337 #define atomic_dec(v)                   atomic_sub(1, v)
 338
 339 #define atomic_inc_and_test(v)          (atomic_add_return(1, v) == 0)
 340 #define atomic_dec_and_test(v)          (atomic_sub_return(1, v) == 0)
 341 #define atomic_inc_return(v)            atomic_add_return(1, (v))
 342 #define atomic_dec_return(v)            atomic_sub_return(1, (v))
 343 #define atomic_sub_and_test(i, v)       (atomic_sub_return(i, v) == 0)
 344
 345 #define atomic_add_negative(i, v)       (atomic_add_return(i, v) < 0)
 346
 347
 348 #ifdef CONFIG_GENERIC_ATOMIC64
 349
 350 #include <asm-generic/atomic64.h>
 351
 352 #else   /* Kconfig ensures this is only enabled with needed h/w assist */
 353
 354 /*
 355  * ARCv2 supports 64-bit exclusive load (LLOCKD) / store (SCONDD)
 356  *  - The address HAS to be 64-bit aligned
 357  *  - There are 2 semantics involved here:
 358  *    = exclusive implies no interim update between load/store to same addr
 359  *    = both words are observed/updated together: this is guaranteed even
 360  *      for regular 64-bit load (LDD) / store (STD). Thus atomic64_set()
 361  *      is NOT required to use LLOCKD+SCONDD, STD suffices
 362  */
 363
 364 typedef struct {
 365         aligned_u64 counter;
 366 } atomic64_t;
 367
 368 #define ATOMIC64_INIT(a) { (a) }
 369
 370 static inline long long atomic64_read(const atomic64_t *v)
 371 {
 372         unsigned long long val;
 373
 374         __asm__ __volatile__(
 375         "       ldd   %0, [%1]  \n"
 376         : "=r"(val)
 377         : "r"(&v->counter));
 378
 379         return val;
 380 }
 381
 382 static inline void atomic64_set(atomic64_t *v, long long a)
 383 {
 384         /*
 385          * This could have been a simple assignment in "C" but would need
 386          * explicit volatile. Otherwise gcc optimizers could elide the store
 387          * which borked atomic64 self-test
 388          * In the inline asm version, memory clobber needed for exact same
 389          * reason, to tell gcc about the store.
 390          *
 391          * This however is not needed for sibling atomic64_add() etc since both
 392          * load/store are explicitly done in inline asm. As long as API is used
 393          * for each access, gcc has no way to optimize away any load/store
 394          */
 395         __asm__ __volatile__(
 396         "       std   %0, [%1]  \n"
 397         :
 398         : "r"(a), "r"(&v->counter)
 399         : "memory");
 400 }
 401
 402 #define ATOMIC64_OP(op, op1, op2)                                       \
 403 static inline void atomic64_##op(long long a, atomic64_t *v)            \
 404 {                                                                       \
 405         unsigned long long val;                                         \
 406                                                                         \
 407         __asm__ __volatile__(                                           \
 408         "1:                             \n"                             \
 409         "       llockd  %0, [%1]        \n"                             \
 410         "       " #op1 " %L0, %L0, %L2  \n"                             \
 411         "       " #op2 " %H0, %H0, %H2  \n"                             \
 412         "       scondd   %0, [%1]       \n"                             \
 413         "       bnz     1b              \n"                             \
 414         : "=&r"(val)                                                    \
 415         : "r"(&v->counter), "ir"(a)                                     \
 416         : "cc");                                                \
 417 }                                                                       \
 418
 419 #define ATOMIC64_OP_RETURN(op, op1, op2)                                \
 420 static inline long long atomic64_##op##_return(long long a, atomic64_t *v)      \
 421 {                                                                       \
 422         unsigned long long val;                                         \
 423                                                                         \
 424         smp_mb();                                                       \
 425                                                                         \
 426         __asm__ __volatile__(                                           \
 427         "1:                             \n"                             \
 428         "       llockd   %0, [%1]       \n"                             \
 429         "       " #op1 " %L0, %L0, %L2  \n"                             \
 430         "       " #op2 " %H0, %H0, %H2  \n"                             \
 431         "       scondd   %0, [%1]       \n"                             \
 432         "       bnz     1b              \n"                             \
 433         : [val] "=&r"(val)                                              \
 434         : "r"(&v->counter), "ir"(a)                                     \
 435         : "cc");        /* memory clobber comes from smp_mb() */        \
 436                                                                         \
 437         smp_mb();                                                       \
 438                                                                         \
 439         return val;                                                     \
 440 }
 441
 442 #define ATOMIC64_FETCH_OP(op, op1, op2)                                 \
 443 static inline long long atomic64_fetch_##op(long long a, atomic64_t *v) \
 444 {                                                                       \
 445         unsigned long long val, orig;                                   \
 446                                                                         \
 447         smp_mb();                                                       \
 448                                                                         \
 449         __asm__ __volatile__(                                           \
 450         "1:                             \n"                             \
 451         "       llockd   %0, [%2]       \n"                             \
 452         "       " #op1 " %L1, %L0, %L3  \n"                             \
 453         "       " #op2 " %H1, %H0, %H3  \n"                             \
 454         "       scondd   %1, [%2]       \n"                             \
 455         "       bnz     1b              \n"                             \
 456         : "=&r"(orig), "=&r"(val)                                       \
 457         : "r"(&v->counter), "ir"(a)                                     \
 458         : "cc");        /* memory clobber comes from smp_mb() */        \
 459                                                                         \
 460         smp_mb();                                                       \
 461                                                                         \
 462         return orig;                                                    \
 463 }
 464
 465 #define ATOMIC64_OPS(op, op1, op2)                                      \
 466         ATOMIC64_OP(op, op1, op2)                                       \
 467         ATOMIC64_OP_RETURN(op, op1, op2)                                \
 468         ATOMIC64_FETCH_OP(op, op1, op2)
 469
 470 #define atomic64_andnot atomic64_andnot
 471
 472 ATOMIC64_OPS(add, add.f, adc)
 473 ATOMIC64_OPS(sub, sub.f, sbc)
 474 ATOMIC64_OPS(and, and, and)
 475 ATOMIC64_OPS(andnot, bic, bic)
 476 ATOMIC64_OPS(or, or, or)
 477 ATOMIC64_OPS(xor, xor, xor)
 478
 479 #undef ATOMIC64_OPS
 480 #undef ATOMIC64_FETCH_OP
 481 #undef ATOMIC64_OP_RETURN
 482 #undef ATOMIC64_OP
 483
 484 static inline long long
 485 atomic64_cmpxchg(atomic64_t *ptr, long long expected, long long new)
 486 {
 487         long long prev;
 488
 489         smp_mb();
 490
 491         __asm__ __volatile__(
 492         "1:     llockd  %0, [%1]        \n"
 493         "       brne    %L0, %L2, 2f    \n"
 494         "       brne    %H0, %H2, 2f    \n"
 495         "       scondd  %3, [%1]        \n"
 496         "       bnz     1b              \n"
 497         "2:                             \n"
 498         : "=&r"(prev)
 499         : "r"(ptr), "ir"(expected), "r"(new)
 500         : "cc");        /* memory clobber comes from smp_mb() */
 501
 502         smp_mb();
 503
 504         return prev;
 505 }
 506
 507 static inline long long atomic64_xchg(atomic64_t *ptr, long long new)
 508 {
 509         long long prev;
 510
 511         smp_mb();
 512
 513         __asm__ __volatile__(
 514         "1:     llockd  %0, [%1]        \n"
 515         "       scondd  %2, [%1]        \n"
 516         "       bnz     1b              \n"
 517         "2:                             \n"
 518         : "=&r"(prev)
 519         : "r"(ptr), "r"(new)
 520         : "cc");        /* memory clobber comes from smp_mb() */
 521
 522         smp_mb();
 523
 524         return prev;
 525 }
 526
 527 /**
 528  * atomic64_dec_if_positive - decrement by 1 if old value positive
 529  * @v: pointer of type atomic64_t
 530  *
 531  * The function returns the old value of *v minus 1, even if
 532  * the atomic variable, v, was not decremented.
 533  */
 534
 535 static inline long long atomic64_dec_if_positive(atomic64_t *v)
 536 {
 537         long long val;
 538
 539         smp_mb();
 540
 541         __asm__ __volatile__(
 542         "1:     llockd  %0, [%1]        \n"
 543         "       sub.f   %L0, %L0, 1     # w0 - 1, set C on borrow\n"
 544         "       sub.c   %H0, %H0, 1     # if C set, w1 - 1\n"
 545         "       brlt    %H0, 0, 2f      \n"
 546         "       scondd  %0, [%1]        \n"
 547         "       bnz     1b              \n"
 548         "2:                             \n"
 549         : "=&r"(val)
 550         : "r"(&v->counter)
 551         : "cc");        /* memory clobber comes from smp_mb() */
 552
 553         smp_mb();
 554
 555         return val;
 556 }
 557
 558 /**
 559  * atomic64_add_unless - add unless the number is a given value
 560  * @v: pointer of type atomic64_t
 561  * @a: the amount to add to v...
 562  * @u: ...unless v is equal to u.
 563  *
 564  * if (v != u) { v += a; ret = 1} else {ret = 0}
 565  * Returns 1 iff @v was not @u (i.e. if add actually happened)
 566  */
 567 static inline int atomic64_add_unless(atomic64_t *v, long long a, long long u)
 568 {
 569         long long val;
 570         int op_done;
 571
 572         smp_mb();
 573
 574         __asm__ __volatile__(
 575         "1:     llockd  %0, [%2]        \n"
 576         "       mov     %1, 1           \n"
 577         "       brne    %L0, %L4, 2f    # continue to add since v != u \n"
 578         "       breq.d  %H0, %H4, 3f    # return since v == u \n"
 579         "       mov     %1, 0           \n"
 580         "2:                             \n"
 581         "       add.f   %L0, %L0, %L3   \n"
 582         "       adc     %H0, %H0, %H3   \n"
 583         "       scondd  %0, [%2]        \n"
 584         "       bnz     1b              \n"
 585         "3:                             \n"
 586         : "=&r"(val), "=&r" (op_done)
 587         : "r"(&v->counter), "r"(a), "r"(u)
 588         : "cc");        /* memory clobber comes from smp_mb() */
 589
 590         smp_mb();
 591
 592         return op_done;
 593 }
 594
 595 #define atomic64_add_negative(a, v)     (atomic64_add_return((a), (v)) < 0)
 596 #define atomic64_inc(v)                 atomic64_add(1LL, (v))
 597 #define atomic64_inc_return(v)          atomic64_add_return(1LL, (v))
 598 #define atomic64_inc_and_test(v)        (atomic64_inc_return(v) == 0)
 599 #define atomic64_sub_and_test(a, v)     (atomic64_sub_return((a), (v)) == 0)
 600 #define atomic64_dec(v)                 atomic64_sub(1LL, (v))
 601 #define atomic64_dec_return(v)          atomic64_sub_return(1LL, (v))
 602 #define atomic64_dec_and_test(v)        (atomic64_dec_return((v)) == 0)
 603 #define atomic64_inc_not_zero(v)        atomic64_add_unless((v), 1LL, 0LL)
 604
 605 #endif  /* !CONFIG_GENERIC_ATOMIC64 */
 606
 607 #endif  /* !__ASSEMBLY__ */
 608
 609 #endif