arch/arm64/include/asm/atomic_lse.h

   1 /*
   2  * Based on arch/arm/include/asm/atomic.h
   3  *
   4  * Copyright (C) 1996 Russell King.
   5  * Copyright (C) 2002 Deep Blue Solutions Ltd.
   6  * Copyright (C) 2012 ARM Ltd.
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License version 2 as
  10  * published by the Free Software Foundation.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19  */
  20
  21 #ifndef __ASM_ATOMIC_LSE_H
  22 #define __ASM_ATOMIC_LSE_H
  23
  24 #ifndef __ARM64_IN_ATOMIC_IMPL
  25 #error "please don't include this file directly"
  26 #endif
  27
  28 #define __LL_SC_ATOMIC(op)      __LL_SC_CALL(atomic_##op)
  29 #define ATOMIC_OP(op, asm_op)                                           \
  30 static inline void atomic_##op(int i, atomic_t *v)                      \
  31 {                                                                       \
  32         register int w0 asm ("w0") = i;                                 \
  33         register atomic_t *x1 asm ("x1") = v;                           \
  34                                                                         \
  35         asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(op),          \
  36 "       " #asm_op "     %w[i], %[v]\n")                                 \
  37         : [i] "+r" (w0), [v] "+Q" (v->counter)                          \
  38         : "r" (x1)                                                      \
  39         : __LL_SC_CLOBBERS);                                            \
  40 }
  41
  42 ATOMIC_OP(andnot, stclr)
  43 ATOMIC_OP(or, stset)
  44 ATOMIC_OP(xor, steor)
  45 ATOMIC_OP(add, stadd)
  46
  47 #undef ATOMIC_OP
  48
  49 #define ATOMIC_OP_ADD_RETURN(name, mb, cl...)                           \
  50 static inline int atomic_add_return##name(int i, atomic_t *v)           \
  51 {                                                                       \
  52         register int w0 asm ("w0") = i;                                 \
  53         register atomic_t *x1 asm ("x1") = v;                           \
  54                                                                         \
  55         asm volatile(ARM64_LSE_ATOMIC_INSN(                             \
  56         /* LL/SC */                                                     \
  57         "       nop\n"                                                  \
  58         __LL_SC_ATOMIC(add_return##name),                               \
  59         /* LSE atomics */                                               \
  60         "       ldadd" #mb "    %w[i], w30, %[v]\n"                     \
  61         "       add     %w[i], %w[i], w30")                             \
  62         : [i] "+r" (w0), [v] "+Q" (v->counter)                          \
  63         : "r" (x1)                                                      \
  64         : __LL_SC_CLOBBERS, ##cl);                                      \
  65                                                                         \
  66         return w0;                                                      \
  67 }
  68
  69 ATOMIC_OP_ADD_RETURN(_relaxed,   )
  70 ATOMIC_OP_ADD_RETURN(_acquire,  a, "memory")
  71 ATOMIC_OP_ADD_RETURN(_release,  l, "memory")
  72 ATOMIC_OP_ADD_RETURN(        , al, "memory")
  73
  74 #undef ATOMIC_OP_ADD_RETURN
  75
  76 static inline void atomic_and(int i, atomic_t *v)
  77 {
  78         register int w0 asm ("w0") = i;
  79         register atomic_t *x1 asm ("x1") = v;
  80
  81         asm volatile(ARM64_LSE_ATOMIC_INSN(
  82         /* LL/SC */
  83         "       nop\n"
  84         __LL_SC_ATOMIC(and),
  85         /* LSE atomics */
  86         "       mvn     %w[i], %w[i]\n"
  87         "       stclr   %w[i], %[v]")
  88         : [i] "+r" (w0), [v] "+Q" (v->counter)
  89         : "r" (x1)
  90         : __LL_SC_CLOBBERS);
  91 }
  92
  93 static inline void atomic_sub(int i, atomic_t *v)
  94 {
  95         register int w0 asm ("w0") = i;
  96         register atomic_t *x1 asm ("x1") = v;
  97
  98         asm volatile(ARM64_LSE_ATOMIC_INSN(
  99         /* LL/SC */
 100         "       nop\n"
 101         __LL_SC_ATOMIC(sub),
 102         /* LSE atomics */
 103         "       neg     %w[i], %w[i]\n"
 104         "       stadd   %w[i], %[v]")
 105         : [i] "+r" (w0), [v] "+Q" (v->counter)
 106         : "r" (x1)
 107         : __LL_SC_CLOBBERS);
 108 }
 109
 110 #define ATOMIC_OP_SUB_RETURN(name, mb, cl...)                           \
 111 static inline int atomic_sub_return##name(int i, atomic_t *v)           \
 112 {                                                                       \
 113         register int w0 asm ("w0") = i;                                 \
 114         register atomic_t *x1 asm ("x1") = v;                           \
 115                                                                         \
 116         asm volatile(ARM64_LSE_ATOMIC_INSN(                             \
 117         /* LL/SC */                                                     \
 118         "       nop\n"                                                  \
 119         __LL_SC_ATOMIC(sub_return##name)                                \
 120         "       nop",                                                   \
 121         /* LSE atomics */                                               \
 122         "       neg     %w[i], %w[i]\n"                                 \
 123         "       ldadd" #mb "    %w[i], w30, %[v]\n"                     \
 124         "       add     %w[i], %w[i], w30")                             \
 125         : [i] "+r" (w0), [v] "+Q" (v->counter)                          \
 126         : "r" (x1)                                                      \
 127         : __LL_SC_CLOBBERS , ##cl);                                     \
 128                                                                         \
 129         return w0;                                                      \
 130 }
 131
 132 ATOMIC_OP_SUB_RETURN(_relaxed,   )
 133 ATOMIC_OP_SUB_RETURN(_acquire,  a, "memory")
 134 ATOMIC_OP_SUB_RETURN(_release,  l, "memory")
 135 ATOMIC_OP_SUB_RETURN(        , al, "memory")
 136
 137 #undef ATOMIC_OP_SUB_RETURN
 138 #undef __LL_SC_ATOMIC
 139
 140 #define __LL_SC_ATOMIC64(op)    __LL_SC_CALL(atomic64_##op)
 141 #define ATOMIC64_OP(op, asm_op)                                         \
 142 static inline void atomic64_##op(long i, atomic64_t *v)                 \
 143 {                                                                       \
 144         register long x0 asm ("x0") = i;                                \
 145         register atomic64_t *x1 asm ("x1") = v;                         \
 146                                                                         \
 147         asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(op),        \
 148 "       " #asm_op "     %[i], %[v]\n")                                  \
 149         : [i] "+r" (x0), [v] "+Q" (v->counter)                          \
 150         : "r" (x1)                                                      \
 151         : __LL_SC_CLOBBERS);                                            \
 152 }
 153
 154 ATOMIC64_OP(andnot, stclr)
 155 ATOMIC64_OP(or, stset)
 156 ATOMIC64_OP(xor, steor)
 157 ATOMIC64_OP(add, stadd)
 158
 159 #undef ATOMIC64_OP
 160
 161 #define ATOMIC64_OP_ADD_RETURN(name, mb, cl...)                         \
 162 static inline long atomic64_add_return##name(long i, atomic64_t *v)     \
 163 {                                                                       \
 164         register long x0 asm ("x0") = i;                                \
 165         register atomic64_t *x1 asm ("x1") = v;                         \
 166                                                                         \
 167         asm volatile(ARM64_LSE_ATOMIC_INSN(                             \
 168         /* LL/SC */                                                     \
 169         "       nop\n"                                                  \
 170         __LL_SC_ATOMIC64(add_return##name),                             \
 171         /* LSE atomics */                                               \
 172         "       ldadd" #mb "    %[i], x30, %[v]\n"                      \
 173         "       add     %[i], %[i], x30")                               \
 174         : [i] "+r" (x0), [v] "+Q" (v->counter)                          \
 175         : "r" (x1)                                                      \
 176         : __LL_SC_CLOBBERS, ##cl);                                      \
 177                                                                         \
 178         return x0;                                                      \
 179 }
 180
 181 ATOMIC64_OP_ADD_RETURN(_relaxed,   )
 182 ATOMIC64_OP_ADD_RETURN(_acquire,  a, "memory")
 183 ATOMIC64_OP_ADD_RETURN(_release,  l, "memory")
 184 ATOMIC64_OP_ADD_RETURN(        , al, "memory")
 185
 186 #undef ATOMIC64_OP_ADD_RETURN
 187
 188 static inline void atomic64_and(long i, atomic64_t *v)
 189 {
 190         register long x0 asm ("x0") = i;
 191         register atomic64_t *x1 asm ("x1") = v;
 192
 193         asm volatile(ARM64_LSE_ATOMIC_INSN(
 194         /* LL/SC */
 195         "       nop\n"
 196         __LL_SC_ATOMIC64(and),
 197         /* LSE atomics */
 198         "       mvn     %[i], %[i]\n"
 199         "       stclr   %[i], %[v]")
 200         : [i] "+r" (x0), [v] "+Q" (v->counter)
 201         : "r" (x1)
 202         : __LL_SC_CLOBBERS);
 203 }
 204
 205 static inline void atomic64_sub(long i, atomic64_t *v)
 206 {
 207         register long x0 asm ("x0") = i;
 208         register atomic64_t *x1 asm ("x1") = v;
 209
 210         asm volatile(ARM64_LSE_ATOMIC_INSN(
 211         /* LL/SC */
 212         "       nop\n"
 213         __LL_SC_ATOMIC64(sub),
 214         /* LSE atomics */
 215         "       neg     %[i], %[i]\n"
 216         "       stadd   %[i], %[v]")
 217         : [i] "+r" (x0), [v] "+Q" (v->counter)
 218         : "r" (x1)
 219         : __LL_SC_CLOBBERS);
 220 }
 221
 222 #define ATOMIC64_OP_SUB_RETURN(name, mb, cl...)                         \
 223 static inline long atomic64_sub_return##name(long i, atomic64_t *v)     \
 224 {                                                                       \
 225         register long x0 asm ("x0") = i;                                \
 226         register atomic64_t *x1 asm ("x1") = v;                         \
 227                                                                         \
 228         asm volatile(ARM64_LSE_ATOMIC_INSN(                             \
 229         /* LL/SC */                                                     \
 230         "       nop\n"                                                  \
 231         __LL_SC_ATOMIC64(sub_return##name)                              \
 232         "       nop",                                                   \
 233         /* LSE atomics */                                               \
 234         "       neg     %[i], %[i]\n"                                   \
 235         "       ldadd" #mb "    %[i], x30, %[v]\n"                      \
 236         "       add     %[i], %[i], x30")                               \
 237         : [i] "+r" (x0), [v] "+Q" (v->counter)                          \
 238         : "r" (x1)                                                      \
 239         : __LL_SC_CLOBBERS, ##cl);                                      \
 240                                                                         \
 241         return x0;                                                      \
 242 }
 243
 244 ATOMIC64_OP_SUB_RETURN(_relaxed,   )
 245 ATOMIC64_OP_SUB_RETURN(_acquire,  a, "memory")
 246 ATOMIC64_OP_SUB_RETURN(_release,  l, "memory")
 247 ATOMIC64_OP_SUB_RETURN(        , al, "memory")
 248
 249 #undef ATOMIC64_OP_SUB_RETURN
 250
 251 static inline long atomic64_dec_if_positive(atomic64_t *v)
 252 {
 253         register long x0 asm ("x0") = (long)v;
 254
 255         asm volatile(ARM64_LSE_ATOMIC_INSN(
 256         /* LL/SC */
 257         "       nop\n"
 258         __LL_SC_ATOMIC64(dec_if_positive)
 259         "       nop\n"
 260         "       nop\n"
 261         "       nop\n"
 262         "       nop\n"
 263         "       nop",
 264         /* LSE atomics */
 265         "1:     ldr     x30, %[v]\n"
 266         "       subs    %[ret], x30, #1\n"
 267         "       b.lt    2f\n"
 268         "       casal   x30, %[ret], %[v]\n"
 269         "       sub     x30, x30, #1\n"
 270         "       sub     x30, x30, %[ret]\n"
 271         "       cbnz    x30, 1b\n"
 272         "2:")
 273         : [ret] "+&r" (x0), [v] "+Q" (v->counter)
 274         :
 275         : __LL_SC_CLOBBERS, "cc", "memory");
 276
 277         return x0;
 278 }
 279
 280 #undef __LL_SC_ATOMIC64
 281
 282 #define __LL_SC_CMPXCHG(op)     __LL_SC_CALL(__cmpxchg_case_##op)
 283
 284 #define __CMPXCHG_CASE(w, sz, name, mb, cl...)                          \
 285 static inline unsigned long __cmpxchg_case_##name(volatile void *ptr,   \
 286                                                   unsigned long old,    \
 287                                                   unsigned long new)    \
 288 {                                                                       \
 289         register unsigned long x0 asm ("x0") = (unsigned long)ptr;      \
 290         register unsigned long x1 asm ("x1") = old;                     \
 291         register unsigned long x2 asm ("x2") = new;                     \
 292                                                                         \
 293         asm volatile(ARM64_LSE_ATOMIC_INSN(                             \
 294         /* LL/SC */                                                     \
 295         "       nop\n"                                                  \
 296                 __LL_SC_CMPXCHG(name)                                   \
 297         "       nop",                                                   \
 298         /* LSE atomics */                                               \
 299         "       mov     " #w "30, %" #w "[old]\n"                       \
 300         "       cas" #mb #sz "\t" #w "30, %" #w "[new], %[v]\n"         \
 301         "       mov     %" #w "[ret], " #w "30")                        \
 302         : [ret] "+r" (x0), [v] "+Q" (*(unsigned long *)ptr)             \
 303         : [old] "r" (x1), [new] "r" (x2)                                \
 304         : __LL_SC_CLOBBERS, ##cl);                                      \
 305                                                                         \
 306         return x0;                                                      \
 307 }
 308
 309 __CMPXCHG_CASE(w, b,     1,   )
 310 __CMPXCHG_CASE(w, h,     2,   )
 311 __CMPXCHG_CASE(w,  ,     4,   )
 312 __CMPXCHG_CASE(x,  ,     8,   )
 313 __CMPXCHG_CASE(w, b, acq_1,  a, "memory")
 314 __CMPXCHG_CASE(w, h, acq_2,  a, "memory")
 315 __CMPXCHG_CASE(w,  , acq_4,  a, "memory")
 316 __CMPXCHG_CASE(x,  , acq_8,  a, "memory")
 317 __CMPXCHG_CASE(w, b, rel_1,  l, "memory")
 318 __CMPXCHG_CASE(w, h, rel_2,  l, "memory")
 319 __CMPXCHG_CASE(w,  , rel_4,  l, "memory")
 320 __CMPXCHG_CASE(x,  , rel_8,  l, "memory")
 321 __CMPXCHG_CASE(w, b,  mb_1, al, "memory")
 322 __CMPXCHG_CASE(w, h,  mb_2, al, "memory")
 323 __CMPXCHG_CASE(w,  ,  mb_4, al, "memory")
 324 __CMPXCHG_CASE(x,  ,  mb_8, al, "memory")
 325
 326 #undef __LL_SC_CMPXCHG
 327 #undef __CMPXCHG_CASE
 328
 329 #define __LL_SC_CMPXCHG_DBL(op) __LL_SC_CALL(__cmpxchg_double##op)
 330
 331 #define __CMPXCHG_DBL(name, mb, cl...)                                  \
 332 static inline long __cmpxchg_double##name(unsigned long old1,           \
 333                                          unsigned long old2,            \
 334                                          unsigned long new1,            \
 335                                          unsigned long new2,            \
 336                                          volatile void *ptr)            \
 337 {                                                                       \
 338         unsigned long oldval1 = old1;                                   \
 339         unsigned long oldval2 = old2;                                   \
 340         register unsigned long x0 asm ("x0") = old1;                    \
 341         register unsigned long x1 asm ("x1") = old2;                    \
 342         register unsigned long x2 asm ("x2") = new1;                    \
 343         register unsigned long x3 asm ("x3") = new2;                    \
 344         register unsigned long x4 asm ("x4") = (unsigned long)ptr;      \
 345                                                                         \
 346         asm volatile(ARM64_LSE_ATOMIC_INSN(                             \
 347         /* LL/SC */                                                     \
 348         "       nop\n"                                                  \
 349         "       nop\n"                                                  \
 350         "       nop\n"                                                  \
 351         __LL_SC_CMPXCHG_DBL(name),                                      \
 352         /* LSE atomics */                                               \
 353         "       casp" #mb "\t%[old1], %[old2], %[new1], %[new2], %[v]\n"\
 354         "       eor     %[old1], %[old1], %[oldval1]\n"                 \
 355         "       eor     %[old2], %[old2], %[oldval2]\n"                 \
 356         "       orr     %[old1], %[old1], %[old2]")                     \
 357         : [old1] "+r" (x0), [old2] "+r" (x1),                           \
 358           [v] "+Q" (*(unsigned long *)ptr)                              \
 359         : [new1] "r" (x2), [new2] "r" (x3), [ptr] "r" (x4),             \
 360           [oldval1] "r" (oldval1), [oldval2] "r" (oldval2)              \
 361         : __LL_SC_CLOBBERS, ##cl);                                      \
 362                                                                         \
 363         return x0;                                                      \
 364 }
 365
 366 __CMPXCHG_DBL(   ,   )
 367 __CMPXCHG_DBL(_mb, al, "memory")
 368
 369 #undef __LL_SC_CMPXCHG_DBL
 370 #undef __CMPXCHG_DBL
 371
 372 #endif  /* __ASM_ATOMIC_LSE_H */