arch/mips/include/asm/uaccess.h

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * Copyright (C) 1996, 1997, 1998, 1999, 2000, 03, 04 by Ralf Baechle
   7  * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
   8  * Copyright (C) 2007  Maciej W. Rozycki
   9  * Copyright (C) 2014, Imagination Technologies Ltd.
  10  */
  11 #ifndef _ASM_UACCESS_H
  12 #define _ASM_UACCESS_H
  13
  14 #include <linux/kernel.h>
  15 #include <linux/errno.h>
  16 #include <linux/thread_info.h>
  17 #include <linux/string.h>
  18 #include <asm/asm-eva.h>
  19
  20 /*
  21  * The fs value determines whether argument validity checking should be
  22  * performed or not.  If get_fs() == USER_DS, checking is performed, with
  23  * get_fs() == KERNEL_DS, checking is bypassed.
  24  *
  25  * For historical reasons, these macros are grossly misnamed.
  26  */
  27 #ifdef CONFIG_32BIT
  28
  29 #ifdef CONFIG_KVM_GUEST
  30 #define __UA_LIMIT 0x40000000UL
  31 #else
  32 #define __UA_LIMIT 0x80000000UL
  33 #endif
  34
  35 #define __UA_ADDR       ".word"
  36 #define __UA_LA         "la"
  37 #define __UA_ADDU       "addu"
  38 #define __UA_t0         "$8"
  39 #define __UA_t1         "$9"
  40
  41 #endif /* CONFIG_32BIT */
  42
  43 #ifdef CONFIG_64BIT
  44
  45 extern u64 __ua_limit;
  46
  47 #define __UA_LIMIT      __ua_limit
  48
  49 #define __UA_ADDR       ".dword"
  50 #define __UA_LA         "dla"
  51 #define __UA_ADDU       "daddu"
  52 #define __UA_t0         "$12"
  53 #define __UA_t1         "$13"
  54
  55 #endif /* CONFIG_64BIT */
  56
  57 /*
  58  * USER_DS is a bitmask that has the bits set that may not be set in a valid
  59  * userspace address.  Note that we limit 32-bit userspace to 0x7fff8000 but
  60  * the arithmetic we're doing only works if the limit is a power of two, so
  61  * we use 0x80000000 here on 32-bit kernels.  If a process passes an invalid
  62  * address in this range it's the process's problem, not ours :-)
  63  */
  64
  65 #ifdef CONFIG_KVM_GUEST
  66 #define KERNEL_DS       ((mm_segment_t) { 0x80000000UL })
  67 #define USER_DS         ((mm_segment_t) { 0xC0000000UL })
  68 #else
  69 #define KERNEL_DS       ((mm_segment_t) { 0UL })
  70 #define USER_DS         ((mm_segment_t) { __UA_LIMIT })
  71 #endif
  72
  73 #define VERIFY_READ    0
  74 #define VERIFY_WRITE   1
  75
  76 #define get_ds()        (KERNEL_DS)
  77 #define get_fs()        (current_thread_info()->addr_limit)
  78 #define set_fs(x)       (current_thread_info()->addr_limit = (x))
  79
  80 #define segment_eq(a, b)        ((a).seg == (b).seg)
  81
  82 /*
  83  * eva_kernel_access() - determine whether kernel memory access on an EVA system
  84  *
  85  * Determines whether memory accesses should be performed to kernel memory
  86  * on a system using Extended Virtual Addressing (EVA).
  87  *
  88  * Return: true if a kernel memory access on an EVA system, else false.
  89  */
  90 static inline bool eva_kernel_access(void)
  91 {
  92         if (!IS_ENABLED(CONFIG_EVA))
  93                 return false;
  94
  95         return segment_eq(get_fs(), get_ds());
  96 }
  97
  98 /*
  99  * Is a address valid? This does a straightforward calculation rather
 100  * than tests.
 101  *
 102  * Address valid if:
 103  *  - "addr" doesn't have any high-bits set
 104  *  - AND "size" doesn't have any high-bits set
 105  *  - AND "addr+size" doesn't have any high-bits set
 106  *  - OR we are in kernel mode.
 107  *
 108  * __ua_size() is a trick to avoid runtime checking of positive constant
 109  * sizes; for those we already know at compile time that the size is ok.
 110  */
 111 #define __ua_size(size)                                                 \
 112         ((__builtin_constant_p(size) && (signed long) (size) > 0) ? 0 : (size))
 113
 114 /*
 115  * access_ok: - Checks if a user space pointer is valid
 116  * @type: Type of access: %VERIFY_READ or %VERIFY_WRITE.  Note that
 117  *        %VERIFY_WRITE is a superset of %VERIFY_READ - if it is safe
 118  *        to write to a block, it is always safe to read from it.
 119  * @addr: User space pointer to start of block to check
 120  * @size: Size of block to check
 121  *
 122  * Context: User context only. This function may sleep if pagefaults are
 123  *          enabled.
 124  *
 125  * Checks if a pointer to a block of memory in user space is valid.
 126  *
 127  * Returns true (nonzero) if the memory block may be valid, false (zero)
 128  * if it is definitely invalid.
 129  *
 130  * Note that, depending on architecture, this function probably just
 131  * checks that the pointer is in the user space range - after calling
 132  * this function, memory access functions may still return -EFAULT.
 133  */
 134
 135 #define __access_mask get_fs().seg
 136
 137 #define __access_ok(addr, size, mask)                                   \
 138 ({                                                                      \
 139         unsigned long __addr = (unsigned long) (addr);                  \
 140         unsigned long __size = size;                                    \
 141         unsigned long __mask = mask;                                    \
 142         unsigned long __ok;                                             \
 143                                                                         \
 144         __chk_user_ptr(addr);                                           \
 145         __ok = (signed long)(__mask & (__addr | (__addr + __size) |     \
 146                 __ua_size(__size)));                                    \
 147         __ok == 0;                                                      \
 148 })
 149
 150 #define access_ok(type, addr, size)                                     \
 151         likely(__access_ok((addr), (size), __access_mask))
 152
 153 /*
 154  * put_user: - Write a simple value into user space.
 155  * @x:   Value to copy to user space.
 156  * @ptr: Destination address, in user space.
 157  *
 158  * Context: User context only. This function may sleep if pagefaults are
 159  *          enabled.
 160  *
 161  * This macro copies a single simple value from kernel space to user
 162  * space.  It supports simple types like char and int, but not larger
 163  * data types like structures or arrays.
 164  *
 165  * @ptr must have pointer-to-simple-variable type, and @x must be assignable
 166  * to the result of dereferencing @ptr.
 167  *
 168  * Returns zero on success, or -EFAULT on error.
 169  */
 170 #define put_user(x,ptr) \
 171         __put_user_check((x), (ptr), sizeof(*(ptr)))
 172
 173 /*
 174  * get_user: - Get a simple variable from user space.
 175  * @x:   Variable to store result.
 176  * @ptr: Source address, in user space.
 177  *
 178  * Context: User context only. This function may sleep if pagefaults are
 179  *          enabled.
 180  *
 181  * This macro copies a single simple variable from user space to kernel
 182  * space.  It supports simple types like char and int, but not larger
 183  * data types like structures or arrays.
 184  *
 185  * @ptr must have pointer-to-simple-variable type, and the result of
 186  * dereferencing @ptr must be assignable to @x without a cast.
 187  *
 188  * Returns zero on success, or -EFAULT on error.
 189  * On error, the variable @x is set to zero.
 190  */
 191 #define get_user(x,ptr) \
 192         __get_user_check((x), (ptr), sizeof(*(ptr)))
 193
 194 /*
 195  * __put_user: - Write a simple value into user space, with less checking.
 196  * @x:   Value to copy to user space.
 197  * @ptr: Destination address, in user space.
 198  *
 199  * Context: User context only. This function may sleep if pagefaults are
 200  *          enabled.
 201  *
 202  * This macro copies a single simple value from kernel space to user
 203  * space.  It supports simple types like char and int, but not larger
 204  * data types like structures or arrays.
 205  *
 206  * @ptr must have pointer-to-simple-variable type, and @x must be assignable
 207  * to the result of dereferencing @ptr.
 208  *
 209  * Caller must check the pointer with access_ok() before calling this
 210  * function.
 211  *
 212  * Returns zero on success, or -EFAULT on error.
 213  */
 214 #define __put_user(x,ptr) \
 215         __put_user_nocheck((x), (ptr), sizeof(*(ptr)))
 216
 217 /*
 218  * __get_user: - Get a simple variable from user space, with less checking.
 219  * @x:   Variable to store result.
 220  * @ptr: Source address, in user space.
 221  *
 222  * Context: User context only. This function may sleep if pagefaults are
 223  *          enabled.
 224  *
 225  * This macro copies a single simple variable from user space to kernel
 226  * space.  It supports simple types like char and int, but not larger
 227  * data types like structures or arrays.
 228  *
 229  * @ptr must have pointer-to-simple-variable type, and the result of
 230  * dereferencing @ptr must be assignable to @x without a cast.
 231  *
 232  * Caller must check the pointer with access_ok() before calling this
 233  * function.
 234  *
 235  * Returns zero on success, or -EFAULT on error.
 236  * On error, the variable @x is set to zero.
 237  */
 238 #define __get_user(x,ptr) \
 239         __get_user_nocheck((x), (ptr), sizeof(*(ptr)))
 240
 241 struct __large_struct { unsigned long buf[100]; };
 242 #define __m(x) (*(struct __large_struct __user *)(x))
 243
 244 /*
 245  * Yuck.  We need two variants, one for 64bit operation and one
 246  * for 32 bit mode and old iron.
 247  */
 248 #ifndef CONFIG_EVA
 249 #define __get_kernel_common(val, size, ptr) __get_user_common(val, size, ptr)
 250 #else
 251 /*
 252  * Kernel specific functions for EVA. We need to use normal load instructions
 253  * to read data from kernel when operating in EVA mode. We use these macros to
 254  * avoid redefining __get_user_asm for EVA.
 255  */
 256 #undef _loadd
 257 #undef _loadw
 258 #undef _loadh
 259 #undef _loadb
 260 #ifdef CONFIG_32BIT
 261 #define _loadd                  _loadw
 262 #else
 263 #define _loadd(reg, addr)       "ld " reg ", " addr
 264 #endif
 265 #define _loadw(reg, addr)       "lw " reg ", " addr
 266 #define _loadh(reg, addr)       "lh " reg ", " addr
 267 #define _loadb(reg, addr)       "lb " reg ", " addr
 268
 269 #define __get_kernel_common(val, size, ptr)                             \
 270 do {                                                                    \
 271         switch (size) {                                                 \
 272         case 1: __get_data_asm(val, _loadb, ptr); break;                \
 273         case 2: __get_data_asm(val, _loadh, ptr); break;                \
 274         case 4: __get_data_asm(val, _loadw, ptr); break;                \
 275         case 8: __GET_DW(val, _loadd, ptr); break;                      \
 276         default: __get_user_unknown(); break;                           \
 277         }                                                               \
 278 } while (0)
 279 #endif
 280
 281 #ifdef CONFIG_32BIT
 282 #define __GET_DW(val, insn, ptr) __get_data_asm_ll32(val, insn, ptr)
 283 #endif
 284 #ifdef CONFIG_64BIT
 285 #define __GET_DW(val, insn, ptr) __get_data_asm(val, insn, ptr)
 286 #endif
 287
 288 extern void __get_user_unknown(void);
 289
 290 #define __get_user_common(val, size, ptr)                               \
 291 do {                                                                    \
 292         switch (size) {                                                 \
 293         case 1: __get_data_asm(val, user_lb, ptr); break;               \
 294         case 2: __get_data_asm(val, user_lh, ptr); break;               \
 295         case 4: __get_data_asm(val, user_lw, ptr); break;               \
 296         case 8: __GET_DW(val, user_ld, ptr); break;                     \
 297         default: __get_user_unknown(); break;                           \
 298         }                                                               \
 299 } while (0)
 300
 301 #define __get_user_nocheck(x, ptr, size)                                \
 302 ({                                                                      \
 303         int __gu_err;                                                   \
 304                                                                         \
 305         if (eva_kernel_access()) {                                      \
 306                 __get_kernel_common((x), size, ptr);                    \
 307         } else {                                                        \
 308                 __chk_user_ptr(ptr);                                    \
 309                 __get_user_common((x), size, ptr);                      \
 310         }                                                               \
 311         __gu_err;                                                       \
 312 })
 313
 314 #define __get_user_check(x, ptr, size)                                  \
 315 ({                                                                      \
 316         int __gu_err = -EFAULT;                                         \
 317         const __typeof__(*(ptr)) __user * __gu_ptr = (ptr);             \
 318                                                                         \
 319         might_fault();                                                  \
 320         if (likely(access_ok(VERIFY_READ,  __gu_ptr, size))) {          \
 321                 if (eva_kernel_access())                                \
 322                         __get_kernel_common((x), size, __gu_ptr);       \
 323                 else                                                    \
 324                         __get_user_common((x), size, __gu_ptr);         \
 325         } else                                                          \
 326                 (x) = 0;                                                \
 327                                                                         \
 328         __gu_err;                                                       \
 329 })
 330
 331 #define __get_data_asm(val, insn, addr)                                 \
 332 {                                                                       \
 333         long __gu_tmp;                                                  \
 334                                                                         \
 335         __asm__ __volatile__(                                           \
 336         "1:     "insn("%1", "%3")"                              \n"     \
 337         "2:                                                     \n"     \
 338         "       .insn                                           \n"     \
 339         "       .section .fixup,\"ax\"                          \n"     \
 340         "3:     li      %0, %4                                  \n"     \
 341         "       move    %1, $0                                  \n"     \
 342         "       j       2b                                      \n"     \
 343         "       .previous                                       \n"     \
 344         "       .section __ex_table,\"a\"                       \n"     \
 345         "       "__UA_ADDR "\t1b, 3b                            \n"     \
 346         "       .previous                                       \n"     \
 347         : "=r" (__gu_err), "=r" (__gu_tmp)                              \
 348         : "0" (0), "o" (__m(addr)), "i" (-EFAULT));                     \
 349                                                                         \
 350         (val) = (__typeof__(*(addr))) __gu_tmp;                         \
 351 }
 352
 353 /*
 354  * Get a long long 64 using 32 bit registers.
 355  */
 356 #define __get_data_asm_ll32(val, insn, addr)                            \
 357 {                                                                       \
 358         union {                                                         \
 359                 unsigned long long      l;                              \
 360                 __typeof__(*(addr))     t;                              \
 361         } __gu_tmp;                                                     \
 362                                                                         \
 363         __asm__ __volatile__(                                           \
 364         "1:     " insn("%1", "(%3)")"                           \n"     \
 365         "2:     " insn("%D1", "4(%3)")"                         \n"     \
 366         "3:                                                     \n"     \
 367         "       .insn                                           \n"     \
 368         "       .section        .fixup,\"ax\"                   \n"     \
 369         "4:     li      %0, %4                                  \n"     \
 370         "       move    %1, $0                                  \n"     \
 371         "       move    %D1, $0                                 \n"     \
 372         "       j       3b                                      \n"     \
 373         "       .previous                                       \n"     \
 374         "       .section        __ex_table,\"a\"                \n"     \
 375         "       " __UA_ADDR "   1b, 4b                          \n"     \
 376         "       " __UA_ADDR "   2b, 4b                          \n"     \
 377         "       .previous                                       \n"     \
 378         : "=r" (__gu_err), "=&r" (__gu_tmp.l)                           \
 379         : "0" (0), "r" (addr), "i" (-EFAULT));                          \
 380                                                                         \
 381         (val) = __gu_tmp.t;                                             \
 382 }
 383
 384 #ifndef CONFIG_EVA
 385 #define __put_kernel_common(ptr, size) __put_user_common(ptr, size)
 386 #else
 387 /*
 388  * Kernel specific functions for EVA. We need to use normal load instructions
 389  * to read data from kernel when operating in EVA mode. We use these macros to
 390  * avoid redefining __get_data_asm for EVA.
 391  */
 392 #undef _stored
 393 #undef _storew
 394 #undef _storeh
 395 #undef _storeb
 396 #ifdef CONFIG_32BIT
 397 #define _stored                 _storew
 398 #else
 399 #define _stored(reg, addr)      "ld " reg ", " addr
 400 #endif
 401
 402 #define _storew(reg, addr)      "sw " reg ", " addr
 403 #define _storeh(reg, addr)      "sh " reg ", " addr
 404 #define _storeb(reg, addr)      "sb " reg ", " addr
 405
 406 #define __put_kernel_common(ptr, size)                                  \
 407 do {                                                                    \
 408         switch (size) {                                                 \
 409         case 1: __put_data_asm(_storeb, ptr); break;                    \
 410         case 2: __put_data_asm(_storeh, ptr); break;                    \
 411         case 4: __put_data_asm(_storew, ptr); break;                    \
 412         case 8: __PUT_DW(_stored, ptr); break;                          \
 413         default: __put_user_unknown(); break;                           \
 414         }                                                               \
 415 } while(0)
 416 #endif
 417
 418 /*
 419  * Yuck.  We need two variants, one for 64bit operation and one
 420  * for 32 bit mode and old iron.
 421  */
 422 #ifdef CONFIG_32BIT
 423 #define __PUT_DW(insn, ptr) __put_data_asm_ll32(insn, ptr)
 424 #endif
 425 #ifdef CONFIG_64BIT
 426 #define __PUT_DW(insn, ptr) __put_data_asm(insn, ptr)
 427 #endif
 428
 429 #define __put_user_common(ptr, size)                                    \
 430 do {                                                                    \
 431         switch (size) {                                                 \
 432         case 1: __put_data_asm(user_sb, ptr); break;                    \
 433         case 2: __put_data_asm(user_sh, ptr); break;                    \
 434         case 4: __put_data_asm(user_sw, ptr); break;                    \
 435         case 8: __PUT_DW(user_sd, ptr); break;                          \
 436         default: __put_user_unknown(); break;                           \
 437         }                                                               \
 438 } while (0)
 439
 440 #define __put_user_nocheck(x, ptr, size)                                \
 441 ({                                                                      \
 442         __typeof__(*(ptr)) __pu_val;                                    \
 443         int __pu_err = 0;                                               \
 444                                                                         \
 445         __pu_val = (x);                                                 \
 446         if (eva_kernel_access()) {                                      \
 447                 __put_kernel_common(ptr, size);                         \
 448         } else {                                                        \
 449                 __chk_user_ptr(ptr);                                    \
 450                 __put_user_common(ptr, size);                           \
 451         }                                                               \
 452         __pu_err;                                                       \
 453 })
 454
 455 #define __put_user_check(x, ptr, size)                                  \
 456 ({                                                                      \
 457         __typeof__(*(ptr)) __user *__pu_addr = (ptr);                   \
 458         __typeof__(*(ptr)) __pu_val = (x);                              \
 459         int __pu_err = -EFAULT;                                         \
 460                                                                         \
 461         might_fault();                                                  \
 462         if (likely(access_ok(VERIFY_WRITE,  __pu_addr, size))) {        \
 463                 if (eva_kernel_access())                                \
 464                         __put_kernel_common(__pu_addr, size);           \
 465                 else                                                    \
 466                         __put_user_common(__pu_addr, size);             \
 467         }                                                               \
 468                                                                         \
 469         __pu_err;                                                       \
 470 })
 471
 472 #define __put_data_asm(insn, ptr)                                       \
 473 {                                                                       \
 474         __asm__ __volatile__(                                           \
 475         "1:     "insn("%z2", "%3")"     # __put_data_asm        \n"     \
 476         "2:                                                     \n"     \
 477         "       .insn                                           \n"     \
 478         "       .section        .fixup,\"ax\"                   \n"     \
 479         "3:     li      %0, %4                                  \n"     \
 480         "       j       2b                                      \n"     \
 481         "       .previous                                       \n"     \
 482         "       .section        __ex_table,\"a\"                \n"     \
 483         "       " __UA_ADDR "   1b, 3b                          \n"     \
 484         "       .previous                                       \n"     \
 485         : "=r" (__pu_err)                                               \
 486         : "0" (0), "Jr" (__pu_val), "o" (__m(ptr)),                     \
 487           "i" (-EFAULT));                                               \
 488 }
 489
 490 #define __put_data_asm_ll32(insn, ptr)                                  \
 491 {                                                                       \
 492         __asm__ __volatile__(                                           \
 493         "1:     "insn("%2", "(%3)")"    # __put_data_asm_ll32   \n"     \
 494         "2:     "insn("%D2", "4(%3)")"                          \n"     \
 495         "3:                                                     \n"     \
 496         "       .insn                                           \n"     \
 497         "       .section        .fixup,\"ax\"                   \n"     \
 498         "4:     li      %0, %4                                  \n"     \
 499         "       j       3b                                      \n"     \
 500         "       .previous                                       \n"     \
 501         "       .section        __ex_table,\"a\"                \n"     \
 502         "       " __UA_ADDR "   1b, 4b                          \n"     \
 503         "       " __UA_ADDR "   2b, 4b                          \n"     \
 504         "       .previous"                                              \
 505         : "=r" (__pu_err)                                               \
 506         : "0" (0), "r" (__pu_val), "r" (ptr),                           \
 507           "i" (-EFAULT));                                               \
 508 }
 509
 510 extern void __put_user_unknown(void);
 511
 512 /*
 513  * ul{b,h,w} are macros and there are no equivalent macros for EVA.
 514  * EVA unaligned access is handled in the ADE exception handler.
 515  */
 516 #ifndef CONFIG_EVA
 517 /*
 518  * put_user_unaligned: - Write a simple value into user space.
 519  * @x:   Value to copy to user space.
 520  * @ptr: Destination address, in user space.
 521  *
 522  * Context: User context only. This function may sleep if pagefaults are
 523  *          enabled.
 524  *
 525  * This macro copies a single simple value from kernel space to user
 526  * space.  It supports simple types like char and int, but not larger
 527  * data types like structures or arrays.
 528  *
 529  * @ptr must have pointer-to-simple-variable type, and @x must be assignable
 530  * to the result of dereferencing @ptr.
 531  *
 532  * Returns zero on success, or -EFAULT on error.
 533  */
 534 #define put_user_unaligned(x,ptr)       \
 535         __put_user_unaligned_check((x),(ptr),sizeof(*(ptr)))
 536
 537 /*
 538  * get_user_unaligned: - Get a simple variable from user space.
 539  * @x:   Variable to store result.
 540  * @ptr: Source address, in user space.
 541  *
 542  * Context: User context only. This function may sleep if pagefaults are
 543  *          enabled.
 544  *
 545  * This macro copies a single simple variable from user space to kernel
 546  * space.  It supports simple types like char and int, but not larger
 547  * data types like structures or arrays.
 548  *
 549  * @ptr must have pointer-to-simple-variable type, and the result of
 550  * dereferencing @ptr must be assignable to @x without a cast.
 551  *
 552  * Returns zero on success, or -EFAULT on error.
 553  * On error, the variable @x is set to zero.
 554  */
 555 #define get_user_unaligned(x,ptr) \
 556         __get_user_unaligned_check((x),(ptr),sizeof(*(ptr)))
 557
 558 /*
 559  * __put_user_unaligned: - Write a simple value into user space, with less checking.
 560  * @x:   Value to copy to user space.
 561  * @ptr: Destination address, in user space.
 562  *
 563  * Context: User context only. This function may sleep if pagefaults are
 564  *          enabled.
 565  *
 566  * This macro copies a single simple value from kernel space to user
 567  * space.  It supports simple types like char and int, but not larger
 568  * data types like structures or arrays.
 569  *
 570  * @ptr must have pointer-to-simple-variable type, and @x must be assignable
 571  * to the result of dereferencing @ptr.
 572  *
 573  * Caller must check the pointer with access_ok() before calling this
 574  * function.
 575  *
 576  * Returns zero on success, or -EFAULT on error.
 577  */
 578 #define __put_user_unaligned(x,ptr) \
 579         __put_user_unaligned_nocheck((x),(ptr),sizeof(*(ptr)))
 580
 581 /*
 582  * __get_user_unaligned: - Get a simple variable from user space, with less checking.
 583  * @x:   Variable to store result.
 584  * @ptr: Source address, in user space.
 585  *
 586  * Context: User context only. This function may sleep if pagefaults are
 587  *          enabled.
 588  *
 589  * This macro copies a single simple variable from user space to kernel
 590  * space.  It supports simple types like char and int, but not larger
 591  * data types like structures or arrays.
 592  *
 593  * @ptr must have pointer-to-simple-variable type, and the result of
 594  * dereferencing @ptr must be assignable to @x without a cast.
 595  *
 596  * Caller must check the pointer with access_ok() before calling this
 597  * function.
 598  *
 599  * Returns zero on success, or -EFAULT on error.
 600  * On error, the variable @x is set to zero.
 601  */
 602 #define __get_user_unaligned(x,ptr) \
 603         __get_user_unaligned_nocheck((x),(ptr),sizeof(*(ptr)))
 604
 605 /*
 606  * Yuck.  We need two variants, one for 64bit operation and one
 607  * for 32 bit mode and old iron.
 608  */
 609 #ifdef CONFIG_32BIT
 610 #define __GET_USER_UNALIGNED_DW(val, ptr)                               \
 611         __get_user_unaligned_asm_ll32(val, ptr)
 612 #endif
 613 #ifdef CONFIG_64BIT
 614 #define __GET_USER_UNALIGNED_DW(val, ptr)                               \
 615         __get_user_unaligned_asm(val, "uld", ptr)
 616 #endif
 617
 618 extern void __get_user_unaligned_unknown(void);
 619
 620 #define __get_user_unaligned_common(val, size, ptr)                     \
 621 do {                                                                    \
 622         switch (size) {                                                 \
 623         case 1: __get_data_asm(val, "lb", ptr); break;                  \
 624         case 2: __get_data_unaligned_asm(val, "ulh", ptr); break;       \
 625         case 4: __get_data_unaligned_asm(val, "ulw", ptr); break;       \
 626         case 8: __GET_USER_UNALIGNED_DW(val, ptr); break;               \
 627         default: __get_user_unaligned_unknown(); break;                 \
 628         }                                                               \
 629 } while (0)
 630
 631 #define __get_user_unaligned_nocheck(x,ptr,size)                        \
 632 ({                                                                      \
 633         int __gu_err;                                                   \
 634                                                                         \
 635         __get_user_unaligned_common((x), size, ptr);                    \
 636         __gu_err;                                                       \
 637 })
 638
 639 #define __get_user_unaligned_check(x,ptr,size)                          \
 640 ({                                                                      \
 641         int __gu_err = -EFAULT;                                         \
 642         const __typeof__(*(ptr)) __user * __gu_ptr = (ptr);             \
 643                                                                         \
 644         if (likely(access_ok(VERIFY_READ,  __gu_ptr, size)))            \
 645                 __get_user_unaligned_common((x), size, __gu_ptr);       \
 646                                                                         \
 647         __gu_err;                                                       \
 648 })
 649
 650 #define __get_data_unaligned_asm(val, insn, addr)                       \
 651 {                                                                       \
 652         long __gu_tmp;                                                  \
 653                                                                         \
 654         __asm__ __volatile__(                                           \
 655         "1:     " insn "        %1, %3                          \n"     \
 656         "2:                                                     \n"     \
 657         "       .insn                                           \n"     \
 658         "       .section .fixup,\"ax\"                          \n"     \
 659         "3:     li      %0, %4                                  \n"     \
 660         "       move    %1, $0                                  \n"     \
 661         "       j       2b                                      \n"     \
 662         "       .previous                                       \n"     \
 663         "       .section __ex_table,\"a\"                       \n"     \
 664         "       "__UA_ADDR "\t1b, 3b                            \n"     \
 665         "       "__UA_ADDR "\t1b + 4, 3b                        \n"     \
 666         "       .previous                                       \n"     \
 667         : "=r" (__gu_err), "=r" (__gu_tmp)                              \
 668         : "0" (0), "o" (__m(addr)), "i" (-EFAULT));                     \
 669                                                                         \
 670         (val) = (__typeof__(*(addr))) __gu_tmp;                         \
 671 }
 672
 673 /*
 674  * Get a long long 64 using 32 bit registers.
 675  */
 676 #define __get_user_unaligned_asm_ll32(val, addr)                        \
 677 {                                                                       \
 678         unsigned long long __gu_tmp;                                    \
 679                                                                         \
 680         __asm__ __volatile__(                                           \
 681         "1:     ulw     %1, (%3)                                \n"     \
 682         "2:     ulw     %D1, 4(%3)                              \n"     \
 683         "       move    %0, $0                                  \n"     \
 684         "3:                                                     \n"     \
 685         "       .insn                                           \n"     \
 686         "       .section        .fixup,\"ax\"                   \n"     \
 687         "4:     li      %0, %4                                  \n"     \
 688         "       move    %1, $0                                  \n"     \
 689         "       move    %D1, $0                                 \n"     \
 690         "       j       3b                                      \n"     \
 691         "       .previous                                       \n"     \
 692         "       .section        __ex_table,\"a\"                \n"     \
 693         "       " __UA_ADDR "   1b, 4b                          \n"     \
 694         "       " __UA_ADDR "   1b + 4, 4b                      \n"     \
 695         "       " __UA_ADDR "   2b, 4b                          \n"     \
 696         "       " __UA_ADDR "   2b + 4, 4b                      \n"     \
 697         "       .previous                                       \n"     \
 698         : "=r" (__gu_err), "=&r" (__gu_tmp)                             \
 699         : "0" (0), "r" (addr), "i" (-EFAULT));                          \
 700         (val) = (__typeof__(*(addr))) __gu_tmp;                         \
 701 }
 702
 703 /*
 704  * Yuck.  We need two variants, one for 64bit operation and one
 705  * for 32 bit mode and old iron.
 706  */
 707 #ifdef CONFIG_32BIT
 708 #define __PUT_USER_UNALIGNED_DW(ptr) __put_user_unaligned_asm_ll32(ptr)
 709 #endif
 710 #ifdef CONFIG_64BIT
 711 #define __PUT_USER_UNALIGNED_DW(ptr) __put_user_unaligned_asm("usd", ptr)
 712 #endif
 713
 714 #define __put_user_unaligned_common(ptr, size)                          \
 715 do {                                                                    \
 716         switch (size) {                                                 \
 717         case 1: __put_data_asm("sb", ptr); break;                       \
 718         case 2: __put_user_unaligned_asm("ush", ptr); break;            \
 719         case 4: __put_user_unaligned_asm("usw", ptr); break;            \
 720         case 8: __PUT_USER_UNALIGNED_DW(ptr); break;                    \
 721         default: __put_user_unaligned_unknown(); break;                 \
 722 } while (0)
 723
 724 #define __put_user_unaligned_nocheck(x,ptr,size)                        \
 725 ({                                                                      \
 726         __typeof__(*(ptr)) __pu_val;                                    \
 727         int __pu_err = 0;                                               \
 728                                                                         \
 729         __pu_val = (x);                                                 \
 730         __put_user_unaligned_common(ptr, size);                         \
 731         __pu_err;                                                       \
 732 })
 733
 734 #define __put_user_unaligned_check(x,ptr,size)                          \
 735 ({                                                                      \
 736         __typeof__(*(ptr)) __user *__pu_addr = (ptr);                   \
 737         __typeof__(*(ptr)) __pu_val = (x);                              \
 738         int __pu_err = -EFAULT;                                         \
 739                                                                         \
 740         if (likely(access_ok(VERIFY_WRITE,  __pu_addr, size)))          \
 741                 __put_user_unaligned_common(__pu_addr, size);           \
 742                                                                         \
 743         __pu_err;                                                       \
 744 })
 745
 746 #define __put_user_unaligned_asm(insn, ptr)                             \
 747 {                                                                       \
 748         __asm__ __volatile__(                                           \
 749         "1:     " insn "        %z2, %3         # __put_user_unaligned_asm\n" \
 750         "2:                                                     \n"     \
 751         "       .insn                                           \n"     \
 752         "       .section        .fixup,\"ax\"                   \n"     \
 753         "3:     li      %0, %4                                  \n"     \
 754         "       j       2b                                      \n"     \
 755         "       .previous                                       \n"     \
 756         "       .section        __ex_table,\"a\"                \n"     \
 757         "       " __UA_ADDR "   1b, 3b                          \n"     \
 758         "       .previous                                       \n"     \
 759         : "=r" (__pu_err)                                               \
 760         : "0" (0), "Jr" (__pu_val), "o" (__m(ptr)),                     \
 761           "i" (-EFAULT));                                               \
 762 }
 763
 764 #define __put_user_unaligned_asm_ll32(ptr)                              \
 765 {                                                                       \
 766         __asm__ __volatile__(                                           \
 767         "1:     sw      %2, (%3)        # __put_user_unaligned_asm_ll32 \n" \
 768         "2:     sw      %D2, 4(%3)                              \n"     \
 769         "3:                                                     \n"     \
 770         "       .insn                                           \n"     \
 771         "       .section        .fixup,\"ax\"                   \n"     \
 772         "4:     li      %0, %4                                  \n"     \
 773         "       j       3b                                      \n"     \
 774         "       .previous                                       \n"     \
 775         "       .section        __ex_table,\"a\"                \n"     \
 776         "       " __UA_ADDR "   1b, 4b                          \n"     \
 777         "       " __UA_ADDR "   1b + 4, 4b                      \n"     \
 778         "       " __UA_ADDR "   2b, 4b                          \n"     \
 779         "       " __UA_ADDR "   2b + 4, 4b                      \n"     \
 780         "       .previous"                                              \
 781         : "=r" (__pu_err)                                               \
 782         : "0" (0), "r" (__pu_val), "r" (ptr),                           \
 783           "i" (-EFAULT));                                               \
 784 }
 785
 786 extern void __put_user_unaligned_unknown(void);
 787 #endif
 788
 789 /*
 790  * We're generating jump to subroutines which will be outside the range of
 791  * jump instructions
 792  */
 793 #ifdef MODULE
 794 #define __MODULE_JAL(destination)                                       \
 795         ".set\tnoat\n\t"                                                \
 796         __UA_LA "\t$1, " #destination "\n\t"                            \
 797         "jalr\t$1\n\t"                                                  \
 798         ".set\tat\n\t"
 799 #else
 800 #define __MODULE_JAL(destination)                                       \
 801         "jal\t" #destination "\n\t"
 802 #endif
 803
 804 #if defined(CONFIG_CPU_DADDI_WORKAROUNDS) || (defined(CONFIG_EVA) &&    \
 805                                               defined(CONFIG_CPU_HAS_PREFETCH))
 806 #define DADDI_SCRATCH "$3"
 807 #else
 808 #define DADDI_SCRATCH "$0"
 809 #endif
 810
 811 extern size_t __copy_user(void *__to, const void *__from, size_t __n);
 812
 813 #ifndef CONFIG_EVA
 814 #define __invoke_copy_to_user(to, from, n)                              \
 815 ({                                                                      \
 816         register void __user *__cu_to_r __asm__("$4");                  \
 817         register const void *__cu_from_r __asm__("$5");                 \
 818         register long __cu_len_r __asm__("$6");                         \
 819                                                                         \
 820         __cu_to_r = (to);                                               \
 821         __cu_from_r = (from);                                           \
 822         __cu_len_r = (n);                                               \
 823         __asm__ __volatile__(                                           \
 824         __MODULE_JAL(__copy_user)                                       \
 825         : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r)       \
 826         :                                                               \
 827         : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31",  \
 828           DADDI_SCRATCH, "memory");                                     \
 829         __cu_len_r;                                                     \
 830 })
 831
 832 #define __invoke_copy_to_kernel(to, from, n)                            \
 833         __invoke_copy_to_user(to, from, n)
 834
 835 #endif
 836
 837 /*
 838  * __copy_to_user: - Copy a block of data into user space, with less checking.
 839  * @to:   Destination address, in user space.
 840  * @from: Source address, in kernel space.
 841  * @n:    Number of bytes to copy.
 842  *
 843  * Context: User context only. This function may sleep if pagefaults are
 844  *          enabled.
 845  *
 846  * Copy data from kernel space to user space.  Caller must check
 847  * the specified block with access_ok() before calling this function.
 848  *
 849  * Returns number of bytes that could not be copied.
 850  * On success, this will be zero.
 851  */
 852 #define __copy_to_user(to, from, n)                                     \
 853 ({                                                                      \
 854         void __user *__cu_to;                                           \
 855         const void *__cu_from;                                          \
 856         long __cu_len;                                                  \
 857                                                                         \
 858         __cu_to = (to);                                                 \
 859         __cu_from = (from);                                             \
 860         __cu_len = (n);                                                 \
 861         might_fault();                                                  \
 862         if (eva_kernel_access())                                        \
 863                 __cu_len = __invoke_copy_to_kernel(__cu_to, __cu_from,  \
 864                                                    __cu_len);           \
 865         else                                                            \
 866                 __cu_len = __invoke_copy_to_user(__cu_to, __cu_from,    \
 867                                                  __cu_len);             \
 868         __cu_len;                                                       \
 869 })
 870
 871 extern size_t __copy_user_inatomic(void *__to, const void *__from, size_t __n);
 872
 873 #define __copy_to_user_inatomic(to, from, n)                            \
 874 ({                                                                      \
 875         void __user *__cu_to;                                           \
 876         const void *__cu_from;                                          \
 877         long __cu_len;                                                  \
 878                                                                         \
 879         __cu_to = (to);                                                 \
 880         __cu_from = (from);                                             \
 881         __cu_len = (n);                                                 \
 882         if (eva_kernel_access())                                        \
 883                 __cu_len = __invoke_copy_to_kernel(__cu_to, __cu_from,  \
 884                                                    __cu_len);           \
 885         else                                                            \
 886                 __cu_len = __invoke_copy_to_user(__cu_to, __cu_from,    \
 887                                                  __cu_len);             \
 888         __cu_len;                                                       \
 889 })
 890
 891 #define __copy_from_user_inatomic(to, from, n)                          \
 892 ({                                                                      \
 893         void *__cu_to;                                                  \
 894         const void __user *__cu_from;                                   \
 895         long __cu_len;                                                  \
 896                                                                         \
 897         __cu_to = (to);                                                 \
 898         __cu_from = (from);                                             \
 899         __cu_len = (n);                                                 \
 900         if (eva_kernel_access())                                        \
 901                 __cu_len = __invoke_copy_from_kernel_inatomic(__cu_to,  \
 902                                                               __cu_from,\
 903                                                               __cu_len);\
 904         else                                                            \
 905                 __cu_len = __invoke_copy_from_user_inatomic(__cu_to,    \
 906                                                             __cu_from,  \
 907                                                             __cu_len);  \
 908         __cu_len;                                                       \
 909 })
 910
 911 /*
 912  * copy_to_user: - Copy a block of data into user space.
 913  * @to:   Destination address, in user space.
 914  * @from: Source address, in kernel space.
 915  * @n:    Number of bytes to copy.
 916  *
 917  * Context: User context only. This function may sleep if pagefaults are
 918  *          enabled.
 919  *
 920  * Copy data from kernel space to user space.
 921  *
 922  * Returns number of bytes that could not be copied.
 923  * On success, this will be zero.
 924  */
 925 #define copy_to_user(to, from, n)                                       \
 926 ({                                                                      \
 927         void __user *__cu_to;                                           \
 928         const void *__cu_from;                                          \
 929         long __cu_len;                                                  \
 930                                                                         \
 931         __cu_to = (to);                                                 \
 932         __cu_from = (from);                                             \
 933         __cu_len = (n);                                                 \
 934         if (eva_kernel_access()) {                                      \
 935                 __cu_len = __invoke_copy_to_kernel(__cu_to,             \
 936                                                    __cu_from,           \
 937                                                    __cu_len);           \
 938         } else {                                                        \
 939                 if (access_ok(VERIFY_WRITE, __cu_to, __cu_len)) {       \
 940                         might_fault();                                  \
 941                         __cu_len = __invoke_copy_to_user(__cu_to,       \
 942                                                          __cu_from,     \
 943                                                          __cu_len);     \
 944                 }                                                       \
 945         }                                                               \
 946         __cu_len;                                                       \
 947 })
 948
 949 #ifndef CONFIG_EVA
 950
 951 #define __invoke_copy_from_user(to, from, n)                            \
 952 ({                                                                      \
 953         register void *__cu_to_r __asm__("$4");                         \
 954         register const void __user *__cu_from_r __asm__("$5");          \
 955         register long __cu_len_r __asm__("$6");                         \
 956                                                                         \
 957         __cu_to_r = (to);                                               \
 958         __cu_from_r = (from);                                           \
 959         __cu_len_r = (n);                                               \
 960         __asm__ __volatile__(                                           \
 961         ".set\tnoreorder\n\t"                                           \
 962         __MODULE_JAL(__copy_user)                                       \
 963         ".set\tnoat\n\t"                                                \
 964         __UA_ADDU "\t$1, %1, %2\n\t"                                    \
 965         ".set\tat\n\t"                                                  \
 966         ".set\treorder"                                                 \
 967         : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r)       \
 968         :                                                               \
 969         : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31",  \
 970           DADDI_SCRATCH, "memory");                                     \
 971         __cu_len_r;                                                     \
 972 })
 973
 974 #define __invoke_copy_from_kernel(to, from, n)                          \
 975         __invoke_copy_from_user(to, from, n)
 976
 977 /* For userland <-> userland operations */
 978 #define ___invoke_copy_in_user(to, from, n)                             \
 979         __invoke_copy_from_user(to, from, n)
 980
 981 /* For kernel <-> kernel operations */
 982 #define ___invoke_copy_in_kernel(to, from, n)                           \
 983         __invoke_copy_from_user(to, from, n)
 984
 985 #define __invoke_copy_from_user_inatomic(to, from, n)                   \
 986 ({                                                                      \
 987         register void *__cu_to_r __asm__("$4");                         \
 988         register const void __user *__cu_from_r __asm__("$5");          \
 989         register long __cu_len_r __asm__("$6");                         \
 990                                                                         \
 991         __cu_to_r = (to);                                               \
 992         __cu_from_r = (from);                                           \
 993         __cu_len_r = (n);                                               \
 994         __asm__ __volatile__(                                           \
 995         ".set\tnoreorder\n\t"                                           \
 996         __MODULE_JAL(__copy_user_inatomic)                              \
 997         ".set\tnoat\n\t"                                                \
 998         __UA_ADDU "\t$1, %1, %2\n\t"                                    \
 999         ".set\tat\n\t"                                                  \
1000         ".set\treorder"                                                 \
1001         : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r)       \
1002         :                                                               \
1003         : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31",  \
1004           DADDI_SCRATCH, "memory");                                     \
1005         __cu_len_r;                                                     \
1006 })
1007
1008 #define __invoke_copy_from_kernel_inatomic(to, from, n)                 \
1009         __invoke_copy_from_user_inatomic(to, from, n)                   \
1010
1011 #else
1012
1013 /* EVA specific functions */
1014
1015 extern size_t __copy_user_inatomic_eva(void *__to, const void *__from,
1016                                        size_t __n);
1017 extern size_t __copy_from_user_eva(void *__to, const void *__from,
1018                                    size_t __n);
1019 extern size_t __copy_to_user_eva(void *__to, const void *__from,
1020                                  size_t __n);
1021 extern size_t __copy_in_user_eva(void *__to, const void *__from, size_t __n);
1022
1023 #define __invoke_copy_from_user_eva_generic(to, from, n, func_ptr)      \
1024 ({                                                                      \
1025         register void *__cu_to_r __asm__("$4");                         \
1026         register const void __user *__cu_from_r __asm__("$5");          \
1027         register long __cu_len_r __asm__("$6");                         \
1028                                                                         \
1029         __cu_to_r = (to);                                               \
1030         __cu_from_r = (from);                                           \
1031         __cu_len_r = (n);                                               \
1032         __asm__ __volatile__(                                           \
1033         ".set\tnoreorder\n\t"                                           \
1034         __MODULE_JAL(func_ptr)                                          \
1035         ".set\tnoat\n\t"                                                \
1036         __UA_ADDU "\t$1, %1, %2\n\t"                                    \
1037         ".set\tat\n\t"                                                  \
1038         ".set\treorder"                                                 \
1039         : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r)       \
1040         :                                                               \
1041         : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31",  \
1042           DADDI_SCRATCH, "memory");                                     \
1043         __cu_len_r;                                                     \
1044 })
1045
1046 #define __invoke_copy_to_user_eva_generic(to, from, n, func_ptr)        \
1047 ({                                                                      \
1048         register void *__cu_to_r __asm__("$4");                         \
1049         register const void __user *__cu_from_r __asm__("$5");          \
1050         register long __cu_len_r __asm__("$6");                         \
1051                                                                         \
1052         __cu_to_r = (to);                                               \
1053         __cu_from_r = (from);                                           \
1054         __cu_len_r = (n);                                               \
1055         __asm__ __volatile__(                                           \
1056         __MODULE_JAL(func_ptr)                                          \
1057         : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r)       \
1058         :                                                               \
1059         : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31",  \
1060           DADDI_SCRATCH, "memory");                                     \
1061         __cu_len_r;                                                     \
1062 })
1063
1064 /*
1065  * Source or destination address is in userland. We need to go through
1066  * the TLB
1067  */
1068 #define __invoke_copy_from_user(to, from, n)                            \
1069         __invoke_copy_from_user_eva_generic(to, from, n, __copy_from_user_eva)
1070
1071 #define __invoke_copy_from_user_inatomic(to, from, n)                   \
1072         __invoke_copy_from_user_eva_generic(to, from, n,                \
1073                                             __copy_user_inatomic_eva)
1074
1075 #define __invoke_copy_to_user(to, from, n)                              \
1076         __invoke_copy_to_user_eva_generic(to, from, n, __copy_to_user_eva)
1077
1078 #define ___invoke_copy_in_user(to, from, n)                             \
1079         __invoke_copy_from_user_eva_generic(to, from, n, __copy_in_user_eva)
1080
1081 /*
1082  * Source or destination address in the kernel. We are not going through
1083  * the TLB
1084  */
1085 #define __invoke_copy_from_kernel(to, from, n)                          \
1086         __invoke_copy_from_user_eva_generic(to, from, n, __copy_user)
1087
1088 #define __invoke_copy_from_kernel_inatomic(to, from, n)                 \
1089         __invoke_copy_from_user_eva_generic(to, from, n, __copy_user_inatomic)
1090
1091 #define __invoke_copy_to_kernel(to, from, n)                            \
1092         __invoke_copy_to_user_eva_generic(to, from, n, __copy_user)
1093
1094 #define ___invoke_copy_in_kernel(to, from, n)                           \
1095         __invoke_copy_from_user_eva_generic(to, from, n, __copy_user)
1096
1097 #endif /* CONFIG_EVA */
1098
1099 /*
1100  * __copy_from_user: - Copy a block of data from user space, with less checking.
1101  * @to:   Destination address, in kernel space.
1102  * @from: Source address, in user space.
1103  * @n:    Number of bytes to copy.
1104  *
1105  * Context: User context only. This function may sleep if pagefaults are
1106  *          enabled.
1107  *
1108  * Copy data from user space to kernel space.  Caller must check
1109  * the specified block with access_ok() before calling this function.
1110  *
1111  * Returns number of bytes that could not be copied.
1112  * On success, this will be zero.
1113  *
1114  * If some data could not be copied, this function will pad the copied
1115  * data to the requested size using zero bytes.
1116  */
1117 #define __copy_from_user(to, from, n)                                   \
1118 ({                                                                      \
1119         void *__cu_to;                                                  \
1120         const void __user *__cu_from;                                   \
1121         long __cu_len;                                                  \
1122                                                                         \
1123         __cu_to = (to);                                                 \
1124         __cu_from = (from);                                             \
1125         __cu_len = (n);                                                 \
1126         if (eva_kernel_access()) {                                      \
1127                 __cu_len = __invoke_copy_from_kernel(__cu_to,           \
1128                                                      __cu_from,         \
1129                                                      __cu_len);         \
1130         } else {                                                        \
1131                 might_fault();                                          \
1132                 __cu_len = __invoke_copy_from_user(__cu_to, __cu_from,  \
1133                                                    __cu_len);           \
1134         }                                                               \
1135         __cu_len;                                                       \
1136 })
1137
1138 /*
1139  * copy_from_user: - Copy a block of data from user space.
1140  * @to:   Destination address, in kernel space.
1141  * @from: Source address, in user space.
1142  * @n:    Number of bytes to copy.
1143  *
1144  * Context: User context only. This function may sleep if pagefaults are
1145  *          enabled.
1146  *
1147  * Copy data from user space to kernel space.
1148  *
1149  * Returns number of bytes that could not be copied.
1150  * On success, this will be zero.
1151  *
1152  * If some data could not be copied, this function will pad the copied
1153  * data to the requested size using zero bytes.
1154  */
1155 #define copy_from_user(to, from, n)                                     \
1156 ({                                                                      \
1157         void *__cu_to;                                                  \
1158         const void __user *__cu_from;                                   \
1159         long __cu_len;                                                  \
1160                                                                         \
1161         __cu_to = (to);                                                 \
1162         __cu_from = (from);                                             \
1163         __cu_len = (n);                                                 \
1164         if (eva_kernel_access()) {                                      \
1165                 __cu_len = __invoke_copy_from_kernel(__cu_to,           \
1166                                                      __cu_from,         \
1167                                                      __cu_len);         \
1168         } else {                                                        \
1169                 if (access_ok(VERIFY_READ, __cu_from, __cu_len)) {      \
1170                         might_fault();                                  \
1171                         __cu_len = __invoke_copy_from_user(__cu_to,     \
1172                                                            __cu_from,   \
1173                                                            __cu_len);   \
1174                 } else {                                                \
1175                         memset(__cu_to, 0, __cu_len);                   \
1176                 }                                                       \
1177         }                                                               \
1178         __cu_len;                                                       \
1179 })
1180
1181 #define __copy_in_user(to, from, n)                                     \
1182 ({                                                                      \
1183         void __user *__cu_to;                                           \
1184         const void __user *__cu_from;                                   \
1185         long __cu_len;                                                  \
1186                                                                         \
1187         __cu_to = (to);                                                 \
1188         __cu_from = (from);                                             \
1189         __cu_len = (n);                                                 \
1190         if (eva_kernel_access()) {                                      \
1191                 __cu_len = ___invoke_copy_in_kernel(__cu_to, __cu_from, \
1192                                                     __cu_len);          \
1193         } else {                                                        \
1194                 might_fault();                                          \
1195                 __cu_len = ___invoke_copy_in_user(__cu_to, __cu_from,   \
1196                                                   __cu_len);            \
1197         }                                                               \
1198         __cu_len;                                                       \
1199 })
1200
1201 #define copy_in_user(to, from, n)                                       \
1202 ({                                                                      \
1203         void __user *__cu_to;                                           \
1204         const void __user *__cu_from;                                   \
1205         long __cu_len;                                                  \
1206                                                                         \
1207         __cu_to = (to);                                                 \
1208         __cu_from = (from);                                             \
1209         __cu_len = (n);                                                 \
1210         if (eva_kernel_access()) {                                      \
1211                 __cu_len = ___invoke_copy_in_kernel(__cu_to,__cu_from,  \
1212                                                     __cu_len);          \
1213         } else {                                                        \
1214                 if (likely(access_ok(VERIFY_READ, __cu_from, __cu_len) &&\
1215                            access_ok(VERIFY_WRITE, __cu_to, __cu_len))) {\
1216                         might_fault();                                  \
1217                         __cu_len = ___invoke_copy_in_user(__cu_to,      \
1218                                                           __cu_from,    \
1219                                                           __cu_len);    \
1220                 }                                                       \
1221         }                                                               \
1222         __cu_len;                                                       \
1223 })
1224
1225 /*
1226  * __clear_user: - Zero a block of memory in user space, with less checking.
1227  * @to:   Destination address, in user space.
1228  * @n:    Number of bytes to zero.
1229  *
1230  * Zero a block of memory in user space.  Caller must check
1231  * the specified block with access_ok() before calling this function.
1232  *
1233  * Returns number of bytes that could not be cleared.
1234  * On success, this will be zero.
1235  */
1236 static inline __kernel_size_t
1237 __clear_user(void __user *addr, __kernel_size_t size)
1238 {
1239         __kernel_size_t res;
1240
1241         if (eva_kernel_access()) {
1242                 __asm__ __volatile__(
1243                         "move\t$4, %1\n\t"
1244                         "move\t$5, $0\n\t"
1245                         "move\t$6, %2\n\t"
1246                         __MODULE_JAL(__bzero_kernel)
1247                         "move\t%0, $6"
1248                         : "=r" (res)
1249                         : "r" (addr), "r" (size)
1250                         : "$4", "$5", "$6", __UA_t0, __UA_t1, "$31");
1251         } else {
1252                 might_fault();
1253                 __asm__ __volatile__(
1254                         "move\t$4, %1\n\t"
1255                         "move\t$5, $0\n\t"
1256                         "move\t$6, %2\n\t"
1257                         __MODULE_JAL(__bzero)
1258                         "move\t%0, $6"
1259                         : "=r" (res)
1260                         : "r" (addr), "r" (size)
1261                         : "$4", "$5", "$6", __UA_t0, __UA_t1, "$31");
1262         }
1263
1264         return res;
1265 }
1266
1267 #define clear_user(addr,n)                                              \
1268 ({                                                                      \
1269         void __user * __cl_addr = (addr);                               \
1270         unsigned long __cl_size = (n);                                  \
1271         if (__cl_size && access_ok(VERIFY_WRITE,                        \
1272                                         __cl_addr, __cl_size))          \
1273                 __cl_size = __clear_user(__cl_addr, __cl_size);         \
1274         __cl_size;                                                      \
1275 })
1276
1277 /*
1278  * __strncpy_from_user: - Copy a NUL terminated string from userspace, with less checking.
1279  * @dst:   Destination address, in kernel space.  This buffer must be at
1280  *         least @count bytes long.
1281  * @src:   Source address, in user space.
1282  * @count: Maximum number of bytes to copy, including the trailing NUL.
1283  *
1284  * Copies a NUL-terminated string from userspace to kernel space.
1285  * Caller must check the specified block with access_ok() before calling
1286  * this function.
1287  *
1288  * On success, returns the length of the string (not including the trailing
1289  * NUL).
1290  *
1291  * If access to userspace fails, returns -EFAULT (some data may have been
1292  * copied).
1293  *
1294  * If @count is smaller than the length of the string, copies @count bytes
1295  * and returns @count.
1296  */
1297 static inline long
1298 __strncpy_from_user(char *__to, const char __user *__from, long __len)
1299 {
1300         long res;
1301
1302         if (eva_kernel_access()) {
1303                 __asm__ __volatile__(
1304                         "move\t$4, %1\n\t"
1305                         "move\t$5, %2\n\t"
1306                         "move\t$6, %3\n\t"
1307                         __MODULE_JAL(__strncpy_from_kernel_nocheck_asm)
1308                         "move\t%0, $2"
1309                         : "=r" (res)
1310                         : "r" (__to), "r" (__from), "r" (__len)
1311                         : "$2", "$3", "$4", "$5", "$6", __UA_t0, "$31", "memory");
1312         } else {
1313                 might_fault();
1314                 __asm__ __volatile__(
1315                         "move\t$4, %1\n\t"
1316                         "move\t$5, %2\n\t"
1317                         "move\t$6, %3\n\t"
1318                         __MODULE_JAL(__strncpy_from_user_nocheck_asm)
1319                         "move\t%0, $2"
1320                         : "=r" (res)
1321                         : "r" (__to), "r" (__from), "r" (__len)
1322                         : "$2", "$3", "$4", "$5", "$6", __UA_t0, "$31", "memory");
1323         }
1324
1325         return res;
1326 }
1327
1328 /*
1329  * strncpy_from_user: - Copy a NUL terminated string from userspace.
1330  * @dst:   Destination address, in kernel space.  This buffer must be at
1331  *         least @count bytes long.
1332  * @src:   Source address, in user space.
1333  * @count: Maximum number of bytes to copy, including the trailing NUL.
1334  *
1335  * Copies a NUL-terminated string from userspace to kernel space.
1336  *
1337  * On success, returns the length of the string (not including the trailing
1338  * NUL).
1339  *
1340  * If access to userspace fails, returns -EFAULT (some data may have been
1341  * copied).
1342  *
1343  * If @count is smaller than the length of the string, copies @count bytes
1344  * and returns @count.
1345  */
1346 static inline long
1347 strncpy_from_user(char *__to, const char __user *__from, long __len)
1348 {
1349         long res;
1350
1351         if (eva_kernel_access()) {
1352                 __asm__ __volatile__(
1353                         "move\t$4, %1\n\t"
1354                         "move\t$5, %2\n\t"
1355                         "move\t$6, %3\n\t"
1356                         __MODULE_JAL(__strncpy_from_kernel_asm)
1357                         "move\t%0, $2"
1358                         : "=r" (res)
1359                         : "r" (__to), "r" (__from), "r" (__len)
1360                         : "$2", "$3", "$4", "$5", "$6", __UA_t0, "$31", "memory");
1361         } else {
1362                 might_fault();
1363                 __asm__ __volatile__(
1364                         "move\t$4, %1\n\t"
1365                         "move\t$5, %2\n\t"
1366                         "move\t$6, %3\n\t"
1367                         __MODULE_JAL(__strncpy_from_user_asm)
1368                         "move\t%0, $2"
1369                         : "=r" (res)
1370                         : "r" (__to), "r" (__from), "r" (__len)
1371                         : "$2", "$3", "$4", "$5", "$6", __UA_t0, "$31", "memory");
1372         }
1373
1374         return res;
1375 }
1376
1377 /*
1378  * strlen_user: - Get the size of a string in user space.
1379  * @str: The string to measure.
1380  *
1381  * Context: User context only. This function may sleep if pagefaults are
1382  *          enabled.
1383  *
1384  * Get the size of a NUL-terminated string in user space.
1385  *
1386  * Returns the size of the string INCLUDING the terminating NUL.
1387  * On exception, returns 0.
1388  *
1389  * If there is a limit on the length of a valid string, you may wish to
1390  * consider using strnlen_user() instead.
1391  */
1392 static inline long strlen_user(const char __user *s)
1393 {
1394         long res;
1395
1396         if (eva_kernel_access()) {
1397                 __asm__ __volatile__(
1398                         "move\t$4, %1\n\t"
1399                         __MODULE_JAL(__strlen_kernel_asm)
1400                         "move\t%0, $2"
1401                         : "=r" (res)
1402                         : "r" (s)
1403                         : "$2", "$4", __UA_t0, "$31");
1404         } else {
1405                 might_fault();
1406                 __asm__ __volatile__(
1407                         "move\t$4, %1\n\t"
1408                         __MODULE_JAL(__strlen_user_asm)
1409                         "move\t%0, $2"
1410                         : "=r" (res)
1411                         : "r" (s)
1412                         : "$2", "$4", __UA_t0, "$31");
1413         }
1414
1415         return res;
1416 }
1417
1418 /* Returns: 0 if bad, string length+1 (memory size) of string if ok */
1419 static inline long __strnlen_user(const char __user *s, long n)
1420 {
1421         long res;
1422
1423         if (eva_kernel_access()) {
1424                 __asm__ __volatile__(
1425                         "move\t$4, %1\n\t"
1426                         "move\t$5, %2\n\t"
1427                         __MODULE_JAL(__strnlen_kernel_nocheck_asm)
1428                         "move\t%0, $2"
1429                         : "=r" (res)
1430                         : "r" (s), "r" (n)
1431                         : "$2", "$4", "$5", __UA_t0, "$31");
1432         } else {
1433                 might_fault();
1434                 __asm__ __volatile__(
1435                         "move\t$4, %1\n\t"
1436                         "move\t$5, %2\n\t"
1437                         __MODULE_JAL(__strnlen_user_nocheck_asm)
1438                         "move\t%0, $2"
1439                         : "=r" (res)
1440                         : "r" (s), "r" (n)
1441                         : "$2", "$4", "$5", __UA_t0, "$31");
1442         }
1443
1444         return res;
1445 }
1446
1447 /*
1448  * strnlen_user: - Get the size of a string in user space.
1449  * @str: The string to measure.
1450  *
1451  * Context: User context only. This function may sleep if pagefaults are
1452  *          enabled.
1453  *
1454  * Get the size of a NUL-terminated string in user space.
1455  *
1456  * Returns the size of the string INCLUDING the terminating NUL.
1457  * On exception, returns 0.
1458  * If the string is too long, returns a value greater than @n.
1459  */
1460 static inline long strnlen_user(const char __user *s, long n)
1461 {
1462         long res;
1463
1464         might_fault();
1465         if (eva_kernel_access()) {
1466                 __asm__ __volatile__(
1467                         "move\t$4, %1\n\t"
1468                         "move\t$5, %2\n\t"
1469                         __MODULE_JAL(__strnlen_kernel_asm)
1470                         "move\t%0, $2"
1471                         : "=r" (res)
1472                         : "r" (s), "r" (n)
1473                         : "$2", "$4", "$5", __UA_t0, "$31");
1474         } else {
1475                 __asm__ __volatile__(
1476                         "move\t$4, %1\n\t"
1477                         "move\t$5, %2\n\t"
1478                         __MODULE_JAL(__strnlen_user_asm)
1479                         "move\t%0, $2"
1480                         : "=r" (res)
1481                         : "r" (s), "r" (n)
1482                         : "$2", "$4", "$5", __UA_t0, "$31");
1483         }
1484
1485         return res;
1486 }
1487
1488 struct exception_table_entry
1489 {
1490         unsigned long insn;
1491         unsigned long nextinsn;
1492 };
1493
1494 extern int fixup_exception(struct pt_regs *regs);
1495
1496 #endif /* _ASM_UACCESS_H */