arch/mips/include/asm/uaccess.h

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * Copyright (C) 1996, 1997, 1998, 1999, 2000, 03, 04 by Ralf Baechle
   7  * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
   8  * Copyright (C) 2007  Maciej W. Rozycki
   9  * Copyright (C) 2014, Imagination Technologies Ltd.
  10  */
  11 #ifndef _ASM_UACCESS_H
  12 #define _ASM_UACCESS_H
  13
  14 #include <linux/kernel.h>
  15 #include <linux/errno.h>
  16 #include <linux/thread_info.h>
  17 #include <linux/string.h>
  18 #include <asm/asm-eva.h>
  19 #include <asm/extable.h>
  20
  21 /*
  22  * The fs value determines whether argument validity checking should be
  23  * performed or not.  If get_fs() == USER_DS, checking is performed, with
  24  * get_fs() == KERNEL_DS, checking is bypassed.
  25  *
  26  * For historical reasons, these macros are grossly misnamed.
  27  */
  28 #ifdef CONFIG_32BIT
  29
  30 #ifdef CONFIG_KVM_GUEST
  31 #define __UA_LIMIT 0x40000000UL
  32 #else
  33 #define __UA_LIMIT 0x80000000UL
  34 #endif
  35
  36 #define __UA_ADDR       ".word"
  37 #define __UA_LA         "la"
  38 #define __UA_ADDU       "addu"
  39 #define __UA_t0         "$8"
  40 #define __UA_t1         "$9"
  41
  42 #endif /* CONFIG_32BIT */
  43
  44 #ifdef CONFIG_64BIT
  45
  46 extern u64 __ua_limit;
  47
  48 #define __UA_LIMIT      __ua_limit
  49
  50 #define __UA_ADDR       ".dword"
  51 #define __UA_LA         "dla"
  52 #define __UA_ADDU       "daddu"
  53 #define __UA_t0         "$12"
  54 #define __UA_t1         "$13"
  55
  56 #endif /* CONFIG_64BIT */
  57
  58 /*
  59  * USER_DS is a bitmask that has the bits set that may not be set in a valid
  60  * userspace address.  Note that we limit 32-bit userspace to 0x7fff8000 but
  61  * the arithmetic we're doing only works if the limit is a power of two, so
  62  * we use 0x80000000 here on 32-bit kernels.  If a process passes an invalid
  63  * address in this range it's the process's problem, not ours :-)
  64  */
  65
  66 #ifdef CONFIG_KVM_GUEST
  67 #define KERNEL_DS       ((mm_segment_t) { 0x80000000UL })
  68 #define USER_DS         ((mm_segment_t) { 0xC0000000UL })
  69 #else
  70 #define KERNEL_DS       ((mm_segment_t) { 0UL })
  71 #define USER_DS         ((mm_segment_t) { __UA_LIMIT })
  72 #endif
  73
  74 #define VERIFY_READ    0
  75 #define VERIFY_WRITE   1
  76
  77 #define get_ds()        (KERNEL_DS)
  78 #define get_fs()        (current_thread_info()->addr_limit)
  79 #define set_fs(x)       (current_thread_info()->addr_limit = (x))
  80
  81 #define segment_eq(a, b)        ((a).seg == (b).seg)
  82
  83 /*
  84  * eva_kernel_access() - determine whether kernel memory access on an EVA system
  85  *
  86  * Determines whether memory accesses should be performed to kernel memory
  87  * on a system using Extended Virtual Addressing (EVA).
  88  *
  89  * Return: true if a kernel memory access on an EVA system, else false.
  90  */
  91 static inline bool eva_kernel_access(void)
  92 {
  93         if (!IS_ENABLED(CONFIG_EVA))
  94                 return false;
  95
  96         return segment_eq(get_fs(), get_ds());
  97 }
  98
  99 /*
 100  * Is a address valid? This does a straightforward calculation rather
 101  * than tests.
 102  *
 103  * Address valid if:
 104  *  - "addr" doesn't have any high-bits set
 105  *  - AND "size" doesn't have any high-bits set
 106  *  - AND "addr+size" doesn't have any high-bits set
 107  *  - OR we are in kernel mode.
 108  *
 109  * __ua_size() is a trick to avoid runtime checking of positive constant
 110  * sizes; for those we already know at compile time that the size is ok.
 111  */
 112 #define __ua_size(size)                                                 \
 113         ((__builtin_constant_p(size) && (signed long) (size) > 0) ? 0 : (size))
 114
 115 /*
 116  * access_ok: - Checks if a user space pointer is valid
 117  * @type: Type of access: %VERIFY_READ or %VERIFY_WRITE.  Note that
 118  *        %VERIFY_WRITE is a superset of %VERIFY_READ - if it is safe
 119  *        to write to a block, it is always safe to read from it.
 120  * @addr: User space pointer to start of block to check
 121  * @size: Size of block to check
 122  *
 123  * Context: User context only. This function may sleep if pagefaults are
 124  *          enabled.
 125  *
 126  * Checks if a pointer to a block of memory in user space is valid.
 127  *
 128  * Returns true (nonzero) if the memory block may be valid, false (zero)
 129  * if it is definitely invalid.
 130  *
 131  * Note that, depending on architecture, this function probably just
 132  * checks that the pointer is in the user space range - after calling
 133  * this function, memory access functions may still return -EFAULT.
 134  */
 135
 136 #define __access_mask get_fs().seg
 137
 138 #define __access_ok(addr, size, mask)                                   \
 139 ({                                                                      \
 140         unsigned long __addr = (unsigned long) (addr);                  \
 141         unsigned long __size = size;                                    \
 142         unsigned long __mask = mask;                                    \
 143         unsigned long __ok;                                             \
 144                                                                         \
 145         __chk_user_ptr(addr);                                           \
 146         __ok = (signed long)(__mask & (__addr | (__addr + __size) |     \
 147                 __ua_size(__size)));                                    \
 148         __ok == 0;                                                      \
 149 })
 150
 151 #define access_ok(type, addr, size)                                     \
 152         likely(__access_ok((addr), (size), __access_mask))
 153
 154 /*
 155  * put_user: - Write a simple value into user space.
 156  * @x:   Value to copy to user space.
 157  * @ptr: Destination address, in user space.
 158  *
 159  * Context: User context only. This function may sleep if pagefaults are
 160  *          enabled.
 161  *
 162  * This macro copies a single simple value from kernel space to user
 163  * space.  It supports simple types like char and int, but not larger
 164  * data types like structures or arrays.
 165  *
 166  * @ptr must have pointer-to-simple-variable type, and @x must be assignable
 167  * to the result of dereferencing @ptr.
 168  *
 169  * Returns zero on success, or -EFAULT on error.
 170  */
 171 #define put_user(x,ptr) \
 172         __put_user_check((x), (ptr), sizeof(*(ptr)))
 173
 174 /*
 175  * get_user: - Get a simple variable from user space.
 176  * @x:   Variable to store result.
 177  * @ptr: Source address, in user space.
 178  *
 179  * Context: User context only. This function may sleep if pagefaults are
 180  *          enabled.
 181  *
 182  * This macro copies a single simple variable from user space to kernel
 183  * space.  It supports simple types like char and int, but not larger
 184  * data types like structures or arrays.
 185  *
 186  * @ptr must have pointer-to-simple-variable type, and the result of
 187  * dereferencing @ptr must be assignable to @x without a cast.
 188  *
 189  * Returns zero on success, or -EFAULT on error.
 190  * On error, the variable @x is set to zero.
 191  */
 192 #define get_user(x,ptr) \
 193         __get_user_check((x), (ptr), sizeof(*(ptr)))
 194
 195 /*
 196  * __put_user: - Write a simple value into user space, with less checking.
 197  * @x:   Value to copy to user space.
 198  * @ptr: Destination address, in user space.
 199  *
 200  * Context: User context only. This function may sleep if pagefaults are
 201  *          enabled.
 202  *
 203  * This macro copies a single simple value from kernel space to user
 204  * space.  It supports simple types like char and int, but not larger
 205  * data types like structures or arrays.
 206  *
 207  * @ptr must have pointer-to-simple-variable type, and @x must be assignable
 208  * to the result of dereferencing @ptr.
 209  *
 210  * Caller must check the pointer with access_ok() before calling this
 211  * function.
 212  *
 213  * Returns zero on success, or -EFAULT on error.
 214  */
 215 #define __put_user(x,ptr) \
 216         __put_user_nocheck((x), (ptr), sizeof(*(ptr)))
 217
 218 /*
 219  * __get_user: - Get a simple variable from user space, with less checking.
 220  * @x:   Variable to store result.
 221  * @ptr: Source address, in user space.
 222  *
 223  * Context: User context only. This function may sleep if pagefaults are
 224  *          enabled.
 225  *
 226  * This macro copies a single simple variable from user space to kernel
 227  * space.  It supports simple types like char and int, but not larger
 228  * data types like structures or arrays.
 229  *
 230  * @ptr must have pointer-to-simple-variable type, and the result of
 231  * dereferencing @ptr must be assignable to @x without a cast.
 232  *
 233  * Caller must check the pointer with access_ok() before calling this
 234  * function.
 235  *
 236  * Returns zero on success, or -EFAULT on error.
 237  * On error, the variable @x is set to zero.
 238  */
 239 #define __get_user(x,ptr) \
 240         __get_user_nocheck((x), (ptr), sizeof(*(ptr)))
 241
 242 struct __large_struct { unsigned long buf[100]; };
 243 #define __m(x) (*(struct __large_struct __user *)(x))
 244
 245 /*
 246  * Yuck.  We need two variants, one for 64bit operation and one
 247  * for 32 bit mode and old iron.
 248  */
 249 #ifndef CONFIG_EVA
 250 #define __get_kernel_common(val, size, ptr) __get_user_common(val, size, ptr)
 251 #else
 252 /*
 253  * Kernel specific functions for EVA. We need to use normal load instructions
 254  * to read data from kernel when operating in EVA mode. We use these macros to
 255  * avoid redefining __get_user_asm for EVA.
 256  */
 257 #undef _loadd
 258 #undef _loadw
 259 #undef _loadh
 260 #undef _loadb
 261 #ifdef CONFIG_32BIT
 262 #define _loadd                  _loadw
 263 #else
 264 #define _loadd(reg, addr)       "ld " reg ", " addr
 265 #endif
 266 #define _loadw(reg, addr)       "lw " reg ", " addr
 267 #define _loadh(reg, addr)       "lh " reg ", " addr
 268 #define _loadb(reg, addr)       "lb " reg ", " addr
 269
 270 #define __get_kernel_common(val, size, ptr)                             \
 271 do {                                                                    \
 272         switch (size) {                                                 \
 273         case 1: __get_data_asm(val, _loadb, ptr); break;                \
 274         case 2: __get_data_asm(val, _loadh, ptr); break;                \
 275         case 4: __get_data_asm(val, _loadw, ptr); break;                \
 276         case 8: __GET_DW(val, _loadd, ptr); break;                      \
 277         default: __get_user_unknown(); break;                           \
 278         }                                                               \
 279 } while (0)
 280 #endif
 281
 282 #ifdef CONFIG_32BIT
 283 #define __GET_DW(val, insn, ptr) __get_data_asm_ll32(val, insn, ptr)
 284 #endif
 285 #ifdef CONFIG_64BIT
 286 #define __GET_DW(val, insn, ptr) __get_data_asm(val, insn, ptr)
 287 #endif
 288
 289 extern void __get_user_unknown(void);
 290
 291 #define __get_user_common(val, size, ptr)                               \
 292 do {                                                                    \
 293         switch (size) {                                                 \
 294         case 1: __get_data_asm(val, user_lb, ptr); break;               \
 295         case 2: __get_data_asm(val, user_lh, ptr); break;               \
 296         case 4: __get_data_asm(val, user_lw, ptr); break;               \
 297         case 8: __GET_DW(val, user_ld, ptr); break;                     \
 298         default: __get_user_unknown(); break;                           \
 299         }                                                               \
 300 } while (0)
 301
 302 #define __get_user_nocheck(x, ptr, size)                                \
 303 ({                                                                      \
 304         int __gu_err;                                                   \
 305                                                                         \
 306         if (eva_kernel_access()) {                                      \
 307                 __get_kernel_common((x), size, ptr);                    \
 308         } else {                                                        \
 309                 __chk_user_ptr(ptr);                                    \
 310                 __get_user_common((x), size, ptr);                      \
 311         }                                                               \
 312         __gu_err;                                                       \
 313 })
 314
 315 #define __get_user_check(x, ptr, size)                                  \
 316 ({                                                                      \
 317         int __gu_err = -EFAULT;                                         \
 318         const __typeof__(*(ptr)) __user * __gu_ptr = (ptr);             \
 319                                                                         \
 320         might_fault();                                                  \
 321         if (likely(access_ok(VERIFY_READ,  __gu_ptr, size))) {          \
 322                 if (eva_kernel_access())                                \
 323                         __get_kernel_common((x), size, __gu_ptr);       \
 324                 else                                                    \
 325                         __get_user_common((x), size, __gu_ptr);         \
 326         } else                                                          \
 327                 (x) = 0;                                                \
 328                                                                         \
 329         __gu_err;                                                       \
 330 })
 331
 332 #define __get_data_asm(val, insn, addr)                                 \
 333 {                                                                       \
 334         long __gu_tmp;                                                  \
 335                                                                         \
 336         __asm__ __volatile__(                                           \
 337         "1:     "insn("%1", "%3")"                              \n"     \
 338         "2:                                                     \n"     \
 339         "       .insn                                           \n"     \
 340         "       .section .fixup,\"ax\"                          \n"     \
 341         "3:     li      %0, %4                                  \n"     \
 342         "       move    %1, $0                                  \n"     \
 343         "       j       2b                                      \n"     \
 344         "       .previous                                       \n"     \
 345         "       .section __ex_table,\"a\"                       \n"     \
 346         "       "__UA_ADDR "\t1b, 3b                            \n"     \
 347         "       .previous                                       \n"     \
 348         : "=r" (__gu_err), "=r" (__gu_tmp)                              \
 349         : "0" (0), "o" (__m(addr)), "i" (-EFAULT));                     \
 350                                                                         \
 351         (val) = (__typeof__(*(addr))) __gu_tmp;                         \
 352 }
 353
 354 /*
 355  * Get a long long 64 using 32 bit registers.
 356  */
 357 #define __get_data_asm_ll32(val, insn, addr)                            \
 358 {                                                                       \
 359         union {                                                         \
 360                 unsigned long long      l;                              \
 361                 __typeof__(*(addr))     t;                              \
 362         } __gu_tmp;                                                     \
 363                                                                         \
 364         __asm__ __volatile__(                                           \
 365         "1:     " insn("%1", "(%3)")"                           \n"     \
 366         "2:     " insn("%D1", "4(%3)")"                         \n"     \
 367         "3:                                                     \n"     \
 368         "       .insn                                           \n"     \
 369         "       .section        .fixup,\"ax\"                   \n"     \
 370         "4:     li      %0, %4                                  \n"     \
 371         "       move    %1, $0                                  \n"     \
 372         "       move    %D1, $0                                 \n"     \
 373         "       j       3b                                      \n"     \
 374         "       .previous                                       \n"     \
 375         "       .section        __ex_table,\"a\"                \n"     \
 376         "       " __UA_ADDR "   1b, 4b                          \n"     \
 377         "       " __UA_ADDR "   2b, 4b                          \n"     \
 378         "       .previous                                       \n"     \
 379         : "=r" (__gu_err), "=&r" (__gu_tmp.l)                           \
 380         : "0" (0), "r" (addr), "i" (-EFAULT));                          \
 381                                                                         \
 382         (val) = __gu_tmp.t;                                             \
 383 }
 384
 385 #ifndef CONFIG_EVA
 386 #define __put_kernel_common(ptr, size) __put_user_common(ptr, size)
 387 #else
 388 /*
 389  * Kernel specific functions for EVA. We need to use normal load instructions
 390  * to read data from kernel when operating in EVA mode. We use these macros to
 391  * avoid redefining __get_data_asm for EVA.
 392  */
 393 #undef _stored
 394 #undef _storew
 395 #undef _storeh
 396 #undef _storeb
 397 #ifdef CONFIG_32BIT
 398 #define _stored                 _storew
 399 #else
 400 #define _stored(reg, addr)      "ld " reg ", " addr
 401 #endif
 402
 403 #define _storew(reg, addr)      "sw " reg ", " addr
 404 #define _storeh(reg, addr)      "sh " reg ", " addr
 405 #define _storeb(reg, addr)      "sb " reg ", " addr
 406
 407 #define __put_kernel_common(ptr, size)                                  \
 408 do {                                                                    \
 409         switch (size) {                                                 \
 410         case 1: __put_data_asm(_storeb, ptr); break;                    \
 411         case 2: __put_data_asm(_storeh, ptr); break;                    \
 412         case 4: __put_data_asm(_storew, ptr); break;                    \
 413         case 8: __PUT_DW(_stored, ptr); break;                          \
 414         default: __put_user_unknown(); break;                           \
 415         }                                                               \
 416 } while(0)
 417 #endif
 418
 419 /*
 420  * Yuck.  We need two variants, one for 64bit operation and one
 421  * for 32 bit mode and old iron.
 422  */
 423 #ifdef CONFIG_32BIT
 424 #define __PUT_DW(insn, ptr) __put_data_asm_ll32(insn, ptr)
 425 #endif
 426 #ifdef CONFIG_64BIT
 427 #define __PUT_DW(insn, ptr) __put_data_asm(insn, ptr)
 428 #endif
 429
 430 #define __put_user_common(ptr, size)                                    \
 431 do {                                                                    \
 432         switch (size) {                                                 \
 433         case 1: __put_data_asm(user_sb, ptr); break;                    \
 434         case 2: __put_data_asm(user_sh, ptr); break;                    \
 435         case 4: __put_data_asm(user_sw, ptr); break;                    \
 436         case 8: __PUT_DW(user_sd, ptr); break;                          \
 437         default: __put_user_unknown(); break;                           \
 438         }                                                               \
 439 } while (0)
 440
 441 #define __put_user_nocheck(x, ptr, size)                                \
 442 ({                                                                      \
 443         __typeof__(*(ptr)) __pu_val;                                    \
 444         int __pu_err = 0;                                               \
 445                                                                         \
 446         __pu_val = (x);                                                 \
 447         if (eva_kernel_access()) {                                      \
 448                 __put_kernel_common(ptr, size);                         \
 449         } else {                                                        \
 450                 __chk_user_ptr(ptr);                                    \
 451                 __put_user_common(ptr, size);                           \
 452         }                                                               \
 453         __pu_err;                                                       \
 454 })
 455
 456 #define __put_user_check(x, ptr, size)                                  \
 457 ({                                                                      \
 458         __typeof__(*(ptr)) __user *__pu_addr = (ptr);                   \
 459         __typeof__(*(ptr)) __pu_val = (x);                              \
 460         int __pu_err = -EFAULT;                                         \
 461                                                                         \
 462         might_fault();                                                  \
 463         if (likely(access_ok(VERIFY_WRITE,  __pu_addr, size))) {        \
 464                 if (eva_kernel_access())                                \
 465                         __put_kernel_common(__pu_addr, size);           \
 466                 else                                                    \
 467                         __put_user_common(__pu_addr, size);             \
 468         }                                                               \
 469                                                                         \
 470         __pu_err;                                                       \
 471 })
 472
 473 #define __put_data_asm(insn, ptr)                                       \
 474 {                                                                       \
 475         __asm__ __volatile__(                                           \
 476         "1:     "insn("%z2", "%3")"     # __put_data_asm        \n"     \
 477         "2:                                                     \n"     \
 478         "       .insn                                           \n"     \
 479         "       .section        .fixup,\"ax\"                   \n"     \
 480         "3:     li      %0, %4                                  \n"     \
 481         "       j       2b                                      \n"     \
 482         "       .previous                                       \n"     \
 483         "       .section        __ex_table,\"a\"                \n"     \
 484         "       " __UA_ADDR "   1b, 3b                          \n"     \
 485         "       .previous                                       \n"     \
 486         : "=r" (__pu_err)                                               \
 487         : "0" (0), "Jr" (__pu_val), "o" (__m(ptr)),                     \
 488           "i" (-EFAULT));                                               \
 489 }
 490
 491 #define __put_data_asm_ll32(insn, ptr)                                  \
 492 {                                                                       \
 493         __asm__ __volatile__(                                           \
 494         "1:     "insn("%2", "(%3)")"    # __put_data_asm_ll32   \n"     \
 495         "2:     "insn("%D2", "4(%3)")"                          \n"     \
 496         "3:                                                     \n"     \
 497         "       .insn                                           \n"     \
 498         "       .section        .fixup,\"ax\"                   \n"     \
 499         "4:     li      %0, %4                                  \n"     \
 500         "       j       3b                                      \n"     \
 501         "       .previous                                       \n"     \
 502         "       .section        __ex_table,\"a\"                \n"     \
 503         "       " __UA_ADDR "   1b, 4b                          \n"     \
 504         "       " __UA_ADDR "   2b, 4b                          \n"     \
 505         "       .previous"                                              \
 506         : "=r" (__pu_err)                                               \
 507         : "0" (0), "r" (__pu_val), "r" (ptr),                           \
 508           "i" (-EFAULT));                                               \
 509 }
 510
 511 extern void __put_user_unknown(void);
 512
 513 /*
 514  * ul{b,h,w} are macros and there are no equivalent macros for EVA.
 515  * EVA unaligned access is handled in the ADE exception handler.
 516  */
 517 #ifndef CONFIG_EVA
 518 /*
 519  * put_user_unaligned: - Write a simple value into user space.
 520  * @x:   Value to copy to user space.
 521  * @ptr: Destination address, in user space.
 522  *
 523  * Context: User context only. This function may sleep if pagefaults are
 524  *          enabled.
 525  *
 526  * This macro copies a single simple value from kernel space to user
 527  * space.  It supports simple types like char and int, but not larger
 528  * data types like structures or arrays.
 529  *
 530  * @ptr must have pointer-to-simple-variable type, and @x must be assignable
 531  * to the result of dereferencing @ptr.
 532  *
 533  * Returns zero on success, or -EFAULT on error.
 534  */
 535 #define put_user_unaligned(x,ptr)       \
 536         __put_user_unaligned_check((x),(ptr),sizeof(*(ptr)))
 537
 538 /*
 539  * get_user_unaligned: - Get a simple variable from user space.
 540  * @x:   Variable to store result.
 541  * @ptr: Source address, in user space.
 542  *
 543  * Context: User context only. This function may sleep if pagefaults are
 544  *          enabled.
 545  *
 546  * This macro copies a single simple variable from user space to kernel
 547  * space.  It supports simple types like char and int, but not larger
 548  * data types like structures or arrays.
 549  *
 550  * @ptr must have pointer-to-simple-variable type, and the result of
 551  * dereferencing @ptr must be assignable to @x without a cast.
 552  *
 553  * Returns zero on success, or -EFAULT on error.
 554  * On error, the variable @x is set to zero.
 555  */
 556 #define get_user_unaligned(x,ptr) \
 557         __get_user_unaligned_check((x),(ptr),sizeof(*(ptr)))
 558
 559 /*
 560  * __put_user_unaligned: - Write a simple value into user space, with less checking.
 561  * @x:   Value to copy to user space.
 562  * @ptr: Destination address, in user space.
 563  *
 564  * Context: User context only. This function may sleep if pagefaults are
 565  *          enabled.
 566  *
 567  * This macro copies a single simple value from kernel space to user
 568  * space.  It supports simple types like char and int, but not larger
 569  * data types like structures or arrays.
 570  *
 571  * @ptr must have pointer-to-simple-variable type, and @x must be assignable
 572  * to the result of dereferencing @ptr.
 573  *
 574  * Caller must check the pointer with access_ok() before calling this
 575  * function.
 576  *
 577  * Returns zero on success, or -EFAULT on error.
 578  */
 579 #define __put_user_unaligned(x,ptr) \
 580         __put_user_unaligned_nocheck((x),(ptr),sizeof(*(ptr)))
 581
 582 /*
 583  * __get_user_unaligned: - Get a simple variable from user space, with less checking.
 584  * @x:   Variable to store result.
 585  * @ptr: Source address, in user space.
 586  *
 587  * Context: User context only. This function may sleep if pagefaults are
 588  *          enabled.
 589  *
 590  * This macro copies a single simple variable from user space to kernel
 591  * space.  It supports simple types like char and int, but not larger
 592  * data types like structures or arrays.
 593  *
 594  * @ptr must have pointer-to-simple-variable type, and the result of
 595  * dereferencing @ptr must be assignable to @x without a cast.
 596  *
 597  * Caller must check the pointer with access_ok() before calling this
 598  * function.
 599  *
 600  * Returns zero on success, or -EFAULT on error.
 601  * On error, the variable @x is set to zero.
 602  */
 603 #define __get_user_unaligned(x,ptr) \
 604         __get_user_unaligned_nocheck((x),(ptr),sizeof(*(ptr)))
 605
 606 /*
 607  * Yuck.  We need two variants, one for 64bit operation and one
 608  * for 32 bit mode and old iron.
 609  */
 610 #ifdef CONFIG_32BIT
 611 #define __GET_USER_UNALIGNED_DW(val, ptr)                               \
 612         __get_user_unaligned_asm_ll32(val, ptr)
 613 #endif
 614 #ifdef CONFIG_64BIT
 615 #define __GET_USER_UNALIGNED_DW(val, ptr)                               \
 616         __get_user_unaligned_asm(val, "uld", ptr)
 617 #endif
 618
 619 extern void __get_user_unaligned_unknown(void);
 620
 621 #define __get_user_unaligned_common(val, size, ptr)                     \
 622 do {                                                                    \
 623         switch (size) {                                                 \
 624         case 1: __get_data_asm(val, "lb", ptr); break;                  \
 625         case 2: __get_data_unaligned_asm(val, "ulh", ptr); break;       \
 626         case 4: __get_data_unaligned_asm(val, "ulw", ptr); break;       \
 627         case 8: __GET_USER_UNALIGNED_DW(val, ptr); break;               \
 628         default: __get_user_unaligned_unknown(); break;                 \
 629         }                                                               \
 630 } while (0)
 631
 632 #define __get_user_unaligned_nocheck(x,ptr,size)                        \
 633 ({                                                                      \
 634         int __gu_err;                                                   \
 635                                                                         \
 636         __get_user_unaligned_common((x), size, ptr);                    \
 637         __gu_err;                                                       \
 638 })
 639
 640 #define __get_user_unaligned_check(x,ptr,size)                          \
 641 ({                                                                      \
 642         int __gu_err = -EFAULT;                                         \
 643         const __typeof__(*(ptr)) __user * __gu_ptr = (ptr);             \
 644                                                                         \
 645         if (likely(access_ok(VERIFY_READ,  __gu_ptr, size)))            \
 646                 __get_user_unaligned_common((x), size, __gu_ptr);       \
 647                                                                         \
 648         __gu_err;                                                       \
 649 })
 650
 651 #define __get_data_unaligned_asm(val, insn, addr)                       \
 652 {                                                                       \
 653         long __gu_tmp;                                                  \
 654                                                                         \
 655         __asm__ __volatile__(                                           \
 656         "1:     " insn "        %1, %3                          \n"     \
 657         "2:                                                     \n"     \
 658         "       .insn                                           \n"     \
 659         "       .section .fixup,\"ax\"                          \n"     \
 660         "3:     li      %0, %4                                  \n"     \
 661         "       move    %1, $0                                  \n"     \
 662         "       j       2b                                      \n"     \
 663         "       .previous                                       \n"     \
 664         "       .section __ex_table,\"a\"                       \n"     \
 665         "       "__UA_ADDR "\t1b, 3b                            \n"     \
 666         "       "__UA_ADDR "\t1b + 4, 3b                        \n"     \
 667         "       .previous                                       \n"     \
 668         : "=r" (__gu_err), "=r" (__gu_tmp)                              \
 669         : "0" (0), "o" (__m(addr)), "i" (-EFAULT));                     \
 670                                                                         \
 671         (val) = (__typeof__(*(addr))) __gu_tmp;                         \
 672 }
 673
 674 /*
 675  * Get a long long 64 using 32 bit registers.
 676  */
 677 #define __get_user_unaligned_asm_ll32(val, addr)                        \
 678 {                                                                       \
 679         unsigned long long __gu_tmp;                                    \
 680                                                                         \
 681         __asm__ __volatile__(                                           \
 682         "1:     ulw     %1, (%3)                                \n"     \
 683         "2:     ulw     %D1, 4(%3)                              \n"     \
 684         "       move    %0, $0                                  \n"     \
 685         "3:                                                     \n"     \
 686         "       .insn                                           \n"     \
 687         "       .section        .fixup,\"ax\"                   \n"     \
 688         "4:     li      %0, %4                                  \n"     \
 689         "       move    %1, $0                                  \n"     \
 690         "       move    %D1, $0                                 \n"     \
 691         "       j       3b                                      \n"     \
 692         "       .previous                                       \n"     \
 693         "       .section        __ex_table,\"a\"                \n"     \
 694         "       " __UA_ADDR "   1b, 4b                          \n"     \
 695         "       " __UA_ADDR "   1b + 4, 4b                      \n"     \
 696         "       " __UA_ADDR "   2b, 4b                          \n"     \
 697         "       " __UA_ADDR "   2b + 4, 4b                      \n"     \
 698         "       .previous                                       \n"     \
 699         : "=r" (__gu_err), "=&r" (__gu_tmp)                             \
 700         : "0" (0), "r" (addr), "i" (-EFAULT));                          \
 701         (val) = (__typeof__(*(addr))) __gu_tmp;                         \
 702 }
 703
 704 /*
 705  * Yuck.  We need two variants, one for 64bit operation and one
 706  * for 32 bit mode and old iron.
 707  */
 708 #ifdef CONFIG_32BIT
 709 #define __PUT_USER_UNALIGNED_DW(ptr) __put_user_unaligned_asm_ll32(ptr)
 710 #endif
 711 #ifdef CONFIG_64BIT
 712 #define __PUT_USER_UNALIGNED_DW(ptr) __put_user_unaligned_asm("usd", ptr)
 713 #endif
 714
 715 #define __put_user_unaligned_common(ptr, size)                          \
 716 do {                                                                    \
 717         switch (size) {                                                 \
 718         case 1: __put_data_asm("sb", ptr); break;                       \
 719         case 2: __put_user_unaligned_asm("ush", ptr); break;            \
 720         case 4: __put_user_unaligned_asm("usw", ptr); break;            \
 721         case 8: __PUT_USER_UNALIGNED_DW(ptr); break;                    \
 722         default: __put_user_unaligned_unknown(); break;                 \
 723 } while (0)
 724
 725 #define __put_user_unaligned_nocheck(x,ptr,size)                        \
 726 ({                                                                      \
 727         __typeof__(*(ptr)) __pu_val;                                    \
 728         int __pu_err = 0;                                               \
 729                                                                         \
 730         __pu_val = (x);                                                 \
 731         __put_user_unaligned_common(ptr, size);                         \
 732         __pu_err;                                                       \
 733 })
 734
 735 #define __put_user_unaligned_check(x,ptr,size)                          \
 736 ({                                                                      \
 737         __typeof__(*(ptr)) __user *__pu_addr = (ptr);                   \
 738         __typeof__(*(ptr)) __pu_val = (x);                              \
 739         int __pu_err = -EFAULT;                                         \
 740                                                                         \
 741         if (likely(access_ok(VERIFY_WRITE,  __pu_addr, size)))          \
 742                 __put_user_unaligned_common(__pu_addr, size);           \
 743                                                                         \
 744         __pu_err;                                                       \
 745 })
 746
 747 #define __put_user_unaligned_asm(insn, ptr)                             \
 748 {                                                                       \
 749         __asm__ __volatile__(                                           \
 750         "1:     " insn "        %z2, %3         # __put_user_unaligned_asm\n" \
 751         "2:                                                     \n"     \
 752         "       .insn                                           \n"     \
 753         "       .section        .fixup,\"ax\"                   \n"     \
 754         "3:     li      %0, %4                                  \n"     \
 755         "       j       2b                                      \n"     \
 756         "       .previous                                       \n"     \
 757         "       .section        __ex_table,\"a\"                \n"     \
 758         "       " __UA_ADDR "   1b, 3b                          \n"     \
 759         "       .previous                                       \n"     \
 760         : "=r" (__pu_err)                                               \
 761         : "0" (0), "Jr" (__pu_val), "o" (__m(ptr)),                     \
 762           "i" (-EFAULT));                                               \
 763 }
 764
 765 #define __put_user_unaligned_asm_ll32(ptr)                              \
 766 {                                                                       \
 767         __asm__ __volatile__(                                           \
 768         "1:     sw      %2, (%3)        # __put_user_unaligned_asm_ll32 \n" \
 769         "2:     sw      %D2, 4(%3)                              \n"     \
 770         "3:                                                     \n"     \
 771         "       .insn                                           \n"     \
 772         "       .section        .fixup,\"ax\"                   \n"     \
 773         "4:     li      %0, %4                                  \n"     \
 774         "       j       3b                                      \n"     \
 775         "       .previous                                       \n"     \
 776         "       .section        __ex_table,\"a\"                \n"     \
 777         "       " __UA_ADDR "   1b, 4b                          \n"     \
 778         "       " __UA_ADDR "   1b + 4, 4b                      \n"     \
 779         "       " __UA_ADDR "   2b, 4b                          \n"     \
 780         "       " __UA_ADDR "   2b + 4, 4b                      \n"     \
 781         "       .previous"                                              \
 782         : "=r" (__pu_err)                                               \
 783         : "0" (0), "r" (__pu_val), "r" (ptr),                           \
 784           "i" (-EFAULT));                                               \
 785 }
 786
 787 extern void __put_user_unaligned_unknown(void);
 788 #endif
 789
 790 /*
 791  * We're generating jump to subroutines which will be outside the range of
 792  * jump instructions
 793  */
 794 #ifdef MODULE
 795 #define __MODULE_JAL(destination)                                       \
 796         ".set\tnoat\n\t"                                                \
 797         __UA_LA "\t$1, " #destination "\n\t"                            \
 798         "jalr\t$1\n\t"                                                  \
 799         ".set\tat\n\t"
 800 #else
 801 #define __MODULE_JAL(destination)                                       \
 802         "jal\t" #destination "\n\t"
 803 #endif
 804
 805 #if defined(CONFIG_CPU_DADDI_WORKAROUNDS) || (defined(CONFIG_EVA) &&    \
 806                                               defined(CONFIG_CPU_HAS_PREFETCH))
 807 #define DADDI_SCRATCH "$3"
 808 #else
 809 #define DADDI_SCRATCH "$0"
 810 #endif
 811
 812 extern size_t __copy_user(void *__to, const void *__from, size_t __n);
 813
 814 #ifndef CONFIG_EVA
 815 #define __invoke_copy_to_user(to, from, n)                              \
 816 ({                                                                      \
 817         register void __user *__cu_to_r __asm__("$4");                  \
 818         register const void *__cu_from_r __asm__("$5");                 \
 819         register long __cu_len_r __asm__("$6");                         \
 820                                                                         \
 821         __cu_to_r = (to);                                               \
 822         __cu_from_r = (from);                                           \
 823         __cu_len_r = (n);                                               \
 824         __asm__ __volatile__(                                           \
 825         __MODULE_JAL(__copy_user)                                       \
 826         : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r)       \
 827         :                                                               \
 828         : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31",  \
 829           DADDI_SCRATCH, "memory");                                     \
 830         __cu_len_r;                                                     \
 831 })
 832
 833 #define __invoke_copy_to_kernel(to, from, n)                            \
 834         __invoke_copy_to_user(to, from, n)
 835
 836 #endif
 837
 838 /*
 839  * __copy_to_user: - Copy a block of data into user space, with less checking.
 840  * @to:   Destination address, in user space.
 841  * @from: Source address, in kernel space.
 842  * @n:    Number of bytes to copy.
 843  *
 844  * Context: User context only. This function may sleep if pagefaults are
 845  *          enabled.
 846  *
 847  * Copy data from kernel space to user space.  Caller must check
 848  * the specified block with access_ok() before calling this function.
 849  *
 850  * Returns number of bytes that could not be copied.
 851  * On success, this will be zero.
 852  */
 853 #define __copy_to_user(to, from, n)                                     \
 854 ({                                                                      \
 855         void __user *__cu_to;                                           \
 856         const void *__cu_from;                                          \
 857         long __cu_len;                                                  \
 858                                                                         \
 859         __cu_to = (to);                                                 \
 860         __cu_from = (from);                                             \
 861         __cu_len = (n);                                                 \
 862         might_fault();                                                  \
 863         if (eva_kernel_access())                                        \
 864                 __cu_len = __invoke_copy_to_kernel(__cu_to, __cu_from,  \
 865                                                    __cu_len);           \
 866         else                                                            \
 867                 __cu_len = __invoke_copy_to_user(__cu_to, __cu_from,    \
 868                                                  __cu_len);             \
 869         __cu_len;                                                       \
 870 })
 871
 872 extern size_t __copy_user_inatomic(void *__to, const void *__from, size_t __n);
 873
 874 #define __copy_to_user_inatomic(to, from, n)                            \
 875 ({                                                                      \
 876         void __user *__cu_to;                                           \
 877         const void *__cu_from;                                          \
 878         long __cu_len;                                                  \
 879                                                                         \
 880         __cu_to = (to);                                                 \
 881         __cu_from = (from);                                             \
 882         __cu_len = (n);                                                 \
 883         if (eva_kernel_access())                                        \
 884                 __cu_len = __invoke_copy_to_kernel(__cu_to, __cu_from,  \
 885                                                    __cu_len);           \
 886         else                                                            \
 887                 __cu_len = __invoke_copy_to_user(__cu_to, __cu_from,    \
 888                                                  __cu_len);             \
 889         __cu_len;                                                       \
 890 })
 891
 892 #define __copy_from_user_inatomic(to, from, n)                          \
 893 ({                                                                      \
 894         void *__cu_to;                                                  \
 895         const void __user *__cu_from;                                   \
 896         long __cu_len;                                                  \
 897                                                                         \
 898         __cu_to = (to);                                                 \
 899         __cu_from = (from);                                             \
 900         __cu_len = (n);                                                 \
 901         if (eva_kernel_access())                                        \
 902                 __cu_len = __invoke_copy_from_kernel_inatomic(__cu_to,  \
 903                                                               __cu_from,\
 904                                                               __cu_len);\
 905         else                                                            \
 906                 __cu_len = __invoke_copy_from_user_inatomic(__cu_to,    \
 907                                                             __cu_from,  \
 908                                                             __cu_len);  \
 909         __cu_len;                                                       \
 910 })
 911
 912 /*
 913  * copy_to_user: - Copy a block of data into user space.
 914  * @to:   Destination address, in user space.
 915  * @from: Source address, in kernel space.
 916  * @n:    Number of bytes to copy.
 917  *
 918  * Context: User context only. This function may sleep if pagefaults are
 919  *          enabled.
 920  *
 921  * Copy data from kernel space to user space.
 922  *
 923  * Returns number of bytes that could not be copied.
 924  * On success, this will be zero.
 925  */
 926 #define copy_to_user(to, from, n)                                       \
 927 ({                                                                      \
 928         void __user *__cu_to;                                           \
 929         const void *__cu_from;                                          \
 930         long __cu_len;                                                  \
 931                                                                         \
 932         __cu_to = (to);                                                 \
 933         __cu_from = (from);                                             \
 934         __cu_len = (n);                                                 \
 935         if (eva_kernel_access()) {                                      \
 936                 __cu_len = __invoke_copy_to_kernel(__cu_to,             \
 937                                                    __cu_from,           \
 938                                                    __cu_len);           \
 939         } else {                                                        \
 940                 if (access_ok(VERIFY_WRITE, __cu_to, __cu_len)) {       \
 941                         might_fault();                                  \
 942                         __cu_len = __invoke_copy_to_user(__cu_to,       \
 943                                                          __cu_from,     \
 944                                                          __cu_len);     \
 945                 }                                                       \
 946         }                                                               \
 947         __cu_len;                                                       \
 948 })
 949
 950 #ifndef CONFIG_EVA
 951
 952 #define __invoke_copy_from_user(to, from, n)                            \
 953 ({                                                                      \
 954         register void *__cu_to_r __asm__("$4");                         \
 955         register const void __user *__cu_from_r __asm__("$5");          \
 956         register long __cu_len_r __asm__("$6");                         \
 957                                                                         \
 958         __cu_to_r = (to);                                               \
 959         __cu_from_r = (from);                                           \
 960         __cu_len_r = (n);                                               \
 961         __asm__ __volatile__(                                           \
 962         ".set\tnoreorder\n\t"                                           \
 963         __MODULE_JAL(__copy_user)                                       \
 964         ".set\tnoat\n\t"                                                \
 965         __UA_ADDU "\t$1, %1, %2\n\t"                                    \
 966         ".set\tat\n\t"                                                  \
 967         ".set\treorder"                                                 \
 968         : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r)       \
 969         :                                                               \
 970         : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31",  \
 971           DADDI_SCRATCH, "memory");                                     \
 972         __cu_len_r;                                                     \
 973 })
 974
 975 #define __invoke_copy_from_kernel(to, from, n)                          \
 976         __invoke_copy_from_user(to, from, n)
 977
 978 /* For userland <-> userland operations */
 979 #define ___invoke_copy_in_user(to, from, n)                             \
 980         __invoke_copy_from_user(to, from, n)
 981
 982 /* For kernel <-> kernel operations */
 983 #define ___invoke_copy_in_kernel(to, from, n)                           \
 984         __invoke_copy_from_user(to, from, n)
 985
 986 #define __invoke_copy_from_user_inatomic(to, from, n)                   \
 987 ({                                                                      \
 988         register void *__cu_to_r __asm__("$4");                         \
 989         register const void __user *__cu_from_r __asm__("$5");          \
 990         register long __cu_len_r __asm__("$6");                         \
 991                                                                         \
 992         __cu_to_r = (to);                                               \
 993         __cu_from_r = (from);                                           \
 994         __cu_len_r = (n);                                               \
 995         __asm__ __volatile__(                                           \
 996         ".set\tnoreorder\n\t"                                           \
 997         __MODULE_JAL(__copy_user_inatomic)                              \
 998         ".set\tnoat\n\t"                                                \
 999         __UA_ADDU "\t$1, %1, %2\n\t"                                    \
1000         ".set\tat\n\t"                                                  \
1001         ".set\treorder"                                                 \
1002         : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r)       \
1003         :                                                               \
1004         : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31",  \
1005           DADDI_SCRATCH, "memory");                                     \
1006         __cu_len_r;                                                     \
1007 })
1008
1009 #define __invoke_copy_from_kernel_inatomic(to, from, n)                 \
1010         __invoke_copy_from_user_inatomic(to, from, n)                   \
1011
1012 #else
1013
1014 /* EVA specific functions */
1015
1016 extern size_t __copy_user_inatomic_eva(void *__to, const void *__from,
1017                                        size_t __n);
1018 extern size_t __copy_from_user_eva(void *__to, const void *__from,
1019                                    size_t __n);
1020 extern size_t __copy_to_user_eva(void *__to, const void *__from,
1021                                  size_t __n);
1022 extern size_t __copy_in_user_eva(void *__to, const void *__from, size_t __n);
1023
1024 #define __invoke_copy_from_user_eva_generic(to, from, n, func_ptr)      \
1025 ({                                                                      \
1026         register void *__cu_to_r __asm__("$4");                         \
1027         register const void __user *__cu_from_r __asm__("$5");          \
1028         register long __cu_len_r __asm__("$6");                         \
1029                                                                         \
1030         __cu_to_r = (to);                                               \
1031         __cu_from_r = (from);                                           \
1032         __cu_len_r = (n);                                               \
1033         __asm__ __volatile__(                                           \
1034         ".set\tnoreorder\n\t"                                           \
1035         __MODULE_JAL(func_ptr)                                          \
1036         ".set\tnoat\n\t"                                                \
1037         __UA_ADDU "\t$1, %1, %2\n\t"                                    \
1038         ".set\tat\n\t"                                                  \
1039         ".set\treorder"                                                 \
1040         : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r)       \
1041         :                                                               \
1042         : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31",  \
1043           DADDI_SCRATCH, "memory");                                     \
1044         __cu_len_r;                                                     \
1045 })
1046
1047 #define __invoke_copy_to_user_eva_generic(to, from, n, func_ptr)        \
1048 ({                                                                      \
1049         register void *__cu_to_r __asm__("$4");                         \
1050         register const void __user *__cu_from_r __asm__("$5");          \
1051         register long __cu_len_r __asm__("$6");                         \
1052                                                                         \
1053         __cu_to_r = (to);                                               \
1054         __cu_from_r = (from);                                           \
1055         __cu_len_r = (n);                                               \
1056         __asm__ __volatile__(                                           \
1057         __MODULE_JAL(func_ptr)                                          \
1058         : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r)       \
1059         :                                                               \
1060         : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31",  \
1061           DADDI_SCRATCH, "memory");                                     \
1062         __cu_len_r;                                                     \
1063 })
1064
1065 /*
1066  * Source or destination address is in userland. We need to go through
1067  * the TLB
1068  */
1069 #define __invoke_copy_from_user(to, from, n)                            \
1070         __invoke_copy_from_user_eva_generic(to, from, n, __copy_from_user_eva)
1071
1072 #define __invoke_copy_from_user_inatomic(to, from, n)                   \
1073         __invoke_copy_from_user_eva_generic(to, from, n,                \
1074                                             __copy_user_inatomic_eva)
1075
1076 #define __invoke_copy_to_user(to, from, n)                              \
1077         __invoke_copy_to_user_eva_generic(to, from, n, __copy_to_user_eva)
1078
1079 #define ___invoke_copy_in_user(to, from, n)                             \
1080         __invoke_copy_from_user_eva_generic(to, from, n, __copy_in_user_eva)
1081
1082 /*
1083  * Source or destination address in the kernel. We are not going through
1084  * the TLB
1085  */
1086 #define __invoke_copy_from_kernel(to, from, n)                          \
1087         __invoke_copy_from_user_eva_generic(to, from, n, __copy_user)
1088
1089 #define __invoke_copy_from_kernel_inatomic(to, from, n)                 \
1090         __invoke_copy_from_user_eva_generic(to, from, n, __copy_user_inatomic)
1091
1092 #define __invoke_copy_to_kernel(to, from, n)                            \
1093         __invoke_copy_to_user_eva_generic(to, from, n, __copy_user)
1094
1095 #define ___invoke_copy_in_kernel(to, from, n)                           \
1096         __invoke_copy_from_user_eva_generic(to, from, n, __copy_user)
1097
1098 #endif /* CONFIG_EVA */
1099
1100 /*
1101  * __copy_from_user: - Copy a block of data from user space, with less checking.
1102  * @to:   Destination address, in kernel space.
1103  * @from: Source address, in user space.
1104  * @n:    Number of bytes to copy.
1105  *
1106  * Context: User context only. This function may sleep if pagefaults are
1107  *          enabled.
1108  *
1109  * Copy data from user space to kernel space.  Caller must check
1110  * the specified block with access_ok() before calling this function.
1111  *
1112  * Returns number of bytes that could not be copied.
1113  * On success, this will be zero.
1114  *
1115  * If some data could not be copied, this function will pad the copied
1116  * data to the requested size using zero bytes.
1117  */
1118 #define __copy_from_user(to, from, n)                                   \
1119 ({                                                                      \
1120         void *__cu_to;                                                  \
1121         const void __user *__cu_from;                                   \
1122         long __cu_len;                                                  \
1123                                                                         \
1124         __cu_to = (to);                                                 \
1125         __cu_from = (from);                                             \
1126         __cu_len = (n);                                                 \
1127         if (eva_kernel_access()) {                                      \
1128                 __cu_len = __invoke_copy_from_kernel(__cu_to,           \
1129                                                      __cu_from,         \
1130                                                      __cu_len);         \
1131         } else {                                                        \
1132                 might_fault();                                          \
1133                 __cu_len = __invoke_copy_from_user(__cu_to, __cu_from,  \
1134                                                    __cu_len);           \
1135         }                                                               \
1136         __cu_len;                                                       \
1137 })
1138
1139 /*
1140  * copy_from_user: - Copy a block of data from user space.
1141  * @to:   Destination address, in kernel space.
1142  * @from: Source address, in user space.
1143  * @n:    Number of bytes to copy.
1144  *
1145  * Context: User context only. This function may sleep if pagefaults are
1146  *          enabled.
1147  *
1148  * Copy data from user space to kernel space.
1149  *
1150  * Returns number of bytes that could not be copied.
1151  * On success, this will be zero.
1152  *
1153  * If some data could not be copied, this function will pad the copied
1154  * data to the requested size using zero bytes.
1155  */
1156 #define copy_from_user(to, from, n)                                     \
1157 ({                                                                      \
1158         void *__cu_to;                                                  \
1159         const void __user *__cu_from;                                   \
1160         long __cu_len;                                                  \
1161                                                                         \
1162         __cu_to = (to);                                                 \
1163         __cu_from = (from);                                             \
1164         __cu_len = (n);                                                 \
1165         if (eva_kernel_access()) {                                      \
1166                 __cu_len = __invoke_copy_from_kernel(__cu_to,           \
1167                                                      __cu_from,         \
1168                                                      __cu_len);         \
1169         } else {                                                        \
1170                 if (access_ok(VERIFY_READ, __cu_from, __cu_len)) {      \
1171                         might_fault();                                  \
1172                         __cu_len = __invoke_copy_from_user(__cu_to,     \
1173                                                            __cu_from,   \
1174                                                            __cu_len);   \
1175                 } else {                                                \
1176                         memset(__cu_to, 0, __cu_len);                   \
1177                 }                                                       \
1178         }                                                               \
1179         __cu_len;                                                       \
1180 })
1181
1182 #define __copy_in_user(to, from, n)                                     \
1183 ({                                                                      \
1184         void __user *__cu_to;                                           \
1185         const void __user *__cu_from;                                   \
1186         long __cu_len;                                                  \
1187                                                                         \
1188         __cu_to = (to);                                                 \
1189         __cu_from = (from);                                             \
1190         __cu_len = (n);                                                 \
1191         if (eva_kernel_access()) {                                      \
1192                 __cu_len = ___invoke_copy_in_kernel(__cu_to, __cu_from, \
1193                                                     __cu_len);          \
1194         } else {                                                        \
1195                 might_fault();                                          \
1196                 __cu_len = ___invoke_copy_in_user(__cu_to, __cu_from,   \
1197                                                   __cu_len);            \
1198         }                                                               \
1199         __cu_len;                                                       \
1200 })
1201
1202 #define copy_in_user(to, from, n)                                       \
1203 ({                                                                      \
1204         void __user *__cu_to;                                           \
1205         const void __user *__cu_from;                                   \
1206         long __cu_len;                                                  \
1207                                                                         \
1208         __cu_to = (to);                                                 \
1209         __cu_from = (from);                                             \
1210         __cu_len = (n);                                                 \
1211         if (eva_kernel_access()) {                                      \
1212                 __cu_len = ___invoke_copy_in_kernel(__cu_to,__cu_from,  \
1213                                                     __cu_len);          \
1214         } else {                                                        \
1215                 if (likely(access_ok(VERIFY_READ, __cu_from, __cu_len) &&\
1216                            access_ok(VERIFY_WRITE, __cu_to, __cu_len))) {\
1217                         might_fault();                                  \
1218                         __cu_len = ___invoke_copy_in_user(__cu_to,      \
1219                                                           __cu_from,    \
1220                                                           __cu_len);    \
1221                 }                                                       \
1222         }                                                               \
1223         __cu_len;                                                       \
1224 })
1225
1226 /*
1227  * __clear_user: - Zero a block of memory in user space, with less checking.
1228  * @to:   Destination address, in user space.
1229  * @n:    Number of bytes to zero.
1230  *
1231  * Zero a block of memory in user space.  Caller must check
1232  * the specified block with access_ok() before calling this function.
1233  *
1234  * Returns number of bytes that could not be cleared.
1235  * On success, this will be zero.
1236  */
1237 static inline __kernel_size_t
1238 __clear_user(void __user *addr, __kernel_size_t size)
1239 {
1240         __kernel_size_t res;
1241
1242         if (eva_kernel_access()) {
1243                 __asm__ __volatile__(
1244                         "move\t$4, %1\n\t"
1245                         "move\t$5, $0\n\t"
1246                         "move\t$6, %2\n\t"
1247                         __MODULE_JAL(__bzero_kernel)
1248                         "move\t%0, $6"
1249                         : "=r" (res)
1250                         : "r" (addr), "r" (size)
1251                         : "$4", "$5", "$6", __UA_t0, __UA_t1, "$31");
1252         } else {
1253                 might_fault();
1254                 __asm__ __volatile__(
1255                         "move\t$4, %1\n\t"
1256                         "move\t$5, $0\n\t"
1257                         "move\t$6, %2\n\t"
1258                         __MODULE_JAL(__bzero)
1259                         "move\t%0, $6"
1260                         : "=r" (res)
1261                         : "r" (addr), "r" (size)
1262                         : "$4", "$5", "$6", __UA_t0, __UA_t1, "$31");
1263         }
1264
1265         return res;
1266 }
1267
1268 #define clear_user(addr,n)                                              \
1269 ({                                                                      \
1270         void __user * __cl_addr = (addr);                               \
1271         unsigned long __cl_size = (n);                                  \
1272         if (__cl_size && access_ok(VERIFY_WRITE,                        \
1273                                         __cl_addr, __cl_size))          \
1274                 __cl_size = __clear_user(__cl_addr, __cl_size);         \
1275         __cl_size;                                                      \
1276 })
1277
1278 /*
1279  * __strncpy_from_user: - Copy a NUL terminated string from userspace, with less checking.
1280  * @dst:   Destination address, in kernel space.  This buffer must be at
1281  *         least @count bytes long.
1282  * @src:   Source address, in user space.
1283  * @count: Maximum number of bytes to copy, including the trailing NUL.
1284  *
1285  * Copies a NUL-terminated string from userspace to kernel space.
1286  * Caller must check the specified block with access_ok() before calling
1287  * this function.
1288  *
1289  * On success, returns the length of the string (not including the trailing
1290  * NUL).
1291  *
1292  * If access to userspace fails, returns -EFAULT (some data may have been
1293  * copied).
1294  *
1295  * If @count is smaller than the length of the string, copies @count bytes
1296  * and returns @count.
1297  */
1298 static inline long
1299 __strncpy_from_user(char *__to, const char __user *__from, long __len)
1300 {
1301         long res;
1302
1303         if (eva_kernel_access()) {
1304                 __asm__ __volatile__(
1305                         "move\t$4, %1\n\t"
1306                         "move\t$5, %2\n\t"
1307                         "move\t$6, %3\n\t"
1308                         __MODULE_JAL(__strncpy_from_kernel_nocheck_asm)
1309                         "move\t%0, $2"
1310                         : "=r" (res)
1311                         : "r" (__to), "r" (__from), "r" (__len)
1312                         : "$2", "$3", "$4", "$5", "$6", __UA_t0, "$31", "memory");
1313         } else {
1314                 might_fault();
1315                 __asm__ __volatile__(
1316                         "move\t$4, %1\n\t"
1317                         "move\t$5, %2\n\t"
1318                         "move\t$6, %3\n\t"
1319                         __MODULE_JAL(__strncpy_from_user_nocheck_asm)
1320                         "move\t%0, $2"
1321                         : "=r" (res)
1322                         : "r" (__to), "r" (__from), "r" (__len)
1323                         : "$2", "$3", "$4", "$5", "$6", __UA_t0, "$31", "memory");
1324         }
1325
1326         return res;
1327 }
1328
1329 /*
1330  * strncpy_from_user: - Copy a NUL terminated string from userspace.
1331  * @dst:   Destination address, in kernel space.  This buffer must be at
1332  *         least @count bytes long.
1333  * @src:   Source address, in user space.
1334  * @count: Maximum number of bytes to copy, including the trailing NUL.
1335  *
1336  * Copies a NUL-terminated string from userspace to kernel space.
1337  *
1338  * On success, returns the length of the string (not including the trailing
1339  * NUL).
1340  *
1341  * If access to userspace fails, returns -EFAULT (some data may have been
1342  * copied).
1343  *
1344  * If @count is smaller than the length of the string, copies @count bytes
1345  * and returns @count.
1346  */
1347 static inline long
1348 strncpy_from_user(char *__to, const char __user *__from, long __len)
1349 {
1350         long res;
1351
1352         if (eva_kernel_access()) {
1353                 __asm__ __volatile__(
1354                         "move\t$4, %1\n\t"
1355                         "move\t$5, %2\n\t"
1356                         "move\t$6, %3\n\t"
1357                         __MODULE_JAL(__strncpy_from_kernel_asm)
1358                         "move\t%0, $2"
1359                         : "=r" (res)
1360                         : "r" (__to), "r" (__from), "r" (__len)
1361                         : "$2", "$3", "$4", "$5", "$6", __UA_t0, "$31", "memory");
1362         } else {
1363                 might_fault();
1364                 __asm__ __volatile__(
1365                         "move\t$4, %1\n\t"
1366                         "move\t$5, %2\n\t"
1367                         "move\t$6, %3\n\t"
1368                         __MODULE_JAL(__strncpy_from_user_asm)
1369                         "move\t%0, $2"
1370                         : "=r" (res)
1371                         : "r" (__to), "r" (__from), "r" (__len)
1372                         : "$2", "$3", "$4", "$5", "$6", __UA_t0, "$31", "memory");
1373         }
1374
1375         return res;
1376 }
1377
1378 /*
1379  * strlen_user: - Get the size of a string in user space.
1380  * @str: The string to measure.
1381  *
1382  * Context: User context only. This function may sleep if pagefaults are
1383  *          enabled.
1384  *
1385  * Get the size of a NUL-terminated string in user space.
1386  *
1387  * Returns the size of the string INCLUDING the terminating NUL.
1388  * On exception, returns 0.
1389  *
1390  * If there is a limit on the length of a valid string, you may wish to
1391  * consider using strnlen_user() instead.
1392  */
1393 static inline long strlen_user(const char __user *s)
1394 {
1395         long res;
1396
1397         if (eva_kernel_access()) {
1398                 __asm__ __volatile__(
1399                         "move\t$4, %1\n\t"
1400                         __MODULE_JAL(__strlen_kernel_asm)
1401                         "move\t%0, $2"
1402                         : "=r" (res)
1403                         : "r" (s)
1404                         : "$2", "$4", __UA_t0, "$31");
1405         } else {
1406                 might_fault();
1407                 __asm__ __volatile__(
1408                         "move\t$4, %1\n\t"
1409                         __MODULE_JAL(__strlen_user_asm)
1410                         "move\t%0, $2"
1411                         : "=r" (res)
1412                         : "r" (s)
1413                         : "$2", "$4", __UA_t0, "$31");
1414         }
1415
1416         return res;
1417 }
1418
1419 /* Returns: 0 if bad, string length+1 (memory size) of string if ok */
1420 static inline long __strnlen_user(const char __user *s, long n)
1421 {
1422         long res;
1423
1424         if (eva_kernel_access()) {
1425                 __asm__ __volatile__(
1426                         "move\t$4, %1\n\t"
1427                         "move\t$5, %2\n\t"
1428                         __MODULE_JAL(__strnlen_kernel_nocheck_asm)
1429                         "move\t%0, $2"
1430                         : "=r" (res)
1431                         : "r" (s), "r" (n)
1432                         : "$2", "$4", "$5", __UA_t0, "$31");
1433         } else {
1434                 might_fault();
1435                 __asm__ __volatile__(
1436                         "move\t$4, %1\n\t"
1437                         "move\t$5, %2\n\t"
1438                         __MODULE_JAL(__strnlen_user_nocheck_asm)
1439                         "move\t%0, $2"
1440                         : "=r" (res)
1441                         : "r" (s), "r" (n)
1442                         : "$2", "$4", "$5", __UA_t0, "$31");
1443         }
1444
1445         return res;
1446 }
1447
1448 /*
1449  * strnlen_user: - Get the size of a string in user space.
1450  * @str: The string to measure.
1451  *
1452  * Context: User context only. This function may sleep if pagefaults are
1453  *          enabled.
1454  *
1455  * Get the size of a NUL-terminated string in user space.
1456  *
1457  * Returns the size of the string INCLUDING the terminating NUL.
1458  * On exception, returns 0.
1459  * If the string is too long, returns a value greater than @n.
1460  */
1461 static inline long strnlen_user(const char __user *s, long n)
1462 {
1463         long res;
1464
1465         might_fault();
1466         if (eva_kernel_access()) {
1467                 __asm__ __volatile__(
1468                         "move\t$4, %1\n\t"
1469                         "move\t$5, %2\n\t"
1470                         __MODULE_JAL(__strnlen_kernel_asm)
1471                         "move\t%0, $2"
1472                         : "=r" (res)
1473                         : "r" (s), "r" (n)
1474                         : "$2", "$4", "$5", __UA_t0, "$31");
1475         } else {
1476                 __asm__ __volatile__(
1477                         "move\t$4, %1\n\t"
1478                         "move\t$5, %2\n\t"
1479                         __MODULE_JAL(__strnlen_user_asm)
1480                         "move\t%0, $2"
1481                         : "=r" (res)
1482                         : "r" (s), "r" (n)
1483                         : "$2", "$4", "$5", __UA_t0, "$31");
1484         }
1485
1486         return res;
1487 }
1488
1489 #endif /* _ASM_UACCESS_H */