Merge branch 'uaccess' (batched user access infrastructure)

Expose an interface to allow users to mark several accesses together as
being user space accesses, allowing batching of the surrounding user
space access markers (SMAP on x86, PAN on arm64, domain register
switching on arm).

This is currently only used for the user string lenth and copying
functions, where the SMAP overhead on x86 drowned the actual user
accesses (only noticeable on newer microarchitectures that support SMAP
in the first place, of course).

* user access batching branch:
  Use the new batched user accesses in generic user string handling
  Add 'unsafe' user access functions for batched accesses
  x86: reorganize SMAP handling in user space accesses

diff --combined arch/x86/include/asm/uaccess.h
index 660458a,ca59e4f..a4a30e4
--- 1/arch/x86/include/asm/uaccess.h
--- 2/arch/x86/include/asm/uaccess.h
+++ b/arch/x86/include/asm/uaccess.h
@@@ -134,6 -134,9 +134,9 @@@ extern int __get_user_4(void)
  extern int __get_user_8(void);
  extern int __get_user_bad(void);
  
+ #define __uaccess_begin() stac()
+ #define __uaccess_end()   clac()
+ 
  /*
   * This is a type: either unsigned long, if the argument fits into
   * that type, or otherwise unsigned long long.
@@@ -193,10 -196,10 +196,10 @@@ __typeof__(__builtin_choose_expr(sizeof
  
  #ifdef CONFIG_X86_32
  #define __put_user_asm_u64(x, addr, err, errret)                      \
-       asm volatile(ASM_STAC "\n"                                      \
+       asm volatile("\n"                                               \
                     "1:        movl %%eax,0(%2)\n"                     \
                     "2:        movl %%edx,4(%2)\n"                     \
-                    "3: " ASM_CLAC "\n"                                \
+                    "3:"                                               \
                     ".section .fixup,\"ax\"\n"                         \
                     "4:        movl %3,%0\n"                           \
                     "  jmp 3b\n"                                       \
@@@ -207,10 -210,10 +210,10 @@@
                     : "A" (x), "r" (addr), "i" (errret), "0" (err))
  
  #define __put_user_asm_ex_u64(x, addr)                                        \
-       asm volatile(ASM_STAC "\n"                                      \
+       asm volatile("\n"                                               \
                     "1:        movl %%eax,0(%1)\n"                     \
                     "2:        movl %%edx,4(%1)\n"                     \
-                    "3: " ASM_CLAC "\n"                                \
+                    "3:"                                               \
                     _ASM_EXTABLE_EX(1b, 2b)                            \
                     _ASM_EXTABLE_EX(2b, 3b)                            \
                     : : "A" (x), "r" (addr))
@@@ -304,6 -307,10 +307,10 @@@ do {                                                                     
        }                                                               \
  } while (0)
  
+ /*
+  * This doesn't do __uaccess_begin/end - the exception handling
+  * around it must do that.
+  */
  #define __put_user_size_ex(x, ptr, size)                              \
  do {                                                                  \
        __chk_user_ptr(ptr);                                            \
@@@ -358,9 -365,9 +365,9 @@@ do {                                                                       
  } while (0)
  
  #define __get_user_asm(x, addr, err, itype, rtype, ltype, errret)     \
-       asm volatile(ASM_STAC "\n"                                      \
+       asm volatile("\n"                                               \
                     "1:        mov"itype" %2,%"rtype"1\n"              \
-                    "2: " ASM_CLAC "\n"                                \
+                    "2:\n"                                             \
                     ".section .fixup,\"ax\"\n"                         \
                     "3:        mov %3,%0\n"                            \
                     "  xor"itype" %"rtype"1,%"rtype"1\n"               \
@@@ -370,6 -377,10 +377,10 @@@
                     : "=r" (err), ltype(x)                             \
                     : "m" (__m(addr)), "i" (errret), "0" (err))
  
+ /*
+  * This doesn't do __uaccess_begin/end - the exception handling
+  * around it must do that.
+  */
  #define __get_user_size_ex(x, ptr, size)                              \
  do {                                                                  \
        __chk_user_ptr(ptr);                                            \
@@@ -400,7 -411,9 +411,9 @@@
  #define __put_user_nocheck(x, ptr, size)                      \
  ({                                                            \
        int __pu_err;                                           \
+       __uaccess_begin();                                      \
        __put_user_size((x), (ptr), (size), __pu_err, -EFAULT); \
+       __uaccess_end();                                        \
        __builtin_expect(__pu_err, 0);                          \
  })
  
@@@ -408,7 -421,9 +421,9 @@@
  ({                                                                    \
        int __gu_err;                                                   \
        unsigned long __gu_val;                                         \
+       __uaccess_begin();                                              \
        __get_user_size(__gu_val, (ptr), (size), __gu_err, -EFAULT);    \
+       __uaccess_end();                                                \
        (x) = (__force __typeof__(*(ptr)))__gu_val;                     \
        __builtin_expect(__gu_err, 0);                                  \
  })
@@@ -423,9 -438,9 +438,9 @@@ struct __large_struct { unsigned long b
   * aliasing issues.
   */
  #define __put_user_asm(x, addr, err, itype, rtype, ltype, errret)     \
-       asm volatile(ASM_STAC "\n"                                      \
+       asm volatile("\n"                                               \
                     "1:        mov"itype" %"rtype"1,%2\n"              \
-                    "2: " ASM_CLAC "\n"                                \
+                    "2:\n"                                             \
                     ".section .fixup,\"ax\"\n"                         \
                     "3:        mov %3,%0\n"                            \
                     "  jmp 2b\n"                                       \
@@@ -445,11 -460,11 +460,11 @@@
   */
  #define uaccess_try   do {                                            \
        current_thread_info()->uaccess_err = 0;                         \
-       stac();                                                         \
+       __uaccess_begin();                                              \
        barrier();
  
  #define uaccess_catch(err)                                            \
-       clac();                                                         \
+       __uaccess_end();                                                \
        (err) |= (current_thread_info()->uaccess_err ? -EFAULT : 0);    \
  } while (0)
  
@@@ -547,12 -562,13 +562,13 @@@ extern void __cmpxchg_wrong_size(void
        __typeof__(ptr) __uval = (uval);                                \
        __typeof__(*(ptr)) __old = (old);                               \
        __typeof__(*(ptr)) __new = (new);                               \
+       __uaccess_begin();                                              \
        switch (size) {                                                 \
        case 1:                                                         \
        {                                                               \
-               asm volatile("\t" ASM_STAC "\n"                         \
+               asm volatile("\n"                                       \
                        "1:\t" LOCK_PREFIX "cmpxchgb %4, %2\n"          \
-                       "2:\t" ASM_CLAC "\n"                            \
+                       "2:\n"                                          \
                        "\t.section .fixup, \"ax\"\n"                   \
                        "3:\tmov     %3, %0\n"                          \
                        "\tjmp     2b\n"                                \
@@@ -566,9 -582,9 +582,9 @@@
        }                                                               \
        case 2:                                                         \
        {                                                               \
-               asm volatile("\t" ASM_STAC "\n"                         \
+               asm volatile("\n"                                       \
                        "1:\t" LOCK_PREFIX "cmpxchgw %4, %2\n"          \
-                       "2:\t" ASM_CLAC "\n"                            \
+                       "2:\n"                                          \
                        "\t.section .fixup, \"ax\"\n"                   \
                        "3:\tmov     %3, %0\n"                          \
                        "\tjmp     2b\n"                                \
@@@ -582,9 -598,9 +598,9 @@@
        }                                                               \
        case 4:                                                         \
        {                                                               \
-               asm volatile("\t" ASM_STAC "\n"                         \
+               asm volatile("\n"                                       \
                        "1:\t" LOCK_PREFIX "cmpxchgl %4, %2\n"          \
-                       "2:\t" ASM_CLAC "\n"                            \
+                       "2:\n"                                          \
                        "\t.section .fixup, \"ax\"\n"                   \
                        "3:\tmov     %3, %0\n"                          \
                        "\tjmp     2b\n"                                \
@@@ -601,9 -617,9 +617,9 @@@
                if (!IS_ENABLED(CONFIG_X86_64))                         \
                        __cmpxchg_wrong_size();                         \
                                                                        \
-               asm volatile("\t" ASM_STAC "\n"                         \
+               asm volatile("\n"                                       \
                        "1:\t" LOCK_PREFIX "cmpxchgq %4, %2\n"          \
-                       "2:\t" ASM_CLAC "\n"                            \
+                       "2:\n"                                          \
                        "\t.section .fixup, \"ax\"\n"                   \
                        "3:\tmov     %3, %0\n"                          \
                        "\tjmp     2b\n"                                \
@@@ -618,6 -634,7 +634,7 @@@
        default:                                                        \
                __cmpxchg_wrong_size();                                 \
        }                                                               \
+       __uaccess_end();                                                \
        *__uval = __old;                                                \
        __ret;                                                          \
  })
@@@ -745,14 -762,30 +762,39 @@@ copy_to_user(void __user *to, const voi
  #undef __copy_from_user_overflow
  #undef __copy_to_user_overflow
  
 +/*
 + * We rely on the nested NMI work to allow atomic faults from the NMI path; the
 + * nested NMI paths are careful to preserve CR2.
 + *
 + * Caller must use pagefault_enable/disable, or run in interrupt context,
 + * and also do a uaccess_ok() check
 + */
 +#define __copy_from_user_nmi __copy_from_user_inatomic
 +
+ /*
+  * The "unsafe" user accesses aren't really "unsafe", but the naming
+  * is a big fat warning: you have to not only do the access_ok()
+  * checking before using them, but you have to surround them with the
+  * user_access_begin/end() pair.
+  */
+ #define user_access_begin()   __uaccess_begin()
+ #define user_access_end()     __uaccess_end()
+ 
+ #define unsafe_put_user(x, ptr)                                               \
+ ({                                                                            \
+       int __pu_err;                                                           \
+       __put_user_size((x), (ptr), sizeof(*(ptr)), __pu_err, -EFAULT);         \
+       __builtin_expect(__pu_err, 0);                                          \
+ })
+ 
+ #define unsafe_get_user(x, ptr)                                               \
+ ({                                                                            \
+       int __gu_err;                                                           \
+       unsigned long __gu_val;                                                 \
+       __get_user_size(__gu_val, (ptr), sizeof(*(ptr)), __gu_err, -EFAULT);    \
+       (x) = (__force __typeof__(*(ptr)))__gu_val;                             \
+       __builtin_expect(__gu_err, 0);                                          \
+ })
+ 
  #endif /* _ASM_X86_UACCESS_H */