arch/x86/include/asm/jump_label.h

   1 #ifndef _ASM_X86_JUMP_LABEL_H
   2 #define _ASM_X86_JUMP_LABEL_H
   3
   4 #ifndef HAVE_JUMP_LABEL
   5 /*
   6  * For better or for worse, if jump labels (the gcc extension) are missing,
   7  * then the entire static branch patching infrastructure is compiled out.
   8  * If that happens, the code in here will malfunction.  Raise a compiler
   9  * error instead.
  10  *
  11  * In theory, jump labels and the static branch patching infrastructure
  12  * could be decoupled to fix this.
  13  */
  14 #error asm/jump_label.h included on a non-jump-label kernel
  15 #endif
  16
  17 #define JUMP_LABEL_NOP_SIZE 5
  18
  19 #ifdef CONFIG_X86_64
  20 # define STATIC_KEY_INIT_NOP P6_NOP5_ATOMIC
  21 #else
  22 # define STATIC_KEY_INIT_NOP GENERIC_NOP5_ATOMIC
  23 #endif
  24
  25 #include <asm/asm.h>
  26 #include <asm/nops.h>
  27
  28 #ifndef __ASSEMBLY__
  29
  30 #include <linux/stringify.h>
  31 #include <linux/types.h>
  32
  33 static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
  34 {
  35         asm_volatile_goto("1:"
  36                 ".byte " __stringify(STATIC_KEY_INIT_NOP) "\n\t"
  37                 ".pushsection __jump_table,  \"aw\" \n\t"
  38                 _ASM_ALIGN "\n\t"
  39                 _ASM_PTR "1b, %l[l_yes], %c0 + %c1 \n\t"
  40                 ".popsection \n\t"
  41                 : :  "i" (key), "i" (branch) : : l_yes);
  42
  43         return false;
  44 l_yes:
  45         return true;
  46 }
  47
  48 static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch)
  49 {
  50         asm_volatile_goto("1:"
  51                 ".byte 0xe9\n\t .long %l[l_yes] - 2f\n\t"
  52                 "2:\n\t"
  53                 ".pushsection __jump_table,  \"aw\" \n\t"
  54                 _ASM_ALIGN "\n\t"
  55                 _ASM_PTR "1b, %l[l_yes], %c0 + %c1 \n\t"
  56                 ".popsection \n\t"
  57                 : :  "i" (key), "i" (branch) : : l_yes);
  58
  59         return false;
  60 l_yes:
  61         return true;
  62 }
  63
  64 #ifdef CONFIG_X86_64
  65 typedef u64 jump_label_t;
  66 #else
  67 typedef u32 jump_label_t;
  68 #endif
  69
  70 struct jump_entry {
  71         jump_label_t code;
  72         jump_label_t target;
  73         jump_label_t key;
  74 };
  75
  76 #else   /* __ASSEMBLY__ */
  77
  78 .macro STATIC_JUMP_IF_TRUE target, key, def
  79 .Lstatic_jump_\@:
  80         .if \def
  81         /* Equivalent to "jmp.d32 \target" */
  82         .byte           0xe9
  83         .long           \target - .Lstatic_jump_after_\@
  84 .Lstatic_jump_after_\@:
  85         .else
  86         .byte           STATIC_KEY_INIT_NOP
  87         .endif
  88         .pushsection __jump_table, "aw"
  89         _ASM_ALIGN
  90         _ASM_PTR        .Lstatic_jump_\@, \target, \key
  91         .popsection
  92 .endm
  93
  94 .macro STATIC_JUMP_IF_FALSE target, key, def
  95 .Lstatic_jump_\@:
  96         .if \def
  97         .byte           STATIC_KEY_INIT_NOP
  98         .else
  99         /* Equivalent to "jmp.d32 \target" */
 100         .byte           0xe9
 101         .long           \target - .Lstatic_jump_after_\@
 102 .Lstatic_jump_after_\@:
 103         .endif
 104         .pushsection __jump_table, "aw"
 105         _ASM_ALIGN
 106         _ASM_PTR        .Lstatic_jump_\@, \target, \key + 1
 107         .popsection
 108 .endm
 109
 110 #endif  /* __ASSEMBLY__ */
 111
 112 #endif