1 #ifndef _ASM_X86_MMU_CONTEXT_H
2 #define _ASM_X86_MMU_CONTEXT_H
5 #include <linux/atomic.h>
6 #include <linux/mm_types.h>
8 #include <trace/events/tlb.h>
10 #include <asm/pgalloc.h>
11 #include <asm/tlbflush.h>
12 #include <asm/paravirt.h>
14 #ifndef CONFIG_PARAVIRT
15 static inline void paravirt_activate_mm(struct mm_struct *prev,
16 struct mm_struct *next)
19 #endif /* !CONFIG_PARAVIRT */
21 #ifdef CONFIG_PERF_EVENTS
22 extern struct static_key rdpmc_always_available;
24 static inline void load_mm_cr4(struct mm_struct *mm)
26 if (static_key_false(&rdpmc_always_available) ||
27 atomic_read(&mm->context.perf_rdpmc_allowed))
28 cr4_set_bits(X86_CR4_PCE);
30 cr4_clear_bits(X86_CR4_PCE);
33 static inline void load_mm_cr4(struct mm_struct *mm) {}
36 #ifdef CONFIG_MODIFY_LDT_SYSCALL
38 * ldt_structs can be allocated, used, and freed, but they are never
39 * modified while live.
43 * Xen requires page-aligned LDTs with special permissions. This is
44 * needed to prevent us from installing evil descriptors such as
45 * call gates. On native, we could merge the ldt_struct and LDT
46 * allocations, but it's not worth trying to optimize.
48 struct desc_struct *entries;
53 * Used for LDT copy/destruction.
55 int init_new_context(struct task_struct *tsk, struct mm_struct *mm);
56 void destroy_context(struct mm_struct *mm);
57 #else /* CONFIG_MODIFY_LDT_SYSCALL */
58 static inline int init_new_context(struct task_struct *tsk,
63 static inline void destroy_context(struct mm_struct *mm) {}
66 static inline void load_mm_ldt(struct mm_struct *mm)
68 #ifdef CONFIG_MODIFY_LDT_SYSCALL
69 struct ldt_struct *ldt;
71 /* lockless_dereference synchronizes with smp_store_release */
72 ldt = lockless_dereference(mm->context.ldt);
75 * Any change to mm->context.ldt is followed by an IPI to all
76 * CPUs with the mm active. The LDT will not be freed until
77 * after the IPI is handled by all such CPUs. This means that,
78 * if the ldt_struct changes before we return, the values we see
79 * will be safe, and the new values will be loaded before we run
82 * NB: don't try to convert this to use RCU without extreme care.
83 * We would still need IRQs off, because we don't want to change
84 * the local LDT after an IPI loaded a newer value than the one
89 set_ldt(ldt->entries, ldt->size);
96 DEBUG_LOCKS_WARN_ON(preemptible());
99 static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
102 if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
103 this_cpu_write(cpu_tlbstate.state, TLBSTATE_LAZY);
107 static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
108 struct task_struct *tsk)
110 unsigned cpu = smp_processor_id();
112 if (likely(prev != next)) {
114 this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK);
115 this_cpu_write(cpu_tlbstate.active_mm, next);
117 cpumask_set_cpu(cpu, mm_cpumask(next));
120 * Re-load page tables.
122 * This logic has an ordering constraint:
124 * CPU 0: Write to a PTE for 'next'
125 * CPU 0: load bit 1 in mm_cpumask. if nonzero, send IPI.
126 * CPU 1: set bit 1 in next's mm_cpumask
127 * CPU 1: load from the PTE that CPU 0 writes (implicit)
129 * We need to prevent an outcome in which CPU 1 observes
130 * the new PTE value and CPU 0 observes bit 1 clear in
131 * mm_cpumask. (If that occurs, then the IPI will never
132 * be sent, and CPU 0's TLB will contain a stale entry.)
134 * The bad outcome can occur if either CPU's load is
135 * reordered before that CPU's store, so both CPUs much
136 * execute full barriers to prevent this from happening.
138 * Thus, switch_mm needs a full barrier between the
139 * store to mm_cpumask and any operation that could load
140 * from next->pgd. This barrier synchronizes with
141 * remote TLB flushers. Fortunately, load_cr3 is
142 * serializing and thus acts as a full barrier.
147 trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL);
149 /* Stop flush ipis for the previous mm */
150 cpumask_clear_cpu(cpu, mm_cpumask(prev));
152 /* Load per-mm CR4 state */
155 #ifdef CONFIG_MODIFY_LDT_SYSCALL
157 * Load the LDT, if the LDT is different.
159 * It's possible that prev->context.ldt doesn't match
160 * the LDT register. This can happen if leave_mm(prev)
161 * was called and then modify_ldt changed
162 * prev->context.ldt but suppressed an IPI to this CPU.
163 * In this case, prev->context.ldt != NULL, because we
164 * never set context.ldt to NULL while the mm still
165 * exists. That means that next->context.ldt !=
166 * prev->context.ldt, because mms never share an LDT.
168 if (unlikely(prev->context.ldt != next->context.ldt))
174 this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK);
175 BUG_ON(this_cpu_read(cpu_tlbstate.active_mm) != next);
177 if (!cpumask_test_cpu(cpu, mm_cpumask(next))) {
179 * On established mms, the mm_cpumask is only changed
180 * from irq context, from ptep_clear_flush() while in
181 * lazy tlb mode, and here. Irqs are blocked during
182 * schedule, protecting us from simultaneous changes.
184 cpumask_set_cpu(cpu, mm_cpumask(next));
187 * We were in lazy tlb mode and leave_mm disabled
188 * tlb flush IPI delivery. We must reload CR3
189 * to make sure to use no freed page tables.
191 * As above, this is a barrier that forces
192 * TLB repopulation to be ordered after the
193 * store to mm_cpumask.
196 trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL);
204 #define activate_mm(prev, next) \
206 paravirt_activate_mm((prev), (next)); \
207 switch_mm((prev), (next), NULL); \
211 #define deactivate_mm(tsk, mm) \
216 #define deactivate_mm(tsk, mm) \
219 loadsegment(fs, 0); \
223 static inline void arch_dup_mmap(struct mm_struct *oldmm,
224 struct mm_struct *mm)
226 paravirt_arch_dup_mmap(oldmm, mm);
229 static inline void arch_exit_mmap(struct mm_struct *mm)
231 paravirt_arch_exit_mmap(mm);
235 static inline bool is_64bit_mm(struct mm_struct *mm)
237 return !config_enabled(CONFIG_IA32_EMULATION) ||
238 !(mm->context.ia32_compat == TIF_IA32);
241 static inline bool is_64bit_mm(struct mm_struct *mm)
247 static inline void arch_bprm_mm_init(struct mm_struct *mm,
248 struct vm_area_struct *vma)
253 static inline void arch_unmap(struct mm_struct *mm, struct vm_area_struct *vma,
254 unsigned long start, unsigned long end)
257 * mpx_notify_unmap() goes and reads a rarely-hot
258 * cacheline in the mm_struct. That can be expensive
259 * enough to be seen in profiles.
261 * The mpx_notify_unmap() call and its contents have been
262 * observed to affect munmap() performance on hardware
263 * where MPX is not present.
265 * The unlikely() optimizes for the fast case: no MPX
266 * in the CPU, or no MPX use in the process. Even if
267 * we get this wrong (in the unlikely event that MPX
268 * is widely enabled on some system) the overhead of
269 * MPX itself (reading bounds tables) is expected to
270 * overwhelm the overhead of getting this unlikely()
271 * consistently wrong.
273 if (unlikely(cpu_feature_enabled(X86_FEATURE_MPX)))
274 mpx_notify_unmap(mm, vma, start, end);
277 #endif /* _ASM_X86_MMU_CONTEXT_H */