2 * Copyright IBM Corp. 2007, 2011
3 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
6 #include <linux/sched.h>
7 #include <linux/kernel.h>
8 #include <linux/errno.h>
11 #include <linux/swap.h>
12 #include <linux/smp.h>
13 #include <linux/spinlock.h>
14 #include <linux/rcupdate.h>
15 #include <linux/slab.h>
16 #include <linux/swapops.h>
17 #include <linux/sysctl.h>
18 #include <linux/ksm.h>
19 #include <linux/mman.h>
21 #include <asm/pgtable.h>
22 #include <asm/pgalloc.h>
24 #include <asm/tlbflush.h>
25 #include <asm/mmu_context.h>
27 static inline pte_t ptep_flush_direct(struct mm_struct *mm,
28 unsigned long addr, pte_t *ptep)
34 if (unlikely(pte_val(old) & _PAGE_INVALID))
36 active = (mm == current->active_mm) ? 1 : 0;
37 count = atomic_add_return(0x10000, &mm->context.attach_count);
38 if (MACHINE_HAS_TLB_LC && (count & 0xffff) <= active &&
39 cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
40 __ptep_ipte_local(addr, ptep);
42 __ptep_ipte(addr, ptep);
43 atomic_sub(0x10000, &mm->context.attach_count);
47 static inline pte_t ptep_flush_lazy(struct mm_struct *mm,
48 unsigned long addr, pte_t *ptep)
54 if (unlikely(pte_val(old) & _PAGE_INVALID))
56 active = (mm == current->active_mm) ? 1 : 0;
57 count = atomic_add_return(0x10000, &mm->context.attach_count);
58 if ((count & 0xffff) <= active) {
59 pte_val(*ptep) |= _PAGE_INVALID;
60 mm->context.flush_mm = 1;
62 __ptep_ipte(addr, ptep);
63 atomic_sub(0x10000, &mm->context.attach_count);
67 static inline pgste_t pgste_get_lock(pte_t *ptep)
69 unsigned long new = 0;
77 " nihh %0,0xff7f\n" /* clear PCL bit in old */
78 " oihh %1,0x0080\n" /* set PCL bit in new */
81 : "=&d" (old), "=&d" (new), "=Q" (ptep[PTRS_PER_PTE])
82 : "Q" (ptep[PTRS_PER_PTE]) : "cc", "memory");
87 static inline void pgste_set_unlock(pte_t *ptep, pgste_t pgste)
91 " nihh %1,0xff7f\n" /* clear PCL bit */
93 : "=Q" (ptep[PTRS_PER_PTE])
94 : "d" (pgste_val(pgste)), "Q" (ptep[PTRS_PER_PTE])
100 static inline pgste_t pgste_get(pte_t *ptep)
102 unsigned long pgste = 0;
104 pgste = *(unsigned long *)(ptep + PTRS_PER_PTE);
106 return __pgste(pgste);
109 static inline void pgste_set(pte_t *ptep, pgste_t pgste)
112 *(pgste_t *)(ptep + PTRS_PER_PTE) = pgste;
116 static inline pgste_t pgste_update_all(pte_t pte, pgste_t pgste,
117 struct mm_struct *mm)
120 unsigned long address, bits, skey;
122 if (!mm_use_skey(mm) || pte_val(pte) & _PAGE_INVALID)
124 address = pte_val(pte) & PAGE_MASK;
125 skey = (unsigned long) page_get_storage_key(address);
126 bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED);
127 /* Transfer page changed & referenced bit to guest bits in pgste */
128 pgste_val(pgste) |= bits << 48; /* GR bit & GC bit */
129 /* Copy page access key and fetch protection bit to pgste */
130 pgste_val(pgste) &= ~(PGSTE_ACC_BITS | PGSTE_FP_BIT);
131 pgste_val(pgste) |= (skey & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56;
137 static inline void pgste_set_key(pte_t *ptep, pgste_t pgste, pte_t entry,
138 struct mm_struct *mm)
141 unsigned long address;
144 if (!mm_use_skey(mm) || pte_val(entry) & _PAGE_INVALID)
146 VM_BUG_ON(!(pte_val(*ptep) & _PAGE_INVALID));
147 address = pte_val(entry) & PAGE_MASK;
149 * Set page access key and fetch protection bit from pgste.
150 * The guest C/R information is still in the PGSTE, set real
153 nkey = (pgste_val(pgste) & (PGSTE_ACC_BITS | PGSTE_FP_BIT)) >> 56;
154 nkey |= (pgste_val(pgste) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 48;
155 page_set_storage_key(address, nkey, 0);
159 static inline pgste_t pgste_set_pte(pte_t *ptep, pgste_t pgste, pte_t entry)
162 if ((pte_val(entry) & _PAGE_PRESENT) &&
163 (pte_val(entry) & _PAGE_WRITE) &&
164 !(pte_val(entry) & _PAGE_INVALID)) {
165 if (!MACHINE_HAS_ESOP) {
167 * Without enhanced suppression-on-protection force
168 * the dirty bit on for all writable ptes.
170 pte_val(entry) |= _PAGE_DIRTY;
171 pte_val(entry) &= ~_PAGE_PROTECT;
173 if (!(pte_val(entry) & _PAGE_PROTECT))
174 /* This pte allows write access, set user-dirty */
175 pgste_val(pgste) |= PGSTE_UC_BIT;
182 static inline pgste_t pgste_pte_notify(struct mm_struct *mm,
184 pte_t *ptep, pgste_t pgste)
189 bits = pgste_val(pgste) & (PGSTE_IN_BIT | PGSTE_VSIE_BIT);
191 pgste_val(pgste) ^= bits;
192 ptep_notify(mm, addr, ptep, bits);
198 static inline pgste_t ptep_xchg_start(struct mm_struct *mm,
199 unsigned long addr, pte_t *ptep)
201 pgste_t pgste = __pgste(0);
203 if (mm_has_pgste(mm)) {
204 pgste = pgste_get_lock(ptep);
205 pgste = pgste_pte_notify(mm, addr, ptep, pgste);
210 static inline void ptep_xchg_commit(struct mm_struct *mm,
211 unsigned long addr, pte_t *ptep,
212 pgste_t pgste, pte_t old, pte_t new)
214 if (mm_has_pgste(mm)) {
215 if (pte_val(old) & _PAGE_INVALID)
216 pgste_set_key(ptep, pgste, new, mm);
217 if (pte_val(new) & _PAGE_INVALID) {
218 pgste = pgste_update_all(old, pgste, mm);
219 if ((pgste_val(pgste) & _PGSTE_GPS_USAGE_MASK) ==
220 _PGSTE_GPS_USAGE_UNUSED)
221 pte_val(old) |= _PAGE_UNUSED;
223 pgste = pgste_set_pte(ptep, pgste, new);
224 pgste_set_unlock(ptep, pgste);
230 pte_t ptep_xchg_direct(struct mm_struct *mm, unsigned long addr,
231 pte_t *ptep, pte_t new)
236 pgste = ptep_xchg_start(mm, addr, ptep);
237 old = ptep_flush_direct(mm, addr, ptep);
238 ptep_xchg_commit(mm, addr, ptep, pgste, old, new);
241 EXPORT_SYMBOL(ptep_xchg_direct);
243 pte_t ptep_xchg_lazy(struct mm_struct *mm, unsigned long addr,
244 pte_t *ptep, pte_t new)
249 pgste = ptep_xchg_start(mm, addr, ptep);
250 old = ptep_flush_lazy(mm, addr, ptep);
251 ptep_xchg_commit(mm, addr, ptep, pgste, old, new);
254 EXPORT_SYMBOL(ptep_xchg_lazy);
256 pte_t ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr,
262 pgste = ptep_xchg_start(mm, addr, ptep);
263 old = ptep_flush_lazy(mm, addr, ptep);
264 if (mm_has_pgste(mm)) {
265 pgste = pgste_update_all(old, pgste, mm);
266 pgste_set(ptep, pgste);
270 EXPORT_SYMBOL(ptep_modify_prot_start);
272 void ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr,
273 pte_t *ptep, pte_t pte)
277 if (mm_has_pgste(mm)) {
278 pgste = pgste_get(ptep);
279 pgste_set_key(ptep, pgste, pte, mm);
280 pgste = pgste_set_pte(ptep, pgste, pte);
281 pgste_set_unlock(ptep, pgste);
286 EXPORT_SYMBOL(ptep_modify_prot_commit);
288 static inline pmd_t pmdp_flush_direct(struct mm_struct *mm,
289 unsigned long addr, pmd_t *pmdp)
295 if (pmd_val(old) & _SEGMENT_ENTRY_INVALID)
297 if (!MACHINE_HAS_IDTE) {
301 active = (mm == current->active_mm) ? 1 : 0;
302 count = atomic_add_return(0x10000, &mm->context.attach_count);
303 if (MACHINE_HAS_TLB_LC && (count & 0xffff) <= active &&
304 cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
305 __pmdp_idte_local(addr, pmdp);
307 __pmdp_idte(addr, pmdp);
308 atomic_sub(0x10000, &mm->context.attach_count);
312 static inline pmd_t pmdp_flush_lazy(struct mm_struct *mm,
313 unsigned long addr, pmd_t *pmdp)
319 if (pmd_val(old) & _SEGMENT_ENTRY_INVALID)
321 active = (mm == current->active_mm) ? 1 : 0;
322 count = atomic_add_return(0x10000, &mm->context.attach_count);
323 if ((count & 0xffff) <= active) {
324 pmd_val(*pmdp) |= _SEGMENT_ENTRY_INVALID;
325 mm->context.flush_mm = 1;
326 } else if (MACHINE_HAS_IDTE)
327 __pmdp_idte(addr, pmdp);
330 atomic_sub(0x10000, &mm->context.attach_count);
334 pmd_t pmdp_xchg_direct(struct mm_struct *mm, unsigned long addr,
335 pmd_t *pmdp, pmd_t new)
339 old = pmdp_flush_direct(mm, addr, pmdp);
343 EXPORT_SYMBOL(pmdp_xchg_direct);
345 pmd_t pmdp_xchg_lazy(struct mm_struct *mm, unsigned long addr,
346 pmd_t *pmdp, pmd_t new)
350 old = pmdp_flush_lazy(mm, addr, pmdp);
354 EXPORT_SYMBOL(pmdp_xchg_lazy);
356 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
357 void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
360 struct list_head *lh = (struct list_head *) pgtable;
362 assert_spin_locked(pmd_lockptr(mm, pmdp));
365 if (!pmd_huge_pte(mm, pmdp))
368 list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp));
369 pmd_huge_pte(mm, pmdp) = pgtable;
372 pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
374 struct list_head *lh;
378 assert_spin_locked(pmd_lockptr(mm, pmdp));
381 pgtable = pmd_huge_pte(mm, pmdp);
382 lh = (struct list_head *) pgtable;
384 pmd_huge_pte(mm, pmdp) = NULL;
386 pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next;
389 ptep = (pte_t *) pgtable;
390 pte_val(*ptep) = _PAGE_INVALID;
392 pte_val(*ptep) = _PAGE_INVALID;
395 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
398 void ptep_set_pte_at(struct mm_struct *mm, unsigned long addr,
399 pte_t *ptep, pte_t entry)
403 /* the mm_has_pgste() check is done in set_pte_at() */
404 pgste = pgste_get_lock(ptep);
405 pgste_val(pgste) &= ~_PGSTE_GPS_ZERO;
406 pgste_set_key(ptep, pgste, entry, mm);
407 pgste = pgste_set_pte(ptep, pgste, entry);
408 pgste_set_unlock(ptep, pgste);
411 void ptep_set_notify(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
415 pgste = pgste_get_lock(ptep);
416 pgste_val(pgste) |= PGSTE_IN_BIT;
417 pgste_set_unlock(ptep, pgste);
421 * ptep_force_prot - change access rights of a locked pte
422 * @mm: pointer to the process mm_struct
423 * @addr: virtual address in the guest address space
424 * @ptep: pointer to the page table entry
425 * @prot: indicates guest access rights: PROT_NONE, PROT_READ or PROT_WRITE
426 * @bit: pgste bit to set (e.g. for notification)
428 * Returns 0 if the access rights were changed and -EAGAIN if the current
429 * and requested access rights are incompatible.
431 int ptep_force_prot(struct mm_struct *mm, unsigned long addr,
432 pte_t *ptep, int prot, unsigned long bit)
438 pgste = pgste_get_lock(ptep);
440 /* Check pte entry after all locks have been acquired */
441 pte_i = pte_val(entry) & _PAGE_INVALID;
442 pte_p = pte_val(entry) & _PAGE_PROTECT;
443 if ((pte_i && (prot != PROT_NONE)) ||
444 (pte_p && (prot & PROT_WRITE))) {
445 pgste_set_unlock(ptep, pgste);
448 /* Change access rights and set pgste bit */
449 if (prot == PROT_NONE && !pte_i) {
450 ptep_flush_direct(mm, addr, ptep);
451 pgste = pgste_update_all(entry, pgste, mm);
452 pte_val(entry) |= _PAGE_INVALID;
454 if (prot == PROT_READ && !pte_p) {
455 ptep_flush_direct(mm, addr, ptep);
456 pte_val(entry) &= ~_PAGE_INVALID;
457 pte_val(entry) |= _PAGE_PROTECT;
459 pgste_val(pgste) |= bit;
460 pgste = pgste_set_pte(ptep, pgste, entry);
461 pgste_set_unlock(ptep, pgste);
465 int ptep_shadow_pte(struct mm_struct *mm, unsigned long saddr,
466 pte_t *sptep, pte_t *tptep, pte_t pte)
468 pgste_t spgste, tpgste;
472 if (!(pte_val(*tptep) & _PAGE_INVALID))
473 return 0; /* already shadowed */
474 spgste = pgste_get_lock(sptep);
476 if (!(pte_val(spte) & _PAGE_INVALID) &&
477 !((pte_val(spte) & _PAGE_PROTECT) &&
478 !(pte_val(pte) & _PAGE_PROTECT))) {
479 pgste_val(spgste) |= PGSTE_VSIE_BIT;
480 tpgste = pgste_get_lock(tptep);
481 pte_val(tpte) = (pte_val(spte) & PAGE_MASK) |
482 (pte_val(pte) & _PAGE_PROTECT);
483 /* don't touch the storage key - it belongs to parent pgste */
484 tpgste = pgste_set_pte(tptep, tpgste, tpte);
485 pgste_set_unlock(tptep, tpgste);
488 pgste_set_unlock(sptep, spgste);
492 void ptep_unshadow_pte(struct mm_struct *mm, unsigned long saddr, pte_t *ptep)
496 pgste = pgste_get_lock(ptep);
497 /* notifier is called by the caller */
498 ptep_flush_direct(mm, saddr, ptep);
499 /* don't touch the storage key - it belongs to parent pgste */
500 pgste = pgste_set_pte(ptep, pgste, __pte(_PAGE_INVALID));
501 pgste_set_unlock(ptep, pgste);
504 static void ptep_zap_swap_entry(struct mm_struct *mm, swp_entry_t entry)
506 if (!non_swap_entry(entry))
507 dec_mm_counter(mm, MM_SWAPENTS);
508 else if (is_migration_entry(entry)) {
509 struct page *page = migration_entry_to_page(entry);
511 dec_mm_counter(mm, mm_counter(page));
513 free_swap_and_cache(entry);
516 void ptep_zap_unused(struct mm_struct *mm, unsigned long addr,
517 pte_t *ptep, int reset)
519 unsigned long pgstev;
523 /* Zap unused and logically-zero pages */
524 pgste = pgste_get_lock(ptep);
525 pgstev = pgste_val(pgste);
528 ((pgstev & _PGSTE_GPS_USAGE_MASK) == _PGSTE_GPS_USAGE_UNUSED ||
529 (pgstev & _PGSTE_GPS_ZERO))) {
530 ptep_zap_swap_entry(mm, pte_to_swp_entry(pte));
531 pte_clear(mm, addr, ptep);
534 pgste_val(pgste) &= ~_PGSTE_GPS_USAGE_MASK;
535 pgste_set_unlock(ptep, pgste);
538 void ptep_zap_key(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
543 /* Clear storage key */
544 pgste = pgste_get_lock(ptep);
545 pgste_val(pgste) &= ~(PGSTE_ACC_BITS | PGSTE_FP_BIT |
546 PGSTE_GR_BIT | PGSTE_GC_BIT);
547 ptev = pte_val(*ptep);
548 if (!(ptev & _PAGE_INVALID) && (ptev & _PAGE_WRITE))
549 page_set_storage_key(ptev & PAGE_MASK, PAGE_DEFAULT_KEY, 1);
550 pgste_set_unlock(ptep, pgste);
554 * Test and reset if a guest page is dirty
556 bool test_and_clear_guest_dirty(struct mm_struct *mm, unsigned long addr)
564 ptep = get_locked_pte(mm, addr, &ptl);
568 pgste = pgste_get_lock(ptep);
569 dirty = !!(pgste_val(pgste) & PGSTE_UC_BIT);
570 pgste_val(pgste) &= ~PGSTE_UC_BIT;
572 if (dirty && (pte_val(pte) & _PAGE_PRESENT)) {
573 pgste = pgste_pte_notify(mm, addr, ptep, pgste);
574 __ptep_ipte(addr, ptep);
575 if (MACHINE_HAS_ESOP || !(pte_val(pte) & _PAGE_WRITE))
576 pte_val(pte) |= _PAGE_PROTECT;
578 pte_val(pte) |= _PAGE_INVALID;
581 pgste_set_unlock(ptep, pgste);
586 EXPORT_SYMBOL_GPL(test_and_clear_guest_dirty);
588 int set_guest_storage_key(struct mm_struct *mm, unsigned long addr,
589 unsigned char key, bool nq)
596 ptep = get_locked_pte(mm, addr, &ptl);
600 new = old = pgste_get_lock(ptep);
601 pgste_val(new) &= ~(PGSTE_GR_BIT | PGSTE_GC_BIT |
602 PGSTE_ACC_BITS | PGSTE_FP_BIT);
603 keyul = (unsigned long) key;
604 pgste_val(new) |= (keyul & (_PAGE_CHANGED | _PAGE_REFERENCED)) << 48;
605 pgste_val(new) |= (keyul & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56;
606 if (!(pte_val(*ptep) & _PAGE_INVALID)) {
607 unsigned long address, bits, skey;
609 address = pte_val(*ptep) & PAGE_MASK;
610 skey = (unsigned long) page_get_storage_key(address);
611 bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED);
612 skey = key & (_PAGE_ACC_BITS | _PAGE_FP_BIT);
613 /* Set storage key ACC and FP */
614 page_set_storage_key(address, skey, !nq);
615 /* Merge host changed & referenced into pgste */
616 pgste_val(new) |= bits << 52;
618 /* changing the guest storage key is considered a change of the page */
619 if ((pgste_val(new) ^ pgste_val(old)) &
620 (PGSTE_ACC_BITS | PGSTE_FP_BIT | PGSTE_GR_BIT | PGSTE_GC_BIT))
621 pgste_val(new) |= PGSTE_UC_BIT;
623 pgste_set_unlock(ptep, new);
624 pte_unmap_unlock(ptep, ptl);
627 EXPORT_SYMBOL(set_guest_storage_key);
630 * Conditionally set a guest storage key (handling csske).
631 * oldkey will be updated when either mr or mc is set and a pointer is given.
633 * Returns 0 if a guests storage key update wasn't necessary, 1 if the guest
634 * storage key was updated and -EFAULT on access errors.
636 int cond_set_guest_storage_key(struct mm_struct *mm, unsigned long addr,
637 unsigned char key, unsigned char *oldkey,
638 bool nq, bool mr, bool mc)
640 unsigned char tmp, mask = _PAGE_ACC_BITS | _PAGE_FP_BIT;
643 /* we can drop the pgste lock between getting and setting the key */
645 rc = get_guest_storage_key(current->mm, addr, &tmp);
651 mask |= _PAGE_REFERENCED;
653 mask |= _PAGE_CHANGED;
654 if (!((tmp ^ key) & mask))
657 rc = set_guest_storage_key(current->mm, addr, key, nq);
658 return rc < 0 ? rc : 1;
660 EXPORT_SYMBOL(cond_set_guest_storage_key);
663 * Reset a guest reference bit (rrbe), returning the reference and changed bit.
665 * Returns < 0 in case of error, otherwise the cc to be reported to the guest.
667 int reset_guest_reference_bit(struct mm_struct *mm, unsigned long addr)
674 ptep = get_locked_pte(mm, addr, &ptl);
678 new = old = pgste_get_lock(ptep);
679 /* Reset guest reference bit only */
680 pgste_val(new) &= ~PGSTE_GR_BIT;
682 if (!(pte_val(*ptep) & _PAGE_INVALID)) {
683 cc = page_reset_referenced(pte_val(*ptep) & PAGE_MASK);
684 /* Merge real referenced bit into host-set */
685 pgste_val(new) |= ((unsigned long) cc << 53) & PGSTE_HR_BIT;
687 /* Reflect guest's logical view, not physical */
688 cc |= (pgste_val(old) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 49;
689 /* Changing the guest storage key is considered a change of the page */
690 if ((pgste_val(new) ^ pgste_val(old)) & PGSTE_GR_BIT)
691 pgste_val(new) |= PGSTE_UC_BIT;
693 pgste_set_unlock(ptep, new);
694 pte_unmap_unlock(ptep, ptl);
697 EXPORT_SYMBOL(reset_guest_reference_bit);
699 int get_guest_storage_key(struct mm_struct *mm, unsigned long addr,
706 ptep = get_locked_pte(mm, addr, &ptl);
710 pgste = pgste_get_lock(ptep);
711 *key = (pgste_val(pgste) & (PGSTE_ACC_BITS | PGSTE_FP_BIT)) >> 56;
712 if (!(pte_val(*ptep) & _PAGE_INVALID))
713 *key = page_get_storage_key(pte_val(*ptep) & PAGE_MASK);
714 /* Reflect guest's logical view, not physical */
715 *key |= (pgste_val(pgste) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 48;
716 pgste_set_unlock(ptep, pgste);
717 pte_unmap_unlock(ptep, ptl);
720 EXPORT_SYMBOL(get_guest_storage_key);