4 * This file contains the various mmu fetch and update operations.
5 * The most important job they must perform is the mapping between the
6 * domain's pfn and the overall machine mfns.
8 * Xen allows guests to directly update the pagetable, in a controlled
9 * fashion. In other words, the guest modifies the same pagetable
10 * that the CPU actually uses, which eliminates the overhead of having
11 * a separate shadow pagetable.
13 * In order to allow this, it falls on the guest domain to map its
14 * notion of a "physical" pfn - which is just a domain-local linear
15 * address - into a real "machine address" which the CPU's MMU can
18 * A pgd_t/pmd_t/pte_t will typically contain an mfn, and so can be
19 * inserted directly into the pagetable. When creating a new
20 * pte/pmd/pgd, it converts the passed pfn into an mfn. Conversely,
21 * when reading the content back with __(pgd|pmd|pte)_val, it converts
22 * the mfn back into a pfn.
24 * The other constraint is that all pages which make up a pagetable
25 * must be mapped read-only in the guest. This prevents uncontrolled
26 * guest updates to the pagetable. Xen strictly enforces this, and
27 * will disallow any pagetable update which will end up mapping a
28 * pagetable page RW, and will disallow using any writable page as a
31 * Naively, when loading %cr3 with the base of a new pagetable, Xen
32 * would need to validate the whole pagetable before going on.
33 * Naturally, this is quite slow. The solution is to "pin" a
34 * pagetable, which enforces all the constraints on the pagetable even
35 * when it is not actively in use. This menas that Xen can be assured
36 * that it is still valid when you do load it into %cr3, and doesn't
37 * need to revalidate it.
39 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
41 #include <linux/sched.h>
42 #include <linux/highmem.h>
43 #include <linux/debugfs.h>
44 #include <linux/bug.h>
45 #include <linux/vmalloc.h>
46 #include <linux/module.h>
47 #include <linux/gfp.h>
48 #include <linux/memblock.h>
49 #include <linux/seq_file.h>
50 #include <linux/crash_dump.h>
52 #include <trace/events/xen.h>
54 #include <asm/pgtable.h>
55 #include <asm/tlbflush.h>
56 #include <asm/fixmap.h>
57 #include <asm/mmu_context.h>
58 #include <asm/setup.h>
59 #include <asm/paravirt.h>
61 #include <asm/linkage.h>
67 #include <asm/xen/hypercall.h>
68 #include <asm/xen/hypervisor.h>
72 #include <xen/interface/xen.h>
73 #include <xen/interface/hvm/hvm_op.h>
74 #include <xen/interface/version.h>
75 #include <xen/interface/memory.h>
76 #include <xen/hvc-console.h>
78 #include "multicalls.h"
83 * Protects atomic reservation decrease/increase against concurrent increases.
84 * Also protects non-atomic updates of current_pages and balloon lists.
86 DEFINE_SPINLOCK(xen_reservation_lock);
90 * Identity map, in addition to plain kernel map. This needs to be
91 * large enough to allocate page table pages to allocate the rest.
92 * Each page can map 2MB.
94 #define LEVEL1_IDENT_ENTRIES (PTRS_PER_PTE * 4)
95 static RESERVE_BRK_ARRAY(pte_t, level1_ident_pgt, LEVEL1_IDENT_ENTRIES);
98 /* l3 pud for userspace vsyscall mapping */
99 static pud_t level3_user_vsyscall[PTRS_PER_PUD] __page_aligned_bss;
100 #endif /* CONFIG_X86_64 */
103 * Note about cr3 (pagetable base) values:
105 * xen_cr3 contains the current logical cr3 value; it contains the
106 * last set cr3. This may not be the current effective cr3, because
107 * its update may be being lazily deferred. However, a vcpu looking
108 * at its own cr3 can use this value knowing that it everything will
109 * be self-consistent.
111 * xen_current_cr3 contains the actual vcpu cr3; it is set once the
112 * hypercall to set the vcpu cr3 is complete (so it may be a little
113 * out of date, but it will never be set early). If one vcpu is
114 * looking at another vcpu's cr3 value, it should use this variable.
116 DEFINE_PER_CPU(unsigned long, xen_cr3); /* cr3 stored as physaddr */
117 DEFINE_PER_CPU(unsigned long, xen_current_cr3); /* actual vcpu cr3 */
121 * Just beyond the highest usermode address. STACK_TOP_MAX has a
122 * redzone above it, so round it up to a PGD boundary.
124 #define USER_LIMIT ((STACK_TOP_MAX + PGDIR_SIZE - 1) & PGDIR_MASK)
126 unsigned long arbitrary_virt_to_mfn(void *vaddr)
128 xmaddr_t maddr = arbitrary_virt_to_machine(vaddr);
130 return PFN_DOWN(maddr.maddr);
133 xmaddr_t arbitrary_virt_to_machine(void *vaddr)
135 unsigned long address = (unsigned long)vaddr;
141 * if the PFN is in the linear mapped vaddr range, we can just use
142 * the (quick) virt_to_machine() p2m lookup
144 if (virt_addr_valid(vaddr))
145 return virt_to_machine(vaddr);
147 /* otherwise we have to do a (slower) full page-table walk */
149 pte = lookup_address(address, &level);
151 offset = address & ~PAGE_MASK;
152 return XMADDR(((phys_addr_t)pte_mfn(*pte) << PAGE_SHIFT) + offset);
154 EXPORT_SYMBOL_GPL(arbitrary_virt_to_machine);
156 void make_lowmem_page_readonly(void *vaddr)
159 unsigned long address = (unsigned long)vaddr;
162 pte = lookup_address(address, &level);
164 return; /* vaddr missing */
166 ptev = pte_wrprotect(*pte);
168 if (HYPERVISOR_update_va_mapping(address, ptev, 0))
172 void make_lowmem_page_readwrite(void *vaddr)
175 unsigned long address = (unsigned long)vaddr;
178 pte = lookup_address(address, &level);
180 return; /* vaddr missing */
182 ptev = pte_mkwrite(*pte);
184 if (HYPERVISOR_update_va_mapping(address, ptev, 0))
189 static bool xen_page_pinned(void *ptr)
191 struct page *page = virt_to_page(ptr);
193 return PagePinned(page);
196 void xen_set_domain_pte(pte_t *ptep, pte_t pteval, unsigned domid)
198 struct multicall_space mcs;
199 struct mmu_update *u;
201 trace_xen_mmu_set_domain_pte(ptep, pteval, domid);
203 mcs = xen_mc_entry(sizeof(*u));
206 /* ptep might be kmapped when using 32-bit HIGHPTE */
207 u->ptr = virt_to_machine(ptep).maddr;
208 u->val = pte_val_ma(pteval);
210 MULTI_mmu_update(mcs.mc, mcs.args, 1, NULL, domid);
212 xen_mc_issue(PARAVIRT_LAZY_MMU);
214 EXPORT_SYMBOL_GPL(xen_set_domain_pte);
216 static void xen_extend_mmu_update(const struct mmu_update *update)
218 struct multicall_space mcs;
219 struct mmu_update *u;
221 mcs = xen_mc_extend_args(__HYPERVISOR_mmu_update, sizeof(*u));
223 if (mcs.mc != NULL) {
226 mcs = __xen_mc_entry(sizeof(*u));
227 MULTI_mmu_update(mcs.mc, mcs.args, 1, NULL, DOMID_SELF);
234 static void xen_extend_mmuext_op(const struct mmuext_op *op)
236 struct multicall_space mcs;
239 mcs = xen_mc_extend_args(__HYPERVISOR_mmuext_op, sizeof(*u));
241 if (mcs.mc != NULL) {
244 mcs = __xen_mc_entry(sizeof(*u));
245 MULTI_mmuext_op(mcs.mc, mcs.args, 1, NULL, DOMID_SELF);
252 static void xen_set_pmd_hyper(pmd_t *ptr, pmd_t val)
260 /* ptr may be ioremapped for 64-bit pagetable setup */
261 u.ptr = arbitrary_virt_to_machine(ptr).maddr;
262 u.val = pmd_val_ma(val);
263 xen_extend_mmu_update(&u);
265 xen_mc_issue(PARAVIRT_LAZY_MMU);
270 static void xen_set_pmd(pmd_t *ptr, pmd_t val)
272 trace_xen_mmu_set_pmd(ptr, val);
274 /* If page is not pinned, we can just update the entry
276 if (!xen_page_pinned(ptr)) {
281 xen_set_pmd_hyper(ptr, val);
285 * Associate a virtual page frame with a given physical page frame
286 * and protection flags for that frame.
288 void set_pte_mfn(unsigned long vaddr, unsigned long mfn, pgprot_t flags)
290 set_pte_vaddr(vaddr, mfn_pte(mfn, flags));
293 static bool xen_batched_set_pte(pte_t *ptep, pte_t pteval)
297 if (paravirt_get_lazy_mode() != PARAVIRT_LAZY_MMU)
302 u.ptr = virt_to_machine(ptep).maddr | MMU_NORMAL_PT_UPDATE;
303 u.val = pte_val_ma(pteval);
304 xen_extend_mmu_update(&u);
306 xen_mc_issue(PARAVIRT_LAZY_MMU);
311 static inline void __xen_set_pte(pte_t *ptep, pte_t pteval)
313 if (!xen_batched_set_pte(ptep, pteval)) {
315 * Could call native_set_pte() here and trap and
316 * emulate the PTE write but with 32-bit guests this
317 * needs two traps (one for each of the two 32-bit
318 * words in the PTE) so do one hypercall directly
323 u.ptr = virt_to_machine(ptep).maddr | MMU_NORMAL_PT_UPDATE;
324 u.val = pte_val_ma(pteval);
325 HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF);
329 static void xen_set_pte(pte_t *ptep, pte_t pteval)
331 trace_xen_mmu_set_pte(ptep, pteval);
332 __xen_set_pte(ptep, pteval);
335 static void xen_set_pte_at(struct mm_struct *mm, unsigned long addr,
336 pte_t *ptep, pte_t pteval)
338 trace_xen_mmu_set_pte_at(mm, addr, ptep, pteval);
339 __xen_set_pte(ptep, pteval);
342 pte_t xen_ptep_modify_prot_start(struct mm_struct *mm,
343 unsigned long addr, pte_t *ptep)
345 /* Just return the pte as-is. We preserve the bits on commit */
346 trace_xen_mmu_ptep_modify_prot_start(mm, addr, ptep, *ptep);
350 void xen_ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr,
351 pte_t *ptep, pte_t pte)
355 trace_xen_mmu_ptep_modify_prot_commit(mm, addr, ptep, pte);
358 u.ptr = virt_to_machine(ptep).maddr | MMU_PT_UPDATE_PRESERVE_AD;
359 u.val = pte_val_ma(pte);
360 xen_extend_mmu_update(&u);
362 xen_mc_issue(PARAVIRT_LAZY_MMU);
365 /* Assume pteval_t is equivalent to all the other *val_t types. */
366 static pteval_t pte_mfn_to_pfn(pteval_t val)
368 if (val & _PAGE_PRESENT) {
369 unsigned long mfn = (val & PTE_PFN_MASK) >> PAGE_SHIFT;
370 unsigned long pfn = mfn_to_pfn(mfn);
372 pteval_t flags = val & PTE_FLAGS_MASK;
373 if (unlikely(pfn == ~0))
374 val = flags & ~_PAGE_PRESENT;
376 val = ((pteval_t)pfn << PAGE_SHIFT) | flags;
382 static pteval_t pte_pfn_to_mfn(pteval_t val)
384 if (val & _PAGE_PRESENT) {
385 unsigned long pfn = (val & PTE_PFN_MASK) >> PAGE_SHIFT;
386 pteval_t flags = val & PTE_FLAGS_MASK;
389 if (!xen_feature(XENFEAT_auto_translated_physmap))
390 mfn = __pfn_to_mfn(pfn);
394 * If there's no mfn for the pfn, then just create an
395 * empty non-present pte. Unfortunately this loses
396 * information about the original pfn, so
397 * pte_mfn_to_pfn is asymmetric.
399 if (unlikely(mfn == INVALID_P2M_ENTRY)) {
403 mfn &= ~(FOREIGN_FRAME_BIT | IDENTITY_FRAME_BIT);
404 val = ((pteval_t)mfn << PAGE_SHIFT) | flags;
410 __visible pteval_t xen_pte_val(pte_t pte)
412 pteval_t pteval = pte.pte;
414 return pte_mfn_to_pfn(pteval);
416 PV_CALLEE_SAVE_REGS_THUNK(xen_pte_val);
418 __visible pgdval_t xen_pgd_val(pgd_t pgd)
420 return pte_mfn_to_pfn(pgd.pgd);
422 PV_CALLEE_SAVE_REGS_THUNK(xen_pgd_val);
424 __visible pte_t xen_make_pte(pteval_t pte)
426 pte = pte_pfn_to_mfn(pte);
428 return native_make_pte(pte);
430 PV_CALLEE_SAVE_REGS_THUNK(xen_make_pte);
432 __visible pgd_t xen_make_pgd(pgdval_t pgd)
434 pgd = pte_pfn_to_mfn(pgd);
435 return native_make_pgd(pgd);
437 PV_CALLEE_SAVE_REGS_THUNK(xen_make_pgd);
439 __visible pmdval_t xen_pmd_val(pmd_t pmd)
441 return pte_mfn_to_pfn(pmd.pmd);
443 PV_CALLEE_SAVE_REGS_THUNK(xen_pmd_val);
445 static void xen_set_pud_hyper(pud_t *ptr, pud_t val)
453 /* ptr may be ioremapped for 64-bit pagetable setup */
454 u.ptr = arbitrary_virt_to_machine(ptr).maddr;
455 u.val = pud_val_ma(val);
456 xen_extend_mmu_update(&u);
458 xen_mc_issue(PARAVIRT_LAZY_MMU);
463 static void xen_set_pud(pud_t *ptr, pud_t val)
465 trace_xen_mmu_set_pud(ptr, val);
467 /* If page is not pinned, we can just update the entry
469 if (!xen_page_pinned(ptr)) {
474 xen_set_pud_hyper(ptr, val);
477 #ifdef CONFIG_X86_PAE
478 static void xen_set_pte_atomic(pte_t *ptep, pte_t pte)
480 trace_xen_mmu_set_pte_atomic(ptep, pte);
481 set_64bit((u64 *)ptep, native_pte_val(pte));
484 static void xen_pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
486 trace_xen_mmu_pte_clear(mm, addr, ptep);
487 if (!xen_batched_set_pte(ptep, native_make_pte(0)))
488 native_pte_clear(mm, addr, ptep);
491 static void xen_pmd_clear(pmd_t *pmdp)
493 trace_xen_mmu_pmd_clear(pmdp);
494 set_pmd(pmdp, __pmd(0));
496 #endif /* CONFIG_X86_PAE */
498 __visible pmd_t xen_make_pmd(pmdval_t pmd)
500 pmd = pte_pfn_to_mfn(pmd);
501 return native_make_pmd(pmd);
503 PV_CALLEE_SAVE_REGS_THUNK(xen_make_pmd);
505 #if PAGETABLE_LEVELS == 4
506 __visible pudval_t xen_pud_val(pud_t pud)
508 return pte_mfn_to_pfn(pud.pud);
510 PV_CALLEE_SAVE_REGS_THUNK(xen_pud_val);
512 __visible pud_t xen_make_pud(pudval_t pud)
514 pud = pte_pfn_to_mfn(pud);
516 return native_make_pud(pud);
518 PV_CALLEE_SAVE_REGS_THUNK(xen_make_pud);
520 static pgd_t *xen_get_user_pgd(pgd_t *pgd)
522 pgd_t *pgd_page = (pgd_t *)(((unsigned long)pgd) & PAGE_MASK);
523 unsigned offset = pgd - pgd_page;
524 pgd_t *user_ptr = NULL;
526 if (offset < pgd_index(USER_LIMIT)) {
527 struct page *page = virt_to_page(pgd_page);
528 user_ptr = (pgd_t *)page->private;
536 static void __xen_set_pgd_hyper(pgd_t *ptr, pgd_t val)
540 u.ptr = virt_to_machine(ptr).maddr;
541 u.val = pgd_val_ma(val);
542 xen_extend_mmu_update(&u);
546 * Raw hypercall-based set_pgd, intended for in early boot before
547 * there's a page structure. This implies:
548 * 1. The only existing pagetable is the kernel's
549 * 2. It is always pinned
550 * 3. It has no user pagetable attached to it
552 static void __init xen_set_pgd_hyper(pgd_t *ptr, pgd_t val)
558 __xen_set_pgd_hyper(ptr, val);
560 xen_mc_issue(PARAVIRT_LAZY_MMU);
565 static void xen_set_pgd(pgd_t *ptr, pgd_t val)
567 pgd_t *user_ptr = xen_get_user_pgd(ptr);
569 trace_xen_mmu_set_pgd(ptr, user_ptr, val);
571 /* If page is not pinned, we can just update the entry
573 if (!xen_page_pinned(ptr)) {
576 WARN_ON(xen_page_pinned(user_ptr));
582 /* If it's pinned, then we can at least batch the kernel and
583 user updates together. */
586 __xen_set_pgd_hyper(ptr, val);
588 __xen_set_pgd_hyper(user_ptr, val);
590 xen_mc_issue(PARAVIRT_LAZY_MMU);
592 #endif /* PAGETABLE_LEVELS == 4 */
595 * (Yet another) pagetable walker. This one is intended for pinning a
596 * pagetable. This means that it walks a pagetable and calls the
597 * callback function on each page it finds making up the page table,
598 * at every level. It walks the entire pagetable, but it only bothers
599 * pinning pte pages which are below limit. In the normal case this
600 * will be STACK_TOP_MAX, but at boot we need to pin up to
603 * For 32-bit the important bit is that we don't pin beyond there,
604 * because then we start getting into Xen's ptes.
606 * For 64-bit, we must skip the Xen hole in the middle of the address
607 * space, just after the big x86-64 virtual hole.
609 static int __xen_pgd_walk(struct mm_struct *mm, pgd_t *pgd,
610 int (*func)(struct mm_struct *mm, struct page *,
615 unsigned hole_low, hole_high;
616 unsigned pgdidx_limit, pudidx_limit, pmdidx_limit;
617 unsigned pgdidx, pudidx, pmdidx;
619 /* The limit is the last byte to be touched */
621 BUG_ON(limit >= FIXADDR_TOP);
623 if (xen_feature(XENFEAT_auto_translated_physmap))
627 * 64-bit has a great big hole in the middle of the address
628 * space, which contains the Xen mappings. On 32-bit these
629 * will end up making a zero-sized hole and so is a no-op.
631 hole_low = pgd_index(USER_LIMIT);
632 hole_high = pgd_index(PAGE_OFFSET);
634 pgdidx_limit = pgd_index(limit);
636 pudidx_limit = pud_index(limit);
641 pmdidx_limit = pmd_index(limit);
646 for (pgdidx = 0; pgdidx <= pgdidx_limit; pgdidx++) {
649 if (pgdidx >= hole_low && pgdidx < hole_high)
652 if (!pgd_val(pgd[pgdidx]))
655 pud = pud_offset(&pgd[pgdidx], 0);
657 if (PTRS_PER_PUD > 1) /* not folded */
658 flush |= (*func)(mm, virt_to_page(pud), PT_PUD);
660 for (pudidx = 0; pudidx < PTRS_PER_PUD; pudidx++) {
663 if (pgdidx == pgdidx_limit &&
664 pudidx > pudidx_limit)
667 if (pud_none(pud[pudidx]))
670 pmd = pmd_offset(&pud[pudidx], 0);
672 if (PTRS_PER_PMD > 1) /* not folded */
673 flush |= (*func)(mm, virt_to_page(pmd), PT_PMD);
675 for (pmdidx = 0; pmdidx < PTRS_PER_PMD; pmdidx++) {
678 if (pgdidx == pgdidx_limit &&
679 pudidx == pudidx_limit &&
680 pmdidx > pmdidx_limit)
683 if (pmd_none(pmd[pmdidx]))
686 pte = pmd_page(pmd[pmdidx]);
687 flush |= (*func)(mm, pte, PT_PTE);
693 /* Do the top level last, so that the callbacks can use it as
694 a cue to do final things like tlb flushes. */
695 flush |= (*func)(mm, virt_to_page(pgd), PT_PGD);
700 static int xen_pgd_walk(struct mm_struct *mm,
701 int (*func)(struct mm_struct *mm, struct page *,
705 return __xen_pgd_walk(mm, mm->pgd, func, limit);
708 /* If we're using split pte locks, then take the page's lock and
709 return a pointer to it. Otherwise return NULL. */
710 static spinlock_t *xen_pte_lock(struct page *page, struct mm_struct *mm)
712 spinlock_t *ptl = NULL;
714 #if USE_SPLIT_PTE_PTLOCKS
715 ptl = ptlock_ptr(page);
716 spin_lock_nest_lock(ptl, &mm->page_table_lock);
722 static void xen_pte_unlock(void *v)
728 static void xen_do_pin(unsigned level, unsigned long pfn)
733 op.arg1.mfn = pfn_to_mfn(pfn);
735 xen_extend_mmuext_op(&op);
738 static int xen_pin_page(struct mm_struct *mm, struct page *page,
741 unsigned pgfl = TestSetPagePinned(page);
745 flush = 0; /* already pinned */
746 else if (PageHighMem(page))
747 /* kmaps need flushing if we found an unpinned
751 void *pt = lowmem_page_address(page);
752 unsigned long pfn = page_to_pfn(page);
753 struct multicall_space mcs = __xen_mc_entry(0);
759 * We need to hold the pagetable lock between the time
760 * we make the pagetable RO and when we actually pin
761 * it. If we don't, then other users may come in and
762 * attempt to update the pagetable by writing it,
763 * which will fail because the memory is RO but not
764 * pinned, so Xen won't do the trap'n'emulate.
766 * If we're using split pte locks, we can't hold the
767 * entire pagetable's worth of locks during the
768 * traverse, because we may wrap the preempt count (8
769 * bits). The solution is to mark RO and pin each PTE
770 * page while holding the lock. This means the number
771 * of locks we end up holding is never more than a
772 * batch size (~32 entries, at present).
774 * If we're not using split pte locks, we needn't pin
775 * the PTE pages independently, because we're
776 * protected by the overall pagetable lock.
780 ptl = xen_pte_lock(page, mm);
782 MULTI_update_va_mapping(mcs.mc, (unsigned long)pt,
783 pfn_pte(pfn, PAGE_KERNEL_RO),
784 level == PT_PGD ? UVMF_TLB_FLUSH : 0);
787 xen_do_pin(MMUEXT_PIN_L1_TABLE, pfn);
789 /* Queue a deferred unlock for when this batch
791 xen_mc_callback(xen_pte_unlock, ptl);
798 /* This is called just after a mm has been created, but it has not
799 been used yet. We need to make sure that its pagetable is all
800 read-only, and can be pinned. */
801 static void __xen_pgd_pin(struct mm_struct *mm, pgd_t *pgd)
803 trace_xen_mmu_pgd_pin(mm, pgd);
807 if (__xen_pgd_walk(mm, pgd, xen_pin_page, USER_LIMIT)) {
808 /* re-enable interrupts for flushing */
818 pgd_t *user_pgd = xen_get_user_pgd(pgd);
820 xen_do_pin(MMUEXT_PIN_L4_TABLE, PFN_DOWN(__pa(pgd)));
823 xen_pin_page(mm, virt_to_page(user_pgd), PT_PGD);
824 xen_do_pin(MMUEXT_PIN_L4_TABLE,
825 PFN_DOWN(__pa(user_pgd)));
828 #else /* CONFIG_X86_32 */
829 #ifdef CONFIG_X86_PAE
830 /* Need to make sure unshared kernel PMD is pinnable */
831 xen_pin_page(mm, pgd_page(pgd[pgd_index(TASK_SIZE)]),
834 xen_do_pin(MMUEXT_PIN_L3_TABLE, PFN_DOWN(__pa(pgd)));
835 #endif /* CONFIG_X86_64 */
839 static void xen_pgd_pin(struct mm_struct *mm)
841 __xen_pgd_pin(mm, mm->pgd);
845 * On save, we need to pin all pagetables to make sure they get their
846 * mfns turned into pfns. Search the list for any unpinned pgds and pin
847 * them (unpinned pgds are not currently in use, probably because the
848 * process is under construction or destruction).
850 * Expected to be called in stop_machine() ("equivalent to taking
851 * every spinlock in the system"), so the locking doesn't really
852 * matter all that much.
854 void xen_mm_pin_all(void)
858 spin_lock(&pgd_lock);
860 list_for_each_entry(page, &pgd_list, lru) {
861 if (!PagePinned(page)) {
862 __xen_pgd_pin(&init_mm, (pgd_t *)page_address(page));
863 SetPageSavePinned(page);
867 spin_unlock(&pgd_lock);
871 * The init_mm pagetable is really pinned as soon as its created, but
872 * that's before we have page structures to store the bits. So do all
873 * the book-keeping now.
875 static int __init xen_mark_pinned(struct mm_struct *mm, struct page *page,
882 static void __init xen_mark_init_mm_pinned(void)
884 xen_pgd_walk(&init_mm, xen_mark_pinned, FIXADDR_TOP);
887 static int xen_unpin_page(struct mm_struct *mm, struct page *page,
890 unsigned pgfl = TestClearPagePinned(page);
892 if (pgfl && !PageHighMem(page)) {
893 void *pt = lowmem_page_address(page);
894 unsigned long pfn = page_to_pfn(page);
895 spinlock_t *ptl = NULL;
896 struct multicall_space mcs;
899 * Do the converse to pin_page. If we're using split
900 * pte locks, we must be holding the lock for while
901 * the pte page is unpinned but still RO to prevent
902 * concurrent updates from seeing it in this
903 * partially-pinned state.
905 if (level == PT_PTE) {
906 ptl = xen_pte_lock(page, mm);
909 xen_do_pin(MMUEXT_UNPIN_TABLE, pfn);
912 mcs = __xen_mc_entry(0);
914 MULTI_update_va_mapping(mcs.mc, (unsigned long)pt,
915 pfn_pte(pfn, PAGE_KERNEL),
916 level == PT_PGD ? UVMF_TLB_FLUSH : 0);
919 /* unlock when batch completed */
920 xen_mc_callback(xen_pte_unlock, ptl);
924 return 0; /* never need to flush on unpin */
927 /* Release a pagetables pages back as normal RW */
928 static void __xen_pgd_unpin(struct mm_struct *mm, pgd_t *pgd)
930 trace_xen_mmu_pgd_unpin(mm, pgd);
934 xen_do_pin(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd)));
938 pgd_t *user_pgd = xen_get_user_pgd(pgd);
941 xen_do_pin(MMUEXT_UNPIN_TABLE,
942 PFN_DOWN(__pa(user_pgd)));
943 xen_unpin_page(mm, virt_to_page(user_pgd), PT_PGD);
948 #ifdef CONFIG_X86_PAE
949 /* Need to make sure unshared kernel PMD is unpinned */
950 xen_unpin_page(mm, pgd_page(pgd[pgd_index(TASK_SIZE)]),
954 __xen_pgd_walk(mm, pgd, xen_unpin_page, USER_LIMIT);
959 static void xen_pgd_unpin(struct mm_struct *mm)
961 __xen_pgd_unpin(mm, mm->pgd);
965 * On resume, undo any pinning done at save, so that the rest of the
966 * kernel doesn't see any unexpected pinned pagetables.
968 void xen_mm_unpin_all(void)
972 spin_lock(&pgd_lock);
974 list_for_each_entry(page, &pgd_list, lru) {
975 if (PageSavePinned(page)) {
976 BUG_ON(!PagePinned(page));
977 __xen_pgd_unpin(&init_mm, (pgd_t *)page_address(page));
978 ClearPageSavePinned(page);
982 spin_unlock(&pgd_lock);
985 static void xen_activate_mm(struct mm_struct *prev, struct mm_struct *next)
987 spin_lock(&next->page_table_lock);
989 spin_unlock(&next->page_table_lock);
992 static void xen_dup_mmap(struct mm_struct *oldmm, struct mm_struct *mm)
994 spin_lock(&mm->page_table_lock);
996 spin_unlock(&mm->page_table_lock);
1001 /* Another cpu may still have their %cr3 pointing at the pagetable, so
1002 we need to repoint it somewhere else before we can unpin it. */
1003 static void drop_other_mm_ref(void *info)
1005 struct mm_struct *mm = info;
1006 struct mm_struct *active_mm;
1008 active_mm = this_cpu_read(cpu_tlbstate.active_mm);
1010 if (active_mm == mm && this_cpu_read(cpu_tlbstate.state) != TLBSTATE_OK)
1011 leave_mm(smp_processor_id());
1013 /* If this cpu still has a stale cr3 reference, then make sure
1014 it has been flushed. */
1015 if (this_cpu_read(xen_current_cr3) == __pa(mm->pgd))
1016 load_cr3(swapper_pg_dir);
1019 static void xen_drop_mm_ref(struct mm_struct *mm)
1024 if (current->active_mm == mm) {
1025 if (current->mm == mm)
1026 load_cr3(swapper_pg_dir);
1028 leave_mm(smp_processor_id());
1031 /* Get the "official" set of cpus referring to our pagetable. */
1032 if (!alloc_cpumask_var(&mask, GFP_ATOMIC)) {
1033 for_each_online_cpu(cpu) {
1034 if (!cpumask_test_cpu(cpu, mm_cpumask(mm))
1035 && per_cpu(xen_current_cr3, cpu) != __pa(mm->pgd))
1037 smp_call_function_single(cpu, drop_other_mm_ref, mm, 1);
1041 cpumask_copy(mask, mm_cpumask(mm));
1043 /* It's possible that a vcpu may have a stale reference to our
1044 cr3, because its in lazy mode, and it hasn't yet flushed
1045 its set of pending hypercalls yet. In this case, we can
1046 look at its actual current cr3 value, and force it to flush
1048 for_each_online_cpu(cpu) {
1049 if (per_cpu(xen_current_cr3, cpu) == __pa(mm->pgd))
1050 cpumask_set_cpu(cpu, mask);
1053 if (!cpumask_empty(mask))
1054 smp_call_function_many(mask, drop_other_mm_ref, mm, 1);
1055 free_cpumask_var(mask);
1058 static void xen_drop_mm_ref(struct mm_struct *mm)
1060 if (current->active_mm == mm)
1061 load_cr3(swapper_pg_dir);
1066 * While a process runs, Xen pins its pagetables, which means that the
1067 * hypervisor forces it to be read-only, and it controls all updates
1068 * to it. This means that all pagetable updates have to go via the
1069 * hypervisor, which is moderately expensive.
1071 * Since we're pulling the pagetable down, we switch to use init_mm,
1072 * unpin old process pagetable and mark it all read-write, which
1073 * allows further operations on it to be simple memory accesses.
1075 * The only subtle point is that another CPU may be still using the
1076 * pagetable because of lazy tlb flushing. This means we need need to
1077 * switch all CPUs off this pagetable before we can unpin it.
1079 static void xen_exit_mmap(struct mm_struct *mm)
1081 get_cpu(); /* make sure we don't move around */
1082 xen_drop_mm_ref(mm);
1085 spin_lock(&mm->page_table_lock);
1087 /* pgd may not be pinned in the error exit path of execve */
1088 if (xen_page_pinned(mm->pgd))
1091 spin_unlock(&mm->page_table_lock);
1094 static void xen_post_allocator_init(void);
1096 #ifdef CONFIG_X86_64
1097 static void __init xen_cleanhighmap(unsigned long vaddr,
1098 unsigned long vaddr_end)
1100 unsigned long kernel_end = roundup((unsigned long)_brk_end, PMD_SIZE) - 1;
1101 pmd_t *pmd = level2_kernel_pgt + pmd_index(vaddr);
1103 /* NOTE: The loop is more greedy than the cleanup_highmap variant.
1104 * We include the PMD passed in on _both_ boundaries. */
1105 for (; vaddr <= vaddr_end && (pmd < (level2_kernel_pgt + PAGE_SIZE));
1106 pmd++, vaddr += PMD_SIZE) {
1109 if (vaddr < (unsigned long) _text || vaddr > kernel_end)
1110 set_pmd(pmd, __pmd(0));
1112 /* In case we did something silly, we should crash in this function
1113 * instead of somewhere later and be confusing. */
1117 static void __init xen_pagetable_p2m_free(void)
1122 size = PAGE_ALIGN(xen_start_info->nr_pages * sizeof(unsigned long));
1124 /* No memory or already called. */
1125 if ((unsigned long)xen_p2m_addr == xen_start_info->mfn_list)
1128 /* using __ka address and sticking INVALID_P2M_ENTRY! */
1129 memset((void *)xen_start_info->mfn_list, 0xff, size);
1131 /* We should be in __ka space. */
1132 BUG_ON(xen_start_info->mfn_list < __START_KERNEL_map);
1133 addr = xen_start_info->mfn_list;
1134 /* We roundup to the PMD, which means that if anybody at this stage is
1135 * using the __ka address of xen_start_info or xen_start_info->shared_info
1136 * they are in going to crash. Fortunatly we have already revectored
1137 * in xen_setup_kernel_pagetable and in xen_setup_shared_info. */
1138 size = roundup(size, PMD_SIZE);
1139 xen_cleanhighmap(addr, addr + size);
1141 size = PAGE_ALIGN(xen_start_info->nr_pages * sizeof(unsigned long));
1142 memblock_free(__pa(xen_start_info->mfn_list), size);
1144 /* At this stage, cleanup_highmap has already cleaned __ka space
1145 * from _brk_limit way up to the max_pfn_mapped (which is the end of
1146 * the ramdisk). We continue on, erasing PMD entries that point to page
1147 * tables - do note that they are accessible at this stage via __va.
1148 * For good measure we also round up to the PMD - which means that if
1149 * anybody is using __ka address to the initial boot-stack - and try
1150 * to use it - they are going to crash. The xen_start_info has been
1151 * taken care of already in xen_setup_kernel_pagetable. */
1152 addr = xen_start_info->pt_base;
1153 size = roundup(xen_start_info->nr_pt_frames * PAGE_SIZE, PMD_SIZE);
1155 xen_cleanhighmap(addr, addr + size);
1156 xen_start_info->pt_base = (unsigned long)__va(__pa(xen_start_info->pt_base));
1158 /* This is superflous and is not neccessary, but you know what
1159 * lets do it. The MODULES_VADDR -> MODULES_END should be clear of
1160 * anything at this stage. */
1161 xen_cleanhighmap(MODULES_VADDR, roundup(MODULES_VADDR, PUD_SIZE) - 1);
1166 static void __init xen_pagetable_p2m_setup(void)
1168 if (xen_feature(XENFEAT_auto_translated_physmap))
1171 xen_vmalloc_p2m_tree();
1173 #ifdef CONFIG_X86_64
1174 xen_pagetable_p2m_free();
1176 /* And revector! Bye bye old array */
1177 xen_start_info->mfn_list = (unsigned long)xen_p2m_addr;
1180 static void __init xen_pagetable_init(void)
1183 xen_post_allocator_init();
1185 xen_pagetable_p2m_setup();
1187 /* Allocate and initialize top and mid mfn levels for p2m structure */
1188 xen_build_mfn_list_list();
1190 /* Remap memory freed due to conflicts with E820 map */
1191 if (!xen_feature(XENFEAT_auto_translated_physmap))
1194 xen_setup_shared_info();
1196 static void xen_write_cr2(unsigned long cr2)
1198 this_cpu_read(xen_vcpu)->arch.cr2 = cr2;
1201 static unsigned long xen_read_cr2(void)
1203 return this_cpu_read(xen_vcpu)->arch.cr2;
1206 unsigned long xen_read_cr2_direct(void)
1208 return this_cpu_read(xen_vcpu_info.arch.cr2);
1211 void xen_flush_tlb_all(void)
1213 struct mmuext_op *op;
1214 struct multicall_space mcs;
1216 trace_xen_mmu_flush_tlb_all(0);
1220 mcs = xen_mc_entry(sizeof(*op));
1223 op->cmd = MMUEXT_TLB_FLUSH_ALL;
1224 MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
1226 xen_mc_issue(PARAVIRT_LAZY_MMU);
1230 static void xen_flush_tlb(void)
1232 struct mmuext_op *op;
1233 struct multicall_space mcs;
1235 trace_xen_mmu_flush_tlb(0);
1239 mcs = xen_mc_entry(sizeof(*op));
1242 op->cmd = MMUEXT_TLB_FLUSH_LOCAL;
1243 MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
1245 xen_mc_issue(PARAVIRT_LAZY_MMU);
1250 static void xen_flush_tlb_single(unsigned long addr)
1252 struct mmuext_op *op;
1253 struct multicall_space mcs;
1255 trace_xen_mmu_flush_tlb_single(addr);
1259 mcs = xen_mc_entry(sizeof(*op));
1261 op->cmd = MMUEXT_INVLPG_LOCAL;
1262 op->arg1.linear_addr = addr & PAGE_MASK;
1263 MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
1265 xen_mc_issue(PARAVIRT_LAZY_MMU);
1270 static void xen_flush_tlb_others(const struct cpumask *cpus,
1271 struct mm_struct *mm, unsigned long start,
1275 struct mmuext_op op;
1277 DECLARE_BITMAP(mask, num_processors);
1279 DECLARE_BITMAP(mask, NR_CPUS);
1282 struct multicall_space mcs;
1284 trace_xen_mmu_flush_tlb_others(cpus, mm, start, end);
1286 if (cpumask_empty(cpus))
1287 return; /* nothing to do */
1289 mcs = xen_mc_entry(sizeof(*args));
1291 args->op.arg2.vcpumask = to_cpumask(args->mask);
1293 /* Remove us, and any offline CPUS. */
1294 cpumask_and(to_cpumask(args->mask), cpus, cpu_online_mask);
1295 cpumask_clear_cpu(smp_processor_id(), to_cpumask(args->mask));
1297 args->op.cmd = MMUEXT_TLB_FLUSH_MULTI;
1298 if (end != TLB_FLUSH_ALL && (end - start) <= PAGE_SIZE) {
1299 args->op.cmd = MMUEXT_INVLPG_MULTI;
1300 args->op.arg1.linear_addr = start;
1303 MULTI_mmuext_op(mcs.mc, &args->op, 1, NULL, DOMID_SELF);
1305 xen_mc_issue(PARAVIRT_LAZY_MMU);
1308 static unsigned long xen_read_cr3(void)
1310 return this_cpu_read(xen_cr3);
1313 static void set_current_cr3(void *v)
1315 this_cpu_write(xen_current_cr3, (unsigned long)v);
1318 static void __xen_write_cr3(bool kernel, unsigned long cr3)
1320 struct mmuext_op op;
1323 trace_xen_mmu_write_cr3(kernel, cr3);
1326 mfn = pfn_to_mfn(PFN_DOWN(cr3));
1330 WARN_ON(mfn == 0 && kernel);
1332 op.cmd = kernel ? MMUEXT_NEW_BASEPTR : MMUEXT_NEW_USER_BASEPTR;
1335 xen_extend_mmuext_op(&op);
1338 this_cpu_write(xen_cr3, cr3);
1340 /* Update xen_current_cr3 once the batch has actually
1342 xen_mc_callback(set_current_cr3, (void *)cr3);
1345 static void xen_write_cr3(unsigned long cr3)
1347 BUG_ON(preemptible());
1349 xen_mc_batch(); /* disables interrupts */
1351 /* Update while interrupts are disabled, so its atomic with
1353 this_cpu_write(xen_cr3, cr3);
1355 __xen_write_cr3(true, cr3);
1357 #ifdef CONFIG_X86_64
1359 pgd_t *user_pgd = xen_get_user_pgd(__va(cr3));
1361 __xen_write_cr3(false, __pa(user_pgd));
1363 __xen_write_cr3(false, 0);
1367 xen_mc_issue(PARAVIRT_LAZY_CPU); /* interrupts restored */
1370 #ifdef CONFIG_X86_64
1372 * At the start of the day - when Xen launches a guest, it has already
1373 * built pagetables for the guest. We diligently look over them
1374 * in xen_setup_kernel_pagetable and graft as appropiate them in the
1375 * init_level4_pgt and its friends. Then when we are happy we load
1376 * the new init_level4_pgt - and continue on.
1378 * The generic code starts (start_kernel) and 'init_mem_mapping' sets
1379 * up the rest of the pagetables. When it has completed it loads the cr3.
1380 * N.B. that baremetal would start at 'start_kernel' (and the early
1381 * #PF handler would create bootstrap pagetables) - so we are running
1382 * with the same assumptions as what to do when write_cr3 is executed
1385 * Since there are no user-page tables at all, we have two variants
1386 * of xen_write_cr3 - the early bootup (this one), and the late one
1387 * (xen_write_cr3). The reason we have to do that is that in 64-bit
1388 * the Linux kernel and user-space are both in ring 3 while the
1389 * hypervisor is in ring 0.
1391 static void __init xen_write_cr3_init(unsigned long cr3)
1393 BUG_ON(preemptible());
1395 xen_mc_batch(); /* disables interrupts */
1397 /* Update while interrupts are disabled, so its atomic with
1399 this_cpu_write(xen_cr3, cr3);
1401 __xen_write_cr3(true, cr3);
1403 xen_mc_issue(PARAVIRT_LAZY_CPU); /* interrupts restored */
1407 static int xen_pgd_alloc(struct mm_struct *mm)
1409 pgd_t *pgd = mm->pgd;
1412 BUG_ON(PagePinned(virt_to_page(pgd)));
1414 #ifdef CONFIG_X86_64
1416 struct page *page = virt_to_page(pgd);
1419 BUG_ON(page->private != 0);
1423 user_pgd = (pgd_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
1424 page->private = (unsigned long)user_pgd;
1426 if (user_pgd != NULL) {
1427 #ifdef CONFIG_X86_VSYSCALL_EMULATION
1428 user_pgd[pgd_index(VSYSCALL_ADDR)] =
1429 __pgd(__pa(level3_user_vsyscall) | _PAGE_TABLE);
1434 BUG_ON(PagePinned(virt_to_page(xen_get_user_pgd(pgd))));
1441 static void xen_pgd_free(struct mm_struct *mm, pgd_t *pgd)
1443 #ifdef CONFIG_X86_64
1444 pgd_t *user_pgd = xen_get_user_pgd(pgd);
1447 free_page((unsigned long)user_pgd);
1451 #ifdef CONFIG_X86_32
1452 static pte_t __init mask_rw_pte(pte_t *ptep, pte_t pte)
1454 /* If there's an existing pte, then don't allow _PAGE_RW to be set */
1455 if (pte_val_ma(*ptep) & _PAGE_PRESENT)
1456 pte = __pte_ma(((pte_val_ma(*ptep) & _PAGE_RW) | ~_PAGE_RW) &
1461 #else /* CONFIG_X86_64 */
1462 static pte_t __init mask_rw_pte(pte_t *ptep, pte_t pte)
1466 #endif /* CONFIG_X86_64 */
1469 * Init-time set_pte while constructing initial pagetables, which
1470 * doesn't allow RO page table pages to be remapped RW.
1472 * If there is no MFN for this PFN then this page is initially
1473 * ballooned out so clear the PTE (as in decrease_reservation() in
1474 * drivers/xen/balloon.c).
1476 * Many of these PTE updates are done on unpinned and writable pages
1477 * and doing a hypercall for these is unnecessary and expensive. At
1478 * this point it is not possible to tell if a page is pinned or not,
1479 * so always write the PTE directly and rely on Xen trapping and
1480 * emulating any updates as necessary.
1482 static void __init xen_set_pte_init(pte_t *ptep, pte_t pte)
1484 if (pte_mfn(pte) != INVALID_P2M_ENTRY)
1485 pte = mask_rw_pte(ptep, pte);
1489 native_set_pte(ptep, pte);
1492 static void __init pin_pagetable_pfn(unsigned cmd, unsigned long pfn)
1494 struct mmuext_op op;
1496 op.arg1.mfn = pfn_to_mfn(pfn);
1497 if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF))
1501 /* Early in boot, while setting up the initial pagetable, assume
1502 everything is pinned. */
1503 static void __init xen_alloc_pte_init(struct mm_struct *mm, unsigned long pfn)
1505 #ifdef CONFIG_FLATMEM
1506 BUG_ON(mem_map); /* should only be used early */
1508 make_lowmem_page_readonly(__va(PFN_PHYS(pfn)));
1509 pin_pagetable_pfn(MMUEXT_PIN_L1_TABLE, pfn);
1512 /* Used for pmd and pud */
1513 static void __init xen_alloc_pmd_init(struct mm_struct *mm, unsigned long pfn)
1515 #ifdef CONFIG_FLATMEM
1516 BUG_ON(mem_map); /* should only be used early */
1518 make_lowmem_page_readonly(__va(PFN_PHYS(pfn)));
1521 /* Early release_pte assumes that all pts are pinned, since there's
1522 only init_mm and anything attached to that is pinned. */
1523 static void __init xen_release_pte_init(unsigned long pfn)
1525 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, pfn);
1526 make_lowmem_page_readwrite(__va(PFN_PHYS(pfn)));
1529 static void __init xen_release_pmd_init(unsigned long pfn)
1531 make_lowmem_page_readwrite(__va(PFN_PHYS(pfn)));
1534 static inline void __pin_pagetable_pfn(unsigned cmd, unsigned long pfn)
1536 struct multicall_space mcs;
1537 struct mmuext_op *op;
1539 mcs = __xen_mc_entry(sizeof(*op));
1542 op->arg1.mfn = pfn_to_mfn(pfn);
1544 MULTI_mmuext_op(mcs.mc, mcs.args, 1, NULL, DOMID_SELF);
1547 static inline void __set_pfn_prot(unsigned long pfn, pgprot_t prot)
1549 struct multicall_space mcs;
1550 unsigned long addr = (unsigned long)__va(pfn << PAGE_SHIFT);
1552 mcs = __xen_mc_entry(0);
1553 MULTI_update_va_mapping(mcs.mc, (unsigned long)addr,
1554 pfn_pte(pfn, prot), 0);
1557 /* This needs to make sure the new pte page is pinned iff its being
1558 attached to a pinned pagetable. */
1559 static inline void xen_alloc_ptpage(struct mm_struct *mm, unsigned long pfn,
1562 bool pinned = PagePinned(virt_to_page(mm->pgd));
1564 trace_xen_mmu_alloc_ptpage(mm, pfn, level, pinned);
1567 struct page *page = pfn_to_page(pfn);
1569 SetPagePinned(page);
1571 if (!PageHighMem(page)) {
1574 __set_pfn_prot(pfn, PAGE_KERNEL_RO);
1576 if (level == PT_PTE && USE_SPLIT_PTE_PTLOCKS)
1577 __pin_pagetable_pfn(MMUEXT_PIN_L1_TABLE, pfn);
1579 xen_mc_issue(PARAVIRT_LAZY_MMU);
1581 /* make sure there are no stray mappings of
1583 kmap_flush_unused();
1588 static void xen_alloc_pte(struct mm_struct *mm, unsigned long pfn)
1590 xen_alloc_ptpage(mm, pfn, PT_PTE);
1593 static void xen_alloc_pmd(struct mm_struct *mm, unsigned long pfn)
1595 xen_alloc_ptpage(mm, pfn, PT_PMD);
1598 /* This should never happen until we're OK to use struct page */
1599 static inline void xen_release_ptpage(unsigned long pfn, unsigned level)
1601 struct page *page = pfn_to_page(pfn);
1602 bool pinned = PagePinned(page);
1604 trace_xen_mmu_release_ptpage(pfn, level, pinned);
1607 if (!PageHighMem(page)) {
1610 if (level == PT_PTE && USE_SPLIT_PTE_PTLOCKS)
1611 __pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, pfn);
1613 __set_pfn_prot(pfn, PAGE_KERNEL);
1615 xen_mc_issue(PARAVIRT_LAZY_MMU);
1617 ClearPagePinned(page);
1621 static void xen_release_pte(unsigned long pfn)
1623 xen_release_ptpage(pfn, PT_PTE);
1626 static void xen_release_pmd(unsigned long pfn)
1628 xen_release_ptpage(pfn, PT_PMD);
1631 #if PAGETABLE_LEVELS == 4
1632 static void xen_alloc_pud(struct mm_struct *mm, unsigned long pfn)
1634 xen_alloc_ptpage(mm, pfn, PT_PUD);
1637 static void xen_release_pud(unsigned long pfn)
1639 xen_release_ptpage(pfn, PT_PUD);
1643 void __init xen_reserve_top(void)
1645 #ifdef CONFIG_X86_32
1646 unsigned long top = HYPERVISOR_VIRT_START;
1647 struct xen_platform_parameters pp;
1649 if (HYPERVISOR_xen_version(XENVER_platform_parameters, &pp) == 0)
1650 top = pp.virt_start;
1652 reserve_top_address(-top);
1653 #endif /* CONFIG_X86_32 */
1657 * Like __va(), but returns address in the kernel mapping (which is
1658 * all we have until the physical memory mapping has been set up.
1660 static void * __init __ka(phys_addr_t paddr)
1662 #ifdef CONFIG_X86_64
1663 return (void *)(paddr + __START_KERNEL_map);
1669 /* Convert a machine address to physical address */
1670 static unsigned long __init m2p(phys_addr_t maddr)
1674 maddr &= PTE_PFN_MASK;
1675 paddr = mfn_to_pfn(maddr >> PAGE_SHIFT) << PAGE_SHIFT;
1680 /* Convert a machine address to kernel virtual */
1681 static void * __init m2v(phys_addr_t maddr)
1683 return __ka(m2p(maddr));
1686 /* Set the page permissions on an identity-mapped pages */
1687 static void __init set_page_prot_flags(void *addr, pgprot_t prot,
1688 unsigned long flags)
1690 unsigned long pfn = __pa(addr) >> PAGE_SHIFT;
1691 pte_t pte = pfn_pte(pfn, prot);
1693 /* For PVH no need to set R/O or R/W to pin them or unpin them. */
1694 if (xen_feature(XENFEAT_auto_translated_physmap))
1697 if (HYPERVISOR_update_va_mapping((unsigned long)addr, pte, flags))
1700 static void __init set_page_prot(void *addr, pgprot_t prot)
1702 return set_page_prot_flags(addr, prot, UVMF_NONE);
1704 #ifdef CONFIG_X86_32
1705 static void __init xen_map_identity_early(pmd_t *pmd, unsigned long max_pfn)
1707 unsigned pmdidx, pteidx;
1711 level1_ident_pgt = extend_brk(sizeof(pte_t) * LEVEL1_IDENT_ENTRIES,
1716 for (pmdidx = 0; pmdidx < PTRS_PER_PMD && pfn < max_pfn; pmdidx++) {
1719 /* Reuse or allocate a page of ptes */
1720 if (pmd_present(pmd[pmdidx]))
1721 pte_page = m2v(pmd[pmdidx].pmd);
1723 /* Check for free pte pages */
1724 if (ident_pte == LEVEL1_IDENT_ENTRIES)
1727 pte_page = &level1_ident_pgt[ident_pte];
1728 ident_pte += PTRS_PER_PTE;
1730 pmd[pmdidx] = __pmd(__pa(pte_page) | _PAGE_TABLE);
1733 /* Install mappings */
1734 for (pteidx = 0; pteidx < PTRS_PER_PTE; pteidx++, pfn++) {
1737 #ifdef CONFIG_X86_32
1738 if (pfn > max_pfn_mapped)
1739 max_pfn_mapped = pfn;
1742 if (!pte_none(pte_page[pteidx]))
1745 pte = pfn_pte(pfn, PAGE_KERNEL_EXEC);
1746 pte_page[pteidx] = pte;
1750 for (pteidx = 0; pteidx < ident_pte; pteidx += PTRS_PER_PTE)
1751 set_page_prot(&level1_ident_pgt[pteidx], PAGE_KERNEL_RO);
1753 set_page_prot(pmd, PAGE_KERNEL_RO);
1756 void __init xen_setup_machphys_mapping(void)
1758 struct xen_machphys_mapping mapping;
1760 if (HYPERVISOR_memory_op(XENMEM_machphys_mapping, &mapping) == 0) {
1761 machine_to_phys_mapping = (unsigned long *)mapping.v_start;
1762 machine_to_phys_nr = mapping.max_mfn + 1;
1764 machine_to_phys_nr = MACH2PHYS_NR_ENTRIES;
1766 #ifdef CONFIG_X86_32
1767 WARN_ON((machine_to_phys_mapping + (machine_to_phys_nr - 1))
1768 < machine_to_phys_mapping);
1772 #ifdef CONFIG_X86_64
1773 static void __init convert_pfn_mfn(void *v)
1778 /* All levels are converted the same way, so just treat them
1780 for (i = 0; i < PTRS_PER_PTE; i++)
1781 pte[i] = xen_make_pte(pte[i].pte);
1783 static void __init check_pt_base(unsigned long *pt_base, unsigned long *pt_end,
1786 if (*pt_base == PFN_DOWN(__pa(addr))) {
1787 set_page_prot_flags((void *)addr, PAGE_KERNEL, UVMF_INVLPG);
1788 clear_page((void *)addr);
1791 if (*pt_end == PFN_DOWN(__pa(addr))) {
1792 set_page_prot_flags((void *)addr, PAGE_KERNEL, UVMF_INVLPG);
1793 clear_page((void *)addr);
1798 * Set up the initial kernel pagetable.
1800 * We can construct this by grafting the Xen provided pagetable into
1801 * head_64.S's preconstructed pagetables. We copy the Xen L2's into
1802 * level2_ident_pgt, and level2_kernel_pgt. This means that only the
1803 * kernel has a physical mapping to start with - but that's enough to
1804 * get __va working. We need to fill in the rest of the physical
1805 * mapping once some sort of allocator has been set up. NOTE: for
1806 * PVH, the page tables are native.
1808 void __init xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn)
1812 unsigned long addr[3];
1813 unsigned long pt_base, pt_end;
1816 /* max_pfn_mapped is the last pfn mapped in the initial memory
1817 * mappings. Considering that on Xen after the kernel mappings we
1818 * have the mappings of some pages that don't exist in pfn space, we
1819 * set max_pfn_mapped to the last real pfn mapped. */
1820 max_pfn_mapped = PFN_DOWN(__pa(xen_start_info->mfn_list));
1822 pt_base = PFN_DOWN(__pa(xen_start_info->pt_base));
1823 pt_end = pt_base + xen_start_info->nr_pt_frames;
1825 /* Zap identity mapping */
1826 init_level4_pgt[0] = __pgd(0);
1828 if (!xen_feature(XENFEAT_auto_translated_physmap)) {
1829 /* Pre-constructed entries are in pfn, so convert to mfn */
1830 /* L4[272] -> level3_ident_pgt
1831 * L4[511] -> level3_kernel_pgt */
1832 convert_pfn_mfn(init_level4_pgt);
1834 /* L3_i[0] -> level2_ident_pgt */
1835 convert_pfn_mfn(level3_ident_pgt);
1836 /* L3_k[510] -> level2_kernel_pgt
1837 * L3_k[511] -> level2_fixmap_pgt */
1838 convert_pfn_mfn(level3_kernel_pgt);
1840 /* L3_k[511][506] -> level1_fixmap_pgt */
1841 convert_pfn_mfn(level2_fixmap_pgt);
1843 /* We get [511][511] and have Xen's version of level2_kernel_pgt */
1844 l3 = m2v(pgd[pgd_index(__START_KERNEL_map)].pgd);
1845 l2 = m2v(l3[pud_index(__START_KERNEL_map)].pud);
1847 addr[0] = (unsigned long)pgd;
1848 addr[1] = (unsigned long)l3;
1849 addr[2] = (unsigned long)l2;
1850 /* Graft it onto L4[272][0]. Note that we creating an aliasing problem:
1851 * Both L4[272][0] and L4[511][510] have entries that point to the same
1852 * L2 (PMD) tables. Meaning that if you modify it in __va space
1853 * it will be also modified in the __ka space! (But if you just
1854 * modify the PMD table to point to other PTE's or none, then you
1855 * are OK - which is what cleanup_highmap does) */
1856 copy_page(level2_ident_pgt, l2);
1857 /* Graft it onto L4[511][510] */
1858 copy_page(level2_kernel_pgt, l2);
1860 if (!xen_feature(XENFEAT_auto_translated_physmap)) {
1861 /* Make pagetable pieces RO */
1862 set_page_prot(init_level4_pgt, PAGE_KERNEL_RO);
1863 set_page_prot(level3_ident_pgt, PAGE_KERNEL_RO);
1864 set_page_prot(level3_kernel_pgt, PAGE_KERNEL_RO);
1865 set_page_prot(level3_user_vsyscall, PAGE_KERNEL_RO);
1866 set_page_prot(level2_ident_pgt, PAGE_KERNEL_RO);
1867 set_page_prot(level2_kernel_pgt, PAGE_KERNEL_RO);
1868 set_page_prot(level2_fixmap_pgt, PAGE_KERNEL_RO);
1869 set_page_prot(level1_fixmap_pgt, PAGE_KERNEL_RO);
1871 /* Pin down new L4 */
1872 pin_pagetable_pfn(MMUEXT_PIN_L4_TABLE,
1873 PFN_DOWN(__pa_symbol(init_level4_pgt)));
1875 /* Unpin Xen-provided one */
1876 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd)));
1879 * At this stage there can be no user pgd, and no page
1880 * structure to attach it to, so make sure we just set kernel
1884 __xen_write_cr3(true, __pa(init_level4_pgt));
1885 xen_mc_issue(PARAVIRT_LAZY_CPU);
1887 native_write_cr3(__pa(init_level4_pgt));
1889 /* We can't that easily rip out L3 and L2, as the Xen pagetables are
1890 * set out this way: [L4], [L1], [L2], [L3], [L1], [L1] ... for
1891 * the initial domain. For guests using the toolstack, they are in:
1892 * [L4], [L3], [L2], [L1], [L1], order .. So for dom0 we can only
1893 * rip out the [L4] (pgd), but for guests we shave off three pages.
1895 for (i = 0; i < ARRAY_SIZE(addr); i++)
1896 check_pt_base(&pt_base, &pt_end, addr[i]);
1898 /* Our (by three pages) smaller Xen pagetable that we are using */
1899 memblock_reserve(PFN_PHYS(pt_base), (pt_end - pt_base) * PAGE_SIZE);
1900 /* Revector the xen_start_info */
1901 xen_start_info = (struct start_info *)__va(__pa(xen_start_info));
1903 #else /* !CONFIG_X86_64 */
1904 static RESERVE_BRK_ARRAY(pmd_t, initial_kernel_pmd, PTRS_PER_PMD);
1905 static RESERVE_BRK_ARRAY(pmd_t, swapper_kernel_pmd, PTRS_PER_PMD);
1907 static void __init xen_write_cr3_init(unsigned long cr3)
1909 unsigned long pfn = PFN_DOWN(__pa(swapper_pg_dir));
1911 BUG_ON(read_cr3() != __pa(initial_page_table));
1912 BUG_ON(cr3 != __pa(swapper_pg_dir));
1915 * We are switching to swapper_pg_dir for the first time (from
1916 * initial_page_table) and therefore need to mark that page
1917 * read-only and then pin it.
1919 * Xen disallows sharing of kernel PMDs for PAE
1920 * guests. Therefore we must copy the kernel PMD from
1921 * initial_page_table into a new kernel PMD to be used in
1924 swapper_kernel_pmd =
1925 extend_brk(sizeof(pmd_t) * PTRS_PER_PMD, PAGE_SIZE);
1926 copy_page(swapper_kernel_pmd, initial_kernel_pmd);
1927 swapper_pg_dir[KERNEL_PGD_BOUNDARY] =
1928 __pgd(__pa(swapper_kernel_pmd) | _PAGE_PRESENT);
1929 set_page_prot(swapper_kernel_pmd, PAGE_KERNEL_RO);
1931 set_page_prot(swapper_pg_dir, PAGE_KERNEL_RO);
1933 pin_pagetable_pfn(MMUEXT_PIN_L3_TABLE, pfn);
1935 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE,
1936 PFN_DOWN(__pa(initial_page_table)));
1937 set_page_prot(initial_page_table, PAGE_KERNEL);
1938 set_page_prot(initial_kernel_pmd, PAGE_KERNEL);
1940 pv_mmu_ops.write_cr3 = &xen_write_cr3;
1943 void __init xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn)
1947 initial_kernel_pmd =
1948 extend_brk(sizeof(pmd_t) * PTRS_PER_PMD, PAGE_SIZE);
1950 max_pfn_mapped = PFN_DOWN(__pa(xen_start_info->pt_base) +
1951 xen_start_info->nr_pt_frames * PAGE_SIZE +
1954 kernel_pmd = m2v(pgd[KERNEL_PGD_BOUNDARY].pgd);
1955 copy_page(initial_kernel_pmd, kernel_pmd);
1957 xen_map_identity_early(initial_kernel_pmd, max_pfn);
1959 copy_page(initial_page_table, pgd);
1960 initial_page_table[KERNEL_PGD_BOUNDARY] =
1961 __pgd(__pa(initial_kernel_pmd) | _PAGE_PRESENT);
1963 set_page_prot(initial_kernel_pmd, PAGE_KERNEL_RO);
1964 set_page_prot(initial_page_table, PAGE_KERNEL_RO);
1965 set_page_prot(empty_zero_page, PAGE_KERNEL_RO);
1967 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd)));
1969 pin_pagetable_pfn(MMUEXT_PIN_L3_TABLE,
1970 PFN_DOWN(__pa(initial_page_table)));
1971 xen_write_cr3(__pa(initial_page_table));
1973 memblock_reserve(__pa(xen_start_info->pt_base),
1974 xen_start_info->nr_pt_frames * PAGE_SIZE);
1976 #endif /* CONFIG_X86_64 */
1978 static unsigned char dummy_mapping[PAGE_SIZE] __page_aligned_bss;
1980 static void xen_set_fixmap(unsigned idx, phys_addr_t phys, pgprot_t prot)
1984 phys >>= PAGE_SHIFT;
1987 case FIX_BTMAP_END ... FIX_BTMAP_BEGIN:
1989 #ifdef CONFIG_X86_32
1991 # ifdef CONFIG_HIGHMEM
1992 case FIX_KMAP_BEGIN ... FIX_KMAP_END:
1994 #elif defined(CONFIG_X86_VSYSCALL_EMULATION)
1997 case FIX_TEXT_POKE0:
1998 case FIX_TEXT_POKE1:
1999 /* All local page mappings */
2000 pte = pfn_pte(phys, prot);
2003 #ifdef CONFIG_X86_LOCAL_APIC
2004 case FIX_APIC_BASE: /* maps dummy local APIC */
2005 pte = pfn_pte(PFN_DOWN(__pa(dummy_mapping)), PAGE_KERNEL);
2009 #ifdef CONFIG_X86_IO_APIC
2010 case FIX_IO_APIC_BASE_0 ... FIX_IO_APIC_BASE_END:
2012 * We just don't map the IO APIC - all access is via
2013 * hypercalls. Keep the address in the pte for reference.
2015 pte = pfn_pte(PFN_DOWN(__pa(dummy_mapping)), PAGE_KERNEL);
2019 case FIX_PARAVIRT_BOOTMAP:
2020 /* This is an MFN, but it isn't an IO mapping from the
2022 pte = mfn_pte(phys, prot);
2026 /* By default, set_fixmap is used for hardware mappings */
2027 pte = mfn_pte(phys, prot);
2031 __native_set_fixmap(idx, pte);
2033 #ifdef CONFIG_X86_VSYSCALL_EMULATION
2034 /* Replicate changes to map the vsyscall page into the user
2035 pagetable vsyscall mapping. */
2036 if (idx == VSYSCALL_PAGE) {
2037 unsigned long vaddr = __fix_to_virt(idx);
2038 set_pte_vaddr_pud(level3_user_vsyscall, vaddr, pte);
2043 static void __init xen_post_allocator_init(void)
2045 if (xen_feature(XENFEAT_auto_translated_physmap))
2048 pv_mmu_ops.set_pte = xen_set_pte;
2049 pv_mmu_ops.set_pmd = xen_set_pmd;
2050 pv_mmu_ops.set_pud = xen_set_pud;
2051 #if PAGETABLE_LEVELS == 4
2052 pv_mmu_ops.set_pgd = xen_set_pgd;
2055 /* This will work as long as patching hasn't happened yet
2056 (which it hasn't) */
2057 pv_mmu_ops.alloc_pte = xen_alloc_pte;
2058 pv_mmu_ops.alloc_pmd = xen_alloc_pmd;
2059 pv_mmu_ops.release_pte = xen_release_pte;
2060 pv_mmu_ops.release_pmd = xen_release_pmd;
2061 #if PAGETABLE_LEVELS == 4
2062 pv_mmu_ops.alloc_pud = xen_alloc_pud;
2063 pv_mmu_ops.release_pud = xen_release_pud;
2066 #ifdef CONFIG_X86_64
2067 pv_mmu_ops.write_cr3 = &xen_write_cr3;
2068 SetPagePinned(virt_to_page(level3_user_vsyscall));
2070 xen_mark_init_mm_pinned();
2073 static void xen_leave_lazy_mmu(void)
2077 paravirt_leave_lazy_mmu();
2081 static const struct pv_mmu_ops xen_mmu_ops __initconst = {
2082 .read_cr2 = xen_read_cr2,
2083 .write_cr2 = xen_write_cr2,
2085 .read_cr3 = xen_read_cr3,
2086 .write_cr3 = xen_write_cr3_init,
2088 .flush_tlb_user = xen_flush_tlb,
2089 .flush_tlb_kernel = xen_flush_tlb,
2090 .flush_tlb_single = xen_flush_tlb_single,
2091 .flush_tlb_others = xen_flush_tlb_others,
2093 .pte_update = paravirt_nop,
2094 .pte_update_defer = paravirt_nop,
2096 .pgd_alloc = xen_pgd_alloc,
2097 .pgd_free = xen_pgd_free,
2099 .alloc_pte = xen_alloc_pte_init,
2100 .release_pte = xen_release_pte_init,
2101 .alloc_pmd = xen_alloc_pmd_init,
2102 .release_pmd = xen_release_pmd_init,
2104 .set_pte = xen_set_pte_init,
2105 .set_pte_at = xen_set_pte_at,
2106 .set_pmd = xen_set_pmd_hyper,
2108 .ptep_modify_prot_start = __ptep_modify_prot_start,
2109 .ptep_modify_prot_commit = __ptep_modify_prot_commit,
2111 .pte_val = PV_CALLEE_SAVE(xen_pte_val),
2112 .pgd_val = PV_CALLEE_SAVE(xen_pgd_val),
2114 .make_pte = PV_CALLEE_SAVE(xen_make_pte),
2115 .make_pgd = PV_CALLEE_SAVE(xen_make_pgd),
2117 #ifdef CONFIG_X86_PAE
2118 .set_pte_atomic = xen_set_pte_atomic,
2119 .pte_clear = xen_pte_clear,
2120 .pmd_clear = xen_pmd_clear,
2121 #endif /* CONFIG_X86_PAE */
2122 .set_pud = xen_set_pud_hyper,
2124 .make_pmd = PV_CALLEE_SAVE(xen_make_pmd),
2125 .pmd_val = PV_CALLEE_SAVE(xen_pmd_val),
2127 #if PAGETABLE_LEVELS == 4
2128 .pud_val = PV_CALLEE_SAVE(xen_pud_val),
2129 .make_pud = PV_CALLEE_SAVE(xen_make_pud),
2130 .set_pgd = xen_set_pgd_hyper,
2132 .alloc_pud = xen_alloc_pmd_init,
2133 .release_pud = xen_release_pmd_init,
2134 #endif /* PAGETABLE_LEVELS == 4 */
2136 .activate_mm = xen_activate_mm,
2137 .dup_mmap = xen_dup_mmap,
2138 .exit_mmap = xen_exit_mmap,
2141 .enter = paravirt_enter_lazy_mmu,
2142 .leave = xen_leave_lazy_mmu,
2143 .flush = paravirt_flush_lazy_mmu,
2146 .set_fixmap = xen_set_fixmap,
2149 void __init xen_init_mmu_ops(void)
2151 x86_init.paging.pagetable_init = xen_pagetable_init;
2153 /* Optimization - we can use the HVM one but it has no idea which
2154 * VCPUs are descheduled - which means that it will needlessly IPI
2155 * them. Xen knows so let it do the job.
2157 if (xen_feature(XENFEAT_auto_translated_physmap)) {
2158 pv_mmu_ops.flush_tlb_others = xen_flush_tlb_others;
2161 pv_mmu_ops = xen_mmu_ops;
2163 memset(dummy_mapping, 0xff, PAGE_SIZE);
2166 /* Protected by xen_reservation_lock. */
2167 #define MAX_CONTIG_ORDER 9 /* 2MB */
2168 static unsigned long discontig_frames[1<<MAX_CONTIG_ORDER];
2170 #define VOID_PTE (mfn_pte(0, __pgprot(0)))
2171 static void xen_zap_pfn_range(unsigned long vaddr, unsigned int order,
2172 unsigned long *in_frames,
2173 unsigned long *out_frames)
2176 struct multicall_space mcs;
2179 for (i = 0; i < (1UL<<order); i++, vaddr += PAGE_SIZE) {
2180 mcs = __xen_mc_entry(0);
2183 in_frames[i] = virt_to_mfn(vaddr);
2185 MULTI_update_va_mapping(mcs.mc, vaddr, VOID_PTE, 0);
2186 __set_phys_to_machine(virt_to_pfn(vaddr), INVALID_P2M_ENTRY);
2189 out_frames[i] = virt_to_pfn(vaddr);
2195 * Update the pfn-to-mfn mappings for a virtual address range, either to
2196 * point to an array of mfns, or contiguously from a single starting
2199 static void xen_remap_exchanged_ptes(unsigned long vaddr, int order,
2200 unsigned long *mfns,
2201 unsigned long first_mfn)
2208 limit = 1u << order;
2209 for (i = 0; i < limit; i++, vaddr += PAGE_SIZE) {
2210 struct multicall_space mcs;
2213 mcs = __xen_mc_entry(0);
2217 mfn = first_mfn + i;
2219 if (i < (limit - 1))
2223 flags = UVMF_INVLPG | UVMF_ALL;
2225 flags = UVMF_TLB_FLUSH | UVMF_ALL;
2228 MULTI_update_va_mapping(mcs.mc, vaddr,
2229 mfn_pte(mfn, PAGE_KERNEL), flags);
2231 set_phys_to_machine(virt_to_pfn(vaddr), mfn);
2238 * Perform the hypercall to exchange a region of our pfns to point to
2239 * memory with the required contiguous alignment. Takes the pfns as
2240 * input, and populates mfns as output.
2242 * Returns a success code indicating whether the hypervisor was able to
2243 * satisfy the request or not.
2245 static int xen_exchange_memory(unsigned long extents_in, unsigned int order_in,
2246 unsigned long *pfns_in,
2247 unsigned long extents_out,
2248 unsigned int order_out,
2249 unsigned long *mfns_out,
2250 unsigned int address_bits)
2255 struct xen_memory_exchange exchange = {
2257 .nr_extents = extents_in,
2258 .extent_order = order_in,
2259 .extent_start = pfns_in,
2263 .nr_extents = extents_out,
2264 .extent_order = order_out,
2265 .extent_start = mfns_out,
2266 .address_bits = address_bits,
2271 BUG_ON(extents_in << order_in != extents_out << order_out);
2273 rc = HYPERVISOR_memory_op(XENMEM_exchange, &exchange);
2274 success = (exchange.nr_exchanged == extents_in);
2276 BUG_ON(!success && ((exchange.nr_exchanged != 0) || (rc == 0)));
2277 BUG_ON(success && (rc != 0));
2282 int xen_create_contiguous_region(phys_addr_t pstart, unsigned int order,
2283 unsigned int address_bits,
2284 dma_addr_t *dma_handle)
2286 unsigned long *in_frames = discontig_frames, out_frame;
2287 unsigned long flags;
2289 unsigned long vstart = (unsigned long)phys_to_virt(pstart);
2292 * Currently an auto-translated guest will not perform I/O, nor will
2293 * it require PAE page directories below 4GB. Therefore any calls to
2294 * this function are redundant and can be ignored.
2297 if (xen_feature(XENFEAT_auto_translated_physmap))
2300 if (unlikely(order > MAX_CONTIG_ORDER))
2303 memset((void *) vstart, 0, PAGE_SIZE << order);
2305 spin_lock_irqsave(&xen_reservation_lock, flags);
2307 /* 1. Zap current PTEs, remembering MFNs. */
2308 xen_zap_pfn_range(vstart, order, in_frames, NULL);
2310 /* 2. Get a new contiguous memory extent. */
2311 out_frame = virt_to_pfn(vstart);
2312 success = xen_exchange_memory(1UL << order, 0, in_frames,
2313 1, order, &out_frame,
2316 /* 3. Map the new extent in place of old pages. */
2318 xen_remap_exchanged_ptes(vstart, order, NULL, out_frame);
2320 xen_remap_exchanged_ptes(vstart, order, in_frames, 0);
2322 spin_unlock_irqrestore(&xen_reservation_lock, flags);
2324 *dma_handle = virt_to_machine(vstart).maddr;
2325 return success ? 0 : -ENOMEM;
2327 EXPORT_SYMBOL_GPL(xen_create_contiguous_region);
2329 void xen_destroy_contiguous_region(phys_addr_t pstart, unsigned int order)
2331 unsigned long *out_frames = discontig_frames, in_frame;
2332 unsigned long flags;
2334 unsigned long vstart;
2336 if (xen_feature(XENFEAT_auto_translated_physmap))
2339 if (unlikely(order > MAX_CONTIG_ORDER))
2342 vstart = (unsigned long)phys_to_virt(pstart);
2343 memset((void *) vstart, 0, PAGE_SIZE << order);
2345 spin_lock_irqsave(&xen_reservation_lock, flags);
2347 /* 1. Find start MFN of contiguous extent. */
2348 in_frame = virt_to_mfn(vstart);
2350 /* 2. Zap current PTEs. */
2351 xen_zap_pfn_range(vstart, order, NULL, out_frames);
2353 /* 3. Do the exchange for non-contiguous MFNs. */
2354 success = xen_exchange_memory(1, order, &in_frame, 1UL << order,
2357 /* 4. Map new pages in place of old pages. */
2359 xen_remap_exchanged_ptes(vstart, order, out_frames, 0);
2361 xen_remap_exchanged_ptes(vstart, order, NULL, in_frame);
2363 spin_unlock_irqrestore(&xen_reservation_lock, flags);
2365 EXPORT_SYMBOL_GPL(xen_destroy_contiguous_region);
2367 #ifdef CONFIG_XEN_PVHVM
2368 #ifdef CONFIG_PROC_VMCORE
2370 * This function is used in two contexts:
2371 * - the kdump kernel has to check whether a pfn of the crashed kernel
2372 * was a ballooned page. vmcore is using this function to decide
2373 * whether to access a pfn of the crashed kernel.
2374 * - the kexec kernel has to check whether a pfn was ballooned by the
2375 * previous kernel. If the pfn is ballooned, handle it properly.
2376 * Returns 0 if the pfn is not backed by a RAM page, the caller may
2377 * handle the pfn special in this case.
2379 static int xen_oldmem_pfn_is_ram(unsigned long pfn)
2381 struct xen_hvm_get_mem_type a = {
2382 .domid = DOMID_SELF,
2387 if (HYPERVISOR_hvm_op(HVMOP_get_mem_type, &a))
2390 switch (a.mem_type) {
2391 case HVMMEM_mmio_dm:
2405 static void xen_hvm_exit_mmap(struct mm_struct *mm)
2407 struct xen_hvm_pagetable_dying a;
2410 a.domid = DOMID_SELF;
2411 a.gpa = __pa(mm->pgd);
2412 rc = HYPERVISOR_hvm_op(HVMOP_pagetable_dying, &a);
2413 WARN_ON_ONCE(rc < 0);
2416 static int is_pagetable_dying_supported(void)
2418 struct xen_hvm_pagetable_dying a;
2421 a.domid = DOMID_SELF;
2423 rc = HYPERVISOR_hvm_op(HVMOP_pagetable_dying, &a);
2425 printk(KERN_DEBUG "HVMOP_pagetable_dying not supported\n");
2431 void __init xen_hvm_init_mmu_ops(void)
2433 if (is_pagetable_dying_supported())
2434 pv_mmu_ops.exit_mmap = xen_hvm_exit_mmap;
2435 #ifdef CONFIG_PROC_VMCORE
2436 register_oldmem_pfn_is_ram(&xen_oldmem_pfn_is_ram);
2441 #ifdef CONFIG_XEN_PVH
2443 * Map foreign gfn (fgfn), to local pfn (lpfn). This for the user
2444 * space creating new guest on pvh dom0 and needing to map domU pages.
2446 static int xlate_add_to_p2m(unsigned long lpfn, unsigned long fgfn,
2450 xen_pfn_t gpfn = lpfn;
2451 xen_ulong_t idx = fgfn;
2453 struct xen_add_to_physmap_range xatp = {
2454 .domid = DOMID_SELF,
2455 .foreign_domid = domid,
2457 .space = XENMAPSPACE_gmfn_foreign,
2459 set_xen_guest_handle(xatp.idxs, &idx);
2460 set_xen_guest_handle(xatp.gpfns, &gpfn);
2461 set_xen_guest_handle(xatp.errs, &err);
2463 rc = HYPERVISOR_memory_op(XENMEM_add_to_physmap_range, &xatp);
2469 static int xlate_remove_from_p2m(unsigned long spfn, int count)
2471 struct xen_remove_from_physmap xrp;
2474 for (i = 0; i < count; i++) {
2475 xrp.domid = DOMID_SELF;
2477 rc = HYPERVISOR_memory_op(XENMEM_remove_from_physmap, &xrp);
2484 struct xlate_remap_data {
2485 unsigned long fgfn; /* foreign domain's gfn */
2489 struct page **pages;
2492 static int xlate_map_pte_fn(pte_t *ptep, pgtable_t token, unsigned long addr,
2496 struct xlate_remap_data *remap = data;
2497 unsigned long pfn = page_to_pfn(remap->pages[remap->index++]);
2498 pte_t pteval = pte_mkspecial(pfn_pte(pfn, remap->prot));
2500 rc = xlate_add_to_p2m(pfn, remap->fgfn, remap->domid);
2503 native_set_pte(ptep, pteval);
2508 static int xlate_remap_gfn_range(struct vm_area_struct *vma,
2509 unsigned long addr, unsigned long mfn,
2510 int nr, pgprot_t prot, unsigned domid,
2511 struct page **pages)
2514 struct xlate_remap_data pvhdata;
2519 pvhdata.prot = prot;
2520 pvhdata.domid = domid;
2522 pvhdata.pages = pages;
2523 err = apply_to_page_range(vma->vm_mm, addr, nr << PAGE_SHIFT,
2524 xlate_map_pte_fn, &pvhdata);
2530 #define REMAP_BATCH_SIZE 16
2535 struct mmu_update *mmu_update;
2538 static int remap_area_mfn_pte_fn(pte_t *ptep, pgtable_t token,
2539 unsigned long addr, void *data)
2541 struct remap_data *rmd = data;
2542 pte_t pte = pte_mkspecial(mfn_pte(rmd->mfn++, rmd->prot));
2544 rmd->mmu_update->ptr = virt_to_machine(ptep).maddr;
2545 rmd->mmu_update->val = pte_val_ma(pte);
2551 int xen_remap_domain_mfn_range(struct vm_area_struct *vma,
2553 xen_pfn_t mfn, int nr,
2554 pgprot_t prot, unsigned domid,
2555 struct page **pages)
2558 struct remap_data rmd;
2559 struct mmu_update mmu_update[REMAP_BATCH_SIZE];
2561 unsigned long range;
2564 BUG_ON(!((vma->vm_flags & (VM_PFNMAP | VM_IO)) == (VM_PFNMAP | VM_IO)));
2566 if (xen_feature(XENFEAT_auto_translated_physmap)) {
2567 #ifdef CONFIG_XEN_PVH
2568 /* We need to update the local page tables and the xen HAP */
2569 return xlate_remap_gfn_range(vma, addr, mfn, nr, prot,
2580 batch = min(REMAP_BATCH_SIZE, nr);
2581 range = (unsigned long)batch << PAGE_SHIFT;
2583 rmd.mmu_update = mmu_update;
2584 err = apply_to_page_range(vma->vm_mm, addr, range,
2585 remap_area_mfn_pte_fn, &rmd);
2589 err = HYPERVISOR_mmu_update(mmu_update, batch, NULL, domid);
2600 xen_flush_tlb_all();
2604 EXPORT_SYMBOL_GPL(xen_remap_domain_mfn_range);
2606 /* Returns: 0 success */
2607 int xen_unmap_domain_mfn_range(struct vm_area_struct *vma,
2608 int numpgs, struct page **pages)
2610 if (!pages || !xen_feature(XENFEAT_auto_translated_physmap))
2613 #ifdef CONFIG_XEN_PVH
2616 * The mmu has already cleaned up the process mmu
2617 * resources at this point (lookup_address will return
2620 unsigned long pfn = page_to_pfn(pages[numpgs]);
2622 xlate_remove_from_p2m(pfn, 1);
2625 * We don't need to flush tlbs because as part of
2626 * xlate_remove_from_p2m, the hypervisor will do tlb flushes
2627 * after removing the p2m entries from the EPT/NPT
2634 EXPORT_SYMBOL_GPL(xen_unmap_domain_mfn_range);