arch/powerpc/kvm/book3s_64_mmu_hv.c

   1 /*
   2  * This program is free software; you can redistribute it and/or modify
   3  * it under the terms of the GNU General Public License, version 2, as
   4  * published by the Free Software Foundation.
   5  *
   6  * This program is distributed in the hope that it will be useful,
   7  * but WITHOUT ANY WARRANTY; without even the implied warranty of
   8  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   9  * GNU General Public License for more details.
  10  *
  11  * You should have received a copy of the GNU General Public License
  12  * along with this program; if not, write to the Free Software
  13  * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  14  *
  15  * Copyright 2010 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
  16  */
  17
  18 #include <linux/types.h>
  19 #include <linux/string.h>
  20 #include <linux/kvm.h>
  21 #include <linux/kvm_host.h>
  22 #include <linux/highmem.h>
  23 #include <linux/gfp.h>
  24 #include <linux/slab.h>
  25 #include <linux/hugetlb.h>
  26 #include <linux/vmalloc.h>
  27 #include <linux/srcu.h>
  28 #include <linux/anon_inodes.h>
  29 #include <linux/file.h>
  30
  31 #include <asm/tlbflush.h>
  32 #include <asm/kvm_ppc.h>
  33 #include <asm/kvm_book3s.h>
  34 #include <asm/mmu-hash64.h>
  35 #include <asm/hvcall.h>
  36 #include <asm/synch.h>
  37 #include <asm/ppc-opcode.h>
  38 #include <asm/cputable.h>
  39
  40 #include "trace_hv.h"
  41
  42 /* POWER7 has 10-bit LPIDs, PPC970 has 6-bit LPIDs */
  43 #define MAX_LPID_970    63
  44
  45 /* Power architecture requires HPT is at least 256kB */
  46 #define PPC_MIN_HPT_ORDER       18
  47
  48 static long kvmppc_virtmode_do_h_enter(struct kvm *kvm, unsigned long flags,
  49                                 long pte_index, unsigned long pteh,
  50                                 unsigned long ptel, unsigned long *pte_idx_ret);
  51 static void kvmppc_rmap_reset(struct kvm *kvm);
  52
  53 long kvmppc_alloc_hpt(struct kvm *kvm, u32 *htab_orderp)
  54 {
  55         unsigned long hpt = 0;
  56         struct revmap_entry *rev;
  57         struct page *page = NULL;
  58         long order = KVM_DEFAULT_HPT_ORDER;
  59
  60         if (htab_orderp) {
  61                 order = *htab_orderp;
  62                 if (order < PPC_MIN_HPT_ORDER)
  63                         order = PPC_MIN_HPT_ORDER;
  64         }
  65
  66         kvm->arch.hpt_cma_alloc = 0;
  67         page = kvm_alloc_hpt(1ul << (order - PAGE_SHIFT));
  68         if (page) {
  69                 hpt = (unsigned long)pfn_to_kaddr(page_to_pfn(page));
  70                 memset((void *)hpt, 0, (1ul << order));
  71                 kvm->arch.hpt_cma_alloc = 1;
  72         }
  73
  74         /* Lastly try successively smaller sizes from the page allocator */
  75         while (!hpt && order > PPC_MIN_HPT_ORDER) {
  76                 hpt = __get_free_pages(GFP_KERNEL|__GFP_ZERO|__GFP_REPEAT|
  77                                        __GFP_NOWARN, order - PAGE_SHIFT);
  78                 if (!hpt)
  79                         --order;
  80         }
  81
  82         if (!hpt)
  83                 return -ENOMEM;
  84
  85         kvm->arch.hpt_virt = hpt;
  86         kvm->arch.hpt_order = order;
  87         /* HPTEs are 2**4 bytes long */
  88         kvm->arch.hpt_npte = 1ul << (order - 4);
  89         /* 128 (2**7) bytes in each HPTEG */
  90         kvm->arch.hpt_mask = (1ul << (order - 7)) - 1;
  91
  92         /* Allocate reverse map array */
  93         rev = vmalloc(sizeof(struct revmap_entry) * kvm->arch.hpt_npte);
  94         if (!rev) {
  95                 pr_err("kvmppc_alloc_hpt: Couldn't alloc reverse map array\n");
  96                 goto out_freehpt;
  97         }
  98         kvm->arch.revmap = rev;
  99         kvm->arch.sdr1 = __pa(hpt) | (order - 18);
 100
 101         pr_info("KVM guest htab at %lx (order %ld), LPID %x\n",
 102                 hpt, order, kvm->arch.lpid);
 103
 104         if (htab_orderp)
 105                 *htab_orderp = order;
 106         return 0;
 107
 108  out_freehpt:
 109         if (kvm->arch.hpt_cma_alloc)
 110                 kvm_release_hpt(page, 1 << (order - PAGE_SHIFT));
 111         else
 112                 free_pages(hpt, order - PAGE_SHIFT);
 113         return -ENOMEM;
 114 }
 115
 116 long kvmppc_alloc_reset_hpt(struct kvm *kvm, u32 *htab_orderp)
 117 {
 118         long err = -EBUSY;
 119         long order;
 120
 121         mutex_lock(&kvm->lock);
 122         if (kvm->arch.rma_setup_done) {
 123                 kvm->arch.rma_setup_done = 0;
 124                 /* order rma_setup_done vs. vcpus_running */
 125                 smp_mb();
 126                 if (atomic_read(&kvm->arch.vcpus_running)) {
 127                         kvm->arch.rma_setup_done = 1;
 128                         goto out;
 129                 }
 130         }
 131         if (kvm->arch.hpt_virt) {
 132                 order = kvm->arch.hpt_order;
 133                 /* Set the entire HPT to 0, i.e. invalid HPTEs */
 134                 memset((void *)kvm->arch.hpt_virt, 0, 1ul << order);
 135                 /*
 136                  * Reset all the reverse-mapping chains for all memslots
 137                  */
 138                 kvmppc_rmap_reset(kvm);
 139                 /* Ensure that each vcpu will flush its TLB on next entry. */
 140                 cpumask_setall(&kvm->arch.need_tlb_flush);
 141                 *htab_orderp = order;
 142                 err = 0;
 143         } else {
 144                 err = kvmppc_alloc_hpt(kvm, htab_orderp);
 145                 order = *htab_orderp;
 146         }
 147  out:
 148         mutex_unlock(&kvm->lock);
 149         return err;
 150 }
 151
 152 void kvmppc_free_hpt(struct kvm *kvm)
 153 {
 154         kvmppc_free_lpid(kvm->arch.lpid);
 155         vfree(kvm->arch.revmap);
 156         if (kvm->arch.hpt_cma_alloc)
 157                 kvm_release_hpt(virt_to_page(kvm->arch.hpt_virt),
 158                                 1 << (kvm->arch.hpt_order - PAGE_SHIFT));
 159         else
 160                 free_pages(kvm->arch.hpt_virt,
 161                            kvm->arch.hpt_order - PAGE_SHIFT);
 162 }
 163
 164 /* Bits in first HPTE dword for pagesize 4k, 64k or 16M */
 165 static inline unsigned long hpte0_pgsize_encoding(unsigned long pgsize)
 166 {
 167         return (pgsize > 0x1000) ? HPTE_V_LARGE : 0;
 168 }
 169
 170 /* Bits in second HPTE dword for pagesize 4k, 64k or 16M */
 171 static inline unsigned long hpte1_pgsize_encoding(unsigned long pgsize)
 172 {
 173         return (pgsize == 0x10000) ? 0x1000 : 0;
 174 }
 175
 176 void kvmppc_map_vrma(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot,
 177                      unsigned long porder)
 178 {
 179         unsigned long i;
 180         unsigned long npages;
 181         unsigned long hp_v, hp_r;
 182         unsigned long addr, hash;
 183         unsigned long psize;
 184         unsigned long hp0, hp1;
 185         unsigned long idx_ret;
 186         long ret;
 187         struct kvm *kvm = vcpu->kvm;
 188
 189         psize = 1ul << porder;
 190         npages = memslot->npages >> (porder - PAGE_SHIFT);
 191
 192         /* VRMA can't be > 1TB */
 193         if (npages > 1ul << (40 - porder))
 194                 npages = 1ul << (40 - porder);
 195         /* Can't use more than 1 HPTE per HPTEG */
 196         if (npages > kvm->arch.hpt_mask + 1)
 197                 npages = kvm->arch.hpt_mask + 1;
 198
 199         hp0 = HPTE_V_1TB_SEG | (VRMA_VSID << (40 - 16)) |
 200                 HPTE_V_BOLTED | hpte0_pgsize_encoding(psize);
 201         hp1 = hpte1_pgsize_encoding(psize) |
 202                 HPTE_R_R | HPTE_R_C | HPTE_R_M | PP_RWXX;
 203
 204         for (i = 0; i < npages; ++i) {
 205                 addr = i << porder;
 206                 /* can't use hpt_hash since va > 64 bits */
 207                 hash = (i ^ (VRMA_VSID ^ (VRMA_VSID << 25))) & kvm->arch.hpt_mask;
 208                 /*
 209                  * We assume that the hash table is empty and no
 210                  * vcpus are using it at this stage.  Since we create
 211                  * at most one HPTE per HPTEG, we just assume entry 7
 212                  * is available and use it.
 213                  */
 214                 hash = (hash << 3) + 7;
 215                 hp_v = hp0 | ((addr >> 16) & ~0x7fUL);
 216                 hp_r = hp1 | addr;
 217                 ret = kvmppc_virtmode_do_h_enter(kvm, H_EXACT, hash, hp_v, hp_r,
 218                                                  &idx_ret);
 219                 if (ret != H_SUCCESS) {
 220                         pr_err("KVM: map_vrma at %lx failed, ret=%ld\n",
 221                                addr, ret);
 222                         break;
 223                 }
 224         }
 225 }
 226
 227 int kvmppc_mmu_hv_init(void)
 228 {
 229         unsigned long host_lpid, rsvd_lpid;
 230
 231         if (!cpu_has_feature(CPU_FTR_HVMODE))
 232                 return -EINVAL;
 233
 234         /* POWER7 has 10-bit LPIDs, PPC970 and e500mc have 6-bit LPIDs */
 235         if (cpu_has_feature(CPU_FTR_ARCH_206)) {
 236                 host_lpid = mfspr(SPRN_LPID);   /* POWER7 */
 237                 rsvd_lpid = LPID_RSVD;
 238         } else {
 239                 host_lpid = 0;                  /* PPC970 */
 240                 rsvd_lpid = MAX_LPID_970;
 241         }
 242
 243         kvmppc_init_lpid(rsvd_lpid + 1);
 244
 245         kvmppc_claim_lpid(host_lpid);
 246         /* rsvd_lpid is reserved for use in partition switching */
 247         kvmppc_claim_lpid(rsvd_lpid);
 248
 249         return 0;
 250 }
 251
 252 static void kvmppc_mmu_book3s_64_hv_reset_msr(struct kvm_vcpu *vcpu)
 253 {
 254         unsigned long msr = vcpu->arch.intr_msr;
 255
 256         /* If transactional, change to suspend mode on IRQ delivery */
 257         if (MSR_TM_TRANSACTIONAL(vcpu->arch.shregs.msr))
 258                 msr |= MSR_TS_S;
 259         else
 260                 msr |= vcpu->arch.shregs.msr & MSR_TS_MASK;
 261         kvmppc_set_msr(vcpu, msr);
 262 }
 263
 264 /*
 265  * This is called to get a reference to a guest page if there isn't
 266  * one already in the memslot->arch.slot_phys[] array.
 267  */
 268 static long kvmppc_get_guest_page(struct kvm *kvm, unsigned long gfn,
 269                                   struct kvm_memory_slot *memslot,
 270                                   unsigned long psize)
 271 {
 272         unsigned long start;
 273         long np, err;
 274         struct page *page, *hpage, *pages[1];
 275         unsigned long s, pgsize;
 276         unsigned long *physp;
 277         unsigned int is_io, got, pgorder;
 278         struct vm_area_struct *vma;
 279         unsigned long pfn, i, npages;
 280
 281         physp = memslot->arch.slot_phys;
 282         if (!physp)
 283                 return -EINVAL;
 284         if (physp[gfn - memslot->base_gfn])
 285                 return 0;
 286
 287         is_io = 0;
 288         got = 0;
 289         page = NULL;
 290         pgsize = psize;
 291         err = -EINVAL;
 292         start = gfn_to_hva_memslot(memslot, gfn);
 293
 294         /* Instantiate and get the page we want access to */
 295         np = get_user_pages_fast(start, 1, 1, pages);
 296         if (np != 1) {
 297                 /* Look up the vma for the page */
 298                 down_read(&current->mm->mmap_sem);
 299                 vma = find_vma(current->mm, start);
 300                 if (!vma || vma->vm_start > start ||
 301                     start + psize > vma->vm_end ||
 302                     !(vma->vm_flags & VM_PFNMAP))
 303                         goto up_err;
 304                 is_io = hpte_cache_bits(pgprot_val(vma->vm_page_prot));
 305                 pfn = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
 306                 /* check alignment of pfn vs. requested page size */
 307                 if (psize > PAGE_SIZE && (pfn & ((psize >> PAGE_SHIFT) - 1)))
 308                         goto up_err;
 309                 up_read(&current->mm->mmap_sem);
 310
 311         } else {
 312                 page = pages[0];
 313                 got = KVMPPC_GOT_PAGE;
 314
 315                 /* See if this is a large page */
 316                 s = PAGE_SIZE;
 317                 if (PageHuge(page)) {
 318                         hpage = compound_head(page);
 319                         s <<= compound_order(hpage);
 320                         /* Get the whole large page if slot alignment is ok */
 321                         if (s > psize && slot_is_aligned(memslot, s) &&
 322                             !(memslot->userspace_addr & (s - 1))) {
 323                                 start &= ~(s - 1);
 324                                 pgsize = s;
 325                                 get_page(hpage);
 326                                 put_page(page);
 327                                 page = hpage;
 328                         }
 329                 }
 330                 if (s < psize)
 331                         goto out;
 332                 pfn = page_to_pfn(page);
 333         }
 334
 335         npages = pgsize >> PAGE_SHIFT;
 336         pgorder = __ilog2(npages);
 337         physp += (gfn - memslot->base_gfn) & ~(npages - 1);
 338         spin_lock(&kvm->arch.slot_phys_lock);
 339         for (i = 0; i < npages; ++i) {
 340                 if (!physp[i]) {
 341                         physp[i] = ((pfn + i) << PAGE_SHIFT) +
 342                                 got + is_io + pgorder;
 343                         got = 0;
 344                 }
 345         }
 346         spin_unlock(&kvm->arch.slot_phys_lock);
 347         err = 0;
 348
 349  out:
 350         if (got)
 351                 put_page(page);
 352         return err;
 353
 354  up_err:
 355         up_read(&current->mm->mmap_sem);
 356         return err;
 357 }
 358
 359 long kvmppc_virtmode_do_h_enter(struct kvm *kvm, unsigned long flags,
 360                                 long pte_index, unsigned long pteh,
 361                                 unsigned long ptel, unsigned long *pte_idx_ret)
 362 {
 363         unsigned long psize, gpa, gfn;
 364         struct kvm_memory_slot *memslot;
 365         long ret;
 366
 367         if (kvm->arch.using_mmu_notifiers)
 368                 goto do_insert;
 369
 370         psize = hpte_page_size(pteh, ptel);
 371         if (!psize)
 372                 return H_PARAMETER;
 373
 374         pteh &= ~(HPTE_V_HVLOCK | HPTE_V_ABSENT | HPTE_V_VALID);
 375
 376         /* Find the memslot (if any) for this address */
 377         gpa = (ptel & HPTE_R_RPN) & ~(psize - 1);
 378         gfn = gpa >> PAGE_SHIFT;
 379         memslot = gfn_to_memslot(kvm, gfn);
 380         if (memslot && !(memslot->flags & KVM_MEMSLOT_INVALID)) {
 381                 if (!slot_is_aligned(memslot, psize))
 382                         return H_PARAMETER;
 383                 if (kvmppc_get_guest_page(kvm, gfn, memslot, psize) < 0)
 384                         return H_PARAMETER;
 385         }
 386
 387  do_insert:
 388         /* Protect linux PTE lookup from page table destruction */
 389         rcu_read_lock_sched();  /* this disables preemption too */
 390         ret = kvmppc_do_h_enter(kvm, flags, pte_index, pteh, ptel,
 391                                 current->mm->pgd, false, pte_idx_ret);
 392         rcu_read_unlock_sched();
 393         if (ret == H_TOO_HARD) {
 394                 /* this can't happen */
 395                 pr_err("KVM: Oops, kvmppc_h_enter returned too hard!\n");
 396                 ret = H_RESOURCE;       /* or something */
 397         }
 398         return ret;
 399
 400 }
 401
 402 /*
 403  * We come here on a H_ENTER call from the guest when we are not
 404  * using mmu notifiers and we don't have the requested page pinned
 405  * already.
 406  */
 407 long kvmppc_virtmode_h_enter(struct kvm_vcpu *vcpu, unsigned long flags,
 408                              long pte_index, unsigned long pteh,
 409                              unsigned long ptel)
 410 {
 411         return kvmppc_virtmode_do_h_enter(vcpu->kvm, flags, pte_index,
 412                                           pteh, ptel, &vcpu->arch.gpr[4]);
 413 }
 414
 415 static struct kvmppc_slb *kvmppc_mmu_book3s_hv_find_slbe(struct kvm_vcpu *vcpu,
 416                                                          gva_t eaddr)
 417 {
 418         u64 mask;
 419         int i;
 420
 421         for (i = 0; i < vcpu->arch.slb_nr; i++) {
 422                 if (!(vcpu->arch.slb[i].orige & SLB_ESID_V))
 423                         continue;
 424
 425                 if (vcpu->arch.slb[i].origv & SLB_VSID_B_1T)
 426                         mask = ESID_MASK_1T;
 427                 else
 428                         mask = ESID_MASK;
 429
 430                 if (((vcpu->arch.slb[i].orige ^ eaddr) & mask) == 0)
 431                         return &vcpu->arch.slb[i];
 432         }
 433         return NULL;
 434 }
 435
 436 static unsigned long kvmppc_mmu_get_real_addr(unsigned long v, unsigned long r,
 437                         unsigned long ea)
 438 {
 439         unsigned long ra_mask;
 440
 441         ra_mask = hpte_page_size(v, r) - 1;
 442         return (r & HPTE_R_RPN & ~ra_mask) | (ea & ra_mask);
 443 }
 444
 445 static int kvmppc_mmu_book3s_64_hv_xlate(struct kvm_vcpu *vcpu, gva_t eaddr,
 446                         struct kvmppc_pte *gpte, bool data, bool iswrite)
 447 {
 448         struct kvm *kvm = vcpu->kvm;
 449         struct kvmppc_slb *slbe;
 450         unsigned long slb_v;
 451         unsigned long pp, key;
 452         unsigned long v, gr;
 453         __be64 *hptep;
 454         int index;
 455         int virtmode = vcpu->arch.shregs.msr & (data ? MSR_DR : MSR_IR);
 456
 457         /* Get SLB entry */
 458         if (virtmode) {
 459                 slbe = kvmppc_mmu_book3s_hv_find_slbe(vcpu, eaddr);
 460                 if (!slbe)
 461                         return -EINVAL;
 462                 slb_v = slbe->origv;
 463         } else {
 464                 /* real mode access */
 465                 slb_v = vcpu->kvm->arch.vrma_slb_v;
 466         }
 467
 468         preempt_disable();
 469         /* Find the HPTE in the hash table */
 470         index = kvmppc_hv_find_lock_hpte(kvm, eaddr, slb_v,
 471                                          HPTE_V_VALID | HPTE_V_ABSENT);
 472         if (index < 0) {
 473                 preempt_enable();
 474                 return -ENOENT;
 475         }
 476         hptep = (__be64 *)(kvm->arch.hpt_virt + (index << 4));
 477         v = be64_to_cpu(hptep[0]) & ~HPTE_V_HVLOCK;
 478         gr = kvm->arch.revmap[index].guest_rpte;
 479
 480         /* Unlock the HPTE */
 481         asm volatile("lwsync" : : : "memory");
 482         hptep[0] = cpu_to_be64(v);
 483         preempt_enable();
 484
 485         gpte->eaddr = eaddr;
 486         gpte->vpage = ((v & HPTE_V_AVPN) << 4) | ((eaddr >> 12) & 0xfff);
 487
 488         /* Get PP bits and key for permission check */
 489         pp = gr & (HPTE_R_PP0 | HPTE_R_PP);
 490         key = (vcpu->arch.shregs.msr & MSR_PR) ? SLB_VSID_KP : SLB_VSID_KS;
 491         key &= slb_v;
 492
 493         /* Calculate permissions */
 494         gpte->may_read = hpte_read_permission(pp, key);
 495         gpte->may_write = hpte_write_permission(pp, key);
 496         gpte->may_execute = gpte->may_read && !(gr & (HPTE_R_N | HPTE_R_G));
 497
 498         /* Storage key permission check for POWER7 */
 499         if (data && virtmode && cpu_has_feature(CPU_FTR_ARCH_206)) {
 500                 int amrfield = hpte_get_skey_perm(gr, vcpu->arch.amr);
 501                 if (amrfield & 1)
 502                         gpte->may_read = 0;
 503                 if (amrfield & 2)
 504                         gpte->may_write = 0;
 505         }
 506
 507         /* Get the guest physical address */
 508         gpte->raddr = kvmppc_mmu_get_real_addr(v, gr, eaddr);
 509         return 0;
 510 }
 511
 512 /*
 513  * Quick test for whether an instruction is a load or a store.
 514  * If the instruction is a load or a store, then this will indicate
 515  * which it is, at least on server processors.  (Embedded processors
 516  * have some external PID instructions that don't follow the rule
 517  * embodied here.)  If the instruction isn't a load or store, then
 518  * this doesn't return anything useful.
 519  */
 520 static int instruction_is_store(unsigned int instr)
 521 {
 522         unsigned int mask;
 523
 524         mask = 0x10000000;
 525         if ((instr & 0xfc000000) == 0x7c000000)
 526                 mask = 0x100;           /* major opcode 31 */
 527         return (instr & mask) != 0;
 528 }
 529
 530 static int kvmppc_hv_emulate_mmio(struct kvm_run *run, struct kvm_vcpu *vcpu,
 531                                   unsigned long gpa, gva_t ea, int is_store)
 532 {
 533         u32 last_inst;
 534
 535         /*
 536          * If we fail, we just return to the guest and try executing it again.
 537          */
 538         if (kvmppc_get_last_inst(vcpu, INST_GENERIC, &last_inst) !=
 539                 EMULATE_DONE)
 540                 return RESUME_GUEST;
 541
 542         /*
 543          * WARNING: We do not know for sure whether the instruction we just
 544          * read from memory is the same that caused the fault in the first
 545          * place.  If the instruction we read is neither an load or a store,
 546          * then it can't access memory, so we don't need to worry about
 547          * enforcing access permissions.  So, assuming it is a load or
 548          * store, we just check that its direction (load or store) is
 549          * consistent with the original fault, since that's what we
 550          * checked the access permissions against.  If there is a mismatch
 551          * we just return and retry the instruction.
 552          */
 553
 554         if (instruction_is_store(last_inst) != !!is_store)
 555                 return RESUME_GUEST;
 556
 557         /*
 558          * Emulated accesses are emulated by looking at the hash for
 559          * translation once, then performing the access later. The
 560          * translation could be invalidated in the meantime in which
 561          * point performing the subsequent memory access on the old
 562          * physical address could possibly be a security hole for the
 563          * guest (but not the host).
 564          *
 565          * This is less of an issue for MMIO stores since they aren't
 566          * globally visible. It could be an issue for MMIO loads to
 567          * a certain extent but we'll ignore it for now.
 568          */
 569
 570         vcpu->arch.paddr_accessed = gpa;
 571         vcpu->arch.vaddr_accessed = ea;
 572         return kvmppc_emulate_mmio(run, vcpu);
 573 }
 574
 575 int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu,
 576                                 unsigned long ea, unsigned long dsisr)
 577 {
 578         struct kvm *kvm = vcpu->kvm;
 579         unsigned long hpte[3], r;
 580         __be64 *hptep;
 581         unsigned long mmu_seq, psize, pte_size;
 582         unsigned long gpa_base, gfn_base;
 583         unsigned long gpa, gfn, hva, pfn;
 584         struct kvm_memory_slot *memslot;
 585         unsigned long *rmap;
 586         struct revmap_entry *rev;
 587         struct page *page, *pages[1];
 588         long index, ret, npages;
 589         unsigned long is_io;
 590         unsigned int writing, write_ok;
 591         struct vm_area_struct *vma;
 592         unsigned long rcbits;
 593
 594         /*
 595          * Real-mode code has already searched the HPT and found the
 596          * entry we're interested in.  Lock the entry and check that
 597          * it hasn't changed.  If it has, just return and re-execute the
 598          * instruction.
 599          */
 600         if (ea != vcpu->arch.pgfault_addr)
 601                 return RESUME_GUEST;
 602         index = vcpu->arch.pgfault_index;
 603         hptep = (__be64 *)(kvm->arch.hpt_virt + (index << 4));
 604         rev = &kvm->arch.revmap[index];
 605         preempt_disable();
 606         while (!try_lock_hpte(hptep, HPTE_V_HVLOCK))
 607                 cpu_relax();
 608         hpte[0] = be64_to_cpu(hptep[0]) & ~HPTE_V_HVLOCK;
 609         hpte[1] = be64_to_cpu(hptep[1]);
 610         hpte[2] = r = rev->guest_rpte;
 611         asm volatile("lwsync" : : : "memory");
 612         hptep[0] = cpu_to_be64(hpte[0]);
 613         preempt_enable();
 614
 615         if (hpte[0] != vcpu->arch.pgfault_hpte[0] ||
 616             hpte[1] != vcpu->arch.pgfault_hpte[1])
 617                 return RESUME_GUEST;
 618
 619         /* Translate the logical address and get the page */
 620         psize = hpte_page_size(hpte[0], r);
 621         gpa_base = r & HPTE_R_RPN & ~(psize - 1);
 622         gfn_base = gpa_base >> PAGE_SHIFT;
 623         gpa = gpa_base | (ea & (psize - 1));
 624         gfn = gpa >> PAGE_SHIFT;
 625         memslot = gfn_to_memslot(kvm, gfn);
 626
 627         trace_kvm_page_fault_enter(vcpu, hpte, memslot, ea, dsisr);
 628
 629         /* No memslot means it's an emulated MMIO region */
 630         if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID))
 631                 return kvmppc_hv_emulate_mmio(run, vcpu, gpa, ea,
 632                                               dsisr & DSISR_ISSTORE);
 633
 634         if (!kvm->arch.using_mmu_notifiers)
 635                 return -EFAULT;         /* should never get here */
 636
 637         /*
 638          * This should never happen, because of the slot_is_aligned()
 639          * check in kvmppc_do_h_enter().
 640          */
 641         if (gfn_base < memslot->base_gfn)
 642                 return -EFAULT;
 643
 644         /* used to check for invalidations in progress */
 645         mmu_seq = kvm->mmu_notifier_seq;
 646         smp_rmb();
 647
 648         ret = -EFAULT;
 649         is_io = 0;
 650         pfn = 0;
 651         page = NULL;
 652         pte_size = PAGE_SIZE;
 653         writing = (dsisr & DSISR_ISSTORE) != 0;
 654         /* If writing != 0, then the HPTE must allow writing, if we get here */
 655         write_ok = writing;
 656         hva = gfn_to_hva_memslot(memslot, gfn);
 657         npages = get_user_pages_fast(hva, 1, writing, pages);
 658         if (npages < 1) {
 659                 /* Check if it's an I/O mapping */
 660                 down_read(&current->mm->mmap_sem);
 661                 vma = find_vma(current->mm, hva);
 662                 if (vma && vma->vm_start <= hva && hva + psize <= vma->vm_end &&
 663                     (vma->vm_flags & VM_PFNMAP)) {
 664                         pfn = vma->vm_pgoff +
 665                                 ((hva - vma->vm_start) >> PAGE_SHIFT);
 666                         pte_size = psize;
 667                         is_io = hpte_cache_bits(pgprot_val(vma->vm_page_prot));
 668                         write_ok = vma->vm_flags & VM_WRITE;
 669                 }
 670                 up_read(&current->mm->mmap_sem);
 671                 if (!pfn)
 672                         goto out_put;
 673         } else {
 674                 page = pages[0];
 675                 pfn = page_to_pfn(page);
 676                 if (PageHuge(page)) {
 677                         page = compound_head(page);
 678                         pte_size <<= compound_order(page);
 679                 }
 680                 /* if the guest wants write access, see if that is OK */
 681                 if (!writing && hpte_is_writable(r)) {
 682                         unsigned int hugepage_shift;
 683                         pte_t *ptep, pte;
 684
 685                         /*
 686                          * We need to protect against page table destruction
 687                          * while looking up and updating the pte.
 688                          */
 689                         rcu_read_lock_sched();
 690                         ptep = find_linux_pte_or_hugepte(current->mm->pgd,
 691                                                          hva, &hugepage_shift);
 692                         if (ptep) {
 693                                 pte = kvmppc_read_update_linux_pte(ptep, 1,
 694                                                            hugepage_shift);
 695                                 if (pte_write(pte))
 696                                         write_ok = 1;
 697                         }
 698                         rcu_read_unlock_sched();
 699                 }
 700         }
 701
 702         if (psize > pte_size)
 703                 goto out_put;
 704
 705         /* Check WIMG vs. the actual page we're accessing */
 706         if (!hpte_cache_flags_ok(r, is_io)) {
 707                 if (is_io)
 708                         goto out_put;
 709
 710                 /*
 711                  * Allow guest to map emulated device memory as
 712                  * uncacheable, but actually make it cacheable.
 713                  */
 714                 r = (r & ~(HPTE_R_W|HPTE_R_I|HPTE_R_G)) | HPTE_R_M;
 715         }
 716
 717         /*
 718          * Set the HPTE to point to pfn.
 719          * Since the pfn is at PAGE_SIZE granularity, make sure we
 720          * don't mask out lower-order bits if psize < PAGE_SIZE.
 721          */
 722         if (psize < PAGE_SIZE)
 723                 psize = PAGE_SIZE;
 724         r = (r & ~(HPTE_R_PP0 - psize)) | ((pfn << PAGE_SHIFT) & ~(psize - 1));
 725         if (hpte_is_writable(r) && !write_ok)
 726                 r = hpte_make_readonly(r);
 727         ret = RESUME_GUEST;
 728         preempt_disable();
 729         while (!try_lock_hpte(hptep, HPTE_V_HVLOCK))
 730                 cpu_relax();
 731         if ((be64_to_cpu(hptep[0]) & ~HPTE_V_HVLOCK) != hpte[0] ||
 732                 be64_to_cpu(hptep[1]) != hpte[1] ||
 733                 rev->guest_rpte != hpte[2])
 734                 /* HPTE has been changed under us; let the guest retry */
 735                 goto out_unlock;
 736         hpte[0] = (hpte[0] & ~HPTE_V_ABSENT) | HPTE_V_VALID;
 737
 738         /* Always put the HPTE in the rmap chain for the page base address */
 739         rmap = &memslot->arch.rmap[gfn_base - memslot->base_gfn];
 740         lock_rmap(rmap);
 741
 742         /* Check if we might have been invalidated; let the guest retry if so */
 743         ret = RESUME_GUEST;
 744         if (mmu_notifier_retry(vcpu->kvm, mmu_seq)) {
 745                 unlock_rmap(rmap);
 746                 goto out_unlock;
 747         }
 748
 749         /* Only set R/C in real HPTE if set in both *rmap and guest_rpte */
 750         rcbits = *rmap >> KVMPPC_RMAP_RC_SHIFT;
 751         r &= rcbits | ~(HPTE_R_R | HPTE_R_C);
 752
 753         if (be64_to_cpu(hptep[0]) & HPTE_V_VALID) {
 754                 /* HPTE was previously valid, so we need to invalidate it */
 755                 unlock_rmap(rmap);
 756                 hptep[0] |= cpu_to_be64(HPTE_V_ABSENT);
 757                 kvmppc_invalidate_hpte(kvm, hptep, index);
 758                 /* don't lose previous R and C bits */
 759                 r |= be64_to_cpu(hptep[1]) & (HPTE_R_R | HPTE_R_C);
 760         } else {
 761                 kvmppc_add_revmap_chain(kvm, rev, rmap, index, 0);
 762         }
 763
 764         hptep[1] = cpu_to_be64(r);
 765         eieio();
 766         hptep[0] = cpu_to_be64(hpte[0]);
 767         asm volatile("ptesync" : : : "memory");
 768         preempt_enable();
 769         if (page && hpte_is_writable(r))
 770                 SetPageDirty(page);
 771
 772  out_put:
 773         trace_kvm_page_fault_exit(vcpu, hpte, ret);
 774
 775         if (page) {
 776                 /*
 777                  * We drop pages[0] here, not page because page might
 778                  * have been set to the head page of a compound, but
 779                  * we have to drop the reference on the correct tail
 780                  * page to match the get inside gup()
 781                  */
 782                 put_page(pages[0]);
 783         }
 784         return ret;
 785
 786  out_unlock:
 787         hptep[0] &= ~cpu_to_be64(HPTE_V_HVLOCK);
 788         preempt_enable();
 789         goto out_put;
 790 }
 791
 792 static void kvmppc_rmap_reset(struct kvm *kvm)
 793 {
 794         struct kvm_memslots *slots;
 795         struct kvm_memory_slot *memslot;
 796         int srcu_idx;
 797
 798         srcu_idx = srcu_read_lock(&kvm->srcu);
 799         slots = kvm->memslots;
 800         kvm_for_each_memslot(memslot, slots) {
 801                 /*
 802                  * This assumes it is acceptable to lose reference and
 803                  * change bits across a reset.
 804                  */
 805                 memset(memslot->arch.rmap, 0,
 806                        memslot->npages * sizeof(*memslot->arch.rmap));
 807         }
 808         srcu_read_unlock(&kvm->srcu, srcu_idx);
 809 }
 810
 811 static int kvm_handle_hva_range(struct kvm *kvm,
 812                                 unsigned long start,
 813                                 unsigned long end,
 814                                 int (*handler)(struct kvm *kvm,
 815                                                unsigned long *rmapp,
 816                                                unsigned long gfn))
 817 {
 818         int ret;
 819         int retval = 0;
 820         struct kvm_memslots *slots;
 821         struct kvm_memory_slot *memslot;
 822
 823         slots = kvm_memslots(kvm);
 824         kvm_for_each_memslot(memslot, slots) {
 825                 unsigned long hva_start, hva_end;
 826                 gfn_t gfn, gfn_end;
 827
 828                 hva_start = max(start, memslot->userspace_addr);
 829                 hva_end = min(end, memslot->userspace_addr +
 830                                         (memslot->npages << PAGE_SHIFT));
 831                 if (hva_start >= hva_end)
 832                         continue;
 833                 /*
 834                  * {gfn(page) | page intersects with [hva_start, hva_end)} =
 835                  * {gfn, gfn+1, ..., gfn_end-1}.
 836                  */
 837                 gfn = hva_to_gfn_memslot(hva_start, memslot);
 838                 gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot);
 839
 840                 for (; gfn < gfn_end; ++gfn) {
 841                         gfn_t gfn_offset = gfn - memslot->base_gfn;
 842
 843                         ret = handler(kvm, &memslot->arch.rmap[gfn_offset], gfn);
 844                         retval |= ret;
 845                 }
 846         }
 847
 848         return retval;
 849 }
 850
 851 static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
 852                           int (*handler)(struct kvm *kvm, unsigned long *rmapp,
 853                                          unsigned long gfn))
 854 {
 855         return kvm_handle_hva_range(kvm, hva, hva + 1, handler);
 856 }
 857
 858 static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp,
 859                            unsigned long gfn)
 860 {
 861         struct revmap_entry *rev = kvm->arch.revmap;
 862         unsigned long h, i, j;
 863         __be64 *hptep;
 864         unsigned long ptel, psize, rcbits;
 865
 866         for (;;) {
 867                 lock_rmap(rmapp);
 868                 if (!(*rmapp & KVMPPC_RMAP_PRESENT)) {
 869                         unlock_rmap(rmapp);
 870                         break;
 871                 }
 872
 873                 /*
 874                  * To avoid an ABBA deadlock with the HPTE lock bit,
 875                  * we can't spin on the HPTE lock while holding the
 876                  * rmap chain lock.
 877                  */
 878                 i = *rmapp & KVMPPC_RMAP_INDEX;
 879                 hptep = (__be64 *) (kvm->arch.hpt_virt + (i << 4));
 880                 if (!try_lock_hpte(hptep, HPTE_V_HVLOCK)) {
 881                         /* unlock rmap before spinning on the HPTE lock */
 882                         unlock_rmap(rmapp);
 883                         while (be64_to_cpu(hptep[0]) & HPTE_V_HVLOCK)
 884                                 cpu_relax();
 885                         continue;
 886                 }
 887                 j = rev[i].forw;
 888                 if (j == i) {
 889                         /* chain is now empty */
 890                         *rmapp &= ~(KVMPPC_RMAP_PRESENT | KVMPPC_RMAP_INDEX);
 891                 } else {
 892                         /* remove i from chain */
 893                         h = rev[i].back;
 894                         rev[h].forw = j;
 895                         rev[j].back = h;
 896                         rev[i].forw = rev[i].back = i;
 897                         *rmapp = (*rmapp & ~KVMPPC_RMAP_INDEX) | j;
 898                 }
 899
 900                 /* Now check and modify the HPTE */
 901                 ptel = rev[i].guest_rpte;
 902                 psize = hpte_page_size(be64_to_cpu(hptep[0]), ptel);
 903                 if ((be64_to_cpu(hptep[0]) & HPTE_V_VALID) &&
 904                     hpte_rpn(ptel, psize) == gfn) {
 905                         if (kvm->arch.using_mmu_notifiers)
 906                                 hptep[0] |= cpu_to_be64(HPTE_V_ABSENT);
 907                         kvmppc_invalidate_hpte(kvm, hptep, i);
 908                         /* Harvest R and C */
 909                         rcbits = be64_to_cpu(hptep[1]) & (HPTE_R_R | HPTE_R_C);
 910                         *rmapp |= rcbits << KVMPPC_RMAP_RC_SHIFT;
 911                         if (rcbits & ~rev[i].guest_rpte) {
 912                                 rev[i].guest_rpte = ptel | rcbits;
 913                                 note_hpte_modification(kvm, &rev[i]);
 914                         }
 915                 }
 916                 unlock_rmap(rmapp);
 917                 hptep[0] &= ~cpu_to_be64(HPTE_V_HVLOCK);
 918         }
 919         return 0;
 920 }
 921
 922 int kvm_unmap_hva_hv(struct kvm *kvm, unsigned long hva)
 923 {
 924         if (kvm->arch.using_mmu_notifiers)
 925                 kvm_handle_hva(kvm, hva, kvm_unmap_rmapp);
 926         return 0;
 927 }
 928
 929 int kvm_unmap_hva_range_hv(struct kvm *kvm, unsigned long start, unsigned long end)
 930 {
 931         if (kvm->arch.using_mmu_notifiers)
 932                 kvm_handle_hva_range(kvm, start, end, kvm_unmap_rmapp);
 933         return 0;
 934 }
 935
 936 void kvmppc_core_flush_memslot_hv(struct kvm *kvm,
 937                                   struct kvm_memory_slot *memslot)
 938 {
 939         unsigned long *rmapp;
 940         unsigned long gfn;
 941         unsigned long n;
 942
 943         rmapp = memslot->arch.rmap;
 944         gfn = memslot->base_gfn;
 945         for (n = memslot->npages; n; --n) {
 946                 /*
 947                  * Testing the present bit without locking is OK because
 948                  * the memslot has been marked invalid already, and hence
 949                  * no new HPTEs referencing this page can be created,
 950                  * thus the present bit can't go from 0 to 1.
 951                  */
 952                 if (*rmapp & KVMPPC_RMAP_PRESENT)
 953                         kvm_unmap_rmapp(kvm, rmapp, gfn);
 954                 ++rmapp;
 955                 ++gfn;
 956         }
 957 }
 958
 959 static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp,
 960                          unsigned long gfn)
 961 {
 962         struct revmap_entry *rev = kvm->arch.revmap;
 963         unsigned long head, i, j;
 964         __be64 *hptep;
 965         int ret = 0;
 966
 967  retry:
 968         lock_rmap(rmapp);
 969         if (*rmapp & KVMPPC_RMAP_REFERENCED) {
 970                 *rmapp &= ~KVMPPC_RMAP_REFERENCED;
 971                 ret = 1;
 972         }
 973         if (!(*rmapp & KVMPPC_RMAP_PRESENT)) {
 974                 unlock_rmap(rmapp);
 975                 return ret;
 976         }
 977
 978         i = head = *rmapp & KVMPPC_RMAP_INDEX;
 979         do {
 980                 hptep = (__be64 *) (kvm->arch.hpt_virt + (i << 4));
 981                 j = rev[i].forw;
 982
 983                 /* If this HPTE isn't referenced, ignore it */
 984                 if (!(be64_to_cpu(hptep[1]) & HPTE_R_R))
 985                         continue;
 986
 987                 if (!try_lock_hpte(hptep, HPTE_V_HVLOCK)) {
 988                         /* unlock rmap before spinning on the HPTE lock */
 989                         unlock_rmap(rmapp);
 990                         while (be64_to_cpu(hptep[0]) & HPTE_V_HVLOCK)
 991                                 cpu_relax();
 992                         goto retry;
 993                 }
 994
 995                 /* Now check and modify the HPTE */
 996                 if ((be64_to_cpu(hptep[0]) & HPTE_V_VALID) &&
 997                     (be64_to_cpu(hptep[1]) & HPTE_R_R)) {
 998                         kvmppc_clear_ref_hpte(kvm, hptep, i);
 999                         if (!(rev[i].guest_rpte & HPTE_R_R)) {
1000                                 rev[i].guest_rpte |= HPTE_R_R;
1001                                 note_hpte_modification(kvm, &rev[i]);
1002                         }
1003                         ret = 1;
1004                 }
1005                 hptep[0] &= ~cpu_to_be64(HPTE_V_HVLOCK);
1006         } while ((i = j) != head);
1007
1008         unlock_rmap(rmapp);
1009         return ret;
1010 }
1011
1012 int kvm_age_hva_hv(struct kvm *kvm, unsigned long start, unsigned long end)
1013 {
1014         if (!kvm->arch.using_mmu_notifiers)
1015                 return 0;
1016         return kvm_handle_hva_range(kvm, start, end, kvm_age_rmapp);
1017 }
1018
1019 static int kvm_test_age_rmapp(struct kvm *kvm, unsigned long *rmapp,
1020                               unsigned long gfn)
1021 {
1022         struct revmap_entry *rev = kvm->arch.revmap;
1023         unsigned long head, i, j;
1024         unsigned long *hp;
1025         int ret = 1;
1026
1027         if (*rmapp & KVMPPC_RMAP_REFERENCED)
1028                 return 1;
1029
1030         lock_rmap(rmapp);
1031         if (*rmapp & KVMPPC_RMAP_REFERENCED)
1032                 goto out;
1033
1034         if (*rmapp & KVMPPC_RMAP_PRESENT) {
1035                 i = head = *rmapp & KVMPPC_RMAP_INDEX;
1036                 do {
1037                         hp = (unsigned long *)(kvm->arch.hpt_virt + (i << 4));
1038                         j = rev[i].forw;
1039                         if (be64_to_cpu(hp[1]) & HPTE_R_R)
1040                                 goto out;
1041                 } while ((i = j) != head);
1042         }
1043         ret = 0;
1044
1045  out:
1046         unlock_rmap(rmapp);
1047         return ret;
1048 }
1049
1050 int kvm_test_age_hva_hv(struct kvm *kvm, unsigned long hva)
1051 {
1052         if (!kvm->arch.using_mmu_notifiers)
1053                 return 0;
1054         return kvm_handle_hva(kvm, hva, kvm_test_age_rmapp);
1055 }
1056
1057 void kvm_set_spte_hva_hv(struct kvm *kvm, unsigned long hva, pte_t pte)
1058 {
1059         if (!kvm->arch.using_mmu_notifiers)
1060                 return;
1061         kvm_handle_hva(kvm, hva, kvm_unmap_rmapp);
1062 }
1063
1064 static int vcpus_running(struct kvm *kvm)
1065 {
1066         return atomic_read(&kvm->arch.vcpus_running) != 0;
1067 }
1068
1069 /*
1070  * Returns the number of system pages that are dirty.
1071  * This can be more than 1 if we find a huge-page HPTE.
1072  */
1073 static int kvm_test_clear_dirty_npages(struct kvm *kvm, unsigned long *rmapp)
1074 {
1075         struct revmap_entry *rev = kvm->arch.revmap;
1076         unsigned long head, i, j;
1077         unsigned long n;
1078         unsigned long v, r;
1079         __be64 *hptep;
1080         int npages_dirty = 0;
1081
1082  retry:
1083         lock_rmap(rmapp);
1084         if (*rmapp & KVMPPC_RMAP_CHANGED) {
1085                 *rmapp &= ~KVMPPC_RMAP_CHANGED;
1086                 npages_dirty = 1;
1087         }
1088         if (!(*rmapp & KVMPPC_RMAP_PRESENT)) {
1089                 unlock_rmap(rmapp);
1090                 return npages_dirty;
1091         }
1092
1093         i = head = *rmapp & KVMPPC_RMAP_INDEX;
1094         do {
1095                 unsigned long hptep1;
1096                 hptep = (__be64 *) (kvm->arch.hpt_virt + (i << 4));
1097                 j = rev[i].forw;
1098
1099                 /*
1100                  * Checking the C (changed) bit here is racy since there
1101                  * is no guarantee about when the hardware writes it back.
1102                  * If the HPTE is not writable then it is stable since the
1103                  * page can't be written to, and we would have done a tlbie
1104                  * (which forces the hardware to complete any writeback)
1105                  * when making the HPTE read-only.
1106                  * If vcpus are running then this call is racy anyway
1107                  * since the page could get dirtied subsequently, so we
1108                  * expect there to be a further call which would pick up
1109                  * any delayed C bit writeback.
1110                  * Otherwise we need to do the tlbie even if C==0 in
1111                  * order to pick up any delayed writeback of C.
1112                  */
1113                 hptep1 = be64_to_cpu(hptep[1]);
1114                 if (!(hptep1 & HPTE_R_C) &&
1115                     (!hpte_is_writable(hptep1) || vcpus_running(kvm)))
1116                         continue;
1117
1118                 if (!try_lock_hpte(hptep, HPTE_V_HVLOCK)) {
1119                         /* unlock rmap before spinning on the HPTE lock */
1120                         unlock_rmap(rmapp);
1121                         while (hptep[0] & cpu_to_be64(HPTE_V_HVLOCK))
1122                                 cpu_relax();
1123                         goto retry;
1124                 }
1125
1126                 /* Now check and modify the HPTE */
1127                 if (!(hptep[0] & cpu_to_be64(HPTE_V_VALID))) {
1128                         /* unlock and continue */
1129                         hptep[0] &= ~cpu_to_be64(HPTE_V_HVLOCK);
1130                         continue;
1131                 }
1132
1133                 /* need to make it temporarily absent so C is stable */
1134                 hptep[0] |= cpu_to_be64(HPTE_V_ABSENT);
1135                 kvmppc_invalidate_hpte(kvm, hptep, i);
1136                 v = be64_to_cpu(hptep[0]);
1137                 r = be64_to_cpu(hptep[1]);
1138                 if (r & HPTE_R_C) {
1139                         hptep[1] = cpu_to_be64(r & ~HPTE_R_C);
1140                         if (!(rev[i].guest_rpte & HPTE_R_C)) {
1141                                 rev[i].guest_rpte |= HPTE_R_C;
1142                                 note_hpte_modification(kvm, &rev[i]);
1143                         }
1144                         n = hpte_page_size(v, r);
1145                         n = (n + PAGE_SIZE - 1) >> PAGE_SHIFT;
1146                         if (n > npages_dirty)
1147                                 npages_dirty = n;
1148                         eieio();
1149                 }
1150                 v &= ~(HPTE_V_ABSENT | HPTE_V_HVLOCK);
1151                 v |= HPTE_V_VALID;
1152                 hptep[0] = cpu_to_be64(v);
1153         } while ((i = j) != head);
1154
1155         unlock_rmap(rmapp);
1156         return npages_dirty;
1157 }
1158
1159 static void harvest_vpa_dirty(struct kvmppc_vpa *vpa,
1160                               struct kvm_memory_slot *memslot,
1161                               unsigned long *map)
1162 {
1163         unsigned long gfn;
1164
1165         if (!vpa->dirty || !vpa->pinned_addr)
1166                 return;
1167         gfn = vpa->gpa >> PAGE_SHIFT;
1168         if (gfn < memslot->base_gfn ||
1169             gfn >= memslot->base_gfn + memslot->npages)
1170                 return;
1171
1172         vpa->dirty = false;
1173         if (map)
1174                 __set_bit_le(gfn - memslot->base_gfn, map);
1175 }
1176
1177 long kvmppc_hv_get_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot,
1178                              unsigned long *map)
1179 {
1180         unsigned long i, j;
1181         unsigned long *rmapp;
1182         struct kvm_vcpu *vcpu;
1183
1184         preempt_disable();
1185         rmapp = memslot->arch.rmap;
1186         for (i = 0; i < memslot->npages; ++i) {
1187                 int npages = kvm_test_clear_dirty_npages(kvm, rmapp);
1188                 /*
1189                  * Note that if npages > 0 then i must be a multiple of npages,
1190                  * since we always put huge-page HPTEs in the rmap chain
1191                  * corresponding to their page base address.
1192                  */
1193                 if (npages && map)
1194                         for (j = i; npages; ++j, --npages)
1195                                 __set_bit_le(j, map);
1196                 ++rmapp;
1197         }
1198
1199         /* Harvest dirty bits from VPA and DTL updates */
1200         /* Note: we never modify the SLB shadow buffer areas */
1201         kvm_for_each_vcpu(i, vcpu, kvm) {
1202                 spin_lock(&vcpu->arch.vpa_update_lock);
1203                 harvest_vpa_dirty(&vcpu->arch.vpa, memslot, map);
1204                 harvest_vpa_dirty(&vcpu->arch.dtl, memslot, map);
1205                 spin_unlock(&vcpu->arch.vpa_update_lock);
1206         }
1207         preempt_enable();
1208         return 0;
1209 }
1210
1211 void *kvmppc_pin_guest_page(struct kvm *kvm, unsigned long gpa,
1212                             unsigned long *nb_ret)
1213 {
1214         struct kvm_memory_slot *memslot;
1215         unsigned long gfn = gpa >> PAGE_SHIFT;
1216         struct page *page, *pages[1];
1217         int npages;
1218         unsigned long hva, offset;
1219         unsigned long pa;
1220         unsigned long *physp;
1221         int srcu_idx;
1222
1223         srcu_idx = srcu_read_lock(&kvm->srcu);
1224         memslot = gfn_to_memslot(kvm, gfn);
1225         if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID))
1226                 goto err;
1227         if (!kvm->arch.using_mmu_notifiers) {
1228                 physp = memslot->arch.slot_phys;
1229                 if (!physp)
1230                         goto err;
1231                 physp += gfn - memslot->base_gfn;
1232                 pa = *physp;
1233                 if (!pa) {
1234                         if (kvmppc_get_guest_page(kvm, gfn, memslot,
1235                                                   PAGE_SIZE) < 0)
1236                                 goto err;
1237                         pa = *physp;
1238                 }
1239                 page = pfn_to_page(pa >> PAGE_SHIFT);
1240                 get_page(page);
1241         } else {
1242                 hva = gfn_to_hva_memslot(memslot, gfn);
1243                 npages = get_user_pages_fast(hva, 1, 1, pages);
1244                 if (npages < 1)
1245                         goto err;
1246                 page = pages[0];
1247         }
1248         srcu_read_unlock(&kvm->srcu, srcu_idx);
1249
1250         offset = gpa & (PAGE_SIZE - 1);
1251         if (nb_ret)
1252                 *nb_ret = PAGE_SIZE - offset;
1253         return page_address(page) + offset;
1254
1255  err:
1256         srcu_read_unlock(&kvm->srcu, srcu_idx);
1257         return NULL;
1258 }
1259
1260 void kvmppc_unpin_guest_page(struct kvm *kvm, void *va, unsigned long gpa,
1261                              bool dirty)
1262 {
1263         struct page *page = virt_to_page(va);
1264         struct kvm_memory_slot *memslot;
1265         unsigned long gfn;
1266         unsigned long *rmap;
1267         int srcu_idx;
1268
1269         put_page(page);
1270
1271         if (!dirty || !kvm->arch.using_mmu_notifiers)
1272                 return;
1273
1274         /* We need to mark this page dirty in the rmap chain */
1275         gfn = gpa >> PAGE_SHIFT;
1276         srcu_idx = srcu_read_lock(&kvm->srcu);
1277         memslot = gfn_to_memslot(kvm, gfn);
1278         if (memslot) {
1279                 rmap = &memslot->arch.rmap[gfn - memslot->base_gfn];
1280                 lock_rmap(rmap);
1281                 *rmap |= KVMPPC_RMAP_CHANGED;
1282                 unlock_rmap(rmap);
1283         }
1284         srcu_read_unlock(&kvm->srcu, srcu_idx);
1285 }
1286
1287 /*
1288  * Functions for reading and writing the hash table via reads and
1289  * writes on a file descriptor.
1290  *
1291  * Reads return the guest view of the hash table, which has to be
1292  * pieced together from the real hash table and the guest_rpte
1293  * values in the revmap array.
1294  *
1295  * On writes, each HPTE written is considered in turn, and if it
1296  * is valid, it is written to the HPT as if an H_ENTER with the
1297  * exact flag set was done.  When the invalid count is non-zero
1298  * in the header written to the stream, the kernel will make
1299  * sure that that many HPTEs are invalid, and invalidate them
1300  * if not.
1301  */
1302
1303 struct kvm_htab_ctx {
1304         unsigned long   index;
1305         unsigned long   flags;
1306         struct kvm      *kvm;
1307         int             first_pass;
1308 };
1309
1310 #define HPTE_SIZE       (2 * sizeof(unsigned long))
1311
1312 /*
1313  * Returns 1 if this HPT entry has been modified or has pending
1314  * R/C bit changes.
1315  */
1316 static int hpte_dirty(struct revmap_entry *revp, __be64 *hptp)
1317 {
1318         unsigned long rcbits_unset;
1319
1320         if (revp->guest_rpte & HPTE_GR_MODIFIED)
1321                 return 1;
1322
1323         /* Also need to consider changes in reference and changed bits */
1324         rcbits_unset = ~revp->guest_rpte & (HPTE_R_R | HPTE_R_C);
1325         if ((be64_to_cpu(hptp[0]) & HPTE_V_VALID) &&
1326             (be64_to_cpu(hptp[1]) & rcbits_unset))
1327                 return 1;
1328
1329         return 0;
1330 }
1331
1332 static long record_hpte(unsigned long flags, __be64 *hptp,
1333                         unsigned long *hpte, struct revmap_entry *revp,
1334                         int want_valid, int first_pass)
1335 {
1336         unsigned long v, r;
1337         unsigned long rcbits_unset;
1338         int ok = 1;
1339         int valid, dirty;
1340
1341         /* Unmodified entries are uninteresting except on the first pass */
1342         dirty = hpte_dirty(revp, hptp);
1343         if (!first_pass && !dirty)
1344                 return 0;
1345
1346         valid = 0;
1347         if (be64_to_cpu(hptp[0]) & (HPTE_V_VALID | HPTE_V_ABSENT)) {
1348                 valid = 1;
1349                 if ((flags & KVM_GET_HTAB_BOLTED_ONLY) &&
1350                     !(be64_to_cpu(hptp[0]) & HPTE_V_BOLTED))
1351                         valid = 0;
1352         }
1353         if (valid != want_valid)
1354                 return 0;
1355
1356         v = r = 0;
1357         if (valid || dirty) {
1358                 /* lock the HPTE so it's stable and read it */
1359                 preempt_disable();
1360                 while (!try_lock_hpte(hptp, HPTE_V_HVLOCK))
1361                         cpu_relax();
1362                 v = be64_to_cpu(hptp[0]);
1363
1364                 /* re-evaluate valid and dirty from synchronized HPTE value */
1365                 valid = !!(v & HPTE_V_VALID);
1366                 dirty = !!(revp->guest_rpte & HPTE_GR_MODIFIED);
1367
1368                 /* Harvest R and C into guest view if necessary */
1369                 rcbits_unset = ~revp->guest_rpte & (HPTE_R_R | HPTE_R_C);
1370                 if (valid && (rcbits_unset & be64_to_cpu(hptp[1]))) {
1371                         revp->guest_rpte |= (be64_to_cpu(hptp[1]) &
1372                                 (HPTE_R_R | HPTE_R_C)) | HPTE_GR_MODIFIED;
1373                         dirty = 1;
1374                 }
1375
1376                 if (v & HPTE_V_ABSENT) {
1377                         v &= ~HPTE_V_ABSENT;
1378                         v |= HPTE_V_VALID;
1379                         valid = 1;
1380                 }
1381                 if ((flags & KVM_GET_HTAB_BOLTED_ONLY) && !(v & HPTE_V_BOLTED))
1382                         valid = 0;
1383
1384                 r = revp->guest_rpte;
1385                 /* only clear modified if this is the right sort of entry */
1386                 if (valid == want_valid && dirty) {
1387                         r &= ~HPTE_GR_MODIFIED;
1388                         revp->guest_rpte = r;
1389                 }
1390                 asm volatile(PPC_RELEASE_BARRIER "" : : : "memory");
1391                 hptp[0] &= ~cpu_to_be64(HPTE_V_HVLOCK);
1392                 preempt_enable();
1393                 if (!(valid == want_valid && (first_pass || dirty)))
1394                         ok = 0;
1395         }
1396         hpte[0] = cpu_to_be64(v);
1397         hpte[1] = cpu_to_be64(r);
1398         return ok;
1399 }
1400
1401 static ssize_t kvm_htab_read(struct file *file, char __user *buf,
1402                              size_t count, loff_t *ppos)
1403 {
1404         struct kvm_htab_ctx *ctx = file->private_data;
1405         struct kvm *kvm = ctx->kvm;
1406         struct kvm_get_htab_header hdr;
1407         __be64 *hptp;
1408         struct revmap_entry *revp;
1409         unsigned long i, nb, nw;
1410         unsigned long __user *lbuf;
1411         struct kvm_get_htab_header __user *hptr;
1412         unsigned long flags;
1413         int first_pass;
1414         unsigned long hpte[2];
1415
1416         if (!access_ok(VERIFY_WRITE, buf, count))
1417                 return -EFAULT;
1418
1419         first_pass = ctx->first_pass;
1420         flags = ctx->flags;
1421
1422         i = ctx->index;
1423         hptp = (__be64 *)(kvm->arch.hpt_virt + (i * HPTE_SIZE));
1424         revp = kvm->arch.revmap + i;
1425         lbuf = (unsigned long __user *)buf;
1426
1427         nb = 0;
1428         while (nb + sizeof(hdr) + HPTE_SIZE < count) {
1429                 /* Initialize header */
1430                 hptr = (struct kvm_get_htab_header __user *)buf;
1431                 hdr.n_valid = 0;
1432                 hdr.n_invalid = 0;
1433                 nw = nb;
1434                 nb += sizeof(hdr);
1435                 lbuf = (unsigned long __user *)(buf + sizeof(hdr));
1436
1437                 /* Skip uninteresting entries, i.e. clean on not-first pass */
1438                 if (!first_pass) {
1439                         while (i < kvm->arch.hpt_npte &&
1440                                !hpte_dirty(revp, hptp)) {
1441                                 ++i;
1442                                 hptp += 2;
1443                                 ++revp;
1444                         }
1445                 }
1446                 hdr.index = i;
1447
1448                 /* Grab a series of valid entries */
1449                 while (i < kvm->arch.hpt_npte &&
1450                        hdr.n_valid < 0xffff &&
1451                        nb + HPTE_SIZE < count &&
1452                        record_hpte(flags, hptp, hpte, revp, 1, first_pass)) {
1453                         /* valid entry, write it out */
1454                         ++hdr.n_valid;
1455                         if (__put_user(hpte[0], lbuf) ||
1456                             __put_user(hpte[1], lbuf + 1))
1457                                 return -EFAULT;
1458                         nb += HPTE_SIZE;
1459                         lbuf += 2;
1460                         ++i;
1461                         hptp += 2;
1462                         ++revp;
1463                 }
1464                 /* Now skip invalid entries while we can */
1465                 while (i < kvm->arch.hpt_npte &&
1466                        hdr.n_invalid < 0xffff &&
1467                        record_hpte(flags, hptp, hpte, revp, 0, first_pass)) {
1468                         /* found an invalid entry */
1469                         ++hdr.n_invalid;
1470                         ++i;
1471                         hptp += 2;
1472                         ++revp;
1473                 }
1474
1475                 if (hdr.n_valid || hdr.n_invalid) {
1476                         /* write back the header */
1477                         if (__copy_to_user(hptr, &hdr, sizeof(hdr)))
1478                                 return -EFAULT;
1479                         nw = nb;
1480                         buf = (char __user *)lbuf;
1481                 } else {
1482                         nb = nw;
1483                 }
1484
1485                 /* Check if we've wrapped around the hash table */
1486                 if (i >= kvm->arch.hpt_npte) {
1487                         i = 0;
1488                         ctx->first_pass = 0;
1489                         break;
1490                 }
1491         }
1492
1493         ctx->index = i;
1494
1495         return nb;
1496 }
1497
1498 static ssize_t kvm_htab_write(struct file *file, const char __user *buf,
1499                               size_t count, loff_t *ppos)
1500 {
1501         struct kvm_htab_ctx *ctx = file->private_data;
1502         struct kvm *kvm = ctx->kvm;
1503         struct kvm_get_htab_header hdr;
1504         unsigned long i, j;
1505         unsigned long v, r;
1506         unsigned long __user *lbuf;
1507         __be64 *hptp;
1508         unsigned long tmp[2];
1509         ssize_t nb;
1510         long int err, ret;
1511         int rma_setup;
1512
1513         if (!access_ok(VERIFY_READ, buf, count))
1514                 return -EFAULT;
1515
1516         /* lock out vcpus from running while we're doing this */
1517         mutex_lock(&kvm->lock);
1518         rma_setup = kvm->arch.rma_setup_done;
1519         if (rma_setup) {
1520                 kvm->arch.rma_setup_done = 0;   /* temporarily */
1521                 /* order rma_setup_done vs. vcpus_running */
1522                 smp_mb();
1523                 if (atomic_read(&kvm->arch.vcpus_running)) {
1524                         kvm->arch.rma_setup_done = 1;
1525                         mutex_unlock(&kvm->lock);
1526                         return -EBUSY;
1527                 }
1528         }
1529
1530         err = 0;
1531         for (nb = 0; nb + sizeof(hdr) <= count; ) {
1532                 err = -EFAULT;
1533                 if (__copy_from_user(&hdr, buf, sizeof(hdr)))
1534                         break;
1535
1536                 err = 0;
1537                 if (nb + hdr.n_valid * HPTE_SIZE > count)
1538                         break;
1539
1540                 nb += sizeof(hdr);
1541                 buf += sizeof(hdr);
1542
1543                 err = -EINVAL;
1544                 i = hdr.index;
1545                 if (i >= kvm->arch.hpt_npte ||
1546                     i + hdr.n_valid + hdr.n_invalid > kvm->arch.hpt_npte)
1547                         break;
1548
1549                 hptp = (__be64 *)(kvm->arch.hpt_virt + (i * HPTE_SIZE));
1550                 lbuf = (unsigned long __user *)buf;
1551                 for (j = 0; j < hdr.n_valid; ++j) {
1552                         __be64 hpte_v;
1553                         __be64 hpte_r;
1554
1555                         err = -EFAULT;
1556                         if (__get_user(hpte_v, lbuf) ||
1557                             __get_user(hpte_r, lbuf + 1))
1558                                 goto out;
1559                         v = be64_to_cpu(hpte_v);
1560                         r = be64_to_cpu(hpte_r);
1561                         err = -EINVAL;
1562                         if (!(v & HPTE_V_VALID))
1563                                 goto out;
1564                         lbuf += 2;
1565                         nb += HPTE_SIZE;
1566
1567                         if (be64_to_cpu(hptp[0]) & (HPTE_V_VALID | HPTE_V_ABSENT))
1568                                 kvmppc_do_h_remove(kvm, 0, i, 0, tmp);
1569                         err = -EIO;
1570                         ret = kvmppc_virtmode_do_h_enter(kvm, H_EXACT, i, v, r,
1571                                                          tmp);
1572                         if (ret != H_SUCCESS) {
1573                                 pr_err("kvm_htab_write ret %ld i=%ld v=%lx "
1574                                        "r=%lx\n", ret, i, v, r);
1575                                 goto out;
1576                         }
1577                         if (!rma_setup && is_vrma_hpte(v)) {
1578                                 unsigned long psize = hpte_base_page_size(v, r);
1579                                 unsigned long senc = slb_pgsize_encoding(psize);
1580                                 unsigned long lpcr;
1581
1582                                 kvm->arch.vrma_slb_v = senc | SLB_VSID_B_1T |
1583                                         (VRMA_VSID << SLB_VSID_SHIFT_1T);
1584                                 lpcr = senc << (LPCR_VRMASD_SH - 4);
1585                                 kvmppc_update_lpcr(kvm, lpcr, LPCR_VRMASD);
1586                                 rma_setup = 1;
1587                         }
1588                         ++i;
1589                         hptp += 2;
1590                 }
1591
1592                 for (j = 0; j < hdr.n_invalid; ++j) {
1593                         if (be64_to_cpu(hptp[0]) & (HPTE_V_VALID | HPTE_V_ABSENT))
1594                                 kvmppc_do_h_remove(kvm, 0, i, 0, tmp);
1595                         ++i;
1596                         hptp += 2;
1597                 }
1598                 err = 0;
1599         }
1600
1601  out:
1602         /* Order HPTE updates vs. rma_setup_done */
1603         smp_wmb();
1604         kvm->arch.rma_setup_done = rma_setup;
1605         mutex_unlock(&kvm->lock);
1606
1607         if (err)
1608                 return err;
1609         return nb;
1610 }
1611
1612 static int kvm_htab_release(struct inode *inode, struct file *filp)
1613 {
1614         struct kvm_htab_ctx *ctx = filp->private_data;
1615
1616         filp->private_data = NULL;
1617         if (!(ctx->flags & KVM_GET_HTAB_WRITE))
1618                 atomic_dec(&ctx->kvm->arch.hpte_mod_interest);
1619         kvm_put_kvm(ctx->kvm);
1620         kfree(ctx);
1621         return 0;
1622 }
1623
1624 static const struct file_operations kvm_htab_fops = {
1625         .read           = kvm_htab_read,
1626         .write          = kvm_htab_write,
1627         .llseek         = default_llseek,
1628         .release        = kvm_htab_release,
1629 };
1630
1631 int kvm_vm_ioctl_get_htab_fd(struct kvm *kvm, struct kvm_get_htab_fd *ghf)
1632 {
1633         int ret;
1634         struct kvm_htab_ctx *ctx;
1635         int rwflag;
1636
1637         /* reject flags we don't recognize */
1638         if (ghf->flags & ~(KVM_GET_HTAB_BOLTED_ONLY | KVM_GET_HTAB_WRITE))
1639                 return -EINVAL;
1640         ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
1641         if (!ctx)
1642                 return -ENOMEM;
1643         kvm_get_kvm(kvm);
1644         ctx->kvm = kvm;
1645         ctx->index = ghf->start_index;
1646         ctx->flags = ghf->flags;
1647         ctx->first_pass = 1;
1648
1649         rwflag = (ghf->flags & KVM_GET_HTAB_WRITE) ? O_WRONLY : O_RDONLY;
1650         ret = anon_inode_getfd("kvm-htab", &kvm_htab_fops, ctx, rwflag | O_CLOEXEC);
1651         if (ret < 0) {
1652                 kvm_put_kvm(kvm);
1653                 return ret;
1654         }
1655
1656         if (rwflag == O_RDONLY) {
1657                 mutex_lock(&kvm->slots_lock);
1658                 atomic_inc(&kvm->arch.hpte_mod_interest);
1659                 /* make sure kvmppc_do_h_enter etc. see the increment */
1660                 synchronize_srcu_expedited(&kvm->srcu);
1661                 mutex_unlock(&kvm->slots_lock);
1662         }
1663
1664         return ret;
1665 }
1666
1667 void kvmppc_mmu_book3s_hv_init(struct kvm_vcpu *vcpu)
1668 {
1669         struct kvmppc_mmu *mmu = &vcpu->arch.mmu;
1670
1671         if (cpu_has_feature(CPU_FTR_ARCH_206))
1672                 vcpu->arch.slb_nr = 32;         /* POWER7 */
1673         else
1674                 vcpu->arch.slb_nr = 64;
1675
1676         mmu->xlate = kvmppc_mmu_book3s_64_hv_xlate;
1677         mmu->reset_msr = kvmppc_mmu_book3s_64_hv_reset_msr;
1678
1679         vcpu->arch.hflags |= BOOK3S_HFLAG_SLB;
1680 }