arch/powerpc/kvm/book3s_hv.c

   1 /*
   2  * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
   3  * Copyright (C) 2009. SUSE Linux Products GmbH. All rights reserved.
   4  *
   5  * Authors:
   6  *    Paul Mackerras <paulus@au1.ibm.com>
   7  *    Alexander Graf <agraf@suse.de>
   8  *    Kevin Wolf <mail@kevin-wolf.de>
   9  *
  10  * Description: KVM functions specific to running on Book 3S
  11  * processors in hypervisor mode (specifically POWER7 and later).
  12  *
  13  * This file is derived from arch/powerpc/kvm/book3s.c,
  14  * by Alexander Graf <agraf@suse.de>.
  15  *
  16  * This program is free software; you can redistribute it and/or modify
  17  * it under the terms of the GNU General Public License, version 2, as
  18  * published by the Free Software Foundation.
  19  */
  20
  21 #include <linux/kvm_host.h>
  22 #include <linux/err.h>
  23 #include <linux/slab.h>
  24 #include <linux/preempt.h>
  25 #include <linux/sched.h>
  26 #include <linux/delay.h>
  27 #include <linux/export.h>
  28 #include <linux/fs.h>
  29 #include <linux/anon_inodes.h>
  30 #include <linux/cpumask.h>
  31 #include <linux/spinlock.h>
  32 #include <linux/page-flags.h>
  33 #include <linux/srcu.h>
  34
  35 #include <asm/reg.h>
  36 #include <asm/cputable.h>
  37 #include <asm/cacheflush.h>
  38 #include <asm/tlbflush.h>
  39 #include <asm/uaccess.h>
  40 #include <asm/io.h>
  41 #include <asm/kvm_ppc.h>
  42 #include <asm/kvm_book3s.h>
  43 #include <asm/mmu_context.h>
  44 #include <asm/lppaca.h>
  45 #include <asm/processor.h>
  46 #include <asm/cputhreads.h>
  47 #include <asm/page.h>
  48 #include <asm/hvcall.h>
  49 #include <asm/switch_to.h>
  50 #include <asm/smp.h>
  51 #include <linux/gfp.h>
  52 #include <linux/vmalloc.h>
  53 #include <linux/highmem.h>
  54 #include <linux/hugetlb.h>
  55
  56 /* #define EXIT_DEBUG */
  57 /* #define EXIT_DEBUG_SIMPLE */
  58 /* #define EXIT_DEBUG_INT */
  59
  60 /* Used to indicate that a guest page fault needs to be handled */
  61 #define RESUME_PAGE_FAULT       (RESUME_GUEST | RESUME_FLAG_ARCH1)
  62
  63 /* Used as a "null" value for timebase values */
  64 #define TB_NIL  (~(u64)0)
  65
  66 static void kvmppc_end_cede(struct kvm_vcpu *vcpu);
  67 static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu);
  68
  69 /*
  70  * We use the vcpu_load/put functions to measure stolen time.
  71  * Stolen time is counted as time when either the vcpu is able to
  72  * run as part of a virtual core, but the task running the vcore
  73  * is preempted or sleeping, or when the vcpu needs something done
  74  * in the kernel by the task running the vcpu, but that task is
  75  * preempted or sleeping.  Those two things have to be counted
  76  * separately, since one of the vcpu tasks will take on the job
  77  * of running the core, and the other vcpu tasks in the vcore will
  78  * sleep waiting for it to do that, but that sleep shouldn't count
  79  * as stolen time.
  80  *
  81  * Hence we accumulate stolen time when the vcpu can run as part of
  82  * a vcore using vc->stolen_tb, and the stolen time when the vcpu
  83  * needs its task to do other things in the kernel (for example,
  84  * service a page fault) in busy_stolen.  We don't accumulate
  85  * stolen time for a vcore when it is inactive, or for a vcpu
  86  * when it is in state RUNNING or NOTREADY.  NOTREADY is a bit of
  87  * a misnomer; it means that the vcpu task is not executing in
  88  * the KVM_VCPU_RUN ioctl, i.e. it is in userspace or elsewhere in
  89  * the kernel.  We don't have any way of dividing up that time
  90  * between time that the vcpu is genuinely stopped, time that
  91  * the task is actively working on behalf of the vcpu, and time
  92  * that the task is preempted, so we don't count any of it as
  93  * stolen.
  94  *
  95  * Updates to busy_stolen are protected by arch.tbacct_lock;
  96  * updates to vc->stolen_tb are protected by the arch.tbacct_lock
  97  * of the vcpu that has taken responsibility for running the vcore
  98  * (i.e. vc->runner).  The stolen times are measured in units of
  99  * timebase ticks.  (Note that the != TB_NIL checks below are
 100  * purely defensive; they should never fail.)
 101  */
 102
 103 void kvmppc_core_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 104 {
 105         struct kvmppc_vcore *vc = vcpu->arch.vcore;
 106
 107         spin_lock(&vcpu->arch.tbacct_lock);
 108         if (vc->runner == vcpu && vc->vcore_state != VCORE_INACTIVE &&
 109             vc->preempt_tb != TB_NIL) {
 110                 vc->stolen_tb += mftb() - vc->preempt_tb;
 111                 vc->preempt_tb = TB_NIL;
 112         }
 113         if (vcpu->arch.state == KVMPPC_VCPU_BUSY_IN_HOST &&
 114             vcpu->arch.busy_preempt != TB_NIL) {
 115                 vcpu->arch.busy_stolen += mftb() - vcpu->arch.busy_preempt;
 116                 vcpu->arch.busy_preempt = TB_NIL;
 117         }
 118         spin_unlock(&vcpu->arch.tbacct_lock);
 119 }
 120
 121 void kvmppc_core_vcpu_put(struct kvm_vcpu *vcpu)
 122 {
 123         struct kvmppc_vcore *vc = vcpu->arch.vcore;
 124
 125         spin_lock(&vcpu->arch.tbacct_lock);
 126         if (vc->runner == vcpu && vc->vcore_state != VCORE_INACTIVE)
 127                 vc->preempt_tb = mftb();
 128         if (vcpu->arch.state == KVMPPC_VCPU_BUSY_IN_HOST)
 129                 vcpu->arch.busy_preempt = mftb();
 130         spin_unlock(&vcpu->arch.tbacct_lock);
 131 }
 132
 133 void kvmppc_set_msr(struct kvm_vcpu *vcpu, u64 msr)
 134 {
 135         vcpu->arch.shregs.msr = msr;
 136         kvmppc_end_cede(vcpu);
 137 }
 138
 139 void kvmppc_set_pvr(struct kvm_vcpu *vcpu, u32 pvr)
 140 {
 141         vcpu->arch.pvr = pvr;
 142 }
 143
 144 void kvmppc_dump_regs(struct kvm_vcpu *vcpu)
 145 {
 146         int r;
 147
 148         pr_err("vcpu %p (%d):\n", vcpu, vcpu->vcpu_id);
 149         pr_err("pc  = %.16lx  msr = %.16llx  trap = %x\n",
 150                vcpu->arch.pc, vcpu->arch.shregs.msr, vcpu->arch.trap);
 151         for (r = 0; r < 16; ++r)
 152                 pr_err("r%2d = %.16lx  r%d = %.16lx\n",
 153                        r, kvmppc_get_gpr(vcpu, r),
 154                        r+16, kvmppc_get_gpr(vcpu, r+16));
 155         pr_err("ctr = %.16lx  lr  = %.16lx\n",
 156                vcpu->arch.ctr, vcpu->arch.lr);
 157         pr_err("srr0 = %.16llx srr1 = %.16llx\n",
 158                vcpu->arch.shregs.srr0, vcpu->arch.shregs.srr1);
 159         pr_err("sprg0 = %.16llx sprg1 = %.16llx\n",
 160                vcpu->arch.shregs.sprg0, vcpu->arch.shregs.sprg1);
 161         pr_err("sprg2 = %.16llx sprg3 = %.16llx\n",
 162                vcpu->arch.shregs.sprg2, vcpu->arch.shregs.sprg3);
 163         pr_err("cr = %.8x  xer = %.16lx  dsisr = %.8x\n",
 164                vcpu->arch.cr, vcpu->arch.xer, vcpu->arch.shregs.dsisr);
 165         pr_err("dar = %.16llx\n", vcpu->arch.shregs.dar);
 166         pr_err("fault dar = %.16lx dsisr = %.8x\n",
 167                vcpu->arch.fault_dar, vcpu->arch.fault_dsisr);
 168         pr_err("SLB (%d entries):\n", vcpu->arch.slb_max);
 169         for (r = 0; r < vcpu->arch.slb_max; ++r)
 170                 pr_err("  ESID = %.16llx VSID = %.16llx\n",
 171                        vcpu->arch.slb[r].orige, vcpu->arch.slb[r].origv);
 172         pr_err("lpcr = %.16lx sdr1 = %.16lx last_inst = %.8x\n",
 173                vcpu->kvm->arch.lpcr, vcpu->kvm->arch.sdr1,
 174                vcpu->arch.last_inst);
 175 }
 176
 177 struct kvm_vcpu *kvmppc_find_vcpu(struct kvm *kvm, int id)
 178 {
 179         int r;
 180         struct kvm_vcpu *v, *ret = NULL;
 181
 182         mutex_lock(&kvm->lock);
 183         kvm_for_each_vcpu(r, v, kvm) {
 184                 if (v->vcpu_id == id) {
 185                         ret = v;
 186                         break;
 187                 }
 188         }
 189         mutex_unlock(&kvm->lock);
 190         return ret;
 191 }
 192
 193 static void init_vpa(struct kvm_vcpu *vcpu, struct lppaca *vpa)
 194 {
 195         vpa->shared_proc = 1;
 196         vpa->yield_count = 1;
 197 }
 198
 199 static int set_vpa(struct kvm_vcpu *vcpu, struct kvmppc_vpa *v,
 200                    unsigned long addr, unsigned long len)
 201 {
 202         /* check address is cacheline aligned */
 203         if (addr & (L1_CACHE_BYTES - 1))
 204                 return -EINVAL;
 205         spin_lock(&vcpu->arch.vpa_update_lock);
 206         if (v->next_gpa != addr || v->len != len) {
 207                 v->next_gpa = addr;
 208                 v->len = addr ? len : 0;
 209                 v->update_pending = 1;
 210         }
 211         spin_unlock(&vcpu->arch.vpa_update_lock);
 212         return 0;
 213 }
 214
 215 /* Length for a per-processor buffer is passed in at offset 4 in the buffer */
 216 struct reg_vpa {
 217         u32 dummy;
 218         union {
 219                 u16 hword;
 220                 u32 word;
 221         } length;
 222 };
 223
 224 static int vpa_is_registered(struct kvmppc_vpa *vpap)
 225 {
 226         if (vpap->update_pending)
 227                 return vpap->next_gpa != 0;
 228         return vpap->pinned_addr != NULL;
 229 }
 230
 231 static unsigned long do_h_register_vpa(struct kvm_vcpu *vcpu,
 232                                        unsigned long flags,
 233                                        unsigned long vcpuid, unsigned long vpa)
 234 {
 235         struct kvm *kvm = vcpu->kvm;
 236         unsigned long len, nb;
 237         void *va;
 238         struct kvm_vcpu *tvcpu;
 239         int err;
 240         int subfunc;
 241         struct kvmppc_vpa *vpap;
 242
 243         tvcpu = kvmppc_find_vcpu(kvm, vcpuid);
 244         if (!tvcpu)
 245                 return H_PARAMETER;
 246
 247         subfunc = (flags >> H_VPA_FUNC_SHIFT) & H_VPA_FUNC_MASK;
 248         if (subfunc == H_VPA_REG_VPA || subfunc == H_VPA_REG_DTL ||
 249             subfunc == H_VPA_REG_SLB) {
 250                 /* Registering new area - address must be cache-line aligned */
 251                 if ((vpa & (L1_CACHE_BYTES - 1)) || !vpa)
 252                         return H_PARAMETER;
 253
 254                 /* convert logical addr to kernel addr and read length */
 255                 va = kvmppc_pin_guest_page(kvm, vpa, &nb);
 256                 if (va == NULL)
 257                         return H_PARAMETER;
 258                 if (subfunc == H_VPA_REG_VPA)
 259                         len = ((struct reg_vpa *)va)->length.hword;
 260                 else
 261                         len = ((struct reg_vpa *)va)->length.word;
 262                 kvmppc_unpin_guest_page(kvm, va);
 263
 264                 /* Check length */
 265                 if (len > nb || len < sizeof(struct reg_vpa))
 266                         return H_PARAMETER;
 267         } else {
 268                 vpa = 0;
 269                 len = 0;
 270         }
 271
 272         err = H_PARAMETER;
 273         vpap = NULL;
 274         spin_lock(&tvcpu->arch.vpa_update_lock);
 275
 276         switch (subfunc) {
 277         case H_VPA_REG_VPA:             /* register VPA */
 278                 if (len < sizeof(struct lppaca))
 279                         break;
 280                 vpap = &tvcpu->arch.vpa;
 281                 err = 0;
 282                 break;
 283
 284         case H_VPA_REG_DTL:             /* register DTL */
 285                 if (len < sizeof(struct dtl_entry))
 286                         break;
 287                 len -= len % sizeof(struct dtl_entry);
 288
 289                 /* Check that they have previously registered a VPA */
 290                 err = H_RESOURCE;
 291                 if (!vpa_is_registered(&tvcpu->arch.vpa))
 292                         break;
 293
 294                 vpap = &tvcpu->arch.dtl;
 295                 err = 0;
 296                 break;
 297
 298         case H_VPA_REG_SLB:             /* register SLB shadow buffer */
 299                 /* Check that they have previously registered a VPA */
 300                 err = H_RESOURCE;
 301                 if (!vpa_is_registered(&tvcpu->arch.vpa))
 302                         break;
 303
 304                 vpap = &tvcpu->arch.slb_shadow;
 305                 err = 0;
 306                 break;
 307
 308         case H_VPA_DEREG_VPA:           /* deregister VPA */
 309                 /* Check they don't still have a DTL or SLB buf registered */
 310                 err = H_RESOURCE;
 311                 if (vpa_is_registered(&tvcpu->arch.dtl) ||
 312                     vpa_is_registered(&tvcpu->arch.slb_shadow))
 313                         break;
 314
 315                 vpap = &tvcpu->arch.vpa;
 316                 err = 0;
 317                 break;
 318
 319         case H_VPA_DEREG_DTL:           /* deregister DTL */
 320                 vpap = &tvcpu->arch.dtl;
 321                 err = 0;
 322                 break;
 323
 324         case H_VPA_DEREG_SLB:           /* deregister SLB shadow buffer */
 325                 vpap = &tvcpu->arch.slb_shadow;
 326                 err = 0;
 327                 break;
 328         }
 329
 330         if (vpap) {
 331                 vpap->next_gpa = vpa;
 332                 vpap->len = len;
 333                 vpap->update_pending = 1;
 334         }
 335
 336         spin_unlock(&tvcpu->arch.vpa_update_lock);
 337
 338         return err;
 339 }
 340
 341 static void kvmppc_update_vpa(struct kvm_vcpu *vcpu, struct kvmppc_vpa *vpap)
 342 {
 343         struct kvm *kvm = vcpu->kvm;
 344         void *va;
 345         unsigned long nb;
 346         unsigned long gpa;
 347
 348         /*
 349          * We need to pin the page pointed to by vpap->next_gpa,
 350          * but we can't call kvmppc_pin_guest_page under the lock
 351          * as it does get_user_pages() and down_read().  So we
 352          * have to drop the lock, pin the page, then get the lock
 353          * again and check that a new area didn't get registered
 354          * in the meantime.
 355          */
 356         for (;;) {
 357                 gpa = vpap->next_gpa;
 358                 spin_unlock(&vcpu->arch.vpa_update_lock);
 359                 va = NULL;
 360                 nb = 0;
 361                 if (gpa)
 362                         va = kvmppc_pin_guest_page(kvm, vpap->next_gpa, &nb);
 363                 spin_lock(&vcpu->arch.vpa_update_lock);
 364                 if (gpa == vpap->next_gpa)
 365                         break;
 366                 /* sigh... unpin that one and try again */
 367                 if (va)
 368                         kvmppc_unpin_guest_page(kvm, va);
 369         }
 370
 371         vpap->update_pending = 0;
 372         if (va && nb < vpap->len) {
 373                 /*
 374                  * If it's now too short, it must be that userspace
 375                  * has changed the mappings underlying guest memory,
 376                  * so unregister the region.
 377                  */
 378                 kvmppc_unpin_guest_page(kvm, va);
 379                 va = NULL;
 380         }
 381         if (vpap->pinned_addr)
 382                 kvmppc_unpin_guest_page(kvm, vpap->pinned_addr);
 383         vpap->pinned_addr = va;
 384         if (va)
 385                 vpap->pinned_end = va + vpap->len;
 386 }
 387
 388 static void kvmppc_update_vpas(struct kvm_vcpu *vcpu)
 389 {
 390         if (!(vcpu->arch.vpa.update_pending ||
 391               vcpu->arch.slb_shadow.update_pending ||
 392               vcpu->arch.dtl.update_pending))
 393                 return;
 394
 395         spin_lock(&vcpu->arch.vpa_update_lock);
 396         if (vcpu->arch.vpa.update_pending) {
 397                 kvmppc_update_vpa(vcpu, &vcpu->arch.vpa);
 398                 if (vcpu->arch.vpa.pinned_addr)
 399                         init_vpa(vcpu, vcpu->arch.vpa.pinned_addr);
 400         }
 401         if (vcpu->arch.dtl.update_pending) {
 402                 kvmppc_update_vpa(vcpu, &vcpu->arch.dtl);
 403                 vcpu->arch.dtl_ptr = vcpu->arch.dtl.pinned_addr;
 404                 vcpu->arch.dtl_index = 0;
 405         }
 406         if (vcpu->arch.slb_shadow.update_pending)
 407                 kvmppc_update_vpa(vcpu, &vcpu->arch.slb_shadow);
 408         spin_unlock(&vcpu->arch.vpa_update_lock);
 409 }
 410
 411 /*
 412  * Return the accumulated stolen time for the vcore up until `now'.
 413  * The caller should hold the vcore lock.
 414  */
 415 static u64 vcore_stolen_time(struct kvmppc_vcore *vc, u64 now)
 416 {
 417         u64 p;
 418
 419         /*
 420          * If we are the task running the vcore, then since we hold
 421          * the vcore lock, we can't be preempted, so stolen_tb/preempt_tb
 422          * can't be updated, so we don't need the tbacct_lock.
 423          * If the vcore is inactive, it can't become active (since we
 424          * hold the vcore lock), so the vcpu load/put functions won't
 425          * update stolen_tb/preempt_tb, and we don't need tbacct_lock.
 426          */
 427         if (vc->vcore_state != VCORE_INACTIVE &&
 428             vc->runner->arch.run_task != current) {
 429                 spin_lock(&vc->runner->arch.tbacct_lock);
 430                 p = vc->stolen_tb;
 431                 if (vc->preempt_tb != TB_NIL)
 432                         p += now - vc->preempt_tb;
 433                 spin_unlock(&vc->runner->arch.tbacct_lock);
 434         } else {
 435                 p = vc->stolen_tb;
 436         }
 437         return p;
 438 }
 439
 440 static void kvmppc_create_dtl_entry(struct kvm_vcpu *vcpu,
 441                                     struct kvmppc_vcore *vc)
 442 {
 443         struct dtl_entry *dt;
 444         struct lppaca *vpa;
 445         unsigned long stolen;
 446         unsigned long core_stolen;
 447         u64 now;
 448
 449         dt = vcpu->arch.dtl_ptr;
 450         vpa = vcpu->arch.vpa.pinned_addr;
 451         now = mftb();
 452         core_stolen = vcore_stolen_time(vc, now);
 453         stolen = core_stolen - vcpu->arch.stolen_logged;
 454         vcpu->arch.stolen_logged = core_stolen;
 455         spin_lock(&vcpu->arch.tbacct_lock);
 456         stolen += vcpu->arch.busy_stolen;
 457         vcpu->arch.busy_stolen = 0;
 458         spin_unlock(&vcpu->arch.tbacct_lock);
 459         if (!dt || !vpa)
 460                 return;
 461         memset(dt, 0, sizeof(struct dtl_entry));
 462         dt->dispatch_reason = 7;
 463         dt->processor_id = vc->pcpu + vcpu->arch.ptid;
 464         dt->timebase = now;
 465         dt->enqueue_to_dispatch_time = stolen;
 466         dt->srr0 = kvmppc_get_pc(vcpu);
 467         dt->srr1 = vcpu->arch.shregs.msr;
 468         ++dt;
 469         if (dt == vcpu->arch.dtl.pinned_end)
 470                 dt = vcpu->arch.dtl.pinned_addr;
 471         vcpu->arch.dtl_ptr = dt;
 472         /* order writing *dt vs. writing vpa->dtl_idx */
 473         smp_wmb();
 474         vpa->dtl_idx = ++vcpu->arch.dtl_index;
 475 }
 476
 477 int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu)
 478 {
 479         unsigned long req = kvmppc_get_gpr(vcpu, 3);
 480         unsigned long target, ret = H_SUCCESS;
 481         struct kvm_vcpu *tvcpu;
 482         int idx;
 483
 484         switch (req) {
 485         case H_ENTER:
 486                 idx = srcu_read_lock(&vcpu->kvm->srcu);
 487                 ret = kvmppc_virtmode_h_enter(vcpu, kvmppc_get_gpr(vcpu, 4),
 488                                               kvmppc_get_gpr(vcpu, 5),
 489                                               kvmppc_get_gpr(vcpu, 6),
 490                                               kvmppc_get_gpr(vcpu, 7));
 491                 srcu_read_unlock(&vcpu->kvm->srcu, idx);
 492                 break;
 493         case H_CEDE:
 494                 break;
 495         case H_PROD:
 496                 target = kvmppc_get_gpr(vcpu, 4);
 497                 tvcpu = kvmppc_find_vcpu(vcpu->kvm, target);
 498                 if (!tvcpu) {
 499                         ret = H_PARAMETER;
 500                         break;
 501                 }
 502                 tvcpu->arch.prodded = 1;
 503                 smp_mb();
 504                 if (vcpu->arch.ceded) {
 505                         if (waitqueue_active(&vcpu->wq)) {
 506                                 wake_up_interruptible(&vcpu->wq);
 507                                 vcpu->stat.halt_wakeup++;
 508                         }
 509                 }
 510                 break;
 511         case H_CONFER:
 512                 break;
 513         case H_REGISTER_VPA:
 514                 ret = do_h_register_vpa(vcpu, kvmppc_get_gpr(vcpu, 4),
 515                                         kvmppc_get_gpr(vcpu, 5),
 516                                         kvmppc_get_gpr(vcpu, 6));
 517                 break;
 518         default:
 519                 return RESUME_HOST;
 520         }
 521         kvmppc_set_gpr(vcpu, 3, ret);
 522         vcpu->arch.hcall_needed = 0;
 523         return RESUME_GUEST;
 524 }
 525
 526 static int kvmppc_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu,
 527                               struct task_struct *tsk)
 528 {
 529         int r = RESUME_HOST;
 530
 531         vcpu->stat.sum_exits++;
 532
 533         run->exit_reason = KVM_EXIT_UNKNOWN;
 534         run->ready_for_interrupt_injection = 1;
 535         switch (vcpu->arch.trap) {
 536         /* We're good on these - the host merely wanted to get our attention */
 537         case BOOK3S_INTERRUPT_HV_DECREMENTER:
 538                 vcpu->stat.dec_exits++;
 539                 r = RESUME_GUEST;
 540                 break;
 541         case BOOK3S_INTERRUPT_EXTERNAL:
 542                 vcpu->stat.ext_intr_exits++;
 543                 r = RESUME_GUEST;
 544                 break;
 545         case BOOK3S_INTERRUPT_PERFMON:
 546                 r = RESUME_GUEST;
 547                 break;
 548         case BOOK3S_INTERRUPT_PROGRAM:
 549         {
 550                 ulong flags;
 551                 /*
 552                  * Normally program interrupts are delivered directly
 553                  * to the guest by the hardware, but we can get here
 554                  * as a result of a hypervisor emulation interrupt
 555                  * (e40) getting turned into a 700 by BML RTAS.
 556                  */
 557                 flags = vcpu->arch.shregs.msr & 0x1f0000ull;
 558                 kvmppc_core_queue_program(vcpu, flags);
 559                 r = RESUME_GUEST;
 560                 break;
 561         }
 562         case BOOK3S_INTERRUPT_SYSCALL:
 563         {
 564                 /* hcall - punt to userspace */
 565                 int i;
 566
 567                 if (vcpu->arch.shregs.msr & MSR_PR) {
 568                         /* sc 1 from userspace - reflect to guest syscall */
 569                         kvmppc_book3s_queue_irqprio(vcpu, BOOK3S_INTERRUPT_SYSCALL);
 570                         r = RESUME_GUEST;
 571                         break;
 572                 }
 573                 run->papr_hcall.nr = kvmppc_get_gpr(vcpu, 3);
 574                 for (i = 0; i < 9; ++i)
 575                         run->papr_hcall.args[i] = kvmppc_get_gpr(vcpu, 4 + i);
 576                 run->exit_reason = KVM_EXIT_PAPR_HCALL;
 577                 vcpu->arch.hcall_needed = 1;
 578                 r = RESUME_HOST;
 579                 break;
 580         }
 581         /*
 582          * We get these next two if the guest accesses a page which it thinks
 583          * it has mapped but which is not actually present, either because
 584          * it is for an emulated I/O device or because the corresonding
 585          * host page has been paged out.  Any other HDSI/HISI interrupts
 586          * have been handled already.
 587          */
 588         case BOOK3S_INTERRUPT_H_DATA_STORAGE:
 589                 r = RESUME_PAGE_FAULT;
 590                 break;
 591         case BOOK3S_INTERRUPT_H_INST_STORAGE:
 592                 vcpu->arch.fault_dar = kvmppc_get_pc(vcpu);
 593                 vcpu->arch.fault_dsisr = 0;
 594                 r = RESUME_PAGE_FAULT;
 595                 break;
 596         /*
 597          * This occurs if the guest executes an illegal instruction.
 598          * We just generate a program interrupt to the guest, since
 599          * we don't emulate any guest instructions at this stage.
 600          */
 601         case BOOK3S_INTERRUPT_H_EMUL_ASSIST:
 602                 kvmppc_core_queue_program(vcpu, 0x80000);
 603                 r = RESUME_GUEST;
 604                 break;
 605         default:
 606                 kvmppc_dump_regs(vcpu);
 607                 printk(KERN_EMERG "trap=0x%x | pc=0x%lx | msr=0x%llx\n",
 608                         vcpu->arch.trap, kvmppc_get_pc(vcpu),
 609                         vcpu->arch.shregs.msr);
 610                 r = RESUME_HOST;
 611                 BUG();
 612                 break;
 613         }
 614
 615         return r;
 616 }
 617
 618 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
 619                                   struct kvm_sregs *sregs)
 620 {
 621         int i;
 622
 623         sregs->pvr = vcpu->arch.pvr;
 624
 625         memset(sregs, 0, sizeof(struct kvm_sregs));
 626         for (i = 0; i < vcpu->arch.slb_max; i++) {
 627                 sregs->u.s.ppc64.slb[i].slbe = vcpu->arch.slb[i].orige;
 628                 sregs->u.s.ppc64.slb[i].slbv = vcpu->arch.slb[i].origv;
 629         }
 630
 631         return 0;
 632 }
 633
 634 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
 635                                   struct kvm_sregs *sregs)
 636 {
 637         int i, j;
 638
 639         kvmppc_set_pvr(vcpu, sregs->pvr);
 640
 641         j = 0;
 642         for (i = 0; i < vcpu->arch.slb_nr; i++) {
 643                 if (sregs->u.s.ppc64.slb[i].slbe & SLB_ESID_V) {
 644                         vcpu->arch.slb[j].orige = sregs->u.s.ppc64.slb[i].slbe;
 645                         vcpu->arch.slb[j].origv = sregs->u.s.ppc64.slb[i].slbv;
 646                         ++j;
 647                 }
 648         }
 649         vcpu->arch.slb_max = j;
 650
 651         return 0;
 652 }
 653
 654 int kvmppc_get_one_reg(struct kvm_vcpu *vcpu, u64 id, union kvmppc_one_reg *val)
 655 {
 656         int r = 0;
 657         long int i;
 658
 659         switch (id) {
 660         case KVM_REG_PPC_HIOR:
 661                 *val = get_reg_val(id, 0);
 662                 break;
 663         case KVM_REG_PPC_DABR:
 664                 *val = get_reg_val(id, vcpu->arch.dabr);
 665                 break;
 666         case KVM_REG_PPC_DSCR:
 667                 *val = get_reg_val(id, vcpu->arch.dscr);
 668                 break;
 669         case KVM_REG_PPC_PURR:
 670                 *val = get_reg_val(id, vcpu->arch.purr);
 671                 break;
 672         case KVM_REG_PPC_SPURR:
 673                 *val = get_reg_val(id, vcpu->arch.spurr);
 674                 break;
 675         case KVM_REG_PPC_AMR:
 676                 *val = get_reg_val(id, vcpu->arch.amr);
 677                 break;
 678         case KVM_REG_PPC_UAMOR:
 679                 *val = get_reg_val(id, vcpu->arch.uamor);
 680                 break;
 681         case KVM_REG_PPC_MMCR0 ... KVM_REG_PPC_MMCRA:
 682                 i = id - KVM_REG_PPC_MMCR0;
 683                 *val = get_reg_val(id, vcpu->arch.mmcr[i]);
 684                 break;
 685         case KVM_REG_PPC_PMC1 ... KVM_REG_PPC_PMC8:
 686                 i = id - KVM_REG_PPC_PMC1;
 687                 *val = get_reg_val(id, vcpu->arch.pmc[i]);
 688                 break;
 689 #ifdef CONFIG_VSX
 690         case KVM_REG_PPC_FPR0 ... KVM_REG_PPC_FPR31:
 691                 if (cpu_has_feature(CPU_FTR_VSX)) {
 692                         /* VSX => FP reg i is stored in arch.vsr[2*i] */
 693                         long int i = id - KVM_REG_PPC_FPR0;
 694                         *val = get_reg_val(id, vcpu->arch.vsr[2 * i]);
 695                 } else {
 696                         /* let generic code handle it */
 697                         r = -EINVAL;
 698                 }
 699                 break;
 700         case KVM_REG_PPC_VSR0 ... KVM_REG_PPC_VSR31:
 701                 if (cpu_has_feature(CPU_FTR_VSX)) {
 702                         long int i = id - KVM_REG_PPC_VSR0;
 703                         val->vsxval[0] = vcpu->arch.vsr[2 * i];
 704                         val->vsxval[1] = vcpu->arch.vsr[2 * i + 1];
 705                 } else {
 706                         r = -ENXIO;
 707                 }
 708                 break;
 709 #endif /* CONFIG_VSX */
 710         case KVM_REG_PPC_VPA_ADDR:
 711                 spin_lock(&vcpu->arch.vpa_update_lock);
 712                 *val = get_reg_val(id, vcpu->arch.vpa.next_gpa);
 713                 spin_unlock(&vcpu->arch.vpa_update_lock);
 714                 break;
 715         case KVM_REG_PPC_VPA_SLB:
 716                 spin_lock(&vcpu->arch.vpa_update_lock);
 717                 val->vpaval.addr = vcpu->arch.slb_shadow.next_gpa;
 718                 val->vpaval.length = vcpu->arch.slb_shadow.len;
 719                 spin_unlock(&vcpu->arch.vpa_update_lock);
 720                 break;
 721         case KVM_REG_PPC_VPA_DTL:
 722                 spin_lock(&vcpu->arch.vpa_update_lock);
 723                 val->vpaval.addr = vcpu->arch.dtl.next_gpa;
 724                 val->vpaval.length = vcpu->arch.dtl.len;
 725                 spin_unlock(&vcpu->arch.vpa_update_lock);
 726                 break;
 727         default:
 728                 r = -EINVAL;
 729                 break;
 730         }
 731
 732         return r;
 733 }
 734
 735 int kvmppc_set_one_reg(struct kvm_vcpu *vcpu, u64 id, union kvmppc_one_reg *val)
 736 {
 737         int r = 0;
 738         long int i;
 739         unsigned long addr, len;
 740
 741         switch (id) {
 742         case KVM_REG_PPC_HIOR:
 743                 /* Only allow this to be set to zero */
 744                 if (set_reg_val(id, *val))
 745                         r = -EINVAL;
 746                 break;
 747         case KVM_REG_PPC_DABR:
 748                 vcpu->arch.dabr = set_reg_val(id, *val);
 749                 break;
 750         case KVM_REG_PPC_DSCR:
 751                 vcpu->arch.dscr = set_reg_val(id, *val);
 752                 break;
 753         case KVM_REG_PPC_PURR:
 754                 vcpu->arch.purr = set_reg_val(id, *val);
 755                 break;
 756         case KVM_REG_PPC_SPURR:
 757                 vcpu->arch.spurr = set_reg_val(id, *val);
 758                 break;
 759         case KVM_REG_PPC_AMR:
 760                 vcpu->arch.amr = set_reg_val(id, *val);
 761                 break;
 762         case KVM_REG_PPC_UAMOR:
 763                 vcpu->arch.uamor = set_reg_val(id, *val);
 764                 break;
 765         case KVM_REG_PPC_MMCR0 ... KVM_REG_PPC_MMCRA:
 766                 i = id - KVM_REG_PPC_MMCR0;
 767                 vcpu->arch.mmcr[i] = set_reg_val(id, *val);
 768                 break;
 769         case KVM_REG_PPC_PMC1 ... KVM_REG_PPC_PMC8:
 770                 i = id - KVM_REG_PPC_PMC1;
 771                 vcpu->arch.pmc[i] = set_reg_val(id, *val);
 772                 break;
 773 #ifdef CONFIG_VSX
 774         case KVM_REG_PPC_FPR0 ... KVM_REG_PPC_FPR31:
 775                 if (cpu_has_feature(CPU_FTR_VSX)) {
 776                         /* VSX => FP reg i is stored in arch.vsr[2*i] */
 777                         long int i = id - KVM_REG_PPC_FPR0;
 778                         vcpu->arch.vsr[2 * i] = set_reg_val(id, *val);
 779                 } else {
 780                         /* let generic code handle it */
 781                         r = -EINVAL;
 782                 }
 783                 break;
 784         case KVM_REG_PPC_VSR0 ... KVM_REG_PPC_VSR31:
 785                 if (cpu_has_feature(CPU_FTR_VSX)) {
 786                         long int i = id - KVM_REG_PPC_VSR0;
 787                         vcpu->arch.vsr[2 * i] = val->vsxval[0];
 788                         vcpu->arch.vsr[2 * i + 1] = val->vsxval[1];
 789                 } else {
 790                         r = -ENXIO;
 791                 }
 792                 break;
 793 #endif /* CONFIG_VSX */
 794         case KVM_REG_PPC_VPA_ADDR:
 795                 addr = set_reg_val(id, *val);
 796                 r = -EINVAL;
 797                 if (!addr && (vcpu->arch.slb_shadow.next_gpa ||
 798                               vcpu->arch.dtl.next_gpa))
 799                         break;
 800                 r = set_vpa(vcpu, &vcpu->arch.vpa, addr, sizeof(struct lppaca));
 801                 break;
 802         case KVM_REG_PPC_VPA_SLB:
 803                 addr = val->vpaval.addr;
 804                 len = val->vpaval.length;
 805                 r = -EINVAL;
 806                 if (addr && !vcpu->arch.vpa.next_gpa)
 807                         break;
 808                 r = set_vpa(vcpu, &vcpu->arch.slb_shadow, addr, len);
 809                 break;
 810         case KVM_REG_PPC_VPA_DTL:
 811                 addr = val->vpaval.addr;
 812                 len = val->vpaval.length;
 813                 r = -EINVAL;
 814                 if (len < sizeof(struct dtl_entry))
 815                         break;
 816                 if (addr && !vcpu->arch.vpa.next_gpa)
 817                         break;
 818                 len -= len % sizeof(struct dtl_entry);
 819                 r = set_vpa(vcpu, &vcpu->arch.dtl, addr, len);
 820                 break;
 821         default:
 822                 r = -EINVAL;
 823                 break;
 824         }
 825
 826         return r;
 827 }
 828
 829 int kvmppc_core_check_processor_compat(void)
 830 {
 831         if (cpu_has_feature(CPU_FTR_HVMODE))
 832                 return 0;
 833         return -EIO;
 834 }
 835
 836 struct kvm_vcpu *kvmppc_core_vcpu_create(struct kvm *kvm, unsigned int id)
 837 {
 838         struct kvm_vcpu *vcpu;
 839         int err = -EINVAL;
 840         int core;
 841         struct kvmppc_vcore *vcore;
 842
 843         core = id / threads_per_core;
 844         if (core >= KVM_MAX_VCORES)
 845                 goto out;
 846
 847         err = -ENOMEM;
 848         vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
 849         if (!vcpu)
 850                 goto out;
 851
 852         err = kvm_vcpu_init(vcpu, kvm, id);
 853         if (err)
 854                 goto free_vcpu;
 855
 856         vcpu->arch.shared = &vcpu->arch.shregs;
 857         vcpu->arch.last_cpu = -1;
 858         vcpu->arch.mmcr[0] = MMCR0_FC;
 859         vcpu->arch.ctrl = CTRL_RUNLATCH;
 860         /* default to host PVR, since we can't spoof it */
 861         vcpu->arch.pvr = mfspr(SPRN_PVR);
 862         kvmppc_set_pvr(vcpu, vcpu->arch.pvr);
 863         spin_lock_init(&vcpu->arch.vpa_update_lock);
 864         spin_lock_init(&vcpu->arch.tbacct_lock);
 865         vcpu->arch.busy_preempt = TB_NIL;
 866
 867         kvmppc_mmu_book3s_hv_init(vcpu);
 868
 869         vcpu->arch.state = KVMPPC_VCPU_NOTREADY;
 870
 871         init_waitqueue_head(&vcpu->arch.cpu_run);
 872
 873         mutex_lock(&kvm->lock);
 874         vcore = kvm->arch.vcores[core];
 875         if (!vcore) {
 876                 vcore = kzalloc(sizeof(struct kvmppc_vcore), GFP_KERNEL);
 877                 if (vcore) {
 878                         INIT_LIST_HEAD(&vcore->runnable_threads);
 879                         spin_lock_init(&vcore->lock);
 880                         init_waitqueue_head(&vcore->wq);
 881                         vcore->preempt_tb = TB_NIL;
 882                 }
 883                 kvm->arch.vcores[core] = vcore;
 884         }
 885         mutex_unlock(&kvm->lock);
 886
 887         if (!vcore)
 888                 goto free_vcpu;
 889
 890         spin_lock(&vcore->lock);
 891         ++vcore->num_threads;
 892         spin_unlock(&vcore->lock);
 893         vcpu->arch.vcore = vcore;
 894
 895         vcpu->arch.cpu_type = KVM_CPU_3S_64;
 896         kvmppc_sanity_check(vcpu);
 897
 898         return vcpu;
 899
 900 free_vcpu:
 901         kmem_cache_free(kvm_vcpu_cache, vcpu);
 902 out:
 903         return ERR_PTR(err);
 904 }
 905
 906 void kvmppc_core_vcpu_free(struct kvm_vcpu *vcpu)
 907 {
 908         spin_lock(&vcpu->arch.vpa_update_lock);
 909         if (vcpu->arch.dtl.pinned_addr)
 910                 kvmppc_unpin_guest_page(vcpu->kvm, vcpu->arch.dtl.pinned_addr);
 911         if (vcpu->arch.slb_shadow.pinned_addr)
 912                 kvmppc_unpin_guest_page(vcpu->kvm, vcpu->arch.slb_shadow.pinned_addr);
 913         if (vcpu->arch.vpa.pinned_addr)
 914                 kvmppc_unpin_guest_page(vcpu->kvm, vcpu->arch.vpa.pinned_addr);
 915         spin_unlock(&vcpu->arch.vpa_update_lock);
 916         kvm_vcpu_uninit(vcpu);
 917         kmem_cache_free(kvm_vcpu_cache, vcpu);
 918 }
 919
 920 static void kvmppc_set_timer(struct kvm_vcpu *vcpu)
 921 {
 922         unsigned long dec_nsec, now;
 923
 924         now = get_tb();
 925         if (now > vcpu->arch.dec_expires) {
 926                 /* decrementer has already gone negative */
 927                 kvmppc_core_queue_dec(vcpu);
 928                 kvmppc_core_prepare_to_enter(vcpu);
 929                 return;
 930         }
 931         dec_nsec = (vcpu->arch.dec_expires - now) * NSEC_PER_SEC
 932                    / tb_ticks_per_sec;
 933         hrtimer_start(&vcpu->arch.dec_timer, ktime_set(0, dec_nsec),
 934                       HRTIMER_MODE_REL);
 935         vcpu->arch.timer_running = 1;
 936 }
 937
 938 static void kvmppc_end_cede(struct kvm_vcpu *vcpu)
 939 {
 940         vcpu->arch.ceded = 0;
 941         if (vcpu->arch.timer_running) {
 942                 hrtimer_try_to_cancel(&vcpu->arch.dec_timer);
 943                 vcpu->arch.timer_running = 0;
 944         }
 945 }
 946
 947 extern int __kvmppc_vcore_entry(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu);
 948 extern void xics_wake_cpu(int cpu);
 949
 950 static void kvmppc_remove_runnable(struct kvmppc_vcore *vc,
 951                                    struct kvm_vcpu *vcpu)
 952 {
 953         u64 now;
 954
 955         if (vcpu->arch.state != KVMPPC_VCPU_RUNNABLE)
 956                 return;
 957         spin_lock(&vcpu->arch.tbacct_lock);
 958         now = mftb();
 959         vcpu->arch.busy_stolen += vcore_stolen_time(vc, now) -
 960                 vcpu->arch.stolen_logged;
 961         vcpu->arch.busy_preempt = now;
 962         vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST;
 963         spin_unlock(&vcpu->arch.tbacct_lock);
 964         --vc->n_runnable;
 965         list_del(&vcpu->arch.run_list);
 966 }
 967
 968 static int kvmppc_grab_hwthread(int cpu)
 969 {
 970         struct paca_struct *tpaca;
 971         long timeout = 1000;
 972
 973         tpaca = &paca[cpu];
 974
 975         /* Ensure the thread won't go into the kernel if it wakes */
 976         tpaca->kvm_hstate.hwthread_req = 1;
 977         tpaca->kvm_hstate.kvm_vcpu = NULL;
 978
 979         /*
 980          * If the thread is already executing in the kernel (e.g. handling
 981          * a stray interrupt), wait for it to get back to nap mode.
 982          * The smp_mb() is to ensure that our setting of hwthread_req
 983          * is visible before we look at hwthread_state, so if this
 984          * races with the code at system_reset_pSeries and the thread
 985          * misses our setting of hwthread_req, we are sure to see its
 986          * setting of hwthread_state, and vice versa.
 987          */
 988         smp_mb();
 989         while (tpaca->kvm_hstate.hwthread_state == KVM_HWTHREAD_IN_KERNEL) {
 990                 if (--timeout <= 0) {
 991                         pr_err("KVM: couldn't grab cpu %d\n", cpu);
 992                         return -EBUSY;
 993                 }
 994                 udelay(1);
 995         }
 996         return 0;
 997 }
 998
 999 static void kvmppc_release_hwthread(int cpu)
1000 {
1001         struct paca_struct *tpaca;
1002
1003         tpaca = &paca[cpu];
1004         tpaca->kvm_hstate.hwthread_req = 0;
1005         tpaca->kvm_hstate.kvm_vcpu = NULL;
1006 }
1007
1008 static void kvmppc_start_thread(struct kvm_vcpu *vcpu)
1009 {
1010         int cpu;
1011         struct paca_struct *tpaca;
1012         struct kvmppc_vcore *vc = vcpu->arch.vcore;
1013
1014         if (vcpu->arch.timer_running) {
1015                 hrtimer_try_to_cancel(&vcpu->arch.dec_timer);
1016                 vcpu->arch.timer_running = 0;
1017         }
1018         cpu = vc->pcpu + vcpu->arch.ptid;
1019         tpaca = &paca[cpu];
1020         tpaca->kvm_hstate.kvm_vcpu = vcpu;
1021         tpaca->kvm_hstate.kvm_vcore = vc;
1022         tpaca->kvm_hstate.napping = 0;
1023         vcpu->cpu = vc->pcpu;
1024         smp_wmb();
1025 #if defined(CONFIG_PPC_ICP_NATIVE) && defined(CONFIG_SMP)
1026         if (vcpu->arch.ptid) {
1027                 xics_wake_cpu(cpu);
1028                 ++vc->n_woken;
1029         }
1030 #endif
1031 }
1032
1033 static void kvmppc_wait_for_nap(struct kvmppc_vcore *vc)
1034 {
1035         int i;
1036
1037         HMT_low();
1038         i = 0;
1039         while (vc->nap_count < vc->n_woken) {
1040                 if (++i >= 1000000) {
1041                         pr_err("kvmppc_wait_for_nap timeout %d %d\n",
1042                                vc->nap_count, vc->n_woken);
1043                         break;
1044                 }
1045                 cpu_relax();
1046         }
1047         HMT_medium();
1048 }
1049
1050 /*
1051  * Check that we are on thread 0 and that any other threads in
1052  * this core are off-line.  Then grab the threads so they can't
1053  * enter the kernel.
1054  */
1055 static int on_primary_thread(void)
1056 {
1057         int cpu = smp_processor_id();
1058         int thr = cpu_thread_in_core(cpu);
1059
1060         if (thr)
1061                 return 0;
1062         while (++thr < threads_per_core)
1063                 if (cpu_online(cpu + thr))
1064                         return 0;
1065
1066         /* Grab all hw threads so they can't go into the kernel */
1067         for (thr = 1; thr < threads_per_core; ++thr) {
1068                 if (kvmppc_grab_hwthread(cpu + thr)) {
1069                         /* Couldn't grab one; let the others go */
1070                         do {
1071                                 kvmppc_release_hwthread(cpu + thr);
1072                         } while (--thr > 0);
1073                         return 0;
1074                 }
1075         }
1076         return 1;
1077 }
1078
1079 /*
1080  * Run a set of guest threads on a physical core.
1081  * Called with vc->lock held.
1082  */
1083 static void kvmppc_run_core(struct kvmppc_vcore *vc)
1084 {
1085         struct kvm_vcpu *vcpu, *vcpu0, *vnext;
1086         long ret;
1087         u64 now;
1088         int ptid, i, need_vpa_update;
1089         int srcu_idx;
1090         struct kvm_vcpu *vcpus_to_update[threads_per_core];
1091
1092         /* don't start if any threads have a signal pending */
1093         need_vpa_update = 0;
1094         list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list) {
1095                 if (signal_pending(vcpu->arch.run_task))
1096                         return;
1097                 if (vcpu->arch.vpa.update_pending ||
1098                     vcpu->arch.slb_shadow.update_pending ||
1099                     vcpu->arch.dtl.update_pending)
1100                         vcpus_to_update[need_vpa_update++] = vcpu;
1101         }
1102
1103         /*
1104          * Initialize *vc, in particular vc->vcore_state, so we can
1105          * drop the vcore lock if necessary.
1106          */
1107         vc->n_woken = 0;
1108         vc->nap_count = 0;
1109         vc->entry_exit_count = 0;
1110         vc->vcore_state = VCORE_STARTING;
1111         vc->in_guest = 0;
1112         vc->napping_threads = 0;
1113
1114         /*
1115          * Updating any of the vpas requires calling kvmppc_pin_guest_page,
1116          * which can't be called with any spinlocks held.
1117          */
1118         if (need_vpa_update) {
1119                 spin_unlock(&vc->lock);
1120                 for (i = 0; i < need_vpa_update; ++i)
1121                         kvmppc_update_vpas(vcpus_to_update[i]);
1122                 spin_lock(&vc->lock);
1123         }
1124
1125         /*
1126          * Assign physical thread IDs, first to non-ceded vcpus
1127          * and then to ceded ones.
1128          */
1129         ptid = 0;
1130         vcpu0 = NULL;
1131         list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list) {
1132                 if (!vcpu->arch.ceded) {
1133                         if (!ptid)
1134                                 vcpu0 = vcpu;
1135                         vcpu->arch.ptid = ptid++;
1136                 }
1137         }
1138         if (!vcpu0)
1139                 goto out;       /* nothing to run; should never happen */
1140         list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list)
1141                 if (vcpu->arch.ceded)
1142                         vcpu->arch.ptid = ptid++;
1143
1144         /*
1145          * Make sure we are running on thread 0, and that
1146          * secondary threads are offline.
1147          */
1148         if (threads_per_core > 1 && !on_primary_thread()) {
1149                 list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list)
1150                         vcpu->arch.ret = -EBUSY;
1151                 goto out;
1152         }
1153
1154         vc->pcpu = smp_processor_id();
1155         list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list) {
1156                 kvmppc_start_thread(vcpu);
1157                 kvmppc_create_dtl_entry(vcpu, vc);
1158         }
1159
1160         vc->vcore_state = VCORE_RUNNING;
1161         preempt_disable();
1162         spin_unlock(&vc->lock);
1163
1164         kvm_guest_enter();
1165
1166         srcu_idx = srcu_read_lock(&vcpu0->kvm->srcu);
1167
1168         __kvmppc_vcore_entry(NULL, vcpu0);
1169
1170         spin_lock(&vc->lock);
1171         /* disable sending of IPIs on virtual external irqs */
1172         list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list)
1173                 vcpu->cpu = -1;
1174         /* wait for secondary threads to finish writing their state to memory */
1175         if (vc->nap_count < vc->n_woken)
1176                 kvmppc_wait_for_nap(vc);
1177         for (i = 0; i < threads_per_core; ++i)
1178                 kvmppc_release_hwthread(vc->pcpu + i);
1179         /* prevent other vcpu threads from doing kvmppc_start_thread() now */
1180         vc->vcore_state = VCORE_EXITING;
1181         spin_unlock(&vc->lock);
1182
1183         srcu_read_unlock(&vcpu0->kvm->srcu, srcu_idx);
1184
1185         /* make sure updates to secondary vcpu structs are visible now */
1186         smp_mb();
1187         kvm_guest_exit();
1188
1189         preempt_enable();
1190         kvm_resched(vcpu);
1191
1192         spin_lock(&vc->lock);
1193         now = get_tb();
1194         list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list) {
1195                 /* cancel pending dec exception if dec is positive */
1196                 if (now < vcpu->arch.dec_expires &&
1197                     kvmppc_core_pending_dec(vcpu))
1198                         kvmppc_core_dequeue_dec(vcpu);
1199
1200                 ret = RESUME_GUEST;
1201                 if (vcpu->arch.trap)
1202                         ret = kvmppc_handle_exit(vcpu->arch.kvm_run, vcpu,
1203                                                  vcpu->arch.run_task);
1204
1205                 vcpu->arch.ret = ret;
1206                 vcpu->arch.trap = 0;
1207
1208                 if (vcpu->arch.ceded) {
1209                         if (ret != RESUME_GUEST)
1210                                 kvmppc_end_cede(vcpu);
1211                         else
1212                                 kvmppc_set_timer(vcpu);
1213                 }
1214         }
1215
1216  out:
1217         vc->vcore_state = VCORE_INACTIVE;
1218         list_for_each_entry_safe(vcpu, vnext, &vc->runnable_threads,
1219                                  arch.run_list) {
1220                 if (vcpu->arch.ret != RESUME_GUEST) {
1221                         kvmppc_remove_runnable(vc, vcpu);
1222                         wake_up(&vcpu->arch.cpu_run);
1223                 }
1224         }
1225 }
1226
1227 /*
1228  * Wait for some other vcpu thread to execute us, and
1229  * wake us up when we need to handle something in the host.
1230  */
1231 static void kvmppc_wait_for_exec(struct kvm_vcpu *vcpu, int wait_state)
1232 {
1233         DEFINE_WAIT(wait);
1234
1235         prepare_to_wait(&vcpu->arch.cpu_run, &wait, wait_state);
1236         if (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE)
1237                 schedule();
1238         finish_wait(&vcpu->arch.cpu_run, &wait);
1239 }
1240
1241 /*
1242  * All the vcpus in this vcore are idle, so wait for a decrementer
1243  * or external interrupt to one of the vcpus.  vc->lock is held.
1244  */
1245 static void kvmppc_vcore_blocked(struct kvmppc_vcore *vc)
1246 {
1247         DEFINE_WAIT(wait);
1248
1249         prepare_to_wait(&vc->wq, &wait, TASK_INTERRUPTIBLE);
1250         vc->vcore_state = VCORE_SLEEPING;
1251         spin_unlock(&vc->lock);
1252         schedule();
1253         finish_wait(&vc->wq, &wait);
1254         spin_lock(&vc->lock);
1255         vc->vcore_state = VCORE_INACTIVE;
1256 }
1257
1258 static int kvmppc_run_vcpu(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
1259 {
1260         int n_ceded;
1261         struct kvmppc_vcore *vc;
1262         struct kvm_vcpu *v, *vn;
1263
1264         kvm_run->exit_reason = 0;
1265         vcpu->arch.ret = RESUME_GUEST;
1266         vcpu->arch.trap = 0;
1267         kvmppc_update_vpas(vcpu);
1268
1269         /*
1270          * Synchronize with other threads in this virtual core
1271          */
1272         vc = vcpu->arch.vcore;
1273         spin_lock(&vc->lock);
1274         vcpu->arch.ceded = 0;
1275         vcpu->arch.run_task = current;
1276         vcpu->arch.kvm_run = kvm_run;
1277         vcpu->arch.stolen_logged = vcore_stolen_time(vc, mftb());
1278         vcpu->arch.state = KVMPPC_VCPU_RUNNABLE;
1279         vcpu->arch.busy_preempt = TB_NIL;
1280         list_add_tail(&vcpu->arch.run_list, &vc->runnable_threads);
1281         ++vc->n_runnable;
1282
1283         /*
1284          * This happens the first time this is called for a vcpu.
1285          * If the vcore is already running, we may be able to start
1286          * this thread straight away and have it join in.
1287          */
1288         if (!signal_pending(current)) {
1289                 if (vc->vcore_state == VCORE_RUNNING &&
1290                     VCORE_EXIT_COUNT(vc) == 0) {
1291                         vcpu->arch.ptid = vc->n_runnable - 1;
1292                         kvmppc_create_dtl_entry(vcpu, vc);
1293                         kvmppc_start_thread(vcpu);
1294                 } else if (vc->vcore_state == VCORE_SLEEPING) {
1295                         wake_up(&vc->wq);
1296                 }
1297
1298         }
1299
1300         while (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE &&
1301                !signal_pending(current)) {
1302                 if (vc->vcore_state != VCORE_INACTIVE) {
1303                         spin_unlock(&vc->lock);
1304                         kvmppc_wait_for_exec(vcpu, TASK_INTERRUPTIBLE);
1305                         spin_lock(&vc->lock);
1306                         continue;
1307                 }
1308                 list_for_each_entry_safe(v, vn, &vc->runnable_threads,
1309                                          arch.run_list) {
1310                         kvmppc_core_prepare_to_enter(v);
1311                         if (signal_pending(v->arch.run_task)) {
1312                                 kvmppc_remove_runnable(vc, v);
1313                                 v->stat.signal_exits++;
1314                                 v->arch.kvm_run->exit_reason = KVM_EXIT_INTR;
1315                                 v->arch.ret = -EINTR;
1316                                 wake_up(&v->arch.cpu_run);
1317                         }
1318                 }
1319                 if (!vc->n_runnable || vcpu->arch.state != KVMPPC_VCPU_RUNNABLE)
1320                         break;
1321                 vc->runner = vcpu;
1322                 n_ceded = 0;
1323                 list_for_each_entry(v, &vc->runnable_threads, arch.run_list)
1324                         if (!v->arch.pending_exceptions)
1325                                 n_ceded += v->arch.ceded;
1326                 if (n_ceded == vc->n_runnable)
1327                         kvmppc_vcore_blocked(vc);
1328                 else
1329                         kvmppc_run_core(vc);
1330                 vc->runner = NULL;
1331         }
1332
1333         while (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE &&
1334                (vc->vcore_state == VCORE_RUNNING ||
1335                 vc->vcore_state == VCORE_EXITING)) {
1336                 spin_unlock(&vc->lock);
1337                 kvmppc_wait_for_exec(vcpu, TASK_UNINTERRUPTIBLE);
1338                 spin_lock(&vc->lock);
1339         }
1340
1341         if (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE) {
1342                 kvmppc_remove_runnable(vc, vcpu);
1343                 vcpu->stat.signal_exits++;
1344                 kvm_run->exit_reason = KVM_EXIT_INTR;
1345                 vcpu->arch.ret = -EINTR;
1346         }
1347
1348         if (vc->n_runnable && vc->vcore_state == VCORE_INACTIVE) {
1349                 /* Wake up some vcpu to run the core */
1350                 v = list_first_entry(&vc->runnable_threads,
1351                                      struct kvm_vcpu, arch.run_list);
1352                 wake_up(&v->arch.cpu_run);
1353         }
1354
1355         spin_unlock(&vc->lock);
1356         return vcpu->arch.ret;
1357 }
1358
1359 int kvmppc_vcpu_run(struct kvm_run *run, struct kvm_vcpu *vcpu)
1360 {
1361         int r;
1362         int srcu_idx;
1363
1364         if (!vcpu->arch.sane) {
1365                 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1366                 return -EINVAL;
1367         }
1368
1369         kvmppc_core_prepare_to_enter(vcpu);
1370
1371         /* No need to go into the guest when all we'll do is come back out */
1372         if (signal_pending(current)) {
1373                 run->exit_reason = KVM_EXIT_INTR;
1374                 return -EINTR;
1375         }
1376
1377         atomic_inc(&vcpu->kvm->arch.vcpus_running);
1378         /* Order vcpus_running vs. rma_setup_done, see kvmppc_alloc_reset_hpt */
1379         smp_mb();
1380
1381         /* On the first time here, set up HTAB and VRMA or RMA */
1382         if (!vcpu->kvm->arch.rma_setup_done) {
1383                 r = kvmppc_hv_setup_htab_rma(vcpu);
1384                 if (r)
1385                         goto out;
1386         }
1387
1388         flush_fp_to_thread(current);
1389         flush_altivec_to_thread(current);
1390         flush_vsx_to_thread(current);
1391         vcpu->arch.wqp = &vcpu->arch.vcore->wq;
1392         vcpu->arch.pgdir = current->mm->pgd;
1393         vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST;
1394
1395         do {
1396                 r = kvmppc_run_vcpu(run, vcpu);
1397
1398                 if (run->exit_reason == KVM_EXIT_PAPR_HCALL &&
1399                     !(vcpu->arch.shregs.msr & MSR_PR)) {
1400                         r = kvmppc_pseries_do_hcall(vcpu);
1401                         kvmppc_core_prepare_to_enter(vcpu);
1402                 } else if (r == RESUME_PAGE_FAULT) {
1403                         srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1404                         r = kvmppc_book3s_hv_page_fault(run, vcpu,
1405                                 vcpu->arch.fault_dar, vcpu->arch.fault_dsisr);
1406                         srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
1407                 }
1408         } while (r == RESUME_GUEST);
1409
1410  out:
1411         vcpu->arch.state = KVMPPC_VCPU_NOTREADY;
1412         atomic_dec(&vcpu->kvm->arch.vcpus_running);
1413         return r;
1414 }
1415
1416
1417 /* Work out RMLS (real mode limit selector) field value for a given RMA size.
1418    Assumes POWER7 or PPC970. */
1419 static inline int lpcr_rmls(unsigned long rma_size)
1420 {
1421         switch (rma_size) {
1422         case 32ul << 20:        /* 32 MB */
1423                 if (cpu_has_feature(CPU_FTR_ARCH_206))
1424                         return 8;       /* only supported on POWER7 */
1425                 return -1;
1426         case 64ul << 20:        /* 64 MB */
1427                 return 3;
1428         case 128ul << 20:       /* 128 MB */
1429                 return 7;
1430         case 256ul << 20:       /* 256 MB */
1431                 return 4;
1432         case 1ul << 30:         /* 1 GB */
1433                 return 2;
1434         case 16ul << 30:        /* 16 GB */
1435                 return 1;
1436         case 256ul << 30:       /* 256 GB */
1437                 return 0;
1438         default:
1439                 return -1;
1440         }
1441 }
1442
1443 static int kvm_rma_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1444 {
1445         struct kvmppc_linear_info *ri = vma->vm_file->private_data;
1446         struct page *page;
1447
1448         if (vmf->pgoff >= ri->npages)
1449                 return VM_FAULT_SIGBUS;
1450
1451         page = pfn_to_page(ri->base_pfn + vmf->pgoff);
1452         get_page(page);
1453         vmf->page = page;
1454         return 0;
1455 }
1456
1457 static const struct vm_operations_struct kvm_rma_vm_ops = {
1458         .fault = kvm_rma_fault,
1459 };
1460
1461 static int kvm_rma_mmap(struct file *file, struct vm_area_struct *vma)
1462 {
1463         vma->vm_flags |= VM_RESERVED;
1464         vma->vm_ops = &kvm_rma_vm_ops;
1465         return 0;
1466 }
1467
1468 static int kvm_rma_release(struct inode *inode, struct file *filp)
1469 {
1470         struct kvmppc_linear_info *ri = filp->private_data;
1471
1472         kvm_release_rma(ri);
1473         return 0;
1474 }
1475
1476 static struct file_operations kvm_rma_fops = {
1477         .mmap           = kvm_rma_mmap,
1478         .release        = kvm_rma_release,
1479 };
1480
1481 long kvm_vm_ioctl_allocate_rma(struct kvm *kvm, struct kvm_allocate_rma *ret)
1482 {
1483         struct kvmppc_linear_info *ri;
1484         long fd;
1485
1486         ri = kvm_alloc_rma();
1487         if (!ri)
1488                 return -ENOMEM;
1489
1490         fd = anon_inode_getfd("kvm-rma", &kvm_rma_fops, ri, O_RDWR);
1491         if (fd < 0)
1492                 kvm_release_rma(ri);
1493
1494         ret->rma_size = ri->npages << PAGE_SHIFT;
1495         return fd;
1496 }
1497
1498 static void kvmppc_add_seg_page_size(struct kvm_ppc_one_seg_page_size **sps,
1499                                      int linux_psize)
1500 {
1501         struct mmu_psize_def *def = &mmu_psize_defs[linux_psize];
1502
1503         if (!def->shift)
1504                 return;
1505         (*sps)->page_shift = def->shift;
1506         (*sps)->slb_enc = def->sllp;
1507         (*sps)->enc[0].page_shift = def->shift;
1508         (*sps)->enc[0].pte_enc = def->penc;
1509         (*sps)++;
1510 }
1511
1512 int kvm_vm_ioctl_get_smmu_info(struct kvm *kvm, struct kvm_ppc_smmu_info *info)
1513 {
1514         struct kvm_ppc_one_seg_page_size *sps;
1515
1516         info->flags = KVM_PPC_PAGE_SIZES_REAL;
1517         if (mmu_has_feature(MMU_FTR_1T_SEGMENT))
1518                 info->flags |= KVM_PPC_1T_SEGMENTS;
1519         info->slb_size = mmu_slb_size;
1520
1521         /* We only support these sizes for now, and no muti-size segments */
1522         sps = &info->sps[0];
1523         kvmppc_add_seg_page_size(&sps, MMU_PAGE_4K);
1524         kvmppc_add_seg_page_size(&sps, MMU_PAGE_64K);
1525         kvmppc_add_seg_page_size(&sps, MMU_PAGE_16M);
1526
1527         return 0;
1528 }
1529
1530 /*
1531  * Get (and clear) the dirty memory log for a memory slot.
1532  */
1533 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log)
1534 {
1535         struct kvm_memory_slot *memslot;
1536         int r;
1537         unsigned long n;
1538
1539         mutex_lock(&kvm->slots_lock);
1540
1541         r = -EINVAL;
1542         if (log->slot >= KVM_MEMORY_SLOTS)
1543                 goto out;
1544
1545         memslot = id_to_memslot(kvm->memslots, log->slot);
1546         r = -ENOENT;
1547         if (!memslot->dirty_bitmap)
1548                 goto out;
1549
1550         n = kvm_dirty_bitmap_bytes(memslot);
1551         memset(memslot->dirty_bitmap, 0, n);
1552
1553         r = kvmppc_hv_get_dirty_log(kvm, memslot, memslot->dirty_bitmap);
1554         if (r)
1555                 goto out;
1556
1557         r = -EFAULT;
1558         if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
1559                 goto out;
1560
1561         r = 0;
1562 out:
1563         mutex_unlock(&kvm->slots_lock);
1564         return r;
1565 }
1566
1567 static unsigned long slb_pgsize_encoding(unsigned long psize)
1568 {
1569         unsigned long senc = 0;
1570
1571         if (psize > 0x1000) {
1572                 senc = SLB_VSID_L;
1573                 if (psize == 0x10000)
1574                         senc |= SLB_VSID_LP_01;
1575         }
1576         return senc;
1577 }
1578
1579 static void unpin_slot(struct kvm_memory_slot *memslot)
1580 {
1581         unsigned long *physp;
1582         unsigned long j, npages, pfn;
1583         struct page *page;
1584
1585         physp = memslot->arch.slot_phys;
1586         npages = memslot->npages;
1587         if (!physp)
1588                 return;
1589         for (j = 0; j < npages; j++) {
1590                 if (!(physp[j] & KVMPPC_GOT_PAGE))
1591                         continue;
1592                 pfn = physp[j] >> PAGE_SHIFT;
1593                 page = pfn_to_page(pfn);
1594                 SetPageDirty(page);
1595                 put_page(page);
1596         }
1597 }
1598
1599 void kvmppc_core_free_memslot(struct kvm_memory_slot *free,
1600                               struct kvm_memory_slot *dont)
1601 {
1602         if (!dont || free->arch.rmap != dont->arch.rmap) {
1603                 vfree(free->arch.rmap);
1604                 free->arch.rmap = NULL;
1605         }
1606         if (!dont || free->arch.slot_phys != dont->arch.slot_phys) {
1607                 unpin_slot(free);
1608                 vfree(free->arch.slot_phys);
1609                 free->arch.slot_phys = NULL;
1610         }
1611 }
1612
1613 int kvmppc_core_create_memslot(struct kvm_memory_slot *slot,
1614                                unsigned long npages)
1615 {
1616         slot->arch.rmap = vzalloc(npages * sizeof(*slot->arch.rmap));
1617         if (!slot->arch.rmap)
1618                 return -ENOMEM;
1619         slot->arch.slot_phys = NULL;
1620
1621         return 0;
1622 }
1623
1624 int kvmppc_core_prepare_memory_region(struct kvm *kvm,
1625                                       struct kvm_memory_slot *memslot,
1626                                       struct kvm_userspace_memory_region *mem)
1627 {
1628         unsigned long *phys;
1629
1630         /* Allocate a slot_phys array if needed */
1631         phys = memslot->arch.slot_phys;
1632         if (!kvm->arch.using_mmu_notifiers && !phys && memslot->npages) {
1633                 phys = vzalloc(memslot->npages * sizeof(unsigned long));
1634                 if (!phys)
1635                         return -ENOMEM;
1636                 memslot->arch.slot_phys = phys;
1637         }
1638
1639         return 0;
1640 }
1641
1642 void kvmppc_core_commit_memory_region(struct kvm *kvm,
1643                                       struct kvm_userspace_memory_region *mem,
1644                                       struct kvm_memory_slot old)
1645 {
1646         unsigned long npages = mem->memory_size >> PAGE_SHIFT;
1647         struct kvm_memory_slot *memslot;
1648
1649         if (npages && old.npages) {
1650                 /*
1651                  * If modifying a memslot, reset all the rmap dirty bits.
1652                  * If this is a new memslot, we don't need to do anything
1653                  * since the rmap array starts out as all zeroes,
1654                  * i.e. no pages are dirty.
1655                  */
1656                 memslot = id_to_memslot(kvm->memslots, mem->slot);
1657                 kvmppc_hv_get_dirty_log(kvm, memslot, NULL);
1658         }
1659 }
1660
1661 static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu)
1662 {
1663         int err = 0;
1664         struct kvm *kvm = vcpu->kvm;
1665         struct kvmppc_linear_info *ri = NULL;
1666         unsigned long hva;
1667         struct kvm_memory_slot *memslot;
1668         struct vm_area_struct *vma;
1669         unsigned long lpcr, senc;
1670         unsigned long psize, porder;
1671         unsigned long rma_size;
1672         unsigned long rmls;
1673         unsigned long *physp;
1674         unsigned long i, npages;
1675         int srcu_idx;
1676
1677         mutex_lock(&kvm->lock);
1678         if (kvm->arch.rma_setup_done)
1679                 goto out;       /* another vcpu beat us to it */
1680
1681         /* Allocate hashed page table (if not done already) and reset it */
1682         if (!kvm->arch.hpt_virt) {
1683                 err = kvmppc_alloc_hpt(kvm, NULL);
1684                 if (err) {
1685                         pr_err("KVM: Couldn't alloc HPT\n");
1686                         goto out;
1687                 }
1688         }
1689
1690         /* Look up the memslot for guest physical address 0 */
1691         srcu_idx = srcu_read_lock(&kvm->srcu);
1692         memslot = gfn_to_memslot(kvm, 0);
1693
1694         /* We must have some memory at 0 by now */
1695         err = -EINVAL;
1696         if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID))
1697                 goto out_srcu;
1698
1699         /* Look up the VMA for the start of this memory slot */
1700         hva = memslot->userspace_addr;
1701         down_read(&current->mm->mmap_sem);
1702         vma = find_vma(current->mm, hva);
1703         if (!vma || vma->vm_start > hva || (vma->vm_flags & VM_IO))
1704                 goto up_out;
1705
1706         psize = vma_kernel_pagesize(vma);
1707         porder = __ilog2(psize);
1708
1709         /* Is this one of our preallocated RMAs? */
1710         if (vma->vm_file && vma->vm_file->f_op == &kvm_rma_fops &&
1711             hva == vma->vm_start)
1712                 ri = vma->vm_file->private_data;
1713
1714         up_read(&current->mm->mmap_sem);
1715
1716         if (!ri) {
1717                 /* On POWER7, use VRMA; on PPC970, give up */
1718                 err = -EPERM;
1719                 if (cpu_has_feature(CPU_FTR_ARCH_201)) {
1720                         pr_err("KVM: CPU requires an RMO\n");
1721                         goto out_srcu;
1722                 }
1723
1724                 /* We can handle 4k, 64k or 16M pages in the VRMA */
1725                 err = -EINVAL;
1726                 if (!(psize == 0x1000 || psize == 0x10000 ||
1727                       psize == 0x1000000))
1728                         goto out_srcu;
1729
1730                 /* Update VRMASD field in the LPCR */
1731                 senc = slb_pgsize_encoding(psize);
1732                 kvm->arch.vrma_slb_v = senc | SLB_VSID_B_1T |
1733                         (VRMA_VSID << SLB_VSID_SHIFT_1T);
1734                 lpcr = kvm->arch.lpcr & ~LPCR_VRMASD;
1735                 lpcr |= senc << (LPCR_VRMASD_SH - 4);
1736                 kvm->arch.lpcr = lpcr;
1737
1738                 /* Create HPTEs in the hash page table for the VRMA */
1739                 kvmppc_map_vrma(vcpu, memslot, porder);
1740
1741         } else {
1742                 /* Set up to use an RMO region */
1743                 rma_size = ri->npages;
1744                 if (rma_size > memslot->npages)
1745                         rma_size = memslot->npages;
1746                 rma_size <<= PAGE_SHIFT;
1747                 rmls = lpcr_rmls(rma_size);
1748                 err = -EINVAL;
1749                 if (rmls < 0) {
1750                         pr_err("KVM: Can't use RMA of 0x%lx bytes\n", rma_size);
1751                         goto out_srcu;
1752                 }
1753                 atomic_inc(&ri->use_count);
1754                 kvm->arch.rma = ri;
1755
1756                 /* Update LPCR and RMOR */
1757                 lpcr = kvm->arch.lpcr;
1758                 if (cpu_has_feature(CPU_FTR_ARCH_201)) {
1759                         /* PPC970; insert RMLS value (split field) in HID4 */
1760                         lpcr &= ~((1ul << HID4_RMLS0_SH) |
1761                                   (3ul << HID4_RMLS2_SH));
1762                         lpcr |= ((rmls >> 2) << HID4_RMLS0_SH) |
1763                                 ((rmls & 3) << HID4_RMLS2_SH);
1764                         /* RMOR is also in HID4 */
1765                         lpcr |= ((ri->base_pfn >> (26 - PAGE_SHIFT)) & 0xffff)
1766                                 << HID4_RMOR_SH;
1767                 } else {
1768                         /* POWER7 */
1769                         lpcr &= ~(LPCR_VPM0 | LPCR_VRMA_L);
1770                         lpcr |= rmls << LPCR_RMLS_SH;
1771                         kvm->arch.rmor = kvm->arch.rma->base_pfn << PAGE_SHIFT;
1772                 }
1773                 kvm->arch.lpcr = lpcr;
1774                 pr_info("KVM: Using RMO at %lx size %lx (LPCR = %lx)\n",
1775                         ri->base_pfn << PAGE_SHIFT, rma_size, lpcr);
1776
1777                 /* Initialize phys addrs of pages in RMO */
1778                 npages = ri->npages;
1779                 porder = __ilog2(npages);
1780                 physp = memslot->arch.slot_phys;
1781                 if (physp) {
1782                         if (npages > memslot->npages)
1783                                 npages = memslot->npages;
1784                         spin_lock(&kvm->arch.slot_phys_lock);
1785                         for (i = 0; i < npages; ++i)
1786                                 physp[i] = ((ri->base_pfn + i) << PAGE_SHIFT) +
1787                                         porder;
1788                         spin_unlock(&kvm->arch.slot_phys_lock);
1789                 }
1790         }
1791
1792         /* Order updates to kvm->arch.lpcr etc. vs. rma_setup_done */
1793         smp_wmb();
1794         kvm->arch.rma_setup_done = 1;
1795         err = 0;
1796  out_srcu:
1797         srcu_read_unlock(&kvm->srcu, srcu_idx);
1798  out:
1799         mutex_unlock(&kvm->lock);
1800         return err;
1801
1802  up_out:
1803         up_read(&current->mm->mmap_sem);
1804         goto out;
1805 }
1806
1807 int kvmppc_core_init_vm(struct kvm *kvm)
1808 {
1809         unsigned long lpcr, lpid;
1810
1811         /* Allocate the guest's logical partition ID */
1812
1813         lpid = kvmppc_alloc_lpid();
1814         if (lpid < 0)
1815                 return -ENOMEM;
1816         kvm->arch.lpid = lpid;
1817
1818         INIT_LIST_HEAD(&kvm->arch.spapr_tce_tables);
1819
1820         kvm->arch.rma = NULL;
1821
1822         kvm->arch.host_sdr1 = mfspr(SPRN_SDR1);
1823
1824         if (cpu_has_feature(CPU_FTR_ARCH_201)) {
1825                 /* PPC970; HID4 is effectively the LPCR */
1826                 kvm->arch.host_lpid = 0;
1827                 kvm->arch.host_lpcr = lpcr = mfspr(SPRN_HID4);
1828                 lpcr &= ~((3 << HID4_LPID1_SH) | (0xful << HID4_LPID5_SH));
1829                 lpcr |= ((lpid >> 4) << HID4_LPID1_SH) |
1830                         ((lpid & 0xf) << HID4_LPID5_SH);
1831         } else {
1832                 /* POWER7; init LPCR for virtual RMA mode */
1833                 kvm->arch.host_lpid = mfspr(SPRN_LPID);
1834                 kvm->arch.host_lpcr = lpcr = mfspr(SPRN_LPCR);
1835                 lpcr &= LPCR_PECE | LPCR_LPES;
1836                 lpcr |= (4UL << LPCR_DPFD_SH) | LPCR_HDICE |
1837                         LPCR_VPM0 | LPCR_VPM1;
1838                 kvm->arch.vrma_slb_v = SLB_VSID_B_1T |
1839                         (VRMA_VSID << SLB_VSID_SHIFT_1T);
1840         }
1841         kvm->arch.lpcr = lpcr;
1842
1843         kvm->arch.using_mmu_notifiers = !!cpu_has_feature(CPU_FTR_ARCH_206);
1844         spin_lock_init(&kvm->arch.slot_phys_lock);
1845
1846         /*
1847          * Don't allow secondary CPU threads to come online
1848          * while any KVM VMs exist.
1849          */
1850         inhibit_secondary_onlining();
1851
1852         return 0;
1853 }
1854
1855 void kvmppc_core_destroy_vm(struct kvm *kvm)
1856 {
1857         uninhibit_secondary_onlining();
1858
1859         if (kvm->arch.rma) {
1860                 kvm_release_rma(kvm->arch.rma);
1861                 kvm->arch.rma = NULL;
1862         }
1863
1864         kvmppc_free_hpt(kvm);
1865         WARN_ON(!list_empty(&kvm->arch.spapr_tce_tables));
1866 }
1867
1868 /* These are stubs for now */
1869 void kvmppc_mmu_pte_pflush(struct kvm_vcpu *vcpu, ulong pa_start, ulong pa_end)
1870 {
1871 }
1872
1873 /* We don't need to emulate any privileged instructions or dcbz */
1874 int kvmppc_core_emulate_op(struct kvm_run *run, struct kvm_vcpu *vcpu,
1875                            unsigned int inst, int *advance)
1876 {
1877         return EMULATE_FAIL;
1878 }
1879
1880 int kvmppc_core_emulate_mtspr(struct kvm_vcpu *vcpu, int sprn, ulong spr_val)
1881 {
1882         return EMULATE_FAIL;
1883 }
1884
1885 int kvmppc_core_emulate_mfspr(struct kvm_vcpu *vcpu, int sprn, ulong *spr_val)
1886 {
1887         return EMULATE_FAIL;
1888 }
1889
1890 static int kvmppc_book3s_hv_init(void)
1891 {
1892         int r;
1893
1894         r = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
1895
1896         if (r)
1897                 return r;
1898
1899         r = kvmppc_mmu_hv_init();
1900
1901         return r;
1902 }
1903
1904 static void kvmppc_book3s_hv_exit(void)
1905 {
1906         kvm_exit();
1907 }
1908
1909 module_init(kvmppc_book3s_hv_init);
1910 module_exit(kvmppc_book3s_hv_exit);