arch/x86/kvm/svm.c

   1 /*
   2  * Kernel-based Virtual Machine driver for Linux
   3  *
   4  * AMD SVM support
   5  *
   6  * Copyright (C) 2006 Qumranet, Inc.
   7  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
   8  *
   9  * Authors:
  10  *   Yaniv Kamay  <yaniv@qumranet.com>
  11  *   Avi Kivity   <avi@qumranet.com>
  12  *
  13  * This work is licensed under the terms of the GNU GPL, version 2.  See
  14  * the COPYING file in the top-level directory.
  15  *
  16  */
  17 #include <linux/kvm_host.h>
  18
  19 #include "irq.h"
  20 #include "mmu.h"
  21 #include "kvm_cache_regs.h"
  22 #include "x86.h"
  23 #include "cpuid.h"
  24 #include "pmu.h"
  25
  26 #include <linux/module.h>
  27 #include <linux/mod_devicetable.h>
  28 #include <linux/kernel.h>
  29 #include <linux/vmalloc.h>
  30 #include <linux/highmem.h>
  31 #include <linux/sched.h>
  32 #include <linux/trace_events.h>
  33 #include <linux/slab.h>
  34
  35 #include <asm/perf_event.h>
  36 #include <asm/tlbflush.h>
  37 #include <asm/desc.h>
  38 #include <asm/debugreg.h>
  39 #include <asm/kvm_para.h>
  40
  41 #include <asm/virtext.h>
  42 #include "trace.h"
  43
  44 #define __ex(x) __kvm_handle_fault_on_reboot(x)
  45
  46 MODULE_AUTHOR("Qumranet");
  47 MODULE_LICENSE("GPL");
  48
  49 static const struct x86_cpu_id svm_cpu_id[] = {
  50         X86_FEATURE_MATCH(X86_FEATURE_SVM),
  51         {}
  52 };
  53 MODULE_DEVICE_TABLE(x86cpu, svm_cpu_id);
  54
  55 #define IOPM_ALLOC_ORDER 2
  56 #define MSRPM_ALLOC_ORDER 1
  57
  58 #define SEG_TYPE_LDT 2
  59 #define SEG_TYPE_BUSY_TSS16 3
  60
  61 #define SVM_FEATURE_NPT            (1 <<  0)
  62 #define SVM_FEATURE_LBRV           (1 <<  1)
  63 #define SVM_FEATURE_SVML           (1 <<  2)
  64 #define SVM_FEATURE_NRIP           (1 <<  3)
  65 #define SVM_FEATURE_TSC_RATE       (1 <<  4)
  66 #define SVM_FEATURE_VMCB_CLEAN     (1 <<  5)
  67 #define SVM_FEATURE_FLUSH_ASID     (1 <<  6)
  68 #define SVM_FEATURE_DECODE_ASSIST  (1 <<  7)
  69 #define SVM_FEATURE_PAUSE_FILTER   (1 << 10)
  70
  71 #define NESTED_EXIT_HOST        0       /* Exit handled on host level */
  72 #define NESTED_EXIT_DONE        1       /* Exit caused nested vmexit  */
  73 #define NESTED_EXIT_CONTINUE    2       /* Further checks needed      */
  74
  75 #define DEBUGCTL_RESERVED_BITS (~(0x3fULL))
  76
  77 #define TSC_RATIO_RSVD          0xffffff0000000000ULL
  78 #define TSC_RATIO_MIN           0x0000000000000001ULL
  79 #define TSC_RATIO_MAX           0x000000ffffffffffULL
  80
  81 static bool erratum_383_found __read_mostly;
  82
  83 static const u32 host_save_user_msrs[] = {
  84 #ifdef CONFIG_X86_64
  85         MSR_STAR, MSR_LSTAR, MSR_CSTAR, MSR_SYSCALL_MASK, MSR_KERNEL_GS_BASE,
  86         MSR_FS_BASE,
  87 #endif
  88         MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
  89         MSR_TSC_AUX,
  90 };
  91
  92 #define NR_HOST_SAVE_USER_MSRS ARRAY_SIZE(host_save_user_msrs)
  93
  94 struct kvm_vcpu;
  95
  96 struct nested_state {
  97         struct vmcb *hsave;
  98         u64 hsave_msr;
  99         u64 vm_cr_msr;
 100         u64 vmcb;
 101
 102         /* These are the merged vectors */
 103         u32 *msrpm;
 104
 105         /* gpa pointers to the real vectors */
 106         u64 vmcb_msrpm;
 107         u64 vmcb_iopm;
 108
 109         /* A VMEXIT is required but not yet emulated */
 110         bool exit_required;
 111
 112         /* cache for intercepts of the guest */
 113         u32 intercept_cr;
 114         u32 intercept_dr;
 115         u32 intercept_exceptions;
 116         u64 intercept;
 117
 118         /* Nested Paging related state */
 119         u64 nested_cr3;
 120 };
 121
 122 #define MSRPM_OFFSETS   16
 123 static u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly;
 124
 125 /*
 126  * Set osvw_len to higher value when updated Revision Guides
 127  * are published and we know what the new status bits are
 128  */
 129 static uint64_t osvw_len = 4, osvw_status;
 130
 131 struct vcpu_svm {
 132         struct kvm_vcpu vcpu;
 133         struct vmcb *vmcb;
 134         unsigned long vmcb_pa;
 135         struct svm_cpu_data *svm_data;
 136         uint64_t asid_generation;
 137         uint64_t sysenter_esp;
 138         uint64_t sysenter_eip;
 139         uint64_t tsc_aux;
 140
 141         u64 next_rip;
 142
 143         u64 host_user_msrs[NR_HOST_SAVE_USER_MSRS];
 144         struct {
 145                 u16 fs;
 146                 u16 gs;
 147                 u16 ldt;
 148                 u64 gs_base;
 149         } host;
 150
 151         u32 *msrpm;
 152
 153         ulong nmi_iret_rip;
 154
 155         struct nested_state nested;
 156
 157         bool nmi_singlestep;
 158
 159         unsigned int3_injected;
 160         unsigned long int3_rip;
 161         u32 apf_reason;
 162
 163         /* cached guest cpuid flags for faster access */
 164         bool nrips_enabled      : 1;
 165 };
 166
 167 static DEFINE_PER_CPU(u64, current_tsc_ratio);
 168 #define TSC_RATIO_DEFAULT       0x0100000000ULL
 169
 170 #define MSR_INVALID                     0xffffffffU
 171
 172 static const struct svm_direct_access_msrs {
 173         u32 index;   /* Index of the MSR */
 174         bool always; /* True if intercept is always on */
 175 } direct_access_msrs[] = {
 176         { .index = MSR_STAR,                            .always = true  },
 177         { .index = MSR_IA32_SYSENTER_CS,                .always = true  },
 178 #ifdef CONFIG_X86_64
 179         { .index = MSR_GS_BASE,                         .always = true  },
 180         { .index = MSR_FS_BASE,                         .always = true  },
 181         { .index = MSR_KERNEL_GS_BASE,                  .always = true  },
 182         { .index = MSR_LSTAR,                           .always = true  },
 183         { .index = MSR_CSTAR,                           .always = true  },
 184         { .index = MSR_SYSCALL_MASK,                    .always = true  },
 185 #endif
 186         { .index = MSR_IA32_LASTBRANCHFROMIP,           .always = false },
 187         { .index = MSR_IA32_LASTBRANCHTOIP,             .always = false },
 188         { .index = MSR_IA32_LASTINTFROMIP,              .always = false },
 189         { .index = MSR_IA32_LASTINTTOIP,                .always = false },
 190         { .index = MSR_INVALID,                         .always = false },
 191 };
 192
 193 /* enable NPT for AMD64 and X86 with PAE */
 194 #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
 195 static bool npt_enabled = true;
 196 #else
 197 static bool npt_enabled;
 198 #endif
 199
 200 /* allow nested paging (virtualized MMU) for all guests */
 201 static int npt = true;
 202 module_param(npt, int, S_IRUGO);
 203
 204 /* allow nested virtualization in KVM/SVM */
 205 static int nested = true;
 206 module_param(nested, int, S_IRUGO);
 207
 208 static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0);
 209 static void svm_flush_tlb(struct kvm_vcpu *vcpu);
 210 static void svm_complete_interrupts(struct vcpu_svm *svm);
 211
 212 static int nested_svm_exit_handled(struct vcpu_svm *svm);
 213 static int nested_svm_intercept(struct vcpu_svm *svm);
 214 static int nested_svm_vmexit(struct vcpu_svm *svm);
 215 static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
 216                                       bool has_error_code, u32 error_code);
 217
 218 enum {
 219         VMCB_INTERCEPTS, /* Intercept vectors, TSC offset,
 220                             pause filter count */
 221         VMCB_PERM_MAP,   /* IOPM Base and MSRPM Base */
 222         VMCB_ASID,       /* ASID */
 223         VMCB_INTR,       /* int_ctl, int_vector */
 224         VMCB_NPT,        /* npt_en, nCR3, gPAT */
 225         VMCB_CR,         /* CR0, CR3, CR4, EFER */
 226         VMCB_DR,         /* DR6, DR7 */
 227         VMCB_DT,         /* GDT, IDT */
 228         VMCB_SEG,        /* CS, DS, SS, ES, CPL */
 229         VMCB_CR2,        /* CR2 only */
 230         VMCB_LBR,        /* DBGCTL, BR_FROM, BR_TO, LAST_EX_FROM, LAST_EX_TO */
 231         VMCB_DIRTY_MAX,
 232 };
 233
 234 /* TPR and CR2 are always written before VMRUN */
 235 #define VMCB_ALWAYS_DIRTY_MASK  ((1U << VMCB_INTR) | (1U << VMCB_CR2))
 236
 237 static inline void mark_all_dirty(struct vmcb *vmcb)
 238 {
 239         vmcb->control.clean = 0;
 240 }
 241
 242 static inline void mark_all_clean(struct vmcb *vmcb)
 243 {
 244         vmcb->control.clean = ((1 << VMCB_DIRTY_MAX) - 1)
 245                                & ~VMCB_ALWAYS_DIRTY_MASK;
 246 }
 247
 248 static inline void mark_dirty(struct vmcb *vmcb, int bit)
 249 {
 250         vmcb->control.clean &= ~(1 << bit);
 251 }
 252
 253 static inline struct vcpu_svm *to_svm(struct kvm_vcpu *vcpu)
 254 {
 255         return container_of(vcpu, struct vcpu_svm, vcpu);
 256 }
 257
 258 static void recalc_intercepts(struct vcpu_svm *svm)
 259 {
 260         struct vmcb_control_area *c, *h;
 261         struct nested_state *g;
 262
 263         mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
 264
 265         if (!is_guest_mode(&svm->vcpu))
 266                 return;
 267
 268         c = &svm->vmcb->control;
 269         h = &svm->nested.hsave->control;
 270         g = &svm->nested;
 271
 272         c->intercept_cr = h->intercept_cr | g->intercept_cr;
 273         c->intercept_dr = h->intercept_dr | g->intercept_dr;
 274         c->intercept_exceptions = h->intercept_exceptions | g->intercept_exceptions;
 275         c->intercept = h->intercept | g->intercept;
 276 }
 277
 278 static inline struct vmcb *get_host_vmcb(struct vcpu_svm *svm)
 279 {
 280         if (is_guest_mode(&svm->vcpu))
 281                 return svm->nested.hsave;
 282         else
 283                 return svm->vmcb;
 284 }
 285
 286 static inline void set_cr_intercept(struct vcpu_svm *svm, int bit)
 287 {
 288         struct vmcb *vmcb = get_host_vmcb(svm);
 289
 290         vmcb->control.intercept_cr |= (1U << bit);
 291
 292         recalc_intercepts(svm);
 293 }
 294
 295 static inline void clr_cr_intercept(struct vcpu_svm *svm, int bit)
 296 {
 297         struct vmcb *vmcb = get_host_vmcb(svm);
 298
 299         vmcb->control.intercept_cr &= ~(1U << bit);
 300
 301         recalc_intercepts(svm);
 302 }
 303
 304 static inline bool is_cr_intercept(struct vcpu_svm *svm, int bit)
 305 {
 306         struct vmcb *vmcb = get_host_vmcb(svm);
 307
 308         return vmcb->control.intercept_cr & (1U << bit);
 309 }
 310
 311 static inline void set_dr_intercepts(struct vcpu_svm *svm)
 312 {
 313         struct vmcb *vmcb = get_host_vmcb(svm);
 314
 315         vmcb->control.intercept_dr = (1 << INTERCEPT_DR0_READ)
 316                 | (1 << INTERCEPT_DR1_READ)
 317                 | (1 << INTERCEPT_DR2_READ)
 318                 | (1 << INTERCEPT_DR3_READ)
 319                 | (1 << INTERCEPT_DR4_READ)
 320                 | (1 << INTERCEPT_DR5_READ)
 321                 | (1 << INTERCEPT_DR6_READ)
 322                 | (1 << INTERCEPT_DR7_READ)
 323                 | (1 << INTERCEPT_DR0_WRITE)
 324                 | (1 << INTERCEPT_DR1_WRITE)
 325                 | (1 << INTERCEPT_DR2_WRITE)
 326                 | (1 << INTERCEPT_DR3_WRITE)
 327                 | (1 << INTERCEPT_DR4_WRITE)
 328                 | (1 << INTERCEPT_DR5_WRITE)
 329                 | (1 << INTERCEPT_DR6_WRITE)
 330                 | (1 << INTERCEPT_DR7_WRITE);
 331
 332         recalc_intercepts(svm);
 333 }
 334
 335 static inline void clr_dr_intercepts(struct vcpu_svm *svm)
 336 {
 337         struct vmcb *vmcb = get_host_vmcb(svm);
 338
 339         vmcb->control.intercept_dr = 0;
 340
 341         recalc_intercepts(svm);
 342 }
 343
 344 static inline void set_exception_intercept(struct vcpu_svm *svm, int bit)
 345 {
 346         struct vmcb *vmcb = get_host_vmcb(svm);
 347
 348         vmcb->control.intercept_exceptions |= (1U << bit);
 349
 350         recalc_intercepts(svm);
 351 }
 352
 353 static inline void clr_exception_intercept(struct vcpu_svm *svm, int bit)
 354 {
 355         struct vmcb *vmcb = get_host_vmcb(svm);
 356
 357         vmcb->control.intercept_exceptions &= ~(1U << bit);
 358
 359         recalc_intercepts(svm);
 360 }
 361
 362 static inline void set_intercept(struct vcpu_svm *svm, int bit)
 363 {
 364         struct vmcb *vmcb = get_host_vmcb(svm);
 365
 366         vmcb->control.intercept |= (1ULL << bit);
 367
 368         recalc_intercepts(svm);
 369 }
 370
 371 static inline void clr_intercept(struct vcpu_svm *svm, int bit)
 372 {
 373         struct vmcb *vmcb = get_host_vmcb(svm);
 374
 375         vmcb->control.intercept &= ~(1ULL << bit);
 376
 377         recalc_intercepts(svm);
 378 }
 379
 380 static inline void enable_gif(struct vcpu_svm *svm)
 381 {
 382         svm->vcpu.arch.hflags |= HF_GIF_MASK;
 383 }
 384
 385 static inline void disable_gif(struct vcpu_svm *svm)
 386 {
 387         svm->vcpu.arch.hflags &= ~HF_GIF_MASK;
 388 }
 389
 390 static inline bool gif_set(struct vcpu_svm *svm)
 391 {
 392         return !!(svm->vcpu.arch.hflags & HF_GIF_MASK);
 393 }
 394
 395 static unsigned long iopm_base;
 396
 397 struct kvm_ldttss_desc {
 398         u16 limit0;
 399         u16 base0;
 400         unsigned base1:8, type:5, dpl:2, p:1;
 401         unsigned limit1:4, zero0:3, g:1, base2:8;
 402         u32 base3;
 403         u32 zero1;
 404 } __attribute__((packed));
 405
 406 struct svm_cpu_data {
 407         int cpu;
 408
 409         u64 asid_generation;
 410         u32 max_asid;
 411         u32 next_asid;
 412         struct kvm_ldttss_desc *tss_desc;
 413
 414         struct page *save_area;
 415 };
 416
 417 static DEFINE_PER_CPU(struct svm_cpu_data *, svm_data);
 418
 419 struct svm_init_data {
 420         int cpu;
 421         int r;
 422 };
 423
 424 static const u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000};
 425
 426 #define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges)
 427 #define MSRS_RANGE_SIZE 2048
 428 #define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2)
 429
 430 static u32 svm_msrpm_offset(u32 msr)
 431 {
 432         u32 offset;
 433         int i;
 434
 435         for (i = 0; i < NUM_MSR_MAPS; i++) {
 436                 if (msr < msrpm_ranges[i] ||
 437                     msr >= msrpm_ranges[i] + MSRS_IN_RANGE)
 438                         continue;
 439
 440                 offset  = (msr - msrpm_ranges[i]) / 4; /* 4 msrs per u8 */
 441                 offset += (i * MSRS_RANGE_SIZE);       /* add range offset */
 442
 443                 /* Now we have the u8 offset - but need the u32 offset */
 444                 return offset / 4;
 445         }
 446
 447         /* MSR not in any range */
 448         return MSR_INVALID;
 449 }
 450
 451 #define MAX_INST_SIZE 15
 452
 453 static inline void clgi(void)
 454 {
 455         asm volatile (__ex(SVM_CLGI));
 456 }
 457
 458 static inline void stgi(void)
 459 {
 460         asm volatile (__ex(SVM_STGI));
 461 }
 462
 463 static inline void invlpga(unsigned long addr, u32 asid)
 464 {
 465         asm volatile (__ex(SVM_INVLPGA) : : "a"(addr), "c"(asid));
 466 }
 467
 468 static int get_npt_level(void)
 469 {
 470 #ifdef CONFIG_X86_64
 471         return PT64_ROOT_LEVEL;
 472 #else
 473         return PT32E_ROOT_LEVEL;
 474 #endif
 475 }
 476
 477 static void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer)
 478 {
 479         vcpu->arch.efer = efer;
 480         if (!npt_enabled && !(efer & EFER_LMA))
 481                 efer &= ~EFER_LME;
 482
 483         to_svm(vcpu)->vmcb->save.efer = efer | EFER_SVME;
 484         mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
 485 }
 486
 487 static int is_external_interrupt(u32 info)
 488 {
 489         info &= SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID;
 490         return info == (SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR);
 491 }
 492
 493 static u32 svm_get_interrupt_shadow(struct kvm_vcpu *vcpu)
 494 {
 495         struct vcpu_svm *svm = to_svm(vcpu);
 496         u32 ret = 0;
 497
 498         if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK)
 499                 ret = KVM_X86_SHADOW_INT_STI | KVM_X86_SHADOW_INT_MOV_SS;
 500         return ret;
 501 }
 502
 503 static void svm_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
 504 {
 505         struct vcpu_svm *svm = to_svm(vcpu);
 506
 507         if (mask == 0)
 508                 svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK;
 509         else
 510                 svm->vmcb->control.int_state |= SVM_INTERRUPT_SHADOW_MASK;
 511
 512 }
 513
 514 static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
 515 {
 516         struct vcpu_svm *svm = to_svm(vcpu);
 517
 518         if (svm->vmcb->control.next_rip != 0) {
 519                 WARN_ON_ONCE(!static_cpu_has(X86_FEATURE_NRIPS));
 520                 svm->next_rip = svm->vmcb->control.next_rip;
 521         }
 522
 523         if (!svm->next_rip) {
 524                 if (emulate_instruction(vcpu, EMULTYPE_SKIP) !=
 525                                 EMULATE_DONE)
 526                         printk(KERN_DEBUG "%s: NOP\n", __func__);
 527                 return;
 528         }
 529         if (svm->next_rip - kvm_rip_read(vcpu) > MAX_INST_SIZE)
 530                 printk(KERN_ERR "%s: ip 0x%lx next 0x%llx\n",
 531                        __func__, kvm_rip_read(vcpu), svm->next_rip);
 532
 533         kvm_rip_write(vcpu, svm->next_rip);
 534         svm_set_interrupt_shadow(vcpu, 0);
 535 }
 536
 537 static void svm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr,
 538                                 bool has_error_code, u32 error_code,
 539                                 bool reinject)
 540 {
 541         struct vcpu_svm *svm = to_svm(vcpu);
 542
 543         /*
 544          * If we are within a nested VM we'd better #VMEXIT and let the guest
 545          * handle the exception
 546          */
 547         if (!reinject &&
 548             nested_svm_check_exception(svm, nr, has_error_code, error_code))
 549                 return;
 550
 551         if (nr == BP_VECTOR && !static_cpu_has(X86_FEATURE_NRIPS)) {
 552                 unsigned long rip, old_rip = kvm_rip_read(&svm->vcpu);
 553
 554                 /*
 555                  * For guest debugging where we have to reinject #BP if some
 556                  * INT3 is guest-owned:
 557                  * Emulate nRIP by moving RIP forward. Will fail if injection
 558                  * raises a fault that is not intercepted. Still better than
 559                  * failing in all cases.
 560                  */
 561                 skip_emulated_instruction(&svm->vcpu);
 562                 rip = kvm_rip_read(&svm->vcpu);
 563                 svm->int3_rip = rip + svm->vmcb->save.cs.base;
 564                 svm->int3_injected = rip - old_rip;
 565         }
 566
 567         svm->vmcb->control.event_inj = nr
 568                 | SVM_EVTINJ_VALID
 569                 | (has_error_code ? SVM_EVTINJ_VALID_ERR : 0)
 570                 | SVM_EVTINJ_TYPE_EXEPT;
 571         svm->vmcb->control.event_inj_err = error_code;
 572 }
 573
 574 static void svm_init_erratum_383(void)
 575 {
 576         u32 low, high;
 577         int err;
 578         u64 val;
 579
 580         if (!static_cpu_has_bug(X86_BUG_AMD_TLB_MMATCH))
 581                 return;
 582
 583         /* Use _safe variants to not break nested virtualization */
 584         val = native_read_msr_safe(MSR_AMD64_DC_CFG, &err);
 585         if (err)
 586                 return;
 587
 588         val |= (1ULL << 47);
 589
 590         low  = lower_32_bits(val);
 591         high = upper_32_bits(val);
 592
 593         native_write_msr_safe(MSR_AMD64_DC_CFG, low, high);
 594
 595         erratum_383_found = true;
 596 }
 597
 598 static void svm_init_osvw(struct kvm_vcpu *vcpu)
 599 {
 600         /*
 601          * Guests should see errata 400 and 415 as fixed (assuming that
 602          * HLT and IO instructions are intercepted).
 603          */
 604         vcpu->arch.osvw.length = (osvw_len >= 3) ? (osvw_len) : 3;
 605         vcpu->arch.osvw.status = osvw_status & ~(6ULL);
 606
 607         /*
 608          * By increasing VCPU's osvw.length to 3 we are telling the guest that
 609          * all osvw.status bits inside that length, including bit 0 (which is
 610          * reserved for erratum 298), are valid. However, if host processor's
 611          * osvw_len is 0 then osvw_status[0] carries no information. We need to
 612          * be conservative here and therefore we tell the guest that erratum 298
 613          * is present (because we really don't know).
 614          */
 615         if (osvw_len == 0 && boot_cpu_data.x86 == 0x10)
 616                 vcpu->arch.osvw.status |= 1;
 617 }
 618
 619 static int has_svm(void)
 620 {
 621         const char *msg;
 622
 623         if (!cpu_has_svm(&msg)) {
 624                 printk(KERN_INFO "has_svm: %s\n", msg);
 625                 return 0;
 626         }
 627
 628         return 1;
 629 }
 630
 631 static void svm_hardware_disable(void)
 632 {
 633         /* Make sure we clean up behind us */
 634         if (static_cpu_has(X86_FEATURE_TSCRATEMSR))
 635                 wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT);
 636
 637         cpu_svm_disable();
 638
 639         amd_pmu_disable_virt();
 640 }
 641
 642 static int svm_hardware_enable(void)
 643 {
 644
 645         struct svm_cpu_data *sd;
 646         uint64_t efer;
 647         struct desc_ptr gdt_descr;
 648         struct desc_struct *gdt;
 649         int me = raw_smp_processor_id();
 650
 651         rdmsrl(MSR_EFER, efer);
 652         if (efer & EFER_SVME)
 653                 return -EBUSY;
 654
 655         if (!has_svm()) {
 656                 pr_err("%s: err EOPNOTSUPP on %d\n", __func__, me);
 657                 return -EINVAL;
 658         }
 659         sd = per_cpu(svm_data, me);
 660         if (!sd) {
 661                 pr_err("%s: svm_data is NULL on %d\n", __func__, me);
 662                 return -EINVAL;
 663         }
 664
 665         sd->asid_generation = 1;
 666         sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1;
 667         sd->next_asid = sd->max_asid + 1;
 668
 669         native_store_gdt(&gdt_descr);
 670         gdt = (struct desc_struct *)gdt_descr.address;
 671         sd->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS);
 672
 673         wrmsrl(MSR_EFER, efer | EFER_SVME);
 674
 675         wrmsrl(MSR_VM_HSAVE_PA, page_to_pfn(sd->save_area) << PAGE_SHIFT);
 676
 677         if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
 678                 wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT);
 679                 __this_cpu_write(current_tsc_ratio, TSC_RATIO_DEFAULT);
 680         }
 681
 682
 683         /*
 684          * Get OSVW bits.
 685          *
 686          * Note that it is possible to have a system with mixed processor
 687          * revisions and therefore different OSVW bits. If bits are not the same
 688          * on different processors then choose the worst case (i.e. if erratum
 689          * is present on one processor and not on another then assume that the
 690          * erratum is present everywhere).
 691          */
 692         if (cpu_has(&boot_cpu_data, X86_FEATURE_OSVW)) {
 693                 uint64_t len, status = 0;
 694                 int err;
 695
 696                 len = native_read_msr_safe(MSR_AMD64_OSVW_ID_LENGTH, &err);
 697                 if (!err)
 698                         status = native_read_msr_safe(MSR_AMD64_OSVW_STATUS,
 699                                                       &err);
 700
 701                 if (err)
 702                         osvw_status = osvw_len = 0;
 703                 else {
 704                         if (len < osvw_len)
 705                                 osvw_len = len;
 706                         osvw_status |= status;
 707                         osvw_status &= (1ULL << osvw_len) - 1;
 708                 }
 709         } else
 710                 osvw_status = osvw_len = 0;
 711
 712         svm_init_erratum_383();
 713
 714         amd_pmu_enable_virt();
 715
 716         return 0;
 717 }
 718
 719 static void svm_cpu_uninit(int cpu)
 720 {
 721         struct svm_cpu_data *sd = per_cpu(svm_data, raw_smp_processor_id());
 722
 723         if (!sd)
 724                 return;
 725
 726         per_cpu(svm_data, raw_smp_processor_id()) = NULL;
 727         __free_page(sd->save_area);
 728         kfree(sd);
 729 }
 730
 731 static int svm_cpu_init(int cpu)
 732 {
 733         struct svm_cpu_data *sd;
 734         int r;
 735
 736         sd = kzalloc(sizeof(struct svm_cpu_data), GFP_KERNEL);
 737         if (!sd)
 738                 return -ENOMEM;
 739         sd->cpu = cpu;
 740         sd->save_area = alloc_page(GFP_KERNEL);
 741         r = -ENOMEM;
 742         if (!sd->save_area)
 743                 goto err_1;
 744
 745         per_cpu(svm_data, cpu) = sd;
 746
 747         return 0;
 748
 749 err_1:
 750         kfree(sd);
 751         return r;
 752
 753 }
 754
 755 static bool valid_msr_intercept(u32 index)
 756 {
 757         int i;
 758
 759         for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++)
 760                 if (direct_access_msrs[i].index == index)
 761                         return true;
 762
 763         return false;
 764 }
 765
 766 static void set_msr_interception(u32 *msrpm, unsigned msr,
 767                                  int read, int write)
 768 {
 769         u8 bit_read, bit_write;
 770         unsigned long tmp;
 771         u32 offset;
 772
 773         /*
 774          * If this warning triggers extend the direct_access_msrs list at the
 775          * beginning of the file
 776          */
 777         WARN_ON(!valid_msr_intercept(msr));
 778
 779         offset    = svm_msrpm_offset(msr);
 780         bit_read  = 2 * (msr & 0x0f);
 781         bit_write = 2 * (msr & 0x0f) + 1;
 782         tmp       = msrpm[offset];
 783
 784         BUG_ON(offset == MSR_INVALID);
 785
 786         read  ? clear_bit(bit_read,  &tmp) : set_bit(bit_read,  &tmp);
 787         write ? clear_bit(bit_write, &tmp) : set_bit(bit_write, &tmp);
 788
 789         msrpm[offset] = tmp;
 790 }
 791
 792 static void svm_vcpu_init_msrpm(u32 *msrpm)
 793 {
 794         int i;
 795
 796         memset(msrpm, 0xff, PAGE_SIZE * (1 << MSRPM_ALLOC_ORDER));
 797
 798         for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
 799                 if (!direct_access_msrs[i].always)
 800                         continue;
 801
 802                 set_msr_interception(msrpm, direct_access_msrs[i].index, 1, 1);
 803         }
 804 }
 805
 806 static void add_msr_offset(u32 offset)
 807 {
 808         int i;
 809
 810         for (i = 0; i < MSRPM_OFFSETS; ++i) {
 811
 812                 /* Offset already in list? */
 813                 if (msrpm_offsets[i] == offset)
 814                         return;
 815
 816                 /* Slot used by another offset? */
 817                 if (msrpm_offsets[i] != MSR_INVALID)
 818                         continue;
 819
 820                 /* Add offset to list */
 821                 msrpm_offsets[i] = offset;
 822
 823                 return;
 824         }
 825
 826         /*
 827          * If this BUG triggers the msrpm_offsets table has an overflow. Just
 828          * increase MSRPM_OFFSETS in this case.
 829          */
 830         BUG();
 831 }
 832
 833 static void init_msrpm_offsets(void)
 834 {
 835         int i;
 836
 837         memset(msrpm_offsets, 0xff, sizeof(msrpm_offsets));
 838
 839         for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
 840                 u32 offset;
 841
 842                 offset = svm_msrpm_offset(direct_access_msrs[i].index);
 843                 BUG_ON(offset == MSR_INVALID);
 844
 845                 add_msr_offset(offset);
 846         }
 847 }
 848
 849 static void svm_enable_lbrv(struct vcpu_svm *svm)
 850 {
 851         u32 *msrpm = svm->msrpm;
 852
 853         svm->vmcb->control.lbr_ctl = 1;
 854         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1);
 855         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1);
 856         set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 1, 1);
 857         set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 1, 1);
 858 }
 859
 860 static void svm_disable_lbrv(struct vcpu_svm *svm)
 861 {
 862         u32 *msrpm = svm->msrpm;
 863
 864         svm->vmcb->control.lbr_ctl = 0;
 865         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 0, 0);
 866         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 0, 0);
 867         set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 0, 0);
 868         set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 0, 0);
 869 }
 870
 871 static __init int svm_hardware_setup(void)
 872 {
 873         int cpu;
 874         struct page *iopm_pages;
 875         void *iopm_va;
 876         int r;
 877
 878         iopm_pages = alloc_pages(GFP_KERNEL, IOPM_ALLOC_ORDER);
 879
 880         if (!iopm_pages)
 881                 return -ENOMEM;
 882
 883         iopm_va = page_address(iopm_pages);
 884         memset(iopm_va, 0xff, PAGE_SIZE * (1 << IOPM_ALLOC_ORDER));
 885         iopm_base = page_to_pfn(iopm_pages) << PAGE_SHIFT;
 886
 887         init_msrpm_offsets();
 888
 889         if (boot_cpu_has(X86_FEATURE_NX))
 890                 kvm_enable_efer_bits(EFER_NX);
 891
 892         if (boot_cpu_has(X86_FEATURE_FXSR_OPT))
 893                 kvm_enable_efer_bits(EFER_FFXSR);
 894
 895         if (boot_cpu_has(X86_FEATURE_TSCRATEMSR)) {
 896                 kvm_has_tsc_control = true;
 897                 kvm_max_tsc_scaling_ratio = TSC_RATIO_MAX;
 898                 kvm_tsc_scaling_ratio_frac_bits = 32;
 899         }
 900
 901         if (nested) {
 902                 printk(KERN_INFO "kvm: Nested Virtualization enabled\n");
 903                 kvm_enable_efer_bits(EFER_SVME | EFER_LMSLE);
 904         }
 905
 906         for_each_possible_cpu(cpu) {
 907                 r = svm_cpu_init(cpu);
 908                 if (r)
 909                         goto err;
 910         }
 911
 912         if (!boot_cpu_has(X86_FEATURE_NPT))
 913                 npt_enabled = false;
 914
 915         if (npt_enabled && !npt) {
 916                 printk(KERN_INFO "kvm: Nested Paging disabled\n");
 917                 npt_enabled = false;
 918         }
 919
 920         if (npt_enabled) {
 921                 printk(KERN_INFO "kvm: Nested Paging enabled\n");
 922                 kvm_enable_tdp();
 923         } else
 924                 kvm_disable_tdp();
 925
 926         return 0;
 927
 928 err:
 929         __free_pages(iopm_pages, IOPM_ALLOC_ORDER);
 930         iopm_base = 0;
 931         return r;
 932 }
 933
 934 static __exit void svm_hardware_unsetup(void)
 935 {
 936         int cpu;
 937
 938         for_each_possible_cpu(cpu)
 939                 svm_cpu_uninit(cpu);
 940
 941         __free_pages(pfn_to_page(iopm_base >> PAGE_SHIFT), IOPM_ALLOC_ORDER);
 942         iopm_base = 0;
 943 }
 944
 945 static void init_seg(struct vmcb_seg *seg)
 946 {
 947         seg->selector = 0;
 948         seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK |
 949                       SVM_SELECTOR_WRITE_MASK; /* Read/Write Data Segment */
 950         seg->limit = 0xffff;
 951         seg->base = 0;
 952 }
 953
 954 static void init_sys_seg(struct vmcb_seg *seg, uint32_t type)
 955 {
 956         seg->selector = 0;
 957         seg->attrib = SVM_SELECTOR_P_MASK | type;
 958         seg->limit = 0xffff;
 959         seg->base = 0;
 960 }
 961
 962 static u64 svm_read_tsc_offset(struct kvm_vcpu *vcpu)
 963 {
 964         struct vcpu_svm *svm = to_svm(vcpu);
 965
 966         return svm->vmcb->control.tsc_offset;
 967 }
 968
 969 static void svm_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
 970 {
 971         struct vcpu_svm *svm = to_svm(vcpu);
 972         u64 g_tsc_offset = 0;
 973
 974         if (is_guest_mode(vcpu)) {
 975                 g_tsc_offset = svm->vmcb->control.tsc_offset -
 976                                svm->nested.hsave->control.tsc_offset;
 977                 svm->nested.hsave->control.tsc_offset = offset;
 978         } else
 979                 trace_kvm_write_tsc_offset(vcpu->vcpu_id,
 980                                            svm->vmcb->control.tsc_offset,
 981                                            offset);
 982
 983         svm->vmcb->control.tsc_offset = offset + g_tsc_offset;
 984
 985         mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
 986 }
 987
 988 static void svm_adjust_tsc_offset_guest(struct kvm_vcpu *vcpu, s64 adjustment)
 989 {
 990         struct vcpu_svm *svm = to_svm(vcpu);
 991
 992         svm->vmcb->control.tsc_offset += adjustment;
 993         if (is_guest_mode(vcpu))
 994                 svm->nested.hsave->control.tsc_offset += adjustment;
 995         else
 996                 trace_kvm_write_tsc_offset(vcpu->vcpu_id,
 997                                      svm->vmcb->control.tsc_offset - adjustment,
 998                                      svm->vmcb->control.tsc_offset);
 999
1000         mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1001 }
1002
1003 static void init_vmcb(struct vcpu_svm *svm)
1004 {
1005         struct vmcb_control_area *control = &svm->vmcb->control;
1006         struct vmcb_save_area *save = &svm->vmcb->save;
1007
1008         svm->vcpu.fpu_active = 1;
1009         svm->vcpu.arch.hflags = 0;
1010
1011         set_cr_intercept(svm, INTERCEPT_CR0_READ);
1012         set_cr_intercept(svm, INTERCEPT_CR3_READ);
1013         set_cr_intercept(svm, INTERCEPT_CR4_READ);
1014         set_cr_intercept(svm, INTERCEPT_CR0_WRITE);
1015         set_cr_intercept(svm, INTERCEPT_CR3_WRITE);
1016         set_cr_intercept(svm, INTERCEPT_CR4_WRITE);
1017         set_cr_intercept(svm, INTERCEPT_CR8_WRITE);
1018
1019         set_dr_intercepts(svm);
1020
1021         set_exception_intercept(svm, PF_VECTOR);
1022         set_exception_intercept(svm, UD_VECTOR);
1023         set_exception_intercept(svm, MC_VECTOR);
1024         set_exception_intercept(svm, AC_VECTOR);
1025         set_exception_intercept(svm, DB_VECTOR);
1026
1027         set_intercept(svm, INTERCEPT_INTR);
1028         set_intercept(svm, INTERCEPT_NMI);
1029         set_intercept(svm, INTERCEPT_SMI);
1030         set_intercept(svm, INTERCEPT_SELECTIVE_CR0);
1031         set_intercept(svm, INTERCEPT_RDPMC);
1032         set_intercept(svm, INTERCEPT_CPUID);
1033         set_intercept(svm, INTERCEPT_INVD);
1034         set_intercept(svm, INTERCEPT_HLT);
1035         set_intercept(svm, INTERCEPT_INVLPG);
1036         set_intercept(svm, INTERCEPT_INVLPGA);
1037         set_intercept(svm, INTERCEPT_IOIO_PROT);
1038         set_intercept(svm, INTERCEPT_MSR_PROT);
1039         set_intercept(svm, INTERCEPT_TASK_SWITCH);
1040         set_intercept(svm, INTERCEPT_SHUTDOWN);
1041         set_intercept(svm, INTERCEPT_VMRUN);
1042         set_intercept(svm, INTERCEPT_VMMCALL);
1043         set_intercept(svm, INTERCEPT_VMLOAD);
1044         set_intercept(svm, INTERCEPT_VMSAVE);
1045         set_intercept(svm, INTERCEPT_STGI);
1046         set_intercept(svm, INTERCEPT_CLGI);
1047         set_intercept(svm, INTERCEPT_SKINIT);
1048         set_intercept(svm, INTERCEPT_WBINVD);
1049         set_intercept(svm, INTERCEPT_MONITOR);
1050         set_intercept(svm, INTERCEPT_MWAIT);
1051         set_intercept(svm, INTERCEPT_XSETBV);
1052
1053         control->iopm_base_pa = iopm_base;
1054         control->msrpm_base_pa = __pa(svm->msrpm);
1055         control->int_ctl = V_INTR_MASKING_MASK;
1056
1057         init_seg(&save->es);
1058         init_seg(&save->ss);
1059         init_seg(&save->ds);
1060         init_seg(&save->fs);
1061         init_seg(&save->gs);
1062
1063         save->cs.selector = 0xf000;
1064         save->cs.base = 0xffff0000;
1065         /* Executable/Readable Code Segment */
1066         save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK |
1067                 SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK;
1068         save->cs.limit = 0xffff;
1069
1070         save->gdtr.limit = 0xffff;
1071         save->idtr.limit = 0xffff;
1072
1073         init_sys_seg(&save->ldtr, SEG_TYPE_LDT);
1074         init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16);
1075
1076         svm_set_efer(&svm->vcpu, 0);
1077         save->dr6 = 0xffff0ff0;
1078         kvm_set_rflags(&svm->vcpu, 2);
1079         save->rip = 0x0000fff0;
1080         svm->vcpu.arch.regs[VCPU_REGS_RIP] = save->rip;
1081
1082         /*
1083          * svm_set_cr0() sets PG and WP and clears NW and CD on save->cr0.
1084          * It also updates the guest-visible cr0 value.
1085          */
1086         svm_set_cr0(&svm->vcpu, X86_CR0_NW | X86_CR0_CD | X86_CR0_ET);
1087         kvm_mmu_reset_context(&svm->vcpu);
1088
1089         save->cr4 = X86_CR4_PAE;
1090         /* rdx = ?? */
1091
1092         if (npt_enabled) {
1093                 /* Setup VMCB for Nested Paging */
1094                 control->nested_ctl = 1;
1095                 clr_intercept(svm, INTERCEPT_INVLPG);
1096                 clr_exception_intercept(svm, PF_VECTOR);
1097                 clr_cr_intercept(svm, INTERCEPT_CR3_READ);
1098                 clr_cr_intercept(svm, INTERCEPT_CR3_WRITE);
1099                 save->g_pat = svm->vcpu.arch.pat;
1100                 save->cr3 = 0;
1101                 save->cr4 = 0;
1102         }
1103         svm->asid_generation = 0;
1104
1105         svm->nested.vmcb = 0;
1106         svm->vcpu.arch.hflags = 0;
1107
1108         if (boot_cpu_has(X86_FEATURE_PAUSEFILTER)) {
1109                 control->pause_filter_count = 3000;
1110                 set_intercept(svm, INTERCEPT_PAUSE);
1111         }
1112
1113         mark_all_dirty(svm->vmcb);
1114
1115         enable_gif(svm);
1116 }
1117
1118 static void svm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
1119 {
1120         struct vcpu_svm *svm = to_svm(vcpu);
1121         u32 dummy;
1122         u32 eax = 1;
1123
1124         if (!init_event) {
1125                 svm->vcpu.arch.apic_base = APIC_DEFAULT_PHYS_BASE |
1126                                            MSR_IA32_APICBASE_ENABLE;
1127                 if (kvm_vcpu_is_reset_bsp(&svm->vcpu))
1128                         svm->vcpu.arch.apic_base |= MSR_IA32_APICBASE_BSP;
1129         }
1130         init_vmcb(svm);
1131
1132         kvm_cpuid(vcpu, &eax, &dummy, &dummy, &dummy);
1133         kvm_register_write(vcpu, VCPU_REGS_RDX, eax);
1134 }
1135
1136 static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id)
1137 {
1138         struct vcpu_svm *svm;
1139         struct page *page;
1140         struct page *msrpm_pages;
1141         struct page *hsave_page;
1142         struct page *nested_msrpm_pages;
1143         int err;
1144
1145         svm = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
1146         if (!svm) {
1147                 err = -ENOMEM;
1148                 goto out;
1149         }
1150
1151         err = kvm_vcpu_init(&svm->vcpu, kvm, id);
1152         if (err)
1153                 goto free_svm;
1154
1155         err = -ENOMEM;
1156         page = alloc_page(GFP_KERNEL);
1157         if (!page)
1158                 goto uninit;
1159
1160         msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER);
1161         if (!msrpm_pages)
1162                 goto free_page1;
1163
1164         nested_msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER);
1165         if (!nested_msrpm_pages)
1166                 goto free_page2;
1167
1168         hsave_page = alloc_page(GFP_KERNEL);
1169         if (!hsave_page)
1170                 goto free_page3;
1171
1172         svm->nested.hsave = page_address(hsave_page);
1173
1174         svm->msrpm = page_address(msrpm_pages);
1175         svm_vcpu_init_msrpm(svm->msrpm);
1176
1177         svm->nested.msrpm = page_address(nested_msrpm_pages);
1178         svm_vcpu_init_msrpm(svm->nested.msrpm);
1179
1180         svm->vmcb = page_address(page);
1181         clear_page(svm->vmcb);
1182         svm->vmcb_pa = page_to_pfn(page) << PAGE_SHIFT;
1183         svm->asid_generation = 0;
1184         init_vmcb(svm);
1185
1186         svm_init_osvw(&svm->vcpu);
1187
1188         return &svm->vcpu;
1189
1190 free_page3:
1191         __free_pages(nested_msrpm_pages, MSRPM_ALLOC_ORDER);
1192 free_page2:
1193         __free_pages(msrpm_pages, MSRPM_ALLOC_ORDER);
1194 free_page1:
1195         __free_page(page);
1196 uninit:
1197         kvm_vcpu_uninit(&svm->vcpu);
1198 free_svm:
1199         kmem_cache_free(kvm_vcpu_cache, svm);
1200 out:
1201         return ERR_PTR(err);
1202 }
1203
1204 static void svm_free_vcpu(struct kvm_vcpu *vcpu)
1205 {
1206         struct vcpu_svm *svm = to_svm(vcpu);
1207
1208         __free_page(pfn_to_page(svm->vmcb_pa >> PAGE_SHIFT));
1209         __free_pages(virt_to_page(svm->msrpm), MSRPM_ALLOC_ORDER);
1210         __free_page(virt_to_page(svm->nested.hsave));
1211         __free_pages(virt_to_page(svm->nested.msrpm), MSRPM_ALLOC_ORDER);
1212         kvm_vcpu_uninit(vcpu);
1213         kmem_cache_free(kvm_vcpu_cache, svm);
1214 }
1215
1216 static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1217 {
1218         struct vcpu_svm *svm = to_svm(vcpu);
1219         int i;
1220
1221         if (unlikely(cpu != vcpu->cpu)) {
1222                 svm->asid_generation = 0;
1223                 mark_all_dirty(svm->vmcb);
1224         }
1225
1226 #ifdef CONFIG_X86_64
1227         rdmsrl(MSR_GS_BASE, to_svm(vcpu)->host.gs_base);
1228 #endif
1229         savesegment(fs, svm->host.fs);
1230         savesegment(gs, svm->host.gs);
1231         svm->host.ldt = kvm_read_ldt();
1232
1233         for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
1234                 rdmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
1235
1236         if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
1237                 u64 tsc_ratio = vcpu->arch.tsc_scaling_ratio;
1238                 if (tsc_ratio != __this_cpu_read(current_tsc_ratio)) {
1239                         __this_cpu_write(current_tsc_ratio, tsc_ratio);
1240                         wrmsrl(MSR_AMD64_TSC_RATIO, tsc_ratio);
1241                 }
1242         }
1243         /* This assumes that the kernel never uses MSR_TSC_AUX */
1244         if (static_cpu_has(X86_FEATURE_RDTSCP))
1245                 wrmsrl(MSR_TSC_AUX, svm->tsc_aux);
1246 }
1247
1248 static void svm_vcpu_put(struct kvm_vcpu *vcpu)
1249 {
1250         struct vcpu_svm *svm = to_svm(vcpu);
1251         int i;
1252
1253         ++vcpu->stat.host_state_reload;
1254         kvm_load_ldt(svm->host.ldt);
1255 #ifdef CONFIG_X86_64
1256         loadsegment(fs, svm->host.fs);
1257         wrmsrl(MSR_KERNEL_GS_BASE, current->thread.gs);
1258         load_gs_index(svm->host.gs);
1259 #else
1260 #ifdef CONFIG_X86_32_LAZY_GS
1261         loadsegment(gs, svm->host.gs);
1262 #endif
1263 #endif
1264         for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
1265                 wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
1266 }
1267
1268 static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu)
1269 {
1270         return to_svm(vcpu)->vmcb->save.rflags;
1271 }
1272
1273 static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
1274 {
1275        /*
1276         * Any change of EFLAGS.VM is accompained by a reload of SS
1277         * (caused by either a task switch or an inter-privilege IRET),
1278         * so we do not need to update the CPL here.
1279         */
1280         to_svm(vcpu)->vmcb->save.rflags = rflags;
1281 }
1282
1283 static u32 svm_get_pkru(struct kvm_vcpu *vcpu)
1284 {
1285         return 0;
1286 }
1287
1288 static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
1289 {
1290         switch (reg) {
1291         case VCPU_EXREG_PDPTR:
1292                 BUG_ON(!npt_enabled);
1293                 load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu));
1294                 break;
1295         default:
1296                 BUG();
1297         }
1298 }
1299
1300 static void svm_set_vintr(struct vcpu_svm *svm)
1301 {
1302         set_intercept(svm, INTERCEPT_VINTR);
1303 }
1304
1305 static void svm_clear_vintr(struct vcpu_svm *svm)
1306 {
1307         clr_intercept(svm, INTERCEPT_VINTR);
1308 }
1309
1310 static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg)
1311 {
1312         struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
1313
1314         switch (seg) {
1315         case VCPU_SREG_CS: return &save->cs;
1316         case VCPU_SREG_DS: return &save->ds;
1317         case VCPU_SREG_ES: return &save->es;
1318         case VCPU_SREG_FS: return &save->fs;
1319         case VCPU_SREG_GS: return &save->gs;
1320         case VCPU_SREG_SS: return &save->ss;
1321         case VCPU_SREG_TR: return &save->tr;
1322         case VCPU_SREG_LDTR: return &save->ldtr;
1323         }
1324         BUG();
1325         return NULL;
1326 }
1327
1328 static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg)
1329 {
1330         struct vmcb_seg *s = svm_seg(vcpu, seg);
1331
1332         return s->base;
1333 }
1334
1335 static void svm_get_segment(struct kvm_vcpu *vcpu,
1336                             struct kvm_segment *var, int seg)
1337 {
1338         struct vmcb_seg *s = svm_seg(vcpu, seg);
1339
1340         var->base = s->base;
1341         var->limit = s->limit;
1342         var->selector = s->selector;
1343         var->type = s->attrib & SVM_SELECTOR_TYPE_MASK;
1344         var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1;
1345         var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3;
1346         var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1;
1347         var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1;
1348         var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1;
1349         var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1;
1350
1351         /*
1352          * AMD CPUs circa 2014 track the G bit for all segments except CS.
1353          * However, the SVM spec states that the G bit is not observed by the
1354          * CPU, and some VMware virtual CPUs drop the G bit for all segments.
1355          * So let's synthesize a legal G bit for all segments, this helps
1356          * running KVM nested. It also helps cross-vendor migration, because
1357          * Intel's vmentry has a check on the 'G' bit.
1358          */
1359         var->g = s->limit > 0xfffff;
1360
1361         /*
1362          * AMD's VMCB does not have an explicit unusable field, so emulate it
1363          * for cross vendor migration purposes by "not present"
1364          */
1365         var->unusable = !var->present || (var->type == 0);
1366
1367         switch (seg) {
1368         case VCPU_SREG_TR:
1369                 /*
1370                  * Work around a bug where the busy flag in the tr selector
1371                  * isn't exposed
1372                  */
1373                 var->type |= 0x2;
1374                 break;
1375         case VCPU_SREG_DS:
1376         case VCPU_SREG_ES:
1377         case VCPU_SREG_FS:
1378         case VCPU_SREG_GS:
1379                 /*
1380                  * The accessed bit must always be set in the segment
1381                  * descriptor cache, although it can be cleared in the
1382                  * descriptor, the cached bit always remains at 1. Since
1383                  * Intel has a check on this, set it here to support
1384                  * cross-vendor migration.
1385                  */
1386                 if (!var->unusable)
1387                         var->type |= 0x1;
1388                 break;
1389         case VCPU_SREG_SS:
1390                 /*
1391                  * On AMD CPUs sometimes the DB bit in the segment
1392                  * descriptor is left as 1, although the whole segment has
1393                  * been made unusable. Clear it here to pass an Intel VMX
1394                  * entry check when cross vendor migrating.
1395                  */
1396                 if (var->unusable)
1397                         var->db = 0;
1398                 var->dpl = to_svm(vcpu)->vmcb->save.cpl;
1399                 break;
1400         }
1401 }
1402
1403 static int svm_get_cpl(struct kvm_vcpu *vcpu)
1404 {
1405         struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
1406
1407         return save->cpl;
1408 }
1409
1410 static void svm_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1411 {
1412         struct vcpu_svm *svm = to_svm(vcpu);
1413
1414         dt->size = svm->vmcb->save.idtr.limit;
1415         dt->address = svm->vmcb->save.idtr.base;
1416 }
1417
1418 static void svm_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1419 {
1420         struct vcpu_svm *svm = to_svm(vcpu);
1421
1422         svm->vmcb->save.idtr.limit = dt->size;
1423         svm->vmcb->save.idtr.base = dt->address ;
1424         mark_dirty(svm->vmcb, VMCB_DT);
1425 }
1426
1427 static void svm_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1428 {
1429         struct vcpu_svm *svm = to_svm(vcpu);
1430
1431         dt->size = svm->vmcb->save.gdtr.limit;
1432         dt->address = svm->vmcb->save.gdtr.base;
1433 }
1434
1435 static void svm_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1436 {
1437         struct vcpu_svm *svm = to_svm(vcpu);
1438
1439         svm->vmcb->save.gdtr.limit = dt->size;
1440         svm->vmcb->save.gdtr.base = dt->address ;
1441         mark_dirty(svm->vmcb, VMCB_DT);
1442 }
1443
1444 static void svm_decache_cr0_guest_bits(struct kvm_vcpu *vcpu)
1445 {
1446 }
1447
1448 static void svm_decache_cr3(struct kvm_vcpu *vcpu)
1449 {
1450 }
1451
1452 static void svm_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
1453 {
1454 }
1455
1456 static void update_cr0_intercept(struct vcpu_svm *svm)
1457 {
1458         ulong gcr0 = svm->vcpu.arch.cr0;
1459         u64 *hcr0 = &svm->vmcb->save.cr0;
1460
1461         if (!svm->vcpu.fpu_active)
1462                 *hcr0 |= SVM_CR0_SELECTIVE_MASK;
1463         else
1464                 *hcr0 = (*hcr0 & ~SVM_CR0_SELECTIVE_MASK)
1465                         | (gcr0 & SVM_CR0_SELECTIVE_MASK);
1466
1467         mark_dirty(svm->vmcb, VMCB_CR);
1468
1469         if (gcr0 == *hcr0 && svm->vcpu.fpu_active) {
1470                 clr_cr_intercept(svm, INTERCEPT_CR0_READ);
1471                 clr_cr_intercept(svm, INTERCEPT_CR0_WRITE);
1472         } else {
1473                 set_cr_intercept(svm, INTERCEPT_CR0_READ);
1474                 set_cr_intercept(svm, INTERCEPT_CR0_WRITE);
1475         }
1476 }
1477
1478 static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
1479 {
1480         struct vcpu_svm *svm = to_svm(vcpu);
1481
1482 #ifdef CONFIG_X86_64
1483         if (vcpu->arch.efer & EFER_LME) {
1484                 if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
1485                         vcpu->arch.efer |= EFER_LMA;
1486                         svm->vmcb->save.efer |= EFER_LMA | EFER_LME;
1487                 }
1488
1489                 if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) {
1490                         vcpu->arch.efer &= ~EFER_LMA;
1491                         svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME);
1492                 }
1493         }
1494 #endif
1495         vcpu->arch.cr0 = cr0;
1496
1497         if (!npt_enabled)
1498                 cr0 |= X86_CR0_PG | X86_CR0_WP;
1499
1500         if (!vcpu->fpu_active)
1501                 cr0 |= X86_CR0_TS;
1502         /*
1503          * re-enable caching here because the QEMU bios
1504          * does not do it - this results in some delay at
1505          * reboot
1506          */
1507         if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
1508                 cr0 &= ~(X86_CR0_CD | X86_CR0_NW);
1509         svm->vmcb->save.cr0 = cr0;
1510         mark_dirty(svm->vmcb, VMCB_CR);
1511         update_cr0_intercept(svm);
1512 }
1513
1514 static int svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1515 {
1516         unsigned long host_cr4_mce = cr4_read_shadow() & X86_CR4_MCE;
1517         unsigned long old_cr4 = to_svm(vcpu)->vmcb->save.cr4;
1518
1519         if (cr4 & X86_CR4_VMXE)
1520                 return 1;
1521
1522         if (npt_enabled && ((old_cr4 ^ cr4) & X86_CR4_PGE))
1523                 svm_flush_tlb(vcpu);
1524
1525         vcpu->arch.cr4 = cr4;
1526         if (!npt_enabled)
1527                 cr4 |= X86_CR4_PAE;
1528         cr4 |= host_cr4_mce;
1529         to_svm(vcpu)->vmcb->save.cr4 = cr4;
1530         mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
1531         return 0;
1532 }
1533
1534 static void svm_set_segment(struct kvm_vcpu *vcpu,
1535                             struct kvm_segment *var, int seg)
1536 {
1537         struct vcpu_svm *svm = to_svm(vcpu);
1538         struct vmcb_seg *s = svm_seg(vcpu, seg);
1539
1540         s->base = var->base;
1541         s->limit = var->limit;
1542         s->selector = var->selector;
1543         if (var->unusable)
1544                 s->attrib = 0;
1545         else {
1546                 s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK);
1547                 s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT;
1548                 s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT;
1549                 s->attrib |= (var->present & 1) << SVM_SELECTOR_P_SHIFT;
1550                 s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT;
1551                 s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT;
1552                 s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT;
1553                 s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT;
1554         }
1555
1556         /*
1557          * This is always accurate, except if SYSRET returned to a segment
1558          * with SS.DPL != 3.  Intel does not have this quirk, and always
1559          * forces SS.DPL to 3 on sysret, so we ignore that case; fixing it
1560          * would entail passing the CPL to userspace and back.
1561          */
1562         if (seg == VCPU_SREG_SS)
1563                 svm->vmcb->save.cpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3;
1564
1565         mark_dirty(svm->vmcb, VMCB_SEG);
1566 }
1567
1568 static void update_bp_intercept(struct kvm_vcpu *vcpu)
1569 {
1570         struct vcpu_svm *svm = to_svm(vcpu);
1571
1572         clr_exception_intercept(svm, BP_VECTOR);
1573
1574         if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) {
1575                 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
1576                         set_exception_intercept(svm, BP_VECTOR);
1577         } else
1578                 vcpu->guest_debug = 0;
1579 }
1580
1581 static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd)
1582 {
1583         if (sd->next_asid > sd->max_asid) {
1584                 ++sd->asid_generation;
1585                 sd->next_asid = 1;
1586                 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID;
1587         }
1588
1589         svm->asid_generation = sd->asid_generation;
1590         svm->vmcb->control.asid = sd->next_asid++;
1591
1592         mark_dirty(svm->vmcb, VMCB_ASID);
1593 }
1594
1595 static u64 svm_get_dr6(struct kvm_vcpu *vcpu)
1596 {
1597         return to_svm(vcpu)->vmcb->save.dr6;
1598 }
1599
1600 static void svm_set_dr6(struct kvm_vcpu *vcpu, unsigned long value)
1601 {
1602         struct vcpu_svm *svm = to_svm(vcpu);
1603
1604         svm->vmcb->save.dr6 = value;
1605         mark_dirty(svm->vmcb, VMCB_DR);
1606 }
1607
1608 static void svm_sync_dirty_debug_regs(struct kvm_vcpu *vcpu)
1609 {
1610         struct vcpu_svm *svm = to_svm(vcpu);
1611
1612         get_debugreg(vcpu->arch.db[0], 0);
1613         get_debugreg(vcpu->arch.db[1], 1);
1614         get_debugreg(vcpu->arch.db[2], 2);
1615         get_debugreg(vcpu->arch.db[3], 3);
1616         vcpu->arch.dr6 = svm_get_dr6(vcpu);
1617         vcpu->arch.dr7 = svm->vmcb->save.dr7;
1618
1619         vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT;
1620         set_dr_intercepts(svm);
1621 }
1622
1623 static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value)
1624 {
1625         struct vcpu_svm *svm = to_svm(vcpu);
1626
1627         svm->vmcb->save.dr7 = value;
1628         mark_dirty(svm->vmcb, VMCB_DR);
1629 }
1630
1631 static int pf_interception(struct vcpu_svm *svm)
1632 {
1633         u64 fault_address = svm->vmcb->control.exit_info_2;
1634         u32 error_code;
1635         int r = 1;
1636
1637         switch (svm->apf_reason) {
1638         default:
1639                 error_code = svm->vmcb->control.exit_info_1;
1640
1641                 trace_kvm_page_fault(fault_address, error_code);
1642                 if (!npt_enabled && kvm_event_needs_reinjection(&svm->vcpu))
1643                         kvm_mmu_unprotect_page_virt(&svm->vcpu, fault_address);
1644                 r = kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code,
1645                         svm->vmcb->control.insn_bytes,
1646                         svm->vmcb->control.insn_len);
1647                 break;
1648         case KVM_PV_REASON_PAGE_NOT_PRESENT:
1649                 svm->apf_reason = 0;
1650                 local_irq_disable();
1651                 kvm_async_pf_task_wait(fault_address);
1652                 local_irq_enable();
1653                 break;
1654         case KVM_PV_REASON_PAGE_READY:
1655                 svm->apf_reason = 0;
1656                 local_irq_disable();
1657                 kvm_async_pf_task_wake(fault_address);
1658                 local_irq_enable();
1659                 break;
1660         }
1661         return r;
1662 }
1663
1664 static int db_interception(struct vcpu_svm *svm)
1665 {
1666         struct kvm_run *kvm_run = svm->vcpu.run;
1667
1668         if (!(svm->vcpu.guest_debug &
1669               (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) &&
1670                 !svm->nmi_singlestep) {
1671                 kvm_queue_exception(&svm->vcpu, DB_VECTOR);
1672                 return 1;
1673         }
1674
1675         if (svm->nmi_singlestep) {
1676                 svm->nmi_singlestep = false;
1677                 if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP))
1678                         svm->vmcb->save.rflags &=
1679                                 ~(X86_EFLAGS_TF | X86_EFLAGS_RF);
1680         }
1681
1682         if (svm->vcpu.guest_debug &
1683             (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) {
1684                 kvm_run->exit_reason = KVM_EXIT_DEBUG;
1685                 kvm_run->debug.arch.pc =
1686                         svm->vmcb->save.cs.base + svm->vmcb->save.rip;
1687                 kvm_run->debug.arch.exception = DB_VECTOR;
1688                 return 0;
1689         }
1690
1691         return 1;
1692 }
1693
1694 static int bp_interception(struct vcpu_svm *svm)
1695 {
1696         struct kvm_run *kvm_run = svm->vcpu.run;
1697
1698         kvm_run->exit_reason = KVM_EXIT_DEBUG;
1699         kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip;
1700         kvm_run->debug.arch.exception = BP_VECTOR;
1701         return 0;
1702 }
1703
1704 static int ud_interception(struct vcpu_svm *svm)
1705 {
1706         int er;
1707
1708         er = emulate_instruction(&svm->vcpu, EMULTYPE_TRAP_UD);
1709         if (er != EMULATE_DONE)
1710                 kvm_queue_exception(&svm->vcpu, UD_VECTOR);
1711         return 1;
1712 }
1713
1714 static int ac_interception(struct vcpu_svm *svm)
1715 {
1716         kvm_queue_exception_e(&svm->vcpu, AC_VECTOR, 0);
1717         return 1;
1718 }
1719
1720 static void svm_fpu_activate(struct kvm_vcpu *vcpu)
1721 {
1722         struct vcpu_svm *svm = to_svm(vcpu);
1723
1724         clr_exception_intercept(svm, NM_VECTOR);
1725
1726         svm->vcpu.fpu_active = 1;
1727         update_cr0_intercept(svm);
1728 }
1729
1730 static int nm_interception(struct vcpu_svm *svm)
1731 {
1732         svm_fpu_activate(&svm->vcpu);
1733         return 1;
1734 }
1735
1736 static bool is_erratum_383(void)
1737 {
1738         int err, i;
1739         u64 value;
1740
1741         if (!erratum_383_found)
1742                 return false;
1743
1744         value = native_read_msr_safe(MSR_IA32_MC0_STATUS, &err);
1745         if (err)
1746                 return false;
1747
1748         /* Bit 62 may or may not be set for this mce */
1749         value &= ~(1ULL << 62);
1750
1751         if (value != 0xb600000000010015ULL)
1752                 return false;
1753
1754         /* Clear MCi_STATUS registers */
1755         for (i = 0; i < 6; ++i)
1756                 native_write_msr_safe(MSR_IA32_MCx_STATUS(i), 0, 0);
1757
1758         value = native_read_msr_safe(MSR_IA32_MCG_STATUS, &err);
1759         if (!err) {
1760                 u32 low, high;
1761
1762                 value &= ~(1ULL << 2);
1763                 low    = lower_32_bits(value);
1764                 high   = upper_32_bits(value);
1765
1766                 native_write_msr_safe(MSR_IA32_MCG_STATUS, low, high);
1767         }
1768
1769         /* Flush tlb to evict multi-match entries */
1770         __flush_tlb_all();
1771
1772         return true;
1773 }
1774
1775 static void svm_handle_mce(struct vcpu_svm *svm)
1776 {
1777         if (is_erratum_383()) {
1778                 /*
1779                  * Erratum 383 triggered. Guest state is corrupt so kill the
1780                  * guest.
1781                  */
1782                 pr_err("KVM: Guest triggered AMD Erratum 383\n");
1783
1784                 kvm_make_request(KVM_REQ_TRIPLE_FAULT, &svm->vcpu);
1785
1786                 return;
1787         }
1788
1789         /*
1790          * On an #MC intercept the MCE handler is not called automatically in
1791          * the host. So do it by hand here.
1792          */
1793         asm volatile (
1794                 "int $0x12\n");
1795         /* not sure if we ever come back to this point */
1796
1797         return;
1798 }
1799
1800 static int mc_interception(struct vcpu_svm *svm)
1801 {
1802         return 1;
1803 }
1804
1805 static int shutdown_interception(struct vcpu_svm *svm)
1806 {
1807         struct kvm_run *kvm_run = svm->vcpu.run;
1808
1809         /*
1810          * VMCB is undefined after a SHUTDOWN intercept
1811          * so reinitialize it.
1812          */
1813         clear_page(svm->vmcb);
1814         init_vmcb(svm);
1815
1816         kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
1817         return 0;
1818 }
1819
1820 static int io_interception(struct vcpu_svm *svm)
1821 {
1822         struct kvm_vcpu *vcpu = &svm->vcpu;
1823         u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */
1824         int size, in, string;
1825         unsigned port;
1826
1827         ++svm->vcpu.stat.io_exits;
1828         string = (io_info & SVM_IOIO_STR_MASK) != 0;
1829         in = (io_info & SVM_IOIO_TYPE_MASK) != 0;
1830         if (string || in)
1831                 return emulate_instruction(vcpu, 0) == EMULATE_DONE;
1832
1833         port = io_info >> 16;
1834         size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT;
1835         svm->next_rip = svm->vmcb->control.exit_info_2;
1836         skip_emulated_instruction(&svm->vcpu);
1837
1838         return kvm_fast_pio_out(vcpu, size, port);
1839 }
1840
1841 static int nmi_interception(struct vcpu_svm *svm)
1842 {
1843         return 1;
1844 }
1845
1846 static int intr_interception(struct vcpu_svm *svm)
1847 {
1848         ++svm->vcpu.stat.irq_exits;
1849         return 1;
1850 }
1851
1852 static int nop_on_interception(struct vcpu_svm *svm)
1853 {
1854         return 1;
1855 }
1856
1857 static int halt_interception(struct vcpu_svm *svm)
1858 {
1859         svm->next_rip = kvm_rip_read(&svm->vcpu) + 1;
1860         return kvm_emulate_halt(&svm->vcpu);
1861 }
1862
1863 static int vmmcall_interception(struct vcpu_svm *svm)
1864 {
1865         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
1866         return kvm_emulate_hypercall(&svm->vcpu);
1867 }
1868
1869 static unsigned long nested_svm_get_tdp_cr3(struct kvm_vcpu *vcpu)
1870 {
1871         struct vcpu_svm *svm = to_svm(vcpu);
1872
1873         return svm->nested.nested_cr3;
1874 }
1875
1876 static u64 nested_svm_get_tdp_pdptr(struct kvm_vcpu *vcpu, int index)
1877 {
1878         struct vcpu_svm *svm = to_svm(vcpu);
1879         u64 cr3 = svm->nested.nested_cr3;
1880         u64 pdpte;
1881         int ret;
1882
1883         ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(cr3), &pdpte,
1884                                        offset_in_page(cr3) + index * 8, 8);
1885         if (ret)
1886                 return 0;
1887         return pdpte;
1888 }
1889
1890 static void nested_svm_set_tdp_cr3(struct kvm_vcpu *vcpu,
1891                                    unsigned long root)
1892 {
1893         struct vcpu_svm *svm = to_svm(vcpu);
1894
1895         svm->vmcb->control.nested_cr3 = root;
1896         mark_dirty(svm->vmcb, VMCB_NPT);
1897         svm_flush_tlb(vcpu);
1898 }
1899
1900 static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu,
1901                                        struct x86_exception *fault)
1902 {
1903         struct vcpu_svm *svm = to_svm(vcpu);
1904
1905         if (svm->vmcb->control.exit_code != SVM_EXIT_NPF) {
1906                 /*
1907                  * TODO: track the cause of the nested page fault, and
1908                  * correctly fill in the high bits of exit_info_1.
1909                  */
1910                 svm->vmcb->control.exit_code = SVM_EXIT_NPF;
1911                 svm->vmcb->control.exit_code_hi = 0;
1912                 svm->vmcb->control.exit_info_1 = (1ULL << 32);
1913                 svm->vmcb->control.exit_info_2 = fault->address;
1914         }
1915
1916         svm->vmcb->control.exit_info_1 &= ~0xffffffffULL;
1917         svm->vmcb->control.exit_info_1 |= fault->error_code;
1918
1919         /*
1920          * The present bit is always zero for page structure faults on real
1921          * hardware.
1922          */
1923         if (svm->vmcb->control.exit_info_1 & (2ULL << 32))
1924                 svm->vmcb->control.exit_info_1 &= ~1;
1925
1926         nested_svm_vmexit(svm);
1927 }
1928
1929 static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu)
1930 {
1931         WARN_ON(mmu_is_nested(vcpu));
1932         kvm_init_shadow_mmu(vcpu);
1933         vcpu->arch.mmu.set_cr3           = nested_svm_set_tdp_cr3;
1934         vcpu->arch.mmu.get_cr3           = nested_svm_get_tdp_cr3;
1935         vcpu->arch.mmu.get_pdptr         = nested_svm_get_tdp_pdptr;
1936         vcpu->arch.mmu.inject_page_fault = nested_svm_inject_npf_exit;
1937         vcpu->arch.mmu.shadow_root_level = get_npt_level();
1938         reset_shadow_zero_bits_mask(vcpu, &vcpu->arch.mmu);
1939         vcpu->arch.walk_mmu              = &vcpu->arch.nested_mmu;
1940 }
1941
1942 static void nested_svm_uninit_mmu_context(struct kvm_vcpu *vcpu)
1943 {
1944         vcpu->arch.walk_mmu = &vcpu->arch.mmu;
1945 }
1946
1947 static int nested_svm_check_permissions(struct vcpu_svm *svm)
1948 {
1949         if (!(svm->vcpu.arch.efer & EFER_SVME)
1950             || !is_paging(&svm->vcpu)) {
1951                 kvm_queue_exception(&svm->vcpu, UD_VECTOR);
1952                 return 1;
1953         }
1954
1955         if (svm->vmcb->save.cpl) {
1956                 kvm_inject_gp(&svm->vcpu, 0);
1957                 return 1;
1958         }
1959
1960        return 0;
1961 }
1962
1963 static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
1964                                       bool has_error_code, u32 error_code)
1965 {
1966         int vmexit;
1967
1968         if (!is_guest_mode(&svm->vcpu))
1969                 return 0;
1970
1971         svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + nr;
1972         svm->vmcb->control.exit_code_hi = 0;
1973         svm->vmcb->control.exit_info_1 = error_code;
1974         svm->vmcb->control.exit_info_2 = svm->vcpu.arch.cr2;
1975
1976         vmexit = nested_svm_intercept(svm);
1977         if (vmexit == NESTED_EXIT_DONE)
1978                 svm->nested.exit_required = true;
1979
1980         return vmexit;
1981 }
1982
1983 /* This function returns true if it is save to enable the irq window */
1984 static inline bool nested_svm_intr(struct vcpu_svm *svm)
1985 {
1986         if (!is_guest_mode(&svm->vcpu))
1987                 return true;
1988
1989         if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
1990                 return true;
1991
1992         if (!(svm->vcpu.arch.hflags & HF_HIF_MASK))
1993                 return false;
1994
1995         /*
1996          * if vmexit was already requested (by intercepted exception
1997          * for instance) do not overwrite it with "external interrupt"
1998          * vmexit.
1999          */
2000         if (svm->nested.exit_required)
2001                 return false;
2002
2003         svm->vmcb->control.exit_code   = SVM_EXIT_INTR;
2004         svm->vmcb->control.exit_info_1 = 0;
2005         svm->vmcb->control.exit_info_2 = 0;
2006
2007         if (svm->nested.intercept & 1ULL) {
2008                 /*
2009                  * The #vmexit can't be emulated here directly because this
2010                  * code path runs with irqs and preemption disabled. A
2011                  * #vmexit emulation might sleep. Only signal request for
2012                  * the #vmexit here.
2013                  */
2014                 svm->nested.exit_required = true;
2015                 trace_kvm_nested_intr_vmexit(svm->vmcb->save.rip);
2016                 return false;
2017         }
2018
2019         return true;
2020 }
2021
2022 /* This function returns true if it is save to enable the nmi window */
2023 static inline bool nested_svm_nmi(struct vcpu_svm *svm)
2024 {
2025         if (!is_guest_mode(&svm->vcpu))
2026                 return true;
2027
2028         if (!(svm->nested.intercept & (1ULL << INTERCEPT_NMI)))
2029                 return true;
2030
2031         svm->vmcb->control.exit_code = SVM_EXIT_NMI;
2032         svm->nested.exit_required = true;
2033
2034         return false;
2035 }
2036
2037 static void *nested_svm_map(struct vcpu_svm *svm, u64 gpa, struct page **_page)
2038 {
2039         struct page *page;
2040
2041         might_sleep();
2042
2043         page = kvm_vcpu_gfn_to_page(&svm->vcpu, gpa >> PAGE_SHIFT);
2044         if (is_error_page(page))
2045                 goto error;
2046
2047         *_page = page;
2048
2049         return kmap(page);
2050
2051 error:
2052         kvm_inject_gp(&svm->vcpu, 0);
2053
2054         return NULL;
2055 }
2056
2057 static void nested_svm_unmap(struct page *page)
2058 {
2059         kunmap(page);
2060         kvm_release_page_dirty(page);
2061 }
2062
2063 static int nested_svm_intercept_ioio(struct vcpu_svm *svm)
2064 {
2065         unsigned port, size, iopm_len;
2066         u16 val, mask;
2067         u8 start_bit;
2068         u64 gpa;
2069
2070         if (!(svm->nested.intercept & (1ULL << INTERCEPT_IOIO_PROT)))
2071                 return NESTED_EXIT_HOST;
2072
2073         port = svm->vmcb->control.exit_info_1 >> 16;
2074         size = (svm->vmcb->control.exit_info_1 & SVM_IOIO_SIZE_MASK) >>
2075                 SVM_IOIO_SIZE_SHIFT;
2076         gpa  = svm->nested.vmcb_iopm + (port / 8);
2077         start_bit = port % 8;
2078         iopm_len = (start_bit + size > 8) ? 2 : 1;
2079         mask = (0xf >> (4 - size)) << start_bit;
2080         val = 0;
2081
2082         if (kvm_vcpu_read_guest(&svm->vcpu, gpa, &val, iopm_len))
2083                 return NESTED_EXIT_DONE;
2084
2085         return (val & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
2086 }
2087
2088 static int nested_svm_exit_handled_msr(struct vcpu_svm *svm)
2089 {
2090         u32 offset, msr, value;
2091         int write, mask;
2092
2093         if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT)))
2094                 return NESTED_EXIT_HOST;
2095
2096         msr    = svm->vcpu.arch.regs[VCPU_REGS_RCX];
2097         offset = svm_msrpm_offset(msr);
2098         write  = svm->vmcb->control.exit_info_1 & 1;
2099         mask   = 1 << ((2 * (msr & 0xf)) + write);
2100
2101         if (offset == MSR_INVALID)
2102                 return NESTED_EXIT_DONE;
2103
2104         /* Offset is in 32 bit units but need in 8 bit units */
2105         offset *= 4;
2106
2107         if (kvm_vcpu_read_guest(&svm->vcpu, svm->nested.vmcb_msrpm + offset, &value, 4))
2108                 return NESTED_EXIT_DONE;
2109
2110         return (value & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
2111 }
2112
2113 static int nested_svm_exit_special(struct vcpu_svm *svm)
2114 {
2115         u32 exit_code = svm->vmcb->control.exit_code;
2116
2117         switch (exit_code) {
2118         case SVM_EXIT_INTR:
2119         case SVM_EXIT_NMI:
2120         case SVM_EXIT_EXCP_BASE + MC_VECTOR:
2121                 return NESTED_EXIT_HOST;
2122         case SVM_EXIT_NPF:
2123                 /* For now we are always handling NPFs when using them */
2124                 if (npt_enabled)
2125                         return NESTED_EXIT_HOST;
2126                 break;
2127         case SVM_EXIT_EXCP_BASE + PF_VECTOR:
2128                 /* When we're shadowing, trap PFs, but not async PF */
2129                 if (!npt_enabled && svm->apf_reason == 0)
2130                         return NESTED_EXIT_HOST;
2131                 break;
2132         case SVM_EXIT_EXCP_BASE + NM_VECTOR:
2133                 nm_interception(svm);
2134                 break;
2135         default:
2136                 break;
2137         }
2138
2139         return NESTED_EXIT_CONTINUE;
2140 }
2141
2142 /*
2143  * If this function returns true, this #vmexit was already handled
2144  */
2145 static int nested_svm_intercept(struct vcpu_svm *svm)
2146 {
2147         u32 exit_code = svm->vmcb->control.exit_code;
2148         int vmexit = NESTED_EXIT_HOST;
2149
2150         switch (exit_code) {
2151         case SVM_EXIT_MSR:
2152                 vmexit = nested_svm_exit_handled_msr(svm);
2153                 break;
2154         case SVM_EXIT_IOIO:
2155                 vmexit = nested_svm_intercept_ioio(svm);
2156                 break;
2157         case SVM_EXIT_READ_CR0 ... SVM_EXIT_WRITE_CR8: {
2158                 u32 bit = 1U << (exit_code - SVM_EXIT_READ_CR0);
2159                 if (svm->nested.intercept_cr & bit)
2160                         vmexit = NESTED_EXIT_DONE;
2161                 break;
2162         }
2163         case SVM_EXIT_READ_DR0 ... SVM_EXIT_WRITE_DR7: {
2164                 u32 bit = 1U << (exit_code - SVM_EXIT_READ_DR0);
2165                 if (svm->nested.intercept_dr & bit)
2166                         vmexit = NESTED_EXIT_DONE;
2167                 break;
2168         }
2169         case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: {
2170                 u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE);
2171                 if (svm->nested.intercept_exceptions & excp_bits)
2172                         vmexit = NESTED_EXIT_DONE;
2173                 /* async page fault always cause vmexit */
2174                 else if ((exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR) &&
2175                          svm->apf_reason != 0)
2176                         vmexit = NESTED_EXIT_DONE;
2177                 break;
2178         }
2179         case SVM_EXIT_ERR: {
2180                 vmexit = NESTED_EXIT_DONE;
2181                 break;
2182         }
2183         default: {
2184                 u64 exit_bits = 1ULL << (exit_code - SVM_EXIT_INTR);
2185                 if (svm->nested.intercept & exit_bits)
2186                         vmexit = NESTED_EXIT_DONE;
2187         }
2188         }
2189
2190         return vmexit;
2191 }
2192
2193 static int nested_svm_exit_handled(struct vcpu_svm *svm)
2194 {
2195         int vmexit;
2196
2197         vmexit = nested_svm_intercept(svm);
2198
2199         if (vmexit == NESTED_EXIT_DONE)
2200                 nested_svm_vmexit(svm);
2201
2202         return vmexit;
2203 }
2204
2205 static inline void copy_vmcb_control_area(struct vmcb *dst_vmcb, struct vmcb *from_vmcb)
2206 {
2207         struct vmcb_control_area *dst  = &dst_vmcb->control;
2208         struct vmcb_control_area *from = &from_vmcb->control;
2209
2210         dst->intercept_cr         = from->intercept_cr;
2211         dst->intercept_dr         = from->intercept_dr;
2212         dst->intercept_exceptions = from->intercept_exceptions;
2213         dst->intercept            = from->intercept;
2214         dst->iopm_base_pa         = from->iopm_base_pa;
2215         dst->msrpm_base_pa        = from->msrpm_base_pa;
2216         dst->tsc_offset           = from->tsc_offset;
2217         dst->asid                 = from->asid;
2218         dst->tlb_ctl              = from->tlb_ctl;
2219         dst->int_ctl              = from->int_ctl;
2220         dst->int_vector           = from->int_vector;
2221         dst->int_state            = from->int_state;
2222         dst->exit_code            = from->exit_code;
2223         dst->exit_code_hi         = from->exit_code_hi;
2224         dst->exit_info_1          = from->exit_info_1;
2225         dst->exit_info_2          = from->exit_info_2;
2226         dst->exit_int_info        = from->exit_int_info;
2227         dst->exit_int_info_err    = from->exit_int_info_err;
2228         dst->nested_ctl           = from->nested_ctl;
2229         dst->event_inj            = from->event_inj;
2230         dst->event_inj_err        = from->event_inj_err;
2231         dst->nested_cr3           = from->nested_cr3;
2232         dst->lbr_ctl              = from->lbr_ctl;
2233 }
2234
2235 static int nested_svm_vmexit(struct vcpu_svm *svm)
2236 {
2237         struct vmcb *nested_vmcb;
2238         struct vmcb *hsave = svm->nested.hsave;
2239         struct vmcb *vmcb = svm->vmcb;
2240         struct page *page;
2241
2242         trace_kvm_nested_vmexit_inject(vmcb->control.exit_code,
2243                                        vmcb->control.exit_info_1,
2244                                        vmcb->control.exit_info_2,
2245                                        vmcb->control.exit_int_info,
2246                                        vmcb->control.exit_int_info_err,
2247                                        KVM_ISA_SVM);
2248
2249         nested_vmcb = nested_svm_map(svm, svm->nested.vmcb, &page);
2250         if (!nested_vmcb)
2251                 return 1;
2252
2253         /* Exit Guest-Mode */
2254         leave_guest_mode(&svm->vcpu);
2255         svm->nested.vmcb = 0;
2256
2257         /* Give the current vmcb to the guest */
2258         disable_gif(svm);
2259
2260         nested_vmcb->save.es     = vmcb->save.es;
2261         nested_vmcb->save.cs     = vmcb->save.cs;
2262         nested_vmcb->save.ss     = vmcb->save.ss;
2263         nested_vmcb->save.ds     = vmcb->save.ds;
2264         nested_vmcb->save.gdtr   = vmcb->save.gdtr;
2265         nested_vmcb->save.idtr   = vmcb->save.idtr;
2266         nested_vmcb->save.efer   = svm->vcpu.arch.efer;
2267         nested_vmcb->save.cr0    = kvm_read_cr0(&svm->vcpu);
2268         nested_vmcb->save.cr3    = kvm_read_cr3(&svm->vcpu);
2269         nested_vmcb->save.cr2    = vmcb->save.cr2;
2270         nested_vmcb->save.cr4    = svm->vcpu.arch.cr4;
2271         nested_vmcb->save.rflags = kvm_get_rflags(&svm->vcpu);
2272         nested_vmcb->save.rip    = vmcb->save.rip;
2273         nested_vmcb->save.rsp    = vmcb->save.rsp;
2274         nested_vmcb->save.rax    = vmcb->save.rax;
2275         nested_vmcb->save.dr7    = vmcb->save.dr7;
2276         nested_vmcb->save.dr6    = vmcb->save.dr6;
2277         nested_vmcb->save.cpl    = vmcb->save.cpl;
2278
2279         nested_vmcb->control.int_ctl           = vmcb->control.int_ctl;
2280         nested_vmcb->control.int_vector        = vmcb->control.int_vector;
2281         nested_vmcb->control.int_state         = vmcb->control.int_state;
2282         nested_vmcb->control.exit_code         = vmcb->control.exit_code;
2283         nested_vmcb->control.exit_code_hi      = vmcb->control.exit_code_hi;
2284         nested_vmcb->control.exit_info_1       = vmcb->control.exit_info_1;
2285         nested_vmcb->control.exit_info_2       = vmcb->control.exit_info_2;
2286         nested_vmcb->control.exit_int_info     = vmcb->control.exit_int_info;
2287         nested_vmcb->control.exit_int_info_err = vmcb->control.exit_int_info_err;
2288
2289         if (svm->nrips_enabled)
2290                 nested_vmcb->control.next_rip  = vmcb->control.next_rip;
2291
2292         /*
2293          * If we emulate a VMRUN/#VMEXIT in the same host #vmexit cycle we have
2294          * to make sure that we do not lose injected events. So check event_inj
2295          * here and copy it to exit_int_info if it is valid.
2296          * Exit_int_info and event_inj can't be both valid because the case
2297          * below only happens on a VMRUN instruction intercept which has
2298          * no valid exit_int_info set.
2299          */
2300         if (vmcb->control.event_inj & SVM_EVTINJ_VALID) {
2301                 struct vmcb_control_area *nc = &nested_vmcb->control;
2302
2303                 nc->exit_int_info     = vmcb->control.event_inj;
2304                 nc->exit_int_info_err = vmcb->control.event_inj_err;
2305         }
2306
2307         nested_vmcb->control.tlb_ctl           = 0;
2308         nested_vmcb->control.event_inj         = 0;
2309         nested_vmcb->control.event_inj_err     = 0;
2310
2311         /* We always set V_INTR_MASKING and remember the old value in hflags */
2312         if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
2313                 nested_vmcb->control.int_ctl &= ~V_INTR_MASKING_MASK;
2314
2315         /* Restore the original control entries */
2316         copy_vmcb_control_area(vmcb, hsave);
2317
2318         kvm_clear_exception_queue(&svm->vcpu);
2319         kvm_clear_interrupt_queue(&svm->vcpu);
2320
2321         svm->nested.nested_cr3 = 0;
2322
2323         /* Restore selected save entries */
2324         svm->vmcb->save.es = hsave->save.es;
2325         svm->vmcb->save.cs = hsave->save.cs;
2326         svm->vmcb->save.ss = hsave->save.ss;
2327         svm->vmcb->save.ds = hsave->save.ds;
2328         svm->vmcb->save.gdtr = hsave->save.gdtr;
2329         svm->vmcb->save.idtr = hsave->save.idtr;
2330         kvm_set_rflags(&svm->vcpu, hsave->save.rflags);
2331         svm_set_efer(&svm->vcpu, hsave->save.efer);
2332         svm_set_cr0(&svm->vcpu, hsave->save.cr0 | X86_CR0_PE);
2333         svm_set_cr4(&svm->vcpu, hsave->save.cr4);
2334         if (npt_enabled) {
2335                 svm->vmcb->save.cr3 = hsave->save.cr3;
2336                 svm->vcpu.arch.cr3 = hsave->save.cr3;
2337         } else {
2338                 (void)kvm_set_cr3(&svm->vcpu, hsave->save.cr3);
2339         }
2340         kvm_register_write(&svm->vcpu, VCPU_REGS_RAX, hsave->save.rax);
2341         kvm_register_write(&svm->vcpu, VCPU_REGS_RSP, hsave->save.rsp);
2342         kvm_register_write(&svm->vcpu, VCPU_REGS_RIP, hsave->save.rip);
2343         svm->vmcb->save.dr7 = 0;
2344         svm->vmcb->save.cpl = 0;
2345         svm->vmcb->control.exit_int_info = 0;
2346
2347         mark_all_dirty(svm->vmcb);
2348
2349         nested_svm_unmap(page);
2350
2351         nested_svm_uninit_mmu_context(&svm->vcpu);
2352         kvm_mmu_reset_context(&svm->vcpu);
2353         kvm_mmu_load(&svm->vcpu);
2354
2355         return 0;
2356 }
2357
2358 static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm)
2359 {
2360         /*
2361          * This function merges the msr permission bitmaps of kvm and the
2362          * nested vmcb. It is optimized in that it only merges the parts where
2363          * the kvm msr permission bitmap may contain zero bits
2364          */
2365         int i;
2366
2367         if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT)))
2368                 return true;
2369
2370         for (i = 0; i < MSRPM_OFFSETS; i++) {
2371                 u32 value, p;
2372                 u64 offset;
2373
2374                 if (msrpm_offsets[i] == 0xffffffff)
2375                         break;
2376
2377                 p      = msrpm_offsets[i];
2378                 offset = svm->nested.vmcb_msrpm + (p * 4);
2379
2380                 if (kvm_vcpu_read_guest(&svm->vcpu, offset, &value, 4))
2381                         return false;
2382
2383                 svm->nested.msrpm[p] = svm->msrpm[p] | value;
2384         }
2385
2386         svm->vmcb->control.msrpm_base_pa = __pa(svm->nested.msrpm);
2387
2388         return true;
2389 }
2390
2391 static bool nested_vmcb_checks(struct vmcb *vmcb)
2392 {
2393         if ((vmcb->control.intercept & (1ULL << INTERCEPT_VMRUN)) == 0)
2394                 return false;
2395
2396         if (vmcb->control.asid == 0)
2397                 return false;
2398
2399         if (vmcb->control.nested_ctl && !npt_enabled)
2400                 return false;
2401
2402         return true;
2403 }
2404
2405 static bool nested_svm_vmrun(struct vcpu_svm *svm)
2406 {
2407         struct vmcb *nested_vmcb;
2408         struct vmcb *hsave = svm->nested.hsave;
2409         struct vmcb *vmcb = svm->vmcb;
2410         struct page *page;
2411         u64 vmcb_gpa;
2412
2413         vmcb_gpa = svm->vmcb->save.rax;
2414
2415         nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
2416         if (!nested_vmcb)
2417                 return false;
2418
2419         if (!nested_vmcb_checks(nested_vmcb)) {
2420                 nested_vmcb->control.exit_code    = SVM_EXIT_ERR;
2421                 nested_vmcb->control.exit_code_hi = 0;
2422                 nested_vmcb->control.exit_info_1  = 0;
2423                 nested_vmcb->control.exit_info_2  = 0;
2424
2425                 nested_svm_unmap(page);
2426
2427                 return false;
2428         }
2429
2430         trace_kvm_nested_vmrun(svm->vmcb->save.rip, vmcb_gpa,
2431                                nested_vmcb->save.rip,
2432                                nested_vmcb->control.int_ctl,
2433                                nested_vmcb->control.event_inj,
2434                                nested_vmcb->control.nested_ctl);
2435
2436         trace_kvm_nested_intercepts(nested_vmcb->control.intercept_cr & 0xffff,
2437                                     nested_vmcb->control.intercept_cr >> 16,
2438                                     nested_vmcb->control.intercept_exceptions,
2439                                     nested_vmcb->control.intercept);
2440
2441         /* Clear internal status */
2442         kvm_clear_exception_queue(&svm->vcpu);
2443         kvm_clear_interrupt_queue(&svm->vcpu);
2444
2445         /*
2446          * Save the old vmcb, so we don't need to pick what we save, but can
2447          * restore everything when a VMEXIT occurs
2448          */
2449         hsave->save.es     = vmcb->save.es;
2450         hsave->save.cs     = vmcb->save.cs;
2451         hsave->save.ss     = vmcb->save.ss;
2452         hsave->save.ds     = vmcb->save.ds;
2453         hsave->save.gdtr   = vmcb->save.gdtr;
2454         hsave->save.idtr   = vmcb->save.idtr;
2455         hsave->save.efer   = svm->vcpu.arch.efer;
2456         hsave->save.cr0    = kvm_read_cr0(&svm->vcpu);
2457         hsave->save.cr4    = svm->vcpu.arch.cr4;
2458         hsave->save.rflags = kvm_get_rflags(&svm->vcpu);
2459         hsave->save.rip    = kvm_rip_read(&svm->vcpu);
2460         hsave->save.rsp    = vmcb->save.rsp;
2461         hsave->save.rax    = vmcb->save.rax;
2462         if (npt_enabled)
2463                 hsave->save.cr3    = vmcb->save.cr3;
2464         else
2465                 hsave->save.cr3    = kvm_read_cr3(&svm->vcpu);
2466
2467         copy_vmcb_control_area(hsave, vmcb);
2468
2469         if (kvm_get_rflags(&svm->vcpu) & X86_EFLAGS_IF)
2470                 svm->vcpu.arch.hflags |= HF_HIF_MASK;
2471         else
2472                 svm->vcpu.arch.hflags &= ~HF_HIF_MASK;
2473
2474         if (nested_vmcb->control.nested_ctl) {
2475                 kvm_mmu_unload(&svm->vcpu);
2476                 svm->nested.nested_cr3 = nested_vmcb->control.nested_cr3;
2477                 nested_svm_init_mmu_context(&svm->vcpu);
2478         }
2479
2480         /* Load the nested guest state */
2481         svm->vmcb->save.es = nested_vmcb->save.es;
2482         svm->vmcb->save.cs = nested_vmcb->save.cs;
2483         svm->vmcb->save.ss = nested_vmcb->save.ss;
2484         svm->vmcb->save.ds = nested_vmcb->save.ds;
2485         svm->vmcb->save.gdtr = nested_vmcb->save.gdtr;
2486         svm->vmcb->save.idtr = nested_vmcb->save.idtr;
2487         kvm_set_rflags(&svm->vcpu, nested_vmcb->save.rflags);
2488         svm_set_efer(&svm->vcpu, nested_vmcb->save.efer);
2489         svm_set_cr0(&svm->vcpu, nested_vmcb->save.cr0);
2490         svm_set_cr4(&svm->vcpu, nested_vmcb->save.cr4);
2491         if (npt_enabled) {
2492                 svm->vmcb->save.cr3 = nested_vmcb->save.cr3;
2493                 svm->vcpu.arch.cr3 = nested_vmcb->save.cr3;
2494         } else
2495                 (void)kvm_set_cr3(&svm->vcpu, nested_vmcb->save.cr3);
2496
2497         /* Guest paging mode is active - reset mmu */
2498         kvm_mmu_reset_context(&svm->vcpu);
2499
2500         svm->vmcb->save.cr2 = svm->vcpu.arch.cr2 = nested_vmcb->save.cr2;
2501         kvm_register_write(&svm->vcpu, VCPU_REGS_RAX, nested_vmcb->save.rax);
2502         kvm_register_write(&svm->vcpu, VCPU_REGS_RSP, nested_vmcb->save.rsp);
2503         kvm_register_write(&svm->vcpu, VCPU_REGS_RIP, nested_vmcb->save.rip);
2504
2505         /* In case we don't even reach vcpu_run, the fields are not updated */
2506         svm->vmcb->save.rax = nested_vmcb->save.rax;
2507         svm->vmcb->save.rsp = nested_vmcb->save.rsp;
2508         svm->vmcb->save.rip = nested_vmcb->save.rip;
2509         svm->vmcb->save.dr7 = nested_vmcb->save.dr7;
2510         svm->vmcb->save.dr6 = nested_vmcb->save.dr6;
2511         svm->vmcb->save.cpl = nested_vmcb->save.cpl;
2512
2513         svm->nested.vmcb_msrpm = nested_vmcb->control.msrpm_base_pa & ~0x0fffULL;
2514         svm->nested.vmcb_iopm  = nested_vmcb->control.iopm_base_pa  & ~0x0fffULL;
2515
2516         /* cache intercepts */
2517         svm->nested.intercept_cr         = nested_vmcb->control.intercept_cr;
2518         svm->nested.intercept_dr         = nested_vmcb->control.intercept_dr;
2519         svm->nested.intercept_exceptions = nested_vmcb->control.intercept_exceptions;
2520         svm->nested.intercept            = nested_vmcb->control.intercept;
2521
2522         svm_flush_tlb(&svm->vcpu);
2523         svm->vmcb->control.int_ctl = nested_vmcb->control.int_ctl | V_INTR_MASKING_MASK;
2524         if (nested_vmcb->control.int_ctl & V_INTR_MASKING_MASK)
2525                 svm->vcpu.arch.hflags |= HF_VINTR_MASK;
2526         else
2527                 svm->vcpu.arch.hflags &= ~HF_VINTR_MASK;
2528
2529         if (svm->vcpu.arch.hflags & HF_VINTR_MASK) {
2530                 /* We only want the cr8 intercept bits of the guest */
2531                 clr_cr_intercept(svm, INTERCEPT_CR8_READ);
2532                 clr_cr_intercept(svm, INTERCEPT_CR8_WRITE);
2533         }
2534
2535         /* We don't want to see VMMCALLs from a nested guest */
2536         clr_intercept(svm, INTERCEPT_VMMCALL);
2537
2538         svm->vmcb->control.lbr_ctl = nested_vmcb->control.lbr_ctl;
2539         svm->vmcb->control.int_vector = nested_vmcb->control.int_vector;
2540         svm->vmcb->control.int_state = nested_vmcb->control.int_state;
2541         svm->vmcb->control.tsc_offset += nested_vmcb->control.tsc_offset;
2542         svm->vmcb->control.event_inj = nested_vmcb->control.event_inj;
2543         svm->vmcb->control.event_inj_err = nested_vmcb->control.event_inj_err;
2544
2545         nested_svm_unmap(page);
2546
2547         /* Enter Guest-Mode */
2548         enter_guest_mode(&svm->vcpu);
2549
2550         /*
2551          * Merge guest and host intercepts - must be called  with vcpu in
2552          * guest-mode to take affect here
2553          */
2554         recalc_intercepts(svm);
2555
2556         svm->nested.vmcb = vmcb_gpa;
2557
2558         enable_gif(svm);
2559
2560         mark_all_dirty(svm->vmcb);
2561
2562         return true;
2563 }
2564
2565 static void nested_svm_vmloadsave(struct vmcb *from_vmcb, struct vmcb *to_vmcb)
2566 {
2567         to_vmcb->save.fs = from_vmcb->save.fs;
2568         to_vmcb->save.gs = from_vmcb->save.gs;
2569         to_vmcb->save.tr = from_vmcb->save.tr;
2570         to_vmcb->save.ldtr = from_vmcb->save.ldtr;
2571         to_vmcb->save.kernel_gs_base = from_vmcb->save.kernel_gs_base;
2572         to_vmcb->save.star = from_vmcb->save.star;
2573         to_vmcb->save.lstar = from_vmcb->save.lstar;
2574         to_vmcb->save.cstar = from_vmcb->save.cstar;
2575         to_vmcb->save.sfmask = from_vmcb->save.sfmask;
2576         to_vmcb->save.sysenter_cs = from_vmcb->save.sysenter_cs;
2577         to_vmcb->save.sysenter_esp = from_vmcb->save.sysenter_esp;
2578         to_vmcb->save.sysenter_eip = from_vmcb->save.sysenter_eip;
2579 }
2580
2581 static int vmload_interception(struct vcpu_svm *svm)
2582 {
2583         struct vmcb *nested_vmcb;
2584         struct page *page;
2585
2586         if (nested_svm_check_permissions(svm))
2587                 return 1;
2588
2589         nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
2590         if (!nested_vmcb)
2591                 return 1;
2592
2593         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2594         skip_emulated_instruction(&svm->vcpu);
2595
2596         nested_svm_vmloadsave(nested_vmcb, svm->vmcb);
2597         nested_svm_unmap(page);
2598
2599         return 1;
2600 }
2601
2602 static int vmsave_interception(struct vcpu_svm *svm)
2603 {
2604         struct vmcb *nested_vmcb;
2605         struct page *page;
2606
2607         if (nested_svm_check_permissions(svm))
2608                 return 1;
2609
2610         nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
2611         if (!nested_vmcb)
2612                 return 1;
2613
2614         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2615         skip_emulated_instruction(&svm->vcpu);
2616
2617         nested_svm_vmloadsave(svm->vmcb, nested_vmcb);
2618         nested_svm_unmap(page);
2619
2620         return 1;
2621 }
2622
2623 static int vmrun_interception(struct vcpu_svm *svm)
2624 {
2625         if (nested_svm_check_permissions(svm))
2626                 return 1;
2627
2628         /* Save rip after vmrun instruction */
2629         kvm_rip_write(&svm->vcpu, kvm_rip_read(&svm->vcpu) + 3);
2630
2631         if (!nested_svm_vmrun(svm))
2632                 return 1;
2633
2634         if (!nested_svm_vmrun_msrpm(svm))
2635                 goto failed;
2636
2637         return 1;
2638
2639 failed:
2640
2641         svm->vmcb->control.exit_code    = SVM_EXIT_ERR;
2642         svm->vmcb->control.exit_code_hi = 0;
2643         svm->vmcb->control.exit_info_1  = 0;
2644         svm->vmcb->control.exit_info_2  = 0;
2645
2646         nested_svm_vmexit(svm);
2647
2648         return 1;
2649 }
2650
2651 static int stgi_interception(struct vcpu_svm *svm)
2652 {
2653         if (nested_svm_check_permissions(svm))
2654                 return 1;
2655
2656         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2657         skip_emulated_instruction(&svm->vcpu);
2658         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
2659
2660         enable_gif(svm);
2661
2662         return 1;
2663 }
2664
2665 static int clgi_interception(struct vcpu_svm *svm)
2666 {
2667         if (nested_svm_check_permissions(svm))
2668                 return 1;
2669
2670         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2671         skip_emulated_instruction(&svm->vcpu);
2672
2673         disable_gif(svm);
2674
2675         /* After a CLGI no interrupts should come */
2676         svm_clear_vintr(svm);
2677         svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
2678
2679         mark_dirty(svm->vmcb, VMCB_INTR);
2680
2681         return 1;
2682 }
2683
2684 static int invlpga_interception(struct vcpu_svm *svm)
2685 {
2686         struct kvm_vcpu *vcpu = &svm->vcpu;
2687
2688         trace_kvm_invlpga(svm->vmcb->save.rip, kvm_register_read(&svm->vcpu, VCPU_REGS_RCX),
2689                           kvm_register_read(&svm->vcpu, VCPU_REGS_RAX));
2690
2691         /* Let's treat INVLPGA the same as INVLPG (can be optimized!) */
2692         kvm_mmu_invlpg(vcpu, kvm_register_read(&svm->vcpu, VCPU_REGS_RAX));
2693
2694         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2695         skip_emulated_instruction(&svm->vcpu);
2696         return 1;
2697 }
2698
2699 static int skinit_interception(struct vcpu_svm *svm)
2700 {
2701         trace_kvm_skinit(svm->vmcb->save.rip, kvm_register_read(&svm->vcpu, VCPU_REGS_RAX));
2702
2703         kvm_queue_exception(&svm->vcpu, UD_VECTOR);
2704         return 1;
2705 }
2706
2707 static int wbinvd_interception(struct vcpu_svm *svm)
2708 {
2709         kvm_emulate_wbinvd(&svm->vcpu);
2710         return 1;
2711 }
2712
2713 static int xsetbv_interception(struct vcpu_svm *svm)
2714 {
2715         u64 new_bv = kvm_read_edx_eax(&svm->vcpu);
2716         u32 index = kvm_register_read(&svm->vcpu, VCPU_REGS_RCX);
2717
2718         if (kvm_set_xcr(&svm->vcpu, index, new_bv) == 0) {
2719                 svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2720                 skip_emulated_instruction(&svm->vcpu);
2721         }
2722
2723         return 1;
2724 }
2725
2726 static int task_switch_interception(struct vcpu_svm *svm)
2727 {
2728         u16 tss_selector;
2729         int reason;
2730         int int_type = svm->vmcb->control.exit_int_info &
2731                 SVM_EXITINTINFO_TYPE_MASK;
2732         int int_vec = svm->vmcb->control.exit_int_info & SVM_EVTINJ_VEC_MASK;
2733         uint32_t type =
2734                 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK;
2735         uint32_t idt_v =
2736                 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID;
2737         bool has_error_code = false;
2738         u32 error_code = 0;
2739
2740         tss_selector = (u16)svm->vmcb->control.exit_info_1;
2741
2742         if (svm->vmcb->control.exit_info_2 &
2743             (1ULL << SVM_EXITINFOSHIFT_TS_REASON_IRET))
2744                 reason = TASK_SWITCH_IRET;
2745         else if (svm->vmcb->control.exit_info_2 &
2746                  (1ULL << SVM_EXITINFOSHIFT_TS_REASON_JMP))
2747                 reason = TASK_SWITCH_JMP;
2748         else if (idt_v)
2749                 reason = TASK_SWITCH_GATE;
2750         else
2751                 reason = TASK_SWITCH_CALL;
2752
2753         if (reason == TASK_SWITCH_GATE) {
2754                 switch (type) {
2755                 case SVM_EXITINTINFO_TYPE_NMI:
2756                         svm->vcpu.arch.nmi_injected = false;
2757                         break;
2758                 case SVM_EXITINTINFO_TYPE_EXEPT:
2759                         if (svm->vmcb->control.exit_info_2 &
2760                             (1ULL << SVM_EXITINFOSHIFT_TS_HAS_ERROR_CODE)) {
2761                                 has_error_code = true;
2762                                 error_code =
2763                                         (u32)svm->vmcb->control.exit_info_2;
2764                         }
2765                         kvm_clear_exception_queue(&svm->vcpu);
2766                         break;
2767                 case SVM_EXITINTINFO_TYPE_INTR:
2768                         kvm_clear_interrupt_queue(&svm->vcpu);
2769                         break;
2770                 default:
2771                         break;
2772                 }
2773         }
2774
2775         if (reason != TASK_SWITCH_GATE ||
2776             int_type == SVM_EXITINTINFO_TYPE_SOFT ||
2777             (int_type == SVM_EXITINTINFO_TYPE_EXEPT &&
2778              (int_vec == OF_VECTOR || int_vec == BP_VECTOR)))
2779                 skip_emulated_instruction(&svm->vcpu);
2780
2781         if (int_type != SVM_EXITINTINFO_TYPE_SOFT)
2782                 int_vec = -1;
2783
2784         if (kvm_task_switch(&svm->vcpu, tss_selector, int_vec, reason,
2785                                 has_error_code, error_code) == EMULATE_FAIL) {
2786                 svm->vcpu.run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
2787                 svm->vcpu.run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
2788                 svm->vcpu.run->internal.ndata = 0;
2789                 return 0;
2790         }
2791         return 1;
2792 }
2793
2794 static int cpuid_interception(struct vcpu_svm *svm)
2795 {
2796         svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
2797         kvm_emulate_cpuid(&svm->vcpu);
2798         return 1;
2799 }
2800
2801 static int iret_interception(struct vcpu_svm *svm)
2802 {
2803         ++svm->vcpu.stat.nmi_window_exits;
2804         clr_intercept(svm, INTERCEPT_IRET);
2805         svm->vcpu.arch.hflags |= HF_IRET_MASK;
2806         svm->nmi_iret_rip = kvm_rip_read(&svm->vcpu);
2807         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
2808         return 1;
2809 }
2810
2811 static int invlpg_interception(struct vcpu_svm *svm)
2812 {
2813         if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
2814                 return emulate_instruction(&svm->vcpu, 0) == EMULATE_DONE;
2815
2816         kvm_mmu_invlpg(&svm->vcpu, svm->vmcb->control.exit_info_1);
2817         skip_emulated_instruction(&svm->vcpu);
2818         return 1;
2819 }
2820
2821 static int emulate_on_interception(struct vcpu_svm *svm)
2822 {
2823         return emulate_instruction(&svm->vcpu, 0) == EMULATE_DONE;
2824 }
2825
2826 static int rdpmc_interception(struct vcpu_svm *svm)
2827 {
2828         int err;
2829
2830         if (!static_cpu_has(X86_FEATURE_NRIPS))
2831                 return emulate_on_interception(svm);
2832
2833         err = kvm_rdpmc(&svm->vcpu);
2834         kvm_complete_insn_gp(&svm->vcpu, err);
2835
2836         return 1;
2837 }
2838
2839 static bool check_selective_cr0_intercepted(struct vcpu_svm *svm,
2840                                             unsigned long val)
2841 {
2842         unsigned long cr0 = svm->vcpu.arch.cr0;
2843         bool ret = false;
2844         u64 intercept;
2845
2846         intercept = svm->nested.intercept;
2847
2848         if (!is_guest_mode(&svm->vcpu) ||
2849             (!(intercept & (1ULL << INTERCEPT_SELECTIVE_CR0))))
2850                 return false;
2851
2852         cr0 &= ~SVM_CR0_SELECTIVE_MASK;
2853         val &= ~SVM_CR0_SELECTIVE_MASK;
2854
2855         if (cr0 ^ val) {
2856                 svm->vmcb->control.exit_code = SVM_EXIT_CR0_SEL_WRITE;
2857                 ret = (nested_svm_exit_handled(svm) == NESTED_EXIT_DONE);
2858         }
2859
2860         return ret;
2861 }
2862
2863 #define CR_VALID (1ULL << 63)
2864
2865 static int cr_interception(struct vcpu_svm *svm)
2866 {
2867         int reg, cr;
2868         unsigned long val;
2869         int err;
2870
2871         if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
2872                 return emulate_on_interception(svm);
2873
2874         if (unlikely((svm->vmcb->control.exit_info_1 & CR_VALID) == 0))
2875                 return emulate_on_interception(svm);
2876
2877         reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
2878         if (svm->vmcb->control.exit_code == SVM_EXIT_CR0_SEL_WRITE)
2879                 cr = SVM_EXIT_WRITE_CR0 - SVM_EXIT_READ_CR0;
2880         else
2881                 cr = svm->vmcb->control.exit_code - SVM_EXIT_READ_CR0;
2882
2883         err = 0;
2884         if (cr >= 16) { /* mov to cr */
2885                 cr -= 16;
2886                 val = kvm_register_read(&svm->vcpu, reg);
2887                 switch (cr) {
2888                 case 0:
2889                         if (!check_selective_cr0_intercepted(svm, val))
2890                                 err = kvm_set_cr0(&svm->vcpu, val);
2891                         else
2892                                 return 1;
2893
2894                         break;
2895                 case 3:
2896                         err = kvm_set_cr3(&svm->vcpu, val);
2897                         break;
2898                 case 4:
2899                         err = kvm_set_cr4(&svm->vcpu, val);
2900                         break;
2901                 case 8:
2902                         err = kvm_set_cr8(&svm->vcpu, val);
2903                         break;
2904                 default:
2905                         WARN(1, "unhandled write to CR%d", cr);
2906                         kvm_queue_exception(&svm->vcpu, UD_VECTOR);
2907                         return 1;
2908                 }
2909         } else { /* mov from cr */
2910                 switch (cr) {
2911                 case 0:
2912                         val = kvm_read_cr0(&svm->vcpu);
2913                         break;
2914                 case 2:
2915                         val = svm->vcpu.arch.cr2;
2916                         break;
2917                 case 3:
2918                         val = kvm_read_cr3(&svm->vcpu);
2919                         break;
2920                 case 4:
2921                         val = kvm_read_cr4(&svm->vcpu);
2922                         break;
2923                 case 8:
2924                         val = kvm_get_cr8(&svm->vcpu);
2925                         break;
2926                 default:
2927                         WARN(1, "unhandled read from CR%d", cr);
2928                         kvm_queue_exception(&svm->vcpu, UD_VECTOR);
2929                         return 1;
2930                 }
2931                 kvm_register_write(&svm->vcpu, reg, val);
2932         }
2933         kvm_complete_insn_gp(&svm->vcpu, err);
2934
2935         return 1;
2936 }
2937
2938 static int dr_interception(struct vcpu_svm *svm)
2939 {
2940         int reg, dr;
2941         unsigned long val;
2942
2943         if (svm->vcpu.guest_debug == 0) {
2944                 /*
2945                  * No more DR vmexits; force a reload of the debug registers
2946                  * and reenter on this instruction.  The next vmexit will
2947                  * retrieve the full state of the debug registers.
2948                  */
2949                 clr_dr_intercepts(svm);
2950                 svm->vcpu.arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT;
2951                 return 1;
2952         }
2953
2954         if (!boot_cpu_has(X86_FEATURE_DECODEASSISTS))
2955                 return emulate_on_interception(svm);
2956
2957         reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
2958         dr = svm->vmcb->control.exit_code - SVM_EXIT_READ_DR0;
2959
2960         if (dr >= 16) { /* mov to DRn */
2961                 if (!kvm_require_dr(&svm->vcpu, dr - 16))
2962                         return 1;
2963                 val = kvm_register_read(&svm->vcpu, reg);
2964                 kvm_set_dr(&svm->vcpu, dr - 16, val);
2965         } else {
2966                 if (!kvm_require_dr(&svm->vcpu, dr))
2967                         return 1;
2968                 kvm_get_dr(&svm->vcpu, dr, &val);
2969                 kvm_register_write(&svm->vcpu, reg, val);
2970         }
2971
2972         skip_emulated_instruction(&svm->vcpu);
2973
2974         return 1;
2975 }
2976
2977 static int cr8_write_interception(struct vcpu_svm *svm)
2978 {
2979         struct kvm_run *kvm_run = svm->vcpu.run;
2980         int r;
2981
2982         u8 cr8_prev = kvm_get_cr8(&svm->vcpu);
2983         /* instruction emulation calls kvm_set_cr8() */
2984         r = cr_interception(svm);
2985         if (lapic_in_kernel(&svm->vcpu))
2986                 return r;
2987         if (cr8_prev <= kvm_get_cr8(&svm->vcpu))
2988                 return r;
2989         kvm_run->exit_reason = KVM_EXIT_SET_TPR;
2990         return 0;
2991 }
2992
2993 static u64 svm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc)
2994 {
2995         struct vmcb *vmcb = get_host_vmcb(to_svm(vcpu));
2996         return vmcb->control.tsc_offset + host_tsc;
2997 }
2998
2999 static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
3000 {
3001         struct vcpu_svm *svm = to_svm(vcpu);
3002
3003         switch (msr_info->index) {
3004         case MSR_IA32_TSC: {
3005                 msr_info->data = svm->vmcb->control.tsc_offset +
3006                         kvm_scale_tsc(vcpu, rdtsc());
3007
3008                 break;
3009         }
3010         case MSR_STAR:
3011                 msr_info->data = svm->vmcb->save.star;
3012                 break;
3013 #ifdef CONFIG_X86_64
3014         case MSR_LSTAR:
3015                 msr_info->data = svm->vmcb->save.lstar;
3016                 break;
3017         case MSR_CSTAR:
3018                 msr_info->data = svm->vmcb->save.cstar;
3019                 break;
3020         case MSR_KERNEL_GS_BASE:
3021                 msr_info->data = svm->vmcb->save.kernel_gs_base;
3022                 break;
3023         case MSR_SYSCALL_MASK:
3024                 msr_info->data = svm->vmcb->save.sfmask;
3025                 break;
3026 #endif
3027         case MSR_IA32_SYSENTER_CS:
3028                 msr_info->data = svm->vmcb->save.sysenter_cs;
3029                 break;
3030         case MSR_IA32_SYSENTER_EIP:
3031                 msr_info->data = svm->sysenter_eip;
3032                 break;
3033         case MSR_IA32_SYSENTER_ESP:
3034                 msr_info->data = svm->sysenter_esp;
3035                 break;
3036         case MSR_TSC_AUX:
3037                 if (!boot_cpu_has(X86_FEATURE_RDTSCP))
3038                         return 1;
3039                 msr_info->data = svm->tsc_aux;
3040                 break;
3041         /*
3042          * Nobody will change the following 5 values in the VMCB so we can
3043          * safely return them on rdmsr. They will always be 0 until LBRV is
3044          * implemented.
3045          */
3046         case MSR_IA32_DEBUGCTLMSR:
3047                 msr_info->data = svm->vmcb->save.dbgctl;
3048                 break;
3049         case MSR_IA32_LASTBRANCHFROMIP:
3050                 msr_info->data = svm->vmcb->save.br_from;
3051                 break;
3052         case MSR_IA32_LASTBRANCHTOIP:
3053                 msr_info->data = svm->vmcb->save.br_to;
3054                 break;
3055         case MSR_IA32_LASTINTFROMIP:
3056                 msr_info->data = svm->vmcb->save.last_excp_from;
3057                 break;
3058         case MSR_IA32_LASTINTTOIP:
3059                 msr_info->data = svm->vmcb->save.last_excp_to;
3060                 break;
3061         case MSR_VM_HSAVE_PA:
3062                 msr_info->data = svm->nested.hsave_msr;
3063                 break;
3064         case MSR_VM_CR:
3065                 msr_info->data = svm->nested.vm_cr_msr;
3066                 break;
3067         case MSR_IA32_UCODE_REV:
3068                 msr_info->data = 0x01000065;
3069                 break;
3070         case MSR_F15H_IC_CFG: {
3071
3072                 int family, model;
3073
3074                 family = guest_cpuid_family(vcpu);
3075                 model  = guest_cpuid_model(vcpu);
3076
3077                 if (family < 0 || model < 0)
3078                         return kvm_get_msr_common(vcpu, msr_info);
3079
3080                 msr_info->data = 0;
3081
3082                 if (family == 0x15 &&
3083                     (model >= 0x2 && model < 0x20))
3084                         msr_info->data = 0x1E;
3085                 }
3086                 break;
3087         default:
3088                 return kvm_get_msr_common(vcpu, msr_info);
3089         }
3090         return 0;
3091 }
3092
3093 static int rdmsr_interception(struct vcpu_svm *svm)
3094 {
3095         u32 ecx = kvm_register_read(&svm->vcpu, VCPU_REGS_RCX);
3096         struct msr_data msr_info;
3097
3098         msr_info.index = ecx;
3099         msr_info.host_initiated = false;
3100         if (svm_get_msr(&svm->vcpu, &msr_info)) {
3101                 trace_kvm_msr_read_ex(ecx);
3102                 kvm_inject_gp(&svm->vcpu, 0);
3103         } else {
3104                 trace_kvm_msr_read(ecx, msr_info.data);
3105
3106                 kvm_register_write(&svm->vcpu, VCPU_REGS_RAX,
3107                                    msr_info.data & 0xffffffff);
3108                 kvm_register_write(&svm->vcpu, VCPU_REGS_RDX,
3109                                    msr_info.data >> 32);
3110                 svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
3111                 skip_emulated_instruction(&svm->vcpu);
3112         }
3113         return 1;
3114 }
3115
3116 static int svm_set_vm_cr(struct kvm_vcpu *vcpu, u64 data)
3117 {
3118         struct vcpu_svm *svm = to_svm(vcpu);
3119         int svm_dis, chg_mask;
3120
3121         if (data & ~SVM_VM_CR_VALID_MASK)
3122                 return 1;
3123
3124         chg_mask = SVM_VM_CR_VALID_MASK;
3125
3126         if (svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK)
3127                 chg_mask &= ~(SVM_VM_CR_SVM_LOCK_MASK | SVM_VM_CR_SVM_DIS_MASK);
3128
3129         svm->nested.vm_cr_msr &= ~chg_mask;
3130         svm->nested.vm_cr_msr |= (data & chg_mask);
3131
3132         svm_dis = svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK;
3133
3134         /* check for svm_disable while efer.svme is set */
3135         if (svm_dis && (vcpu->arch.efer & EFER_SVME))
3136                 return 1;
3137
3138         return 0;
3139 }
3140
3141 static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr)
3142 {
3143         struct vcpu_svm *svm = to_svm(vcpu);
3144
3145         u32 ecx = msr->index;
3146         u64 data = msr->data;
3147         switch (ecx) {
3148         case MSR_IA32_TSC:
3149                 kvm_write_tsc(vcpu, msr);
3150                 break;
3151         case MSR_STAR:
3152                 svm->vmcb->save.star = data;
3153                 break;
3154 #ifdef CONFIG_X86_64
3155         case MSR_LSTAR:
3156                 svm->vmcb->save.lstar = data;
3157                 break;
3158         case MSR_CSTAR:
3159                 svm->vmcb->save.cstar = data;
3160                 break;
3161         case MSR_KERNEL_GS_BASE:
3162                 svm->vmcb->save.kernel_gs_base = data;
3163                 break;
3164         case MSR_SYSCALL_MASK:
3165                 svm->vmcb->save.sfmask = data;
3166                 break;
3167 #endif
3168         case MSR_IA32_SYSENTER_CS:
3169                 svm->vmcb->save.sysenter_cs = data;
3170                 break;
3171         case MSR_IA32_SYSENTER_EIP:
3172                 svm->sysenter_eip = data;
3173                 svm->vmcb->save.sysenter_eip = data;
3174                 break;
3175         case MSR_IA32_SYSENTER_ESP:
3176                 svm->sysenter_esp = data;
3177                 svm->vmcb->save.sysenter_esp = data;
3178                 break;
3179         case MSR_TSC_AUX:
3180                 if (!boot_cpu_has(X86_FEATURE_RDTSCP))
3181                         return 1;
3182
3183                 /*
3184                  * This is rare, so we update the MSR here instead of using
3185                  * direct_access_msrs.  Doing that would require a rdmsr in
3186                  * svm_vcpu_put.
3187                  */
3188                 svm->tsc_aux = data;
3189                 wrmsrl(MSR_TSC_AUX, svm->tsc_aux);
3190                 break;
3191         case MSR_IA32_DEBUGCTLMSR:
3192                 if (!boot_cpu_has(X86_FEATURE_LBRV)) {
3193                         vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTL 0x%llx, nop\n",
3194                                     __func__, data);
3195                         break;
3196                 }
3197                 if (data & DEBUGCTL_RESERVED_BITS)
3198                         return 1;
3199
3200                 svm->vmcb->save.dbgctl = data;
3201                 mark_dirty(svm->vmcb, VMCB_LBR);
3202                 if (data & (1ULL<<0))
3203                         svm_enable_lbrv(svm);
3204                 else
3205                         svm_disable_lbrv(svm);
3206                 break;
3207         case MSR_VM_HSAVE_PA:
3208                 svm->nested.hsave_msr = data;
3209                 break;
3210         case MSR_VM_CR:
3211                 return svm_set_vm_cr(vcpu, data);
3212         case MSR_VM_IGNNE:
3213                 vcpu_unimpl(vcpu, "unimplemented wrmsr: 0x%x data 0x%llx\n", ecx, data);
3214                 break;
3215         default:
3216                 return kvm_set_msr_common(vcpu, msr);
3217         }
3218         return 0;
3219 }
3220
3221 static int wrmsr_interception(struct vcpu_svm *svm)
3222 {
3223         struct msr_data msr;
3224         u32 ecx = kvm_register_read(&svm->vcpu, VCPU_REGS_RCX);
3225         u64 data = kvm_read_edx_eax(&svm->vcpu);
3226
3227         msr.data = data;
3228         msr.index = ecx;
3229         msr.host_initiated = false;
3230
3231         svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
3232         if (kvm_set_msr(&svm->vcpu, &msr)) {
3233                 trace_kvm_msr_write_ex(ecx, data);
3234                 kvm_inject_gp(&svm->vcpu, 0);
3235         } else {
3236                 trace_kvm_msr_write(ecx, data);
3237                 skip_emulated_instruction(&svm->vcpu);
3238         }
3239         return 1;
3240 }
3241
3242 static int msr_interception(struct vcpu_svm *svm)
3243 {
3244         if (svm->vmcb->control.exit_info_1)
3245                 return wrmsr_interception(svm);
3246         else
3247                 return rdmsr_interception(svm);
3248 }
3249
3250 static int interrupt_window_interception(struct vcpu_svm *svm)
3251 {
3252         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
3253         svm_clear_vintr(svm);
3254         svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
3255         mark_dirty(svm->vmcb, VMCB_INTR);
3256         ++svm->vcpu.stat.irq_window_exits;
3257         return 1;
3258 }
3259
3260 static int pause_interception(struct vcpu_svm *svm)
3261 {
3262         kvm_vcpu_on_spin(&(svm->vcpu));
3263         return 1;
3264 }
3265
3266 static int nop_interception(struct vcpu_svm *svm)
3267 {
3268         skip_emulated_instruction(&(svm->vcpu));
3269         return 1;
3270 }
3271
3272 static int monitor_interception(struct vcpu_svm *svm)
3273 {
3274         printk_once(KERN_WARNING "kvm: MONITOR instruction emulated as NOP!\n");
3275         return nop_interception(svm);
3276 }
3277
3278 static int mwait_interception(struct vcpu_svm *svm)
3279 {
3280         printk_once(KERN_WARNING "kvm: MWAIT instruction emulated as NOP!\n");
3281         return nop_interception(svm);
3282 }
3283
3284 static int (*const svm_exit_handlers[])(struct vcpu_svm *svm) = {
3285         [SVM_EXIT_READ_CR0]                     = cr_interception,
3286         [SVM_EXIT_READ_CR3]                     = cr_interception,
3287         [SVM_EXIT_READ_CR4]                     = cr_interception,
3288         [SVM_EXIT_READ_CR8]                     = cr_interception,
3289         [SVM_EXIT_CR0_SEL_WRITE]                = cr_interception,
3290         [SVM_EXIT_WRITE_CR0]                    = cr_interception,
3291         [SVM_EXIT_WRITE_CR3]                    = cr_interception,
3292         [SVM_EXIT_WRITE_CR4]                    = cr_interception,
3293         [SVM_EXIT_WRITE_CR8]                    = cr8_write_interception,
3294         [SVM_EXIT_READ_DR0]                     = dr_interception,
3295         [SVM_EXIT_READ_DR1]                     = dr_interception,
3296         [SVM_EXIT_READ_DR2]                     = dr_interception,
3297         [SVM_EXIT_READ_DR3]                     = dr_interception,
3298         [SVM_EXIT_READ_DR4]                     = dr_interception,
3299         [SVM_EXIT_READ_DR5]                     = dr_interception,
3300         [SVM_EXIT_READ_DR6]                     = dr_interception,
3301         [SVM_EXIT_READ_DR7]                     = dr_interception,
3302         [SVM_EXIT_WRITE_DR0]                    = dr_interception,
3303         [SVM_EXIT_WRITE_DR1]                    = dr_interception,
3304         [SVM_EXIT_WRITE_DR2]                    = dr_interception,
3305         [SVM_EXIT_WRITE_DR3]                    = dr_interception,
3306         [SVM_EXIT_WRITE_DR4]                    = dr_interception,
3307         [SVM_EXIT_WRITE_DR5]                    = dr_interception,
3308         [SVM_EXIT_WRITE_DR6]                    = dr_interception,
3309         [SVM_EXIT_WRITE_DR7]                    = dr_interception,
3310         [SVM_EXIT_EXCP_BASE + DB_VECTOR]        = db_interception,
3311         [SVM_EXIT_EXCP_BASE + BP_VECTOR]        = bp_interception,
3312         [SVM_EXIT_EXCP_BASE + UD_VECTOR]        = ud_interception,
3313         [SVM_EXIT_EXCP_BASE + PF_VECTOR]        = pf_interception,
3314         [SVM_EXIT_EXCP_BASE + NM_VECTOR]        = nm_interception,
3315         [SVM_EXIT_EXCP_BASE + MC_VECTOR]        = mc_interception,
3316         [SVM_EXIT_EXCP_BASE + AC_VECTOR]        = ac_interception,
3317         [SVM_EXIT_INTR]                         = intr_interception,
3318         [SVM_EXIT_NMI]                          = nmi_interception,
3319         [SVM_EXIT_SMI]                          = nop_on_interception,
3320         [SVM_EXIT_INIT]                         = nop_on_interception,
3321         [SVM_EXIT_VINTR]                        = interrupt_window_interception,
3322         [SVM_EXIT_RDPMC]                        = rdpmc_interception,
3323         [SVM_EXIT_CPUID]                        = cpuid_interception,
3324         [SVM_EXIT_IRET]                         = iret_interception,
3325         [SVM_EXIT_INVD]                         = emulate_on_interception,
3326         [SVM_EXIT_PAUSE]                        = pause_interception,
3327         [SVM_EXIT_HLT]                          = halt_interception,
3328         [SVM_EXIT_INVLPG]                       = invlpg_interception,
3329         [SVM_EXIT_INVLPGA]                      = invlpga_interception,
3330         [SVM_EXIT_IOIO]                         = io_interception,
3331         [SVM_EXIT_MSR]                          = msr_interception,
3332         [SVM_EXIT_TASK_SWITCH]                  = task_switch_interception,
3333         [SVM_EXIT_SHUTDOWN]                     = shutdown_interception,
3334         [SVM_EXIT_VMRUN]                        = vmrun_interception,
3335         [SVM_EXIT_VMMCALL]                      = vmmcall_interception,
3336         [SVM_EXIT_VMLOAD]                       = vmload_interception,
3337         [SVM_EXIT_VMSAVE]                       = vmsave_interception,
3338         [SVM_EXIT_STGI]                         = stgi_interception,
3339         [SVM_EXIT_CLGI]                         = clgi_interception,
3340         [SVM_EXIT_SKINIT]                       = skinit_interception,
3341         [SVM_EXIT_WBINVD]                       = wbinvd_interception,
3342         [SVM_EXIT_MONITOR]                      = monitor_interception,
3343         [SVM_EXIT_MWAIT]                        = mwait_interception,
3344         [SVM_EXIT_XSETBV]                       = xsetbv_interception,
3345         [SVM_EXIT_NPF]                          = pf_interception,
3346         [SVM_EXIT_RSM]                          = emulate_on_interception,
3347 };
3348
3349 static void dump_vmcb(struct kvm_vcpu *vcpu)
3350 {
3351         struct vcpu_svm *svm = to_svm(vcpu);
3352         struct vmcb_control_area *control = &svm->vmcb->control;
3353         struct vmcb_save_area *save = &svm->vmcb->save;
3354
3355         pr_err("VMCB Control Area:\n");
3356         pr_err("%-20s%04x\n", "cr_read:", control->intercept_cr & 0xffff);
3357         pr_err("%-20s%04x\n", "cr_write:", control->intercept_cr >> 16);
3358         pr_err("%-20s%04x\n", "dr_read:", control->intercept_dr & 0xffff);
3359         pr_err("%-20s%04x\n", "dr_write:", control->intercept_dr >> 16);
3360         pr_err("%-20s%08x\n", "exceptions:", control->intercept_exceptions);
3361         pr_err("%-20s%016llx\n", "intercepts:", control->intercept);
3362         pr_err("%-20s%d\n", "pause filter count:", control->pause_filter_count);
3363         pr_err("%-20s%016llx\n", "iopm_base_pa:", control->iopm_base_pa);
3364         pr_err("%-20s%016llx\n", "msrpm_base_pa:", control->msrpm_base_pa);
3365         pr_err("%-20s%016llx\n", "tsc_offset:", control->tsc_offset);
3366         pr_err("%-20s%d\n", "asid:", control->asid);
3367         pr_err("%-20s%d\n", "tlb_ctl:", control->tlb_ctl);
3368         pr_err("%-20s%08x\n", "int_ctl:", control->int_ctl);
3369         pr_err("%-20s%08x\n", "int_vector:", control->int_vector);
3370         pr_err("%-20s%08x\n", "int_state:", control->int_state);
3371         pr_err("%-20s%08x\n", "exit_code:", control->exit_code);
3372         pr_err("%-20s%016llx\n", "exit_info1:", control->exit_info_1);
3373         pr_err("%-20s%016llx\n", "exit_info2:", control->exit_info_2);
3374         pr_err("%-20s%08x\n", "exit_int_info:", control->exit_int_info);
3375         pr_err("%-20s%08x\n", "exit_int_info_err:", control->exit_int_info_err);
3376         pr_err("%-20s%lld\n", "nested_ctl:", control->nested_ctl);
3377         pr_err("%-20s%016llx\n", "nested_cr3:", control->nested_cr3);
3378         pr_err("%-20s%08x\n", "event_inj:", control->event_inj);
3379         pr_err("%-20s%08x\n", "event_inj_err:", control->event_inj_err);
3380         pr_err("%-20s%lld\n", "lbr_ctl:", control->lbr_ctl);
3381         pr_err("%-20s%016llx\n", "next_rip:", control->next_rip);
3382         pr_err("VMCB State Save Area:\n");
3383         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3384                "es:",
3385                save->es.selector, save->es.attrib,
3386                save->es.limit, save->es.base);
3387         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3388                "cs:",
3389                save->cs.selector, save->cs.attrib,
3390                save->cs.limit, save->cs.base);
3391         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3392                "ss:",
3393                save->ss.selector, save->ss.attrib,
3394                save->ss.limit, save->ss.base);
3395         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3396                "ds:",
3397                save->ds.selector, save->ds.attrib,
3398                save->ds.limit, save->ds.base);
3399         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3400                "fs:",
3401                save->fs.selector, save->fs.attrib,
3402                save->fs.limit, save->fs.base);
3403         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3404                "gs:",
3405                save->gs.selector, save->gs.attrib,
3406                save->gs.limit, save->gs.base);
3407         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3408                "gdtr:",
3409                save->gdtr.selector, save->gdtr.attrib,
3410                save->gdtr.limit, save->gdtr.base);
3411         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3412                "ldtr:",
3413                save->ldtr.selector, save->ldtr.attrib,
3414                save->ldtr.limit, save->ldtr.base);
3415         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3416                "idtr:",
3417                save->idtr.selector, save->idtr.attrib,
3418                save->idtr.limit, save->idtr.base);
3419         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3420                "tr:",
3421                save->tr.selector, save->tr.attrib,
3422                save->tr.limit, save->tr.base);
3423         pr_err("cpl:            %d                efer:         %016llx\n",
3424                 save->cpl, save->efer);
3425         pr_err("%-15s %016llx %-13s %016llx\n",
3426                "cr0:", save->cr0, "cr2:", save->cr2);
3427         pr_err("%-15s %016llx %-13s %016llx\n",
3428                "cr3:", save->cr3, "cr4:", save->cr4);
3429         pr_err("%-15s %016llx %-13s %016llx\n",
3430                "dr6:", save->dr6, "dr7:", save->dr7);
3431         pr_err("%-15s %016llx %-13s %016llx\n",
3432                "rip:", save->rip, "rflags:", save->rflags);
3433         pr_err("%-15s %016llx %-13s %016llx\n",
3434                "rsp:", save->rsp, "rax:", save->rax);
3435         pr_err("%-15s %016llx %-13s %016llx\n",
3436                "star:", save->star, "lstar:", save->lstar);
3437         pr_err("%-15s %016llx %-13s %016llx\n",
3438                "cstar:", save->cstar, "sfmask:", save->sfmask);
3439         pr_err("%-15s %016llx %-13s %016llx\n",
3440                "kernel_gs_base:", save->kernel_gs_base,
3441                "sysenter_cs:", save->sysenter_cs);
3442         pr_err("%-15s %016llx %-13s %016llx\n",
3443                "sysenter_esp:", save->sysenter_esp,
3444                "sysenter_eip:", save->sysenter_eip);
3445         pr_err("%-15s %016llx %-13s %016llx\n",
3446                "gpat:", save->g_pat, "dbgctl:", save->dbgctl);
3447         pr_err("%-15s %016llx %-13s %016llx\n",
3448                "br_from:", save->br_from, "br_to:", save->br_to);
3449         pr_err("%-15s %016llx %-13s %016llx\n",
3450                "excp_from:", save->last_excp_from,
3451                "excp_to:", save->last_excp_to);
3452 }
3453
3454 static void svm_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2)
3455 {
3456         struct vmcb_control_area *control = &to_svm(vcpu)->vmcb->control;
3457
3458         *info1 = control->exit_info_1;
3459         *info2 = control->exit_info_2;
3460 }
3461
3462 static int handle_exit(struct kvm_vcpu *vcpu)
3463 {
3464         struct vcpu_svm *svm = to_svm(vcpu);
3465         struct kvm_run *kvm_run = vcpu->run;
3466         u32 exit_code = svm->vmcb->control.exit_code;
3467
3468         trace_kvm_exit(exit_code, vcpu, KVM_ISA_SVM);
3469
3470         if (!is_cr_intercept(svm, INTERCEPT_CR0_WRITE))
3471                 vcpu->arch.cr0 = svm->vmcb->save.cr0;
3472         if (npt_enabled)
3473                 vcpu->arch.cr3 = svm->vmcb->save.cr3;
3474
3475         if (unlikely(svm->nested.exit_required)) {
3476                 nested_svm_vmexit(svm);
3477                 svm->nested.exit_required = false;
3478
3479                 return 1;
3480         }
3481
3482         if (is_guest_mode(vcpu)) {
3483                 int vmexit;
3484
3485                 trace_kvm_nested_vmexit(svm->vmcb->save.rip, exit_code,
3486                                         svm->vmcb->control.exit_info_1,
3487                                         svm->vmcb->control.exit_info_2,
3488                                         svm->vmcb->control.exit_int_info,
3489                                         svm->vmcb->control.exit_int_info_err,
3490                                         KVM_ISA_SVM);
3491
3492                 vmexit = nested_svm_exit_special(svm);
3493
3494                 if (vmexit == NESTED_EXIT_CONTINUE)
3495                         vmexit = nested_svm_exit_handled(svm);
3496
3497                 if (vmexit == NESTED_EXIT_DONE)
3498                         return 1;
3499         }
3500
3501         svm_complete_interrupts(svm);
3502
3503         if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) {
3504                 kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
3505                 kvm_run->fail_entry.hardware_entry_failure_reason
3506                         = svm->vmcb->control.exit_code;
3507                 pr_err("KVM: FAILED VMRUN WITH VMCB:\n");
3508                 dump_vmcb(vcpu);
3509                 return 0;
3510         }
3511
3512         if (is_external_interrupt(svm->vmcb->control.exit_int_info) &&
3513             exit_code != SVM_EXIT_EXCP_BASE + PF_VECTOR &&
3514             exit_code != SVM_EXIT_NPF && exit_code != SVM_EXIT_TASK_SWITCH &&
3515             exit_code != SVM_EXIT_INTR && exit_code != SVM_EXIT_NMI)
3516                 printk(KERN_ERR "%s: unexpected exit_int_info 0x%x "
3517                        "exit_code 0x%x\n",
3518                        __func__, svm->vmcb->control.exit_int_info,
3519                        exit_code);
3520
3521         if (exit_code >= ARRAY_SIZE(svm_exit_handlers)
3522             || !svm_exit_handlers[exit_code]) {
3523                 WARN_ONCE(1, "svm: unexpected exit reason 0x%x\n", exit_code);
3524                 kvm_queue_exception(vcpu, UD_VECTOR);
3525                 return 1;
3526         }
3527
3528         return svm_exit_handlers[exit_code](svm);
3529 }
3530
3531 static void reload_tss(struct kvm_vcpu *vcpu)
3532 {
3533         int cpu = raw_smp_processor_id();
3534
3535         struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
3536         sd->tss_desc->type = 9; /* available 32/64-bit TSS */
3537         load_TR_desc();
3538 }
3539
3540 static void pre_svm_run(struct vcpu_svm *svm)
3541 {
3542         int cpu = raw_smp_processor_id();
3543
3544         struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
3545
3546         /* FIXME: handle wraparound of asid_generation */
3547         if (svm->asid_generation != sd->asid_generation)
3548                 new_asid(svm, sd);
3549 }
3550
3551 static void svm_inject_nmi(struct kvm_vcpu *vcpu)
3552 {
3553         struct vcpu_svm *svm = to_svm(vcpu);
3554
3555         svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI;
3556         vcpu->arch.hflags |= HF_NMI_MASK;
3557         set_intercept(svm, INTERCEPT_IRET);
3558         ++vcpu->stat.nmi_injections;
3559 }
3560
3561 static inline void svm_inject_irq(struct vcpu_svm *svm, int irq)
3562 {
3563         struct vmcb_control_area *control;
3564
3565         control = &svm->vmcb->control;
3566         control->int_vector = irq;
3567         control->int_ctl &= ~V_INTR_PRIO_MASK;
3568         control->int_ctl |= V_IRQ_MASK |
3569                 ((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT);
3570         mark_dirty(svm->vmcb, VMCB_INTR);
3571 }
3572
3573 static void svm_set_irq(struct kvm_vcpu *vcpu)
3574 {
3575         struct vcpu_svm *svm = to_svm(vcpu);
3576
3577         BUG_ON(!(gif_set(svm)));
3578
3579         trace_kvm_inj_virq(vcpu->arch.interrupt.nr);
3580         ++vcpu->stat.irq_injections;
3581
3582         svm->vmcb->control.event_inj = vcpu->arch.interrupt.nr |
3583                 SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR;
3584 }
3585
3586 static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
3587 {
3588         struct vcpu_svm *svm = to_svm(vcpu);
3589
3590         if (is_guest_mode(vcpu) && (vcpu->arch.hflags & HF_VINTR_MASK))
3591                 return;
3592
3593         clr_cr_intercept(svm, INTERCEPT_CR8_WRITE);
3594
3595         if (irr == -1)
3596                 return;
3597
3598         if (tpr >= irr)
3599                 set_cr_intercept(svm, INTERCEPT_CR8_WRITE);
3600 }
3601
3602 static void svm_set_virtual_x2apic_mode(struct kvm_vcpu *vcpu, bool set)
3603 {
3604         return;
3605 }
3606
3607 static bool svm_get_enable_apicv(void)
3608 {
3609         return false;
3610 }
3611
3612 static void svm_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu)
3613 {
3614 }
3615
3616 static void svm_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap)
3617 {
3618         return;
3619 }
3620
3621 static void svm_sync_pir_to_irr(struct kvm_vcpu *vcpu)
3622 {
3623         return;
3624 }
3625
3626 static int svm_nmi_allowed(struct kvm_vcpu *vcpu)
3627 {
3628         struct vcpu_svm *svm = to_svm(vcpu);
3629         struct vmcb *vmcb = svm->vmcb;
3630         int ret;
3631         ret = !(vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) &&
3632               !(svm->vcpu.arch.hflags & HF_NMI_MASK);
3633         ret = ret && gif_set(svm) && nested_svm_nmi(svm);
3634
3635         return ret;
3636 }
3637
3638 static bool svm_get_nmi_mask(struct kvm_vcpu *vcpu)
3639 {
3640         struct vcpu_svm *svm = to_svm(vcpu);
3641
3642         return !!(svm->vcpu.arch.hflags & HF_NMI_MASK);
3643 }
3644
3645 static void svm_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
3646 {
3647         struct vcpu_svm *svm = to_svm(vcpu);
3648
3649         if (masked) {
3650                 svm->vcpu.arch.hflags |= HF_NMI_MASK;
3651                 set_intercept(svm, INTERCEPT_IRET);
3652         } else {
3653                 svm->vcpu.arch.hflags &= ~HF_NMI_MASK;
3654                 clr_intercept(svm, INTERCEPT_IRET);
3655         }
3656 }
3657
3658 static int svm_interrupt_allowed(struct kvm_vcpu *vcpu)
3659 {
3660         struct vcpu_svm *svm = to_svm(vcpu);
3661         struct vmcb *vmcb = svm->vmcb;
3662         int ret;
3663
3664         if (!gif_set(svm) ||
3665              (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK))
3666                 return 0;
3667
3668         ret = !!(kvm_get_rflags(vcpu) & X86_EFLAGS_IF);
3669
3670         if (is_guest_mode(vcpu))
3671                 return ret && !(svm->vcpu.arch.hflags & HF_VINTR_MASK);
3672
3673         return ret;
3674 }
3675
3676 static void enable_irq_window(struct kvm_vcpu *vcpu)
3677 {
3678         struct vcpu_svm *svm = to_svm(vcpu);
3679
3680         /*
3681          * In case GIF=0 we can't rely on the CPU to tell us when GIF becomes
3682          * 1, because that's a separate STGI/VMRUN intercept.  The next time we
3683          * get that intercept, this function will be called again though and
3684          * we'll get the vintr intercept.
3685          */
3686         if (gif_set(svm) && nested_svm_intr(svm)) {
3687                 svm_set_vintr(svm);
3688                 svm_inject_irq(svm, 0x0);
3689         }
3690 }
3691
3692 static void enable_nmi_window(struct kvm_vcpu *vcpu)
3693 {
3694         struct vcpu_svm *svm = to_svm(vcpu);
3695
3696         if ((svm->vcpu.arch.hflags & (HF_NMI_MASK | HF_IRET_MASK))
3697             == HF_NMI_MASK)
3698                 return; /* IRET will cause a vm exit */
3699
3700         /*
3701          * Something prevents NMI from been injected. Single step over possible
3702          * problem (IRET or exception injection or interrupt shadow)
3703          */
3704         svm->nmi_singlestep = true;
3705         svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
3706 }
3707
3708 static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr)
3709 {
3710         return 0;
3711 }
3712
3713 static void svm_flush_tlb(struct kvm_vcpu *vcpu)
3714 {
3715         struct vcpu_svm *svm = to_svm(vcpu);
3716
3717         if (static_cpu_has(X86_FEATURE_FLUSHBYASID))
3718                 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID;
3719         else
3720                 svm->asid_generation--;
3721 }
3722
3723 static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu)
3724 {
3725 }
3726
3727 static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu)
3728 {
3729         struct vcpu_svm *svm = to_svm(vcpu);
3730
3731         if (is_guest_mode(vcpu) && (vcpu->arch.hflags & HF_VINTR_MASK))
3732                 return;
3733
3734         if (!is_cr_intercept(svm, INTERCEPT_CR8_WRITE)) {
3735                 int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK;
3736                 kvm_set_cr8(vcpu, cr8);
3737         }
3738 }
3739
3740 static inline void sync_lapic_to_cr8(struct kvm_vcpu *vcpu)
3741 {
3742         struct vcpu_svm *svm = to_svm(vcpu);
3743         u64 cr8;
3744
3745         if (is_guest_mode(vcpu) && (vcpu->arch.hflags & HF_VINTR_MASK))
3746                 return;
3747
3748         cr8 = kvm_get_cr8(vcpu);
3749         svm->vmcb->control.int_ctl &= ~V_TPR_MASK;
3750         svm->vmcb->control.int_ctl |= cr8 & V_TPR_MASK;
3751 }
3752
3753 static void svm_complete_interrupts(struct vcpu_svm *svm)
3754 {
3755         u8 vector;
3756         int type;
3757         u32 exitintinfo = svm->vmcb->control.exit_int_info;
3758         unsigned int3_injected = svm->int3_injected;
3759
3760         svm->int3_injected = 0;
3761
3762         /*
3763          * If we've made progress since setting HF_IRET_MASK, we've
3764          * executed an IRET and can allow NMI injection.
3765          */
3766         if ((svm->vcpu.arch.hflags & HF_IRET_MASK)
3767             && kvm_rip_read(&svm->vcpu) != svm->nmi_iret_rip) {
3768                 svm->vcpu.arch.hflags &= ~(HF_NMI_MASK | HF_IRET_MASK);
3769                 kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
3770         }
3771
3772         svm->vcpu.arch.nmi_injected = false;
3773         kvm_clear_exception_queue(&svm->vcpu);
3774         kvm_clear_interrupt_queue(&svm->vcpu);
3775
3776         if (!(exitintinfo & SVM_EXITINTINFO_VALID))
3777                 return;
3778
3779         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
3780
3781         vector = exitintinfo & SVM_EXITINTINFO_VEC_MASK;
3782         type = exitintinfo & SVM_EXITINTINFO_TYPE_MASK;
3783
3784         switch (type) {
3785         case SVM_EXITINTINFO_TYPE_NMI:
3786                 svm->vcpu.arch.nmi_injected = true;
3787                 break;
3788         case SVM_EXITINTINFO_TYPE_EXEPT:
3789                 /*
3790                  * In case of software exceptions, do not reinject the vector,
3791                  * but re-execute the instruction instead. Rewind RIP first
3792                  * if we emulated INT3 before.
3793                  */
3794                 if (kvm_exception_is_soft(vector)) {
3795                         if (vector == BP_VECTOR && int3_injected &&
3796                             kvm_is_linear_rip(&svm->vcpu, svm->int3_rip))
3797                                 kvm_rip_write(&svm->vcpu,
3798                                               kvm_rip_read(&svm->vcpu) -
3799                                               int3_injected);
3800                         break;
3801                 }
3802                 if (exitintinfo & SVM_EXITINTINFO_VALID_ERR) {
3803                         u32 err = svm->vmcb->control.exit_int_info_err;
3804                         kvm_requeue_exception_e(&svm->vcpu, vector, err);
3805
3806                 } else
3807                         kvm_requeue_exception(&svm->vcpu, vector);
3808                 break;
3809         case SVM_EXITINTINFO_TYPE_INTR:
3810                 kvm_queue_interrupt(&svm->vcpu, vector, false);
3811                 break;
3812         default:
3813                 break;
3814         }
3815 }
3816
3817 static void svm_cancel_injection(struct kvm_vcpu *vcpu)
3818 {
3819         struct vcpu_svm *svm = to_svm(vcpu);
3820         struct vmcb_control_area *control = &svm->vmcb->control;
3821
3822         control->exit_int_info = control->event_inj;
3823         control->exit_int_info_err = control->event_inj_err;
3824         control->event_inj = 0;
3825         svm_complete_interrupts(svm);
3826 }
3827
3828 static void svm_vcpu_run(struct kvm_vcpu *vcpu)
3829 {
3830         struct vcpu_svm *svm = to_svm(vcpu);
3831
3832         svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
3833         svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
3834         svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
3835
3836         /*
3837          * A vmexit emulation is required before the vcpu can be executed
3838          * again.
3839          */
3840         if (unlikely(svm->nested.exit_required))
3841                 return;
3842
3843         pre_svm_run(svm);
3844
3845         sync_lapic_to_cr8(vcpu);
3846
3847         svm->vmcb->save.cr2 = vcpu->arch.cr2;
3848
3849         clgi();
3850
3851         local_irq_enable();
3852
3853         asm volatile (
3854                 "push %%" _ASM_BP "; \n\t"
3855                 "mov %c[rbx](%[svm]), %%" _ASM_BX " \n\t"
3856                 "mov %c[rcx](%[svm]), %%" _ASM_CX " \n\t"
3857                 "mov %c[rdx](%[svm]), %%" _ASM_DX " \n\t"
3858                 "mov %c[rsi](%[svm]), %%" _ASM_SI " \n\t"
3859                 "mov %c[rdi](%[svm]), %%" _ASM_DI " \n\t"
3860                 "mov %c[rbp](%[svm]), %%" _ASM_BP " \n\t"
3861 #ifdef CONFIG_X86_64
3862                 "mov %c[r8](%[svm]),  %%r8  \n\t"
3863                 "mov %c[r9](%[svm]),  %%r9  \n\t"
3864                 "mov %c[r10](%[svm]), %%r10 \n\t"
3865                 "mov %c[r11](%[svm]), %%r11 \n\t"
3866                 "mov %c[r12](%[svm]), %%r12 \n\t"
3867                 "mov %c[r13](%[svm]), %%r13 \n\t"
3868                 "mov %c[r14](%[svm]), %%r14 \n\t"
3869                 "mov %c[r15](%[svm]), %%r15 \n\t"
3870 #endif
3871
3872                 /* Enter guest mode */
3873                 "push %%" _ASM_AX " \n\t"
3874                 "mov %c[vmcb](%[svm]), %%" _ASM_AX " \n\t"
3875                 __ex(SVM_VMLOAD) "\n\t"
3876                 __ex(SVM_VMRUN) "\n\t"
3877                 __ex(SVM_VMSAVE) "\n\t"
3878                 "pop %%" _ASM_AX " \n\t"
3879
3880                 /* Save guest registers, load host registers */
3881                 "mov %%" _ASM_BX ", %c[rbx](%[svm]) \n\t"
3882                 "mov %%" _ASM_CX ", %c[rcx](%[svm]) \n\t"
3883                 "mov %%" _ASM_DX ", %c[rdx](%[svm]) \n\t"
3884                 "mov %%" _ASM_SI ", %c[rsi](%[svm]) \n\t"
3885                 "mov %%" _ASM_DI ", %c[rdi](%[svm]) \n\t"
3886                 "mov %%" _ASM_BP ", %c[rbp](%[svm]) \n\t"
3887 #ifdef CONFIG_X86_64
3888                 "mov %%r8,  %c[r8](%[svm]) \n\t"
3889                 "mov %%r9,  %c[r9](%[svm]) \n\t"
3890                 "mov %%r10, %c[r10](%[svm]) \n\t"
3891                 "mov %%r11, %c[r11](%[svm]) \n\t"
3892                 "mov %%r12, %c[r12](%[svm]) \n\t"
3893                 "mov %%r13, %c[r13](%[svm]) \n\t"
3894                 "mov %%r14, %c[r14](%[svm]) \n\t"
3895                 "mov %%r15, %c[r15](%[svm]) \n\t"
3896 #endif
3897                 "pop %%" _ASM_BP
3898                 :
3899                 : [svm]"a"(svm),
3900                   [vmcb]"i"(offsetof(struct vcpu_svm, vmcb_pa)),
3901                   [rbx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBX])),
3902                   [rcx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RCX])),
3903                   [rdx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDX])),
3904                   [rsi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RSI])),
3905                   [rdi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDI])),
3906                   [rbp]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBP]))
3907 #ifdef CONFIG_X86_64
3908                   , [r8]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R8])),
3909                   [r9]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R9])),
3910                   [r10]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R10])),
3911                   [r11]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R11])),
3912                   [r12]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R12])),
3913                   [r13]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R13])),
3914                   [r14]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R14])),
3915                   [r15]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R15]))
3916 #endif
3917                 : "cc", "memory"
3918 #ifdef CONFIG_X86_64
3919                 , "rbx", "rcx", "rdx", "rsi", "rdi"
3920                 , "r8", "r9", "r10", "r11" , "r12", "r13", "r14", "r15"
3921 #else
3922                 , "ebx", "ecx", "edx", "esi", "edi"
3923 #endif
3924                 );
3925
3926 #ifdef CONFIG_X86_64
3927         wrmsrl(MSR_GS_BASE, svm->host.gs_base);
3928 #else
3929         loadsegment(fs, svm->host.fs);
3930 #ifndef CONFIG_X86_32_LAZY_GS
3931         loadsegment(gs, svm->host.gs);
3932 #endif
3933 #endif
3934
3935         reload_tss(vcpu);
3936
3937         local_irq_disable();
3938
3939         vcpu->arch.cr2 = svm->vmcb->save.cr2;
3940         vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax;
3941         vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;
3942         vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip;
3943
3944         if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
3945                 kvm_before_handle_nmi(&svm->vcpu);
3946
3947         stgi();
3948
3949         /* Any pending NMI will happen here */
3950
3951         if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
3952                 kvm_after_handle_nmi(&svm->vcpu);
3953
3954         sync_cr8_to_lapic(vcpu);
3955
3956         svm->next_rip = 0;
3957
3958         svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
3959
3960         /* if exit due to PF check for async PF */
3961         if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR)
3962                 svm->apf_reason = kvm_read_and_reset_pf_reason();
3963
3964         if (npt_enabled) {
3965                 vcpu->arch.regs_avail &= ~(1 << VCPU_EXREG_PDPTR);
3966                 vcpu->arch.regs_dirty &= ~(1 << VCPU_EXREG_PDPTR);
3967         }
3968
3969         /*
3970          * We need to handle MC intercepts here before the vcpu has a chance to
3971          * change the physical cpu
3972          */
3973         if (unlikely(svm->vmcb->control.exit_code ==
3974                      SVM_EXIT_EXCP_BASE + MC_VECTOR))
3975                 svm_handle_mce(svm);
3976
3977         mark_all_clean(svm->vmcb);
3978 }
3979
3980 static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root)
3981 {
3982         struct vcpu_svm *svm = to_svm(vcpu);
3983
3984         svm->vmcb->save.cr3 = root;
3985         mark_dirty(svm->vmcb, VMCB_CR);
3986         svm_flush_tlb(vcpu);
3987 }
3988
3989 static void set_tdp_cr3(struct kvm_vcpu *vcpu, unsigned long root)
3990 {
3991         struct vcpu_svm *svm = to_svm(vcpu);
3992
3993         svm->vmcb->control.nested_cr3 = root;
3994         mark_dirty(svm->vmcb, VMCB_NPT);
3995
3996         /* Also sync guest cr3 here in case we live migrate */
3997         svm->vmcb->save.cr3 = kvm_read_cr3(vcpu);
3998         mark_dirty(svm->vmcb, VMCB_CR);
3999
4000         svm_flush_tlb(vcpu);
4001 }
4002
4003 static int is_disabled(void)
4004 {
4005         u64 vm_cr;
4006
4007         rdmsrl(MSR_VM_CR, vm_cr);
4008         if (vm_cr & (1 << SVM_VM_CR_SVM_DISABLE))
4009                 return 1;
4010
4011         return 0;
4012 }
4013
4014 static void
4015 svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
4016 {
4017         /*
4018          * Patch in the VMMCALL instruction:
4019          */
4020         hypercall[0] = 0x0f;
4021         hypercall[1] = 0x01;
4022         hypercall[2] = 0xd9;
4023 }
4024
4025 static void svm_check_processor_compat(void *rtn)
4026 {
4027         *(int *)rtn = 0;
4028 }
4029
4030 static bool svm_cpu_has_accelerated_tpr(void)
4031 {
4032         return false;
4033 }
4034
4035 static bool svm_has_high_real_mode_segbase(void)
4036 {
4037         return true;
4038 }
4039
4040 static u64 svm_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
4041 {
4042         return 0;
4043 }
4044
4045 static void svm_cpuid_update(struct kvm_vcpu *vcpu)
4046 {
4047         struct vcpu_svm *svm = to_svm(vcpu);
4048
4049         /* Update nrips enabled cache */
4050         svm->nrips_enabled = !!guest_cpuid_has_nrips(&svm->vcpu);
4051 }
4052
4053 static void svm_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry)
4054 {
4055         switch (func) {
4056         case 0x80000001:
4057                 if (nested)
4058                         entry->ecx |= (1 << 2); /* Set SVM bit */
4059                 break;
4060         case 0x8000000A:
4061                 entry->eax = 1; /* SVM revision 1 */
4062                 entry->ebx = 8; /* Lets support 8 ASIDs in case we add proper
4063                                    ASID emulation to nested SVM */
4064                 entry->ecx = 0; /* Reserved */
4065                 entry->edx = 0; /* Per default do not support any
4066                                    additional features */
4067
4068                 /* Support next_rip if host supports it */
4069                 if (boot_cpu_has(X86_FEATURE_NRIPS))
4070                         entry->edx |= SVM_FEATURE_NRIP;
4071
4072                 /* Support NPT for the guest if enabled */
4073                 if (npt_enabled)
4074                         entry->edx |= SVM_FEATURE_NPT;
4075
4076                 break;
4077         }
4078 }
4079
4080 static int svm_get_lpage_level(void)
4081 {
4082         return PT_PDPE_LEVEL;
4083 }
4084
4085 static bool svm_rdtscp_supported(void)
4086 {
4087         return boot_cpu_has(X86_FEATURE_RDTSCP);
4088 }
4089
4090 static bool svm_invpcid_supported(void)
4091 {
4092         return false;
4093 }
4094
4095 static bool svm_mpx_supported(void)
4096 {
4097         return false;
4098 }
4099
4100 static bool svm_xsaves_supported(void)
4101 {
4102         return false;
4103 }
4104
4105 static bool svm_has_wbinvd_exit(void)
4106 {
4107         return true;
4108 }
4109
4110 static void svm_fpu_deactivate(struct kvm_vcpu *vcpu)
4111 {
4112         struct vcpu_svm *svm = to_svm(vcpu);
4113
4114         set_exception_intercept(svm, NM_VECTOR);
4115         update_cr0_intercept(svm);
4116 }
4117
4118 #define PRE_EX(exit)  { .exit_code = (exit), \
4119                         .stage = X86_ICPT_PRE_EXCEPT, }
4120 #define POST_EX(exit) { .exit_code = (exit), \
4121                         .stage = X86_ICPT_POST_EXCEPT, }
4122 #define POST_MEM(exit) { .exit_code = (exit), \
4123                         .stage = X86_ICPT_POST_MEMACCESS, }
4124
4125 static const struct __x86_intercept {
4126         u32 exit_code;
4127         enum x86_intercept_stage stage;
4128 } x86_intercept_map[] = {
4129         [x86_intercept_cr_read]         = POST_EX(SVM_EXIT_READ_CR0),
4130         [x86_intercept_cr_write]        = POST_EX(SVM_EXIT_WRITE_CR0),
4131         [x86_intercept_clts]            = POST_EX(SVM_EXIT_WRITE_CR0),
4132         [x86_intercept_lmsw]            = POST_EX(SVM_EXIT_WRITE_CR0),
4133         [x86_intercept_smsw]            = POST_EX(SVM_EXIT_READ_CR0),
4134         [x86_intercept_dr_read]         = POST_EX(SVM_EXIT_READ_DR0),
4135         [x86_intercept_dr_write]        = POST_EX(SVM_EXIT_WRITE_DR0),
4136         [x86_intercept_sldt]            = POST_EX(SVM_EXIT_LDTR_READ),
4137         [x86_intercept_str]             = POST_EX(SVM_EXIT_TR_READ),
4138         [x86_intercept_lldt]            = POST_EX(SVM_EXIT_LDTR_WRITE),
4139         [x86_intercept_ltr]             = POST_EX(SVM_EXIT_TR_WRITE),
4140         [x86_intercept_sgdt]            = POST_EX(SVM_EXIT_GDTR_READ),
4141         [x86_intercept_sidt]            = POST_EX(SVM_EXIT_IDTR_READ),
4142         [x86_intercept_lgdt]            = POST_EX(SVM_EXIT_GDTR_WRITE),
4143         [x86_intercept_lidt]            = POST_EX(SVM_EXIT_IDTR_WRITE),
4144         [x86_intercept_vmrun]           = POST_EX(SVM_EXIT_VMRUN),
4145         [x86_intercept_vmmcall]         = POST_EX(SVM_EXIT_VMMCALL),
4146         [x86_intercept_vmload]          = POST_EX(SVM_EXIT_VMLOAD),
4147         [x86_intercept_vmsave]          = POST_EX(SVM_EXIT_VMSAVE),
4148         [x86_intercept_stgi]            = POST_EX(SVM_EXIT_STGI),
4149         [x86_intercept_clgi]            = POST_EX(SVM_EXIT_CLGI),
4150         [x86_intercept_skinit]          = POST_EX(SVM_EXIT_SKINIT),
4151         [x86_intercept_invlpga]         = POST_EX(SVM_EXIT_INVLPGA),
4152         [x86_intercept_rdtscp]          = POST_EX(SVM_EXIT_RDTSCP),
4153         [x86_intercept_monitor]         = POST_MEM(SVM_EXIT_MONITOR),
4154         [x86_intercept_mwait]           = POST_EX(SVM_EXIT_MWAIT),
4155         [x86_intercept_invlpg]          = POST_EX(SVM_EXIT_INVLPG),
4156         [x86_intercept_invd]            = POST_EX(SVM_EXIT_INVD),
4157         [x86_intercept_wbinvd]          = POST_EX(SVM_EXIT_WBINVD),
4158         [x86_intercept_wrmsr]           = POST_EX(SVM_EXIT_MSR),
4159         [x86_intercept_rdtsc]           = POST_EX(SVM_EXIT_RDTSC),
4160         [x86_intercept_rdmsr]           = POST_EX(SVM_EXIT_MSR),
4161         [x86_intercept_rdpmc]           = POST_EX(SVM_EXIT_RDPMC),
4162         [x86_intercept_cpuid]           = PRE_EX(SVM_EXIT_CPUID),
4163         [x86_intercept_rsm]             = PRE_EX(SVM_EXIT_RSM),
4164         [x86_intercept_pause]           = PRE_EX(SVM_EXIT_PAUSE),
4165         [x86_intercept_pushf]           = PRE_EX(SVM_EXIT_PUSHF),
4166         [x86_intercept_popf]            = PRE_EX(SVM_EXIT_POPF),
4167         [x86_intercept_intn]            = PRE_EX(SVM_EXIT_SWINT),
4168         [x86_intercept_iret]            = PRE_EX(SVM_EXIT_IRET),
4169         [x86_intercept_icebp]           = PRE_EX(SVM_EXIT_ICEBP),
4170         [x86_intercept_hlt]             = POST_EX(SVM_EXIT_HLT),
4171         [x86_intercept_in]              = POST_EX(SVM_EXIT_IOIO),
4172         [x86_intercept_ins]             = POST_EX(SVM_EXIT_IOIO),
4173         [x86_intercept_out]             = POST_EX(SVM_EXIT_IOIO),
4174         [x86_intercept_outs]            = POST_EX(SVM_EXIT_IOIO),
4175 };
4176
4177 #undef PRE_EX
4178 #undef POST_EX
4179 #undef POST_MEM
4180
4181 static int svm_check_intercept(struct kvm_vcpu *vcpu,
4182                                struct x86_instruction_info *info,
4183                                enum x86_intercept_stage stage)
4184 {
4185         struct vcpu_svm *svm = to_svm(vcpu);
4186         int vmexit, ret = X86EMUL_CONTINUE;
4187         struct __x86_intercept icpt_info;
4188         struct vmcb *vmcb = svm->vmcb;
4189
4190         if (info->intercept >= ARRAY_SIZE(x86_intercept_map))
4191                 goto out;
4192
4193         icpt_info = x86_intercept_map[info->intercept];
4194
4195         if (stage != icpt_info.stage)
4196                 goto out;
4197
4198         switch (icpt_info.exit_code) {
4199         case SVM_EXIT_READ_CR0:
4200                 if (info->intercept == x86_intercept_cr_read)
4201                         icpt_info.exit_code += info->modrm_reg;
4202                 break;
4203         case SVM_EXIT_WRITE_CR0: {
4204                 unsigned long cr0, val;
4205                 u64 intercept;
4206
4207                 if (info->intercept == x86_intercept_cr_write)
4208                         icpt_info.exit_code += info->modrm_reg;
4209
4210                 if (icpt_info.exit_code != SVM_EXIT_WRITE_CR0 ||
4211                     info->intercept == x86_intercept_clts)
4212                         break;
4213
4214                 intercept = svm->nested.intercept;
4215
4216                 if (!(intercept & (1ULL << INTERCEPT_SELECTIVE_CR0)))
4217                         break;
4218
4219                 cr0 = vcpu->arch.cr0 & ~SVM_CR0_SELECTIVE_MASK;
4220                 val = info->src_val  & ~SVM_CR0_SELECTIVE_MASK;
4221
4222                 if (info->intercept == x86_intercept_lmsw) {
4223                         cr0 &= 0xfUL;
4224                         val &= 0xfUL;
4225                         /* lmsw can't clear PE - catch this here */
4226                         if (cr0 & X86_CR0_PE)
4227                                 val |= X86_CR0_PE;
4228                 }
4229
4230                 if (cr0 ^ val)
4231                         icpt_info.exit_code = SVM_EXIT_CR0_SEL_WRITE;
4232
4233                 break;
4234         }
4235         case SVM_EXIT_READ_DR0:
4236         case SVM_EXIT_WRITE_DR0:
4237                 icpt_info.exit_code += info->modrm_reg;
4238                 break;
4239         case SVM_EXIT_MSR:
4240                 if (info->intercept == x86_intercept_wrmsr)
4241                         vmcb->control.exit_info_1 = 1;
4242                 else
4243                         vmcb->control.exit_info_1 = 0;
4244                 break;
4245         case SVM_EXIT_PAUSE:
4246                 /*
4247                  * We get this for NOP only, but pause
4248                  * is rep not, check this here
4249                  */
4250                 if (info->rep_prefix != REPE_PREFIX)
4251                         goto out;
4252         case SVM_EXIT_IOIO: {
4253                 u64 exit_info;
4254                 u32 bytes;
4255
4256                 if (info->intercept == x86_intercept_in ||
4257                     info->intercept == x86_intercept_ins) {
4258                         exit_info = ((info->src_val & 0xffff) << 16) |
4259                                 SVM_IOIO_TYPE_MASK;
4260                         bytes = info->dst_bytes;
4261                 } else {
4262                         exit_info = (info->dst_val & 0xffff) << 16;
4263                         bytes = info->src_bytes;
4264                 }
4265
4266                 if (info->intercept == x86_intercept_outs ||
4267                     info->intercept == x86_intercept_ins)
4268                         exit_info |= SVM_IOIO_STR_MASK;
4269
4270                 if (info->rep_prefix)
4271                         exit_info |= SVM_IOIO_REP_MASK;
4272
4273                 bytes = min(bytes, 4u);
4274
4275                 exit_info |= bytes << SVM_IOIO_SIZE_SHIFT;
4276
4277                 exit_info |= (u32)info->ad_bytes << (SVM_IOIO_ASIZE_SHIFT - 1);
4278
4279                 vmcb->control.exit_info_1 = exit_info;
4280                 vmcb->control.exit_info_2 = info->next_rip;
4281
4282                 break;
4283         }
4284         default:
4285                 break;
4286         }
4287
4288         /* TODO: Advertise NRIPS to guest hypervisor unconditionally */
4289         if (static_cpu_has(X86_FEATURE_NRIPS))
4290                 vmcb->control.next_rip  = info->next_rip;
4291         vmcb->control.exit_code = icpt_info.exit_code;
4292         vmexit = nested_svm_exit_handled(svm);
4293
4294         ret = (vmexit == NESTED_EXIT_DONE) ? X86EMUL_INTERCEPTED
4295                                            : X86EMUL_CONTINUE;
4296
4297 out:
4298         return ret;
4299 }
4300
4301 static void svm_handle_external_intr(struct kvm_vcpu *vcpu)
4302 {
4303         local_irq_enable();
4304 }
4305
4306 static void svm_sched_in(struct kvm_vcpu *vcpu, int cpu)
4307 {
4308 }
4309
4310 static struct kvm_x86_ops svm_x86_ops = {
4311         .cpu_has_kvm_support = has_svm,
4312         .disabled_by_bios = is_disabled,
4313         .hardware_setup = svm_hardware_setup,
4314         .hardware_unsetup = svm_hardware_unsetup,
4315         .check_processor_compatibility = svm_check_processor_compat,
4316         .hardware_enable = svm_hardware_enable,
4317         .hardware_disable = svm_hardware_disable,
4318         .cpu_has_accelerated_tpr = svm_cpu_has_accelerated_tpr,
4319         .cpu_has_high_real_mode_segbase = svm_has_high_real_mode_segbase,
4320
4321         .vcpu_create = svm_create_vcpu,
4322         .vcpu_free = svm_free_vcpu,
4323         .vcpu_reset = svm_vcpu_reset,
4324
4325         .prepare_guest_switch = svm_prepare_guest_switch,
4326         .vcpu_load = svm_vcpu_load,
4327         .vcpu_put = svm_vcpu_put,
4328
4329         .update_bp_intercept = update_bp_intercept,
4330         .get_msr = svm_get_msr,
4331         .set_msr = svm_set_msr,
4332         .get_segment_base = svm_get_segment_base,
4333         .get_segment = svm_get_segment,
4334         .set_segment = svm_set_segment,
4335         .get_cpl = svm_get_cpl,
4336         .get_cs_db_l_bits = kvm_get_cs_db_l_bits,
4337         .decache_cr0_guest_bits = svm_decache_cr0_guest_bits,
4338         .decache_cr3 = svm_decache_cr3,
4339         .decache_cr4_guest_bits = svm_decache_cr4_guest_bits,
4340         .set_cr0 = svm_set_cr0,
4341         .set_cr3 = svm_set_cr3,
4342         .set_cr4 = svm_set_cr4,
4343         .set_efer = svm_set_efer,
4344         .get_idt = svm_get_idt,
4345         .set_idt = svm_set_idt,
4346         .get_gdt = svm_get_gdt,
4347         .set_gdt = svm_set_gdt,
4348         .get_dr6 = svm_get_dr6,
4349         .set_dr6 = svm_set_dr6,
4350         .set_dr7 = svm_set_dr7,
4351         .sync_dirty_debug_regs = svm_sync_dirty_debug_regs,
4352         .cache_reg = svm_cache_reg,
4353         .get_rflags = svm_get_rflags,
4354         .set_rflags = svm_set_rflags,
4355
4356         .get_pkru = svm_get_pkru,
4357
4358         .fpu_activate = svm_fpu_activate,
4359         .fpu_deactivate = svm_fpu_deactivate,
4360
4361         .tlb_flush = svm_flush_tlb,
4362
4363         .run = svm_vcpu_run,
4364         .handle_exit = handle_exit,
4365         .skip_emulated_instruction = skip_emulated_instruction,
4366         .set_interrupt_shadow = svm_set_interrupt_shadow,
4367         .get_interrupt_shadow = svm_get_interrupt_shadow,
4368         .patch_hypercall = svm_patch_hypercall,
4369         .set_irq = svm_set_irq,
4370         .set_nmi = svm_inject_nmi,
4371         .queue_exception = svm_queue_exception,
4372         .cancel_injection = svm_cancel_injection,
4373         .interrupt_allowed = svm_interrupt_allowed,
4374         .nmi_allowed = svm_nmi_allowed,
4375         .get_nmi_mask = svm_get_nmi_mask,
4376         .set_nmi_mask = svm_set_nmi_mask,
4377         .enable_nmi_window = enable_nmi_window,
4378         .enable_irq_window = enable_irq_window,
4379         .update_cr8_intercept = update_cr8_intercept,
4380         .set_virtual_x2apic_mode = svm_set_virtual_x2apic_mode,
4381         .get_enable_apicv = svm_get_enable_apicv,
4382         .refresh_apicv_exec_ctrl = svm_refresh_apicv_exec_ctrl,
4383         .load_eoi_exitmap = svm_load_eoi_exitmap,
4384         .sync_pir_to_irr = svm_sync_pir_to_irr,
4385
4386         .set_tss_addr = svm_set_tss_addr,
4387         .get_tdp_level = get_npt_level,
4388         .get_mt_mask = svm_get_mt_mask,
4389
4390         .get_exit_info = svm_get_exit_info,
4391
4392         .get_lpage_level = svm_get_lpage_level,
4393
4394         .cpuid_update = svm_cpuid_update,
4395
4396         .rdtscp_supported = svm_rdtscp_supported,
4397         .invpcid_supported = svm_invpcid_supported,
4398         .mpx_supported = svm_mpx_supported,
4399         .xsaves_supported = svm_xsaves_supported,
4400
4401         .set_supported_cpuid = svm_set_supported_cpuid,
4402
4403         .has_wbinvd_exit = svm_has_wbinvd_exit,
4404
4405         .read_tsc_offset = svm_read_tsc_offset,
4406         .write_tsc_offset = svm_write_tsc_offset,
4407         .adjust_tsc_offset_guest = svm_adjust_tsc_offset_guest,
4408         .read_l1_tsc = svm_read_l1_tsc,
4409
4410         .set_tdp_cr3 = set_tdp_cr3,
4411
4412         .check_intercept = svm_check_intercept,
4413         .handle_external_intr = svm_handle_external_intr,
4414
4415         .sched_in = svm_sched_in,
4416
4417         .pmu_ops = &amd_pmu_ops,
4418 };
4419
4420 static int __init svm_init(void)
4421 {
4422         return kvm_init(&svm_x86_ops, sizeof(struct vcpu_svm),
4423                         __alignof__(struct vcpu_svm), THIS_MODULE);
4424 }
4425
4426 static void __exit svm_exit(void)
4427 {
4428         kvm_exit();
4429 }
4430
4431 module_init(svm_init)
4432 module_exit(svm_exit)