drivers/kvm/vmx.c

   1 /*
   2  * Kernel-based Virtual Machine driver for Linux
   3  *
   4  * This module enables machines with Intel VT-x extensions to run virtual
   5  * machines without emulation or binary translation.
   6  *
   7  * Copyright (C) 2006 Qumranet, Inc.
   8  *
   9  * Authors:
  10  *   Avi Kivity   <avi@qumranet.com>
  11  *   Yaniv Kamay  <yaniv@qumranet.com>
  12  *
  13  * This work is licensed under the terms of the GNU GPL, version 2.  See
  14  * the COPYING file in the top-level directory.
  15  *
  16  */
  17
  18 #include "kvm.h"
  19 #include "vmx.h"
  20 #include <linux/module.h>
  21 #include <linux/kernel.h>
  22 #include <linux/mm.h>
  23 #include <linux/highmem.h>
  24 #include <linux/profile.h>
  25 #include <asm/io.h>
  26 #include <asm/desc.h>
  27
  28 #include "segment_descriptor.h"
  29
  30 MODULE_AUTHOR("Qumranet");
  31 MODULE_LICENSE("GPL");
  32
  33 static DEFINE_PER_CPU(struct vmcs *, vmxarea);
  34 static DEFINE_PER_CPU(struct vmcs *, current_vmcs);
  35
  36 #ifdef CONFIG_X86_64
  37 #define HOST_IS_64 1
  38 #else
  39 #define HOST_IS_64 0
  40 #endif
  41
  42 static struct vmcs_descriptor {
  43         int size;
  44         int order;
  45         u32 revision_id;
  46 } vmcs_descriptor;
  47
  48 #define VMX_SEGMENT_FIELD(seg)                                  \
  49         [VCPU_SREG_##seg] = {                                   \
  50                 .selector = GUEST_##seg##_SELECTOR,             \
  51                 .base = GUEST_##seg##_BASE,                     \
  52                 .limit = GUEST_##seg##_LIMIT,                   \
  53                 .ar_bytes = GUEST_##seg##_AR_BYTES,             \
  54         }
  55
  56 static struct kvm_vmx_segment_field {
  57         unsigned selector;
  58         unsigned base;
  59         unsigned limit;
  60         unsigned ar_bytes;
  61 } kvm_vmx_segment_fields[] = {
  62         VMX_SEGMENT_FIELD(CS),
  63         VMX_SEGMENT_FIELD(DS),
  64         VMX_SEGMENT_FIELD(ES),
  65         VMX_SEGMENT_FIELD(FS),
  66         VMX_SEGMENT_FIELD(GS),
  67         VMX_SEGMENT_FIELD(SS),
  68         VMX_SEGMENT_FIELD(TR),
  69         VMX_SEGMENT_FIELD(LDTR),
  70 };
  71
  72 /*
  73  * Keep MSR_K6_STAR at the end, as setup_msrs() will try to optimize it
  74  * away by decrementing the array size.
  75  */
  76 static const u32 vmx_msr_index[] = {
  77 #ifdef CONFIG_X86_64
  78         MSR_SYSCALL_MASK, MSR_LSTAR, MSR_CSTAR, MSR_KERNEL_GS_BASE,
  79 #endif
  80         MSR_EFER, MSR_K6_STAR,
  81 };
  82 #define NR_VMX_MSR ARRAY_SIZE(vmx_msr_index)
  83
  84 #ifdef CONFIG_X86_64
  85 static unsigned msr_offset_kernel_gs_base;
  86 #define NR_64BIT_MSRS 4
  87 /*
  88  * avoid save/load MSR_SYSCALL_MASK and MSR_LSTAR by std vt
  89  * mechanism (cpu bug AA24)
  90  */
  91 #define NR_BAD_MSRS 2
  92 #else
  93 #define NR_64BIT_MSRS 0
  94 #define NR_BAD_MSRS 0
  95 #endif
  96
  97 static inline int is_page_fault(u32 intr_info)
  98 {
  99         return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
 100                              INTR_INFO_VALID_MASK)) ==
 101                 (INTR_TYPE_EXCEPTION | PF_VECTOR | INTR_INFO_VALID_MASK);
 102 }
 103
 104 static inline int is_no_device(u32 intr_info)
 105 {
 106         return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
 107                              INTR_INFO_VALID_MASK)) ==
 108                 (INTR_TYPE_EXCEPTION | NM_VECTOR | INTR_INFO_VALID_MASK);
 109 }
 110
 111 static inline int is_external_interrupt(u32 intr_info)
 112 {
 113         return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VALID_MASK))
 114                 == (INTR_TYPE_EXT_INTR | INTR_INFO_VALID_MASK);
 115 }
 116
 117 static struct vmx_msr_entry *find_msr_entry(struct kvm_vcpu *vcpu, u32 msr)
 118 {
 119         int i;
 120
 121         for (i = 0; i < vcpu->nmsrs; ++i)
 122                 if (vcpu->guest_msrs[i].index == msr)
 123                         return &vcpu->guest_msrs[i];
 124         return NULL;
 125 }
 126
 127 static void vmcs_clear(struct vmcs *vmcs)
 128 {
 129         u64 phys_addr = __pa(vmcs);
 130         u8 error;
 131
 132         asm volatile (ASM_VMX_VMCLEAR_RAX "; setna %0"
 133                       : "=g"(error) : "a"(&phys_addr), "m"(phys_addr)
 134                       : "cc", "memory");
 135         if (error)
 136                 printk(KERN_ERR "kvm: vmclear fail: %p/%llx\n",
 137                        vmcs, phys_addr);
 138 }
 139
 140 static void __vcpu_clear(void *arg)
 141 {
 142         struct kvm_vcpu *vcpu = arg;
 143         int cpu = raw_smp_processor_id();
 144
 145         if (vcpu->cpu == cpu)
 146                 vmcs_clear(vcpu->vmcs);
 147         if (per_cpu(current_vmcs, cpu) == vcpu->vmcs)
 148                 per_cpu(current_vmcs, cpu) = NULL;
 149 }
 150
 151 static void vcpu_clear(struct kvm_vcpu *vcpu)
 152 {
 153         if (vcpu->cpu != raw_smp_processor_id() && vcpu->cpu != -1)
 154                 smp_call_function_single(vcpu->cpu, __vcpu_clear, vcpu, 0, 1);
 155         else
 156                 __vcpu_clear(vcpu);
 157         vcpu->launched = 0;
 158 }
 159
 160 static unsigned long vmcs_readl(unsigned long field)
 161 {
 162         unsigned long value;
 163
 164         asm volatile (ASM_VMX_VMREAD_RDX_RAX
 165                       : "=a"(value) : "d"(field) : "cc");
 166         return value;
 167 }
 168
 169 static u16 vmcs_read16(unsigned long field)
 170 {
 171         return vmcs_readl(field);
 172 }
 173
 174 static u32 vmcs_read32(unsigned long field)
 175 {
 176         return vmcs_readl(field);
 177 }
 178
 179 static u64 vmcs_read64(unsigned long field)
 180 {
 181 #ifdef CONFIG_X86_64
 182         return vmcs_readl(field);
 183 #else
 184         return vmcs_readl(field) | ((u64)vmcs_readl(field+1) << 32);
 185 #endif
 186 }
 187
 188 static noinline void vmwrite_error(unsigned long field, unsigned long value)
 189 {
 190         printk(KERN_ERR "vmwrite error: reg %lx value %lx (err %d)\n",
 191                field, value, vmcs_read32(VM_INSTRUCTION_ERROR));
 192         dump_stack();
 193 }
 194
 195 static void vmcs_writel(unsigned long field, unsigned long value)
 196 {
 197         u8 error;
 198
 199         asm volatile (ASM_VMX_VMWRITE_RAX_RDX "; setna %0"
 200                        : "=q"(error) : "a"(value), "d"(field) : "cc" );
 201         if (unlikely(error))
 202                 vmwrite_error(field, value);
 203 }
 204
 205 static void vmcs_write16(unsigned long field, u16 value)
 206 {
 207         vmcs_writel(field, value);
 208 }
 209
 210 static void vmcs_write32(unsigned long field, u32 value)
 211 {
 212         vmcs_writel(field, value);
 213 }
 214
 215 static void vmcs_write64(unsigned long field, u64 value)
 216 {
 217 #ifdef CONFIG_X86_64
 218         vmcs_writel(field, value);
 219 #else
 220         vmcs_writel(field, value);
 221         asm volatile ("");
 222         vmcs_writel(field+1, value >> 32);
 223 #endif
 224 }
 225
 226 static void vmcs_clear_bits(unsigned long field, u32 mask)
 227 {
 228         vmcs_writel(field, vmcs_readl(field) & ~mask);
 229 }
 230
 231 static void vmcs_set_bits(unsigned long field, u32 mask)
 232 {
 233         vmcs_writel(field, vmcs_readl(field) | mask);
 234 }
 235
 236 /*
 237  * Switches to specified vcpu, until a matching vcpu_put(), but assumes
 238  * vcpu mutex is already taken.
 239  */
 240 static void vmx_vcpu_load(struct kvm_vcpu *vcpu)
 241 {
 242         u64 phys_addr = __pa(vcpu->vmcs);
 243         int cpu;
 244
 245         cpu = get_cpu();
 246
 247         if (vcpu->cpu != cpu)
 248                 vcpu_clear(vcpu);
 249
 250         if (per_cpu(current_vmcs, cpu) != vcpu->vmcs) {
 251                 u8 error;
 252
 253                 per_cpu(current_vmcs, cpu) = vcpu->vmcs;
 254                 asm volatile (ASM_VMX_VMPTRLD_RAX "; setna %0"
 255                               : "=g"(error) : "a"(&phys_addr), "m"(phys_addr)
 256                               : "cc");
 257                 if (error)
 258                         printk(KERN_ERR "kvm: vmptrld %p/%llx fail\n",
 259                                vcpu->vmcs, phys_addr);
 260         }
 261
 262         if (vcpu->cpu != cpu) {
 263                 struct descriptor_table dt;
 264                 unsigned long sysenter_esp;
 265
 266                 vcpu->cpu = cpu;
 267                 /*
 268                  * Linux uses per-cpu TSS and GDT, so set these when switching
 269                  * processors.
 270                  */
 271                 vmcs_writel(HOST_TR_BASE, read_tr_base()); /* 22.2.4 */
 272                 get_gdt(&dt);
 273                 vmcs_writel(HOST_GDTR_BASE, dt.base);   /* 22.2.4 */
 274
 275                 rdmsrl(MSR_IA32_SYSENTER_ESP, sysenter_esp);
 276                 vmcs_writel(HOST_IA32_SYSENTER_ESP, sysenter_esp); /* 22.2.3 */
 277         }
 278 }
 279
 280 static void vmx_vcpu_put(struct kvm_vcpu *vcpu)
 281 {
 282         put_cpu();
 283 }
 284
 285 static void vmx_vcpu_decache(struct kvm_vcpu *vcpu)
 286 {
 287         vcpu_clear(vcpu);
 288 }
 289
 290 static unsigned long vmx_get_rflags(struct kvm_vcpu *vcpu)
 291 {
 292         return vmcs_readl(GUEST_RFLAGS);
 293 }
 294
 295 static void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
 296 {
 297         vmcs_writel(GUEST_RFLAGS, rflags);
 298 }
 299
 300 static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
 301 {
 302         unsigned long rip;
 303         u32 interruptibility;
 304
 305         rip = vmcs_readl(GUEST_RIP);
 306         rip += vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
 307         vmcs_writel(GUEST_RIP, rip);
 308
 309         /*
 310          * We emulated an instruction, so temporary interrupt blocking
 311          * should be removed, if set.
 312          */
 313         interruptibility = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
 314         if (interruptibility & 3)
 315                 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO,
 316                              interruptibility & ~3);
 317         vcpu->interrupt_window_open = 1;
 318 }
 319
 320 static void vmx_inject_gp(struct kvm_vcpu *vcpu, unsigned error_code)
 321 {
 322         printk(KERN_DEBUG "inject_general_protection: rip 0x%lx\n",
 323                vmcs_readl(GUEST_RIP));
 324         vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, error_code);
 325         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
 326                      GP_VECTOR |
 327                      INTR_TYPE_EXCEPTION |
 328                      INTR_INFO_DELIEVER_CODE_MASK |
 329                      INTR_INFO_VALID_MASK);
 330 }
 331
 332 /*
 333  * Set up the vmcs to automatically save and restore system
 334  * msrs.  Don't touch the 64-bit msrs if the guest is in legacy
 335  * mode, as fiddling with msrs is very expensive.
 336  */
 337 static void setup_msrs(struct kvm_vcpu *vcpu)
 338 {
 339         int nr_skip, nr_good_msrs;
 340
 341         if (is_long_mode(vcpu))
 342                 nr_skip = NR_BAD_MSRS;
 343         else
 344                 nr_skip = NR_64BIT_MSRS;
 345         nr_good_msrs = vcpu->nmsrs - nr_skip;
 346
 347         /*
 348          * MSR_K6_STAR is only needed on long mode guests, and only
 349          * if efer.sce is enabled.
 350          */
 351         if (find_msr_entry(vcpu, MSR_K6_STAR)) {
 352                 --nr_good_msrs;
 353 #ifdef CONFIG_X86_64
 354                 if (is_long_mode(vcpu) && (vcpu->shadow_efer & EFER_SCE))
 355                         ++nr_good_msrs;
 356 #endif
 357         }
 358
 359         vmcs_writel(VM_ENTRY_MSR_LOAD_ADDR,
 360                     virt_to_phys(vcpu->guest_msrs + nr_skip));
 361         vmcs_writel(VM_EXIT_MSR_STORE_ADDR,
 362                     virt_to_phys(vcpu->guest_msrs + nr_skip));
 363         vmcs_writel(VM_EXIT_MSR_LOAD_ADDR,
 364                     virt_to_phys(vcpu->host_msrs + nr_skip));
 365         vmcs_write32(VM_EXIT_MSR_STORE_COUNT, nr_good_msrs); /* 22.2.2 */
 366         vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, nr_good_msrs);  /* 22.2.2 */
 367         vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, nr_good_msrs); /* 22.2.2 */
 368 }
 369
 370 /*
 371  * reads and returns guest's timestamp counter "register"
 372  * guest_tsc = host_tsc + tsc_offset    -- 21.3
 373  */
 374 static u64 guest_read_tsc(void)
 375 {
 376         u64 host_tsc, tsc_offset;
 377
 378         rdtscll(host_tsc);
 379         tsc_offset = vmcs_read64(TSC_OFFSET);
 380         return host_tsc + tsc_offset;
 381 }
 382
 383 /*
 384  * writes 'guest_tsc' into guest's timestamp counter "register"
 385  * guest_tsc = host_tsc + tsc_offset ==> tsc_offset = guest_tsc - host_tsc
 386  */
 387 static void guest_write_tsc(u64 guest_tsc)
 388 {
 389         u64 host_tsc;
 390
 391         rdtscll(host_tsc);
 392         vmcs_write64(TSC_OFFSET, guest_tsc - host_tsc);
 393 }
 394
 395 static void reload_tss(void)
 396 {
 397 #ifndef CONFIG_X86_64
 398
 399         /*
 400          * VT restores TR but not its size.  Useless.
 401          */
 402         struct descriptor_table gdt;
 403         struct segment_descriptor *descs;
 404
 405         get_gdt(&gdt);
 406         descs = (void *)gdt.base;
 407         descs[GDT_ENTRY_TSS].type = 9; /* available TSS */
 408         load_TR_desc();
 409 #endif
 410 }
 411
 412 /*
 413  * Reads an msr value (of 'msr_index') into 'pdata'.
 414  * Returns 0 on success, non-0 otherwise.
 415  * Assumes vcpu_load() was already called.
 416  */
 417 static int vmx_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
 418 {
 419         u64 data;
 420         struct vmx_msr_entry *msr;
 421
 422         if (!pdata) {
 423                 printk(KERN_ERR "BUG: get_msr called with NULL pdata\n");
 424                 return -EINVAL;
 425         }
 426
 427         switch (msr_index) {
 428 #ifdef CONFIG_X86_64
 429         case MSR_FS_BASE:
 430                 data = vmcs_readl(GUEST_FS_BASE);
 431                 break;
 432         case MSR_GS_BASE:
 433                 data = vmcs_readl(GUEST_GS_BASE);
 434                 break;
 435         case MSR_EFER:
 436                 return kvm_get_msr_common(vcpu, msr_index, pdata);
 437 #endif
 438         case MSR_IA32_TIME_STAMP_COUNTER:
 439                 data = guest_read_tsc();
 440                 break;
 441         case MSR_IA32_SYSENTER_CS:
 442                 data = vmcs_read32(GUEST_SYSENTER_CS);
 443                 break;
 444         case MSR_IA32_SYSENTER_EIP:
 445                 data = vmcs_readl(GUEST_SYSENTER_EIP);
 446                 break;
 447         case MSR_IA32_SYSENTER_ESP:
 448                 data = vmcs_readl(GUEST_SYSENTER_ESP);
 449                 break;
 450         default:
 451                 msr = find_msr_entry(vcpu, msr_index);
 452                 if (msr) {
 453                         data = msr->data;
 454                         break;
 455                 }
 456                 return kvm_get_msr_common(vcpu, msr_index, pdata);
 457         }
 458
 459         *pdata = data;
 460         return 0;
 461 }
 462
 463 /*
 464  * Writes msr value into into the appropriate "register".
 465  * Returns 0 on success, non-0 otherwise.
 466  * Assumes vcpu_load() was already called.
 467  */
 468 static int vmx_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
 469 {
 470         struct vmx_msr_entry *msr;
 471         switch (msr_index) {
 472 #ifdef CONFIG_X86_64
 473         case MSR_EFER:
 474                 return kvm_set_msr_common(vcpu, msr_index, data);
 475         case MSR_FS_BASE:
 476                 vmcs_writel(GUEST_FS_BASE, data);
 477                 break;
 478         case MSR_GS_BASE:
 479                 vmcs_writel(GUEST_GS_BASE, data);
 480                 break;
 481 #endif
 482         case MSR_IA32_SYSENTER_CS:
 483                 vmcs_write32(GUEST_SYSENTER_CS, data);
 484                 break;
 485         case MSR_IA32_SYSENTER_EIP:
 486                 vmcs_writel(GUEST_SYSENTER_EIP, data);
 487                 break;
 488         case MSR_IA32_SYSENTER_ESP:
 489                 vmcs_writel(GUEST_SYSENTER_ESP, data);
 490                 break;
 491         case MSR_IA32_TIME_STAMP_COUNTER:
 492                 guest_write_tsc(data);
 493                 break;
 494         default:
 495                 msr = find_msr_entry(vcpu, msr_index);
 496                 if (msr) {
 497                         msr->data = data;
 498                         break;
 499                 }
 500                 return kvm_set_msr_common(vcpu, msr_index, data);
 501                 msr->data = data;
 502                 break;
 503         }
 504
 505         return 0;
 506 }
 507
 508 /*
 509  * Sync the rsp and rip registers into the vcpu structure.  This allows
 510  * registers to be accessed by indexing vcpu->regs.
 511  */
 512 static void vcpu_load_rsp_rip(struct kvm_vcpu *vcpu)
 513 {
 514         vcpu->regs[VCPU_REGS_RSP] = vmcs_readl(GUEST_RSP);
 515         vcpu->rip = vmcs_readl(GUEST_RIP);
 516 }
 517
 518 /*
 519  * Syncs rsp and rip back into the vmcs.  Should be called after possible
 520  * modification.
 521  */
 522 static void vcpu_put_rsp_rip(struct kvm_vcpu *vcpu)
 523 {
 524         vmcs_writel(GUEST_RSP, vcpu->regs[VCPU_REGS_RSP]);
 525         vmcs_writel(GUEST_RIP, vcpu->rip);
 526 }
 527
 528 static int set_guest_debug(struct kvm_vcpu *vcpu, struct kvm_debug_guest *dbg)
 529 {
 530         unsigned long dr7 = 0x400;
 531         u32 exception_bitmap;
 532         int old_singlestep;
 533
 534         exception_bitmap = vmcs_read32(EXCEPTION_BITMAP);
 535         old_singlestep = vcpu->guest_debug.singlestep;
 536
 537         vcpu->guest_debug.enabled = dbg->enabled;
 538         if (vcpu->guest_debug.enabled) {
 539                 int i;
 540
 541                 dr7 |= 0x200;  /* exact */
 542                 for (i = 0; i < 4; ++i) {
 543                         if (!dbg->breakpoints[i].enabled)
 544                                 continue;
 545                         vcpu->guest_debug.bp[i] = dbg->breakpoints[i].address;
 546                         dr7 |= 2 << (i*2);    /* global enable */
 547                         dr7 |= 0 << (i*4+16); /* execution breakpoint */
 548                 }
 549
 550                 exception_bitmap |= (1u << 1);  /* Trap debug exceptions */
 551
 552                 vcpu->guest_debug.singlestep = dbg->singlestep;
 553         } else {
 554                 exception_bitmap &= ~(1u << 1); /* Ignore debug exceptions */
 555                 vcpu->guest_debug.singlestep = 0;
 556         }
 557
 558         if (old_singlestep && !vcpu->guest_debug.singlestep) {
 559                 unsigned long flags;
 560
 561                 flags = vmcs_readl(GUEST_RFLAGS);
 562                 flags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF);
 563                 vmcs_writel(GUEST_RFLAGS, flags);
 564         }
 565
 566         vmcs_write32(EXCEPTION_BITMAP, exception_bitmap);
 567         vmcs_writel(GUEST_DR7, dr7);
 568
 569         return 0;
 570 }
 571
 572 static __init int cpu_has_kvm_support(void)
 573 {
 574         unsigned long ecx = cpuid_ecx(1);
 575         return test_bit(5, &ecx); /* CPUID.1:ECX.VMX[bit 5] -> VT */
 576 }
 577
 578 static __init int vmx_disabled_by_bios(void)
 579 {
 580         u64 msr;
 581
 582         rdmsrl(MSR_IA32_FEATURE_CONTROL, msr);
 583         return (msr & 5) == 1; /* locked but not enabled */
 584 }
 585
 586 static void hardware_enable(void *garbage)
 587 {
 588         int cpu = raw_smp_processor_id();
 589         u64 phys_addr = __pa(per_cpu(vmxarea, cpu));
 590         u64 old;
 591
 592         rdmsrl(MSR_IA32_FEATURE_CONTROL, old);
 593         if ((old & 5) != 5)
 594                 /* enable and lock */
 595                 wrmsrl(MSR_IA32_FEATURE_CONTROL, old | 5);
 596         write_cr4(read_cr4() | CR4_VMXE); /* FIXME: not cpu hotplug safe */
 597         asm volatile (ASM_VMX_VMXON_RAX : : "a"(&phys_addr), "m"(phys_addr)
 598                       : "memory", "cc");
 599 }
 600
 601 static void hardware_disable(void *garbage)
 602 {
 603         asm volatile (ASM_VMX_VMXOFF : : : "cc");
 604 }
 605
 606 static __init void setup_vmcs_descriptor(void)
 607 {
 608         u32 vmx_msr_low, vmx_msr_high;
 609
 610         rdmsr(MSR_IA32_VMX_BASIC, vmx_msr_low, vmx_msr_high);
 611         vmcs_descriptor.size = vmx_msr_high & 0x1fff;
 612         vmcs_descriptor.order = get_order(vmcs_descriptor.size);
 613         vmcs_descriptor.revision_id = vmx_msr_low;
 614 }
 615
 616 static struct vmcs *alloc_vmcs_cpu(int cpu)
 617 {
 618         int node = cpu_to_node(cpu);
 619         struct page *pages;
 620         struct vmcs *vmcs;
 621
 622         pages = alloc_pages_node(node, GFP_KERNEL, vmcs_descriptor.order);
 623         if (!pages)
 624                 return NULL;
 625         vmcs = page_address(pages);
 626         memset(vmcs, 0, vmcs_descriptor.size);
 627         vmcs->revision_id = vmcs_descriptor.revision_id; /* vmcs revision id */
 628         return vmcs;
 629 }
 630
 631 static struct vmcs *alloc_vmcs(void)
 632 {
 633         return alloc_vmcs_cpu(raw_smp_processor_id());
 634 }
 635
 636 static void free_vmcs(struct vmcs *vmcs)
 637 {
 638         free_pages((unsigned long)vmcs, vmcs_descriptor.order);
 639 }
 640
 641 static __exit void free_kvm_area(void)
 642 {
 643         int cpu;
 644
 645         for_each_online_cpu(cpu)
 646                 free_vmcs(per_cpu(vmxarea, cpu));
 647 }
 648
 649 extern struct vmcs *alloc_vmcs_cpu(int cpu);
 650
 651 static __init int alloc_kvm_area(void)
 652 {
 653         int cpu;
 654
 655         for_each_online_cpu(cpu) {
 656                 struct vmcs *vmcs;
 657
 658                 vmcs = alloc_vmcs_cpu(cpu);
 659                 if (!vmcs) {
 660                         free_kvm_area();
 661                         return -ENOMEM;
 662                 }
 663
 664                 per_cpu(vmxarea, cpu) = vmcs;
 665         }
 666         return 0;
 667 }
 668
 669 static __init int hardware_setup(void)
 670 {
 671         setup_vmcs_descriptor();
 672         return alloc_kvm_area();
 673 }
 674
 675 static __exit void hardware_unsetup(void)
 676 {
 677         free_kvm_area();
 678 }
 679
 680 static void update_exception_bitmap(struct kvm_vcpu *vcpu)
 681 {
 682         if (vcpu->rmode.active)
 683                 vmcs_write32(EXCEPTION_BITMAP, ~0);
 684         else
 685                 vmcs_write32(EXCEPTION_BITMAP, 1 << PF_VECTOR);
 686 }
 687
 688 static void fix_pmode_dataseg(int seg, struct kvm_save_segment *save)
 689 {
 690         struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
 691
 692         if (vmcs_readl(sf->base) == save->base && (save->base & AR_S_MASK)) {
 693                 vmcs_write16(sf->selector, save->selector);
 694                 vmcs_writel(sf->base, save->base);
 695                 vmcs_write32(sf->limit, save->limit);
 696                 vmcs_write32(sf->ar_bytes, save->ar);
 697         } else {
 698                 u32 dpl = (vmcs_read16(sf->selector) & SELECTOR_RPL_MASK)
 699                         << AR_DPL_SHIFT;
 700                 vmcs_write32(sf->ar_bytes, 0x93 | dpl);
 701         }
 702 }
 703
 704 static void enter_pmode(struct kvm_vcpu *vcpu)
 705 {
 706         unsigned long flags;
 707
 708         vcpu->rmode.active = 0;
 709
 710         vmcs_writel(GUEST_TR_BASE, vcpu->rmode.tr.base);
 711         vmcs_write32(GUEST_TR_LIMIT, vcpu->rmode.tr.limit);
 712         vmcs_write32(GUEST_TR_AR_BYTES, vcpu->rmode.tr.ar);
 713
 714         flags = vmcs_readl(GUEST_RFLAGS);
 715         flags &= ~(IOPL_MASK | X86_EFLAGS_VM);
 716         flags |= (vcpu->rmode.save_iopl << IOPL_SHIFT);
 717         vmcs_writel(GUEST_RFLAGS, flags);
 718
 719         vmcs_writel(GUEST_CR4, (vmcs_readl(GUEST_CR4) & ~CR4_VME_MASK) |
 720                         (vmcs_readl(CR4_READ_SHADOW) & CR4_VME_MASK));
 721
 722         update_exception_bitmap(vcpu);
 723
 724         fix_pmode_dataseg(VCPU_SREG_ES, &vcpu->rmode.es);
 725         fix_pmode_dataseg(VCPU_SREG_DS, &vcpu->rmode.ds);
 726         fix_pmode_dataseg(VCPU_SREG_GS, &vcpu->rmode.gs);
 727         fix_pmode_dataseg(VCPU_SREG_FS, &vcpu->rmode.fs);
 728
 729         vmcs_write16(GUEST_SS_SELECTOR, 0);
 730         vmcs_write32(GUEST_SS_AR_BYTES, 0x93);
 731
 732         vmcs_write16(GUEST_CS_SELECTOR,
 733                      vmcs_read16(GUEST_CS_SELECTOR) & ~SELECTOR_RPL_MASK);
 734         vmcs_write32(GUEST_CS_AR_BYTES, 0x9b);
 735 }
 736
 737 static int rmode_tss_base(struct kvm* kvm)
 738 {
 739         gfn_t base_gfn = kvm->memslots[0].base_gfn + kvm->memslots[0].npages - 3;
 740         return base_gfn << PAGE_SHIFT;
 741 }
 742
 743 static void fix_rmode_seg(int seg, struct kvm_save_segment *save)
 744 {
 745         struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
 746
 747         save->selector = vmcs_read16(sf->selector);
 748         save->base = vmcs_readl(sf->base);
 749         save->limit = vmcs_read32(sf->limit);
 750         save->ar = vmcs_read32(sf->ar_bytes);
 751         vmcs_write16(sf->selector, vmcs_readl(sf->base) >> 4);
 752         vmcs_write32(sf->limit, 0xffff);
 753         vmcs_write32(sf->ar_bytes, 0xf3);
 754 }
 755
 756 static void enter_rmode(struct kvm_vcpu *vcpu)
 757 {
 758         unsigned long flags;
 759
 760         vcpu->rmode.active = 1;
 761
 762         vcpu->rmode.tr.base = vmcs_readl(GUEST_TR_BASE);
 763         vmcs_writel(GUEST_TR_BASE, rmode_tss_base(vcpu->kvm));
 764
 765         vcpu->rmode.tr.limit = vmcs_read32(GUEST_TR_LIMIT);
 766         vmcs_write32(GUEST_TR_LIMIT, RMODE_TSS_SIZE - 1);
 767
 768         vcpu->rmode.tr.ar = vmcs_read32(GUEST_TR_AR_BYTES);
 769         vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);
 770
 771         flags = vmcs_readl(GUEST_RFLAGS);
 772         vcpu->rmode.save_iopl = (flags & IOPL_MASK) >> IOPL_SHIFT;
 773
 774         flags |= IOPL_MASK | X86_EFLAGS_VM;
 775
 776         vmcs_writel(GUEST_RFLAGS, flags);
 777         vmcs_writel(GUEST_CR4, vmcs_readl(GUEST_CR4) | CR4_VME_MASK);
 778         update_exception_bitmap(vcpu);
 779
 780         vmcs_write16(GUEST_SS_SELECTOR, vmcs_readl(GUEST_SS_BASE) >> 4);
 781         vmcs_write32(GUEST_SS_LIMIT, 0xffff);
 782         vmcs_write32(GUEST_SS_AR_BYTES, 0xf3);
 783
 784         vmcs_write32(GUEST_CS_AR_BYTES, 0xf3);
 785         vmcs_write32(GUEST_CS_LIMIT, 0xffff);
 786         if (vmcs_readl(GUEST_CS_BASE) == 0xffff0000)
 787                 vmcs_writel(GUEST_CS_BASE, 0xf0000);
 788         vmcs_write16(GUEST_CS_SELECTOR, vmcs_readl(GUEST_CS_BASE) >> 4);
 789
 790         fix_rmode_seg(VCPU_SREG_ES, &vcpu->rmode.es);
 791         fix_rmode_seg(VCPU_SREG_DS, &vcpu->rmode.ds);
 792         fix_rmode_seg(VCPU_SREG_GS, &vcpu->rmode.gs);
 793         fix_rmode_seg(VCPU_SREG_FS, &vcpu->rmode.fs);
 794 }
 795
 796 #ifdef CONFIG_X86_64
 797
 798 static void enter_lmode(struct kvm_vcpu *vcpu)
 799 {
 800         u32 guest_tr_ar;
 801
 802         guest_tr_ar = vmcs_read32(GUEST_TR_AR_BYTES);
 803         if ((guest_tr_ar & AR_TYPE_MASK) != AR_TYPE_BUSY_64_TSS) {
 804                 printk(KERN_DEBUG "%s: tss fixup for long mode. \n",
 805                        __FUNCTION__);
 806                 vmcs_write32(GUEST_TR_AR_BYTES,
 807                              (guest_tr_ar & ~AR_TYPE_MASK)
 808                              | AR_TYPE_BUSY_64_TSS);
 809         }
 810
 811         vcpu->shadow_efer |= EFER_LMA;
 812
 813         find_msr_entry(vcpu, MSR_EFER)->data |= EFER_LMA | EFER_LME;
 814         vmcs_write32(VM_ENTRY_CONTROLS,
 815                      vmcs_read32(VM_ENTRY_CONTROLS)
 816                      | VM_ENTRY_CONTROLS_IA32E_MASK);
 817 }
 818
 819 static void exit_lmode(struct kvm_vcpu *vcpu)
 820 {
 821         vcpu->shadow_efer &= ~EFER_LMA;
 822
 823         vmcs_write32(VM_ENTRY_CONTROLS,
 824                      vmcs_read32(VM_ENTRY_CONTROLS)
 825                      & ~VM_ENTRY_CONTROLS_IA32E_MASK);
 826 }
 827
 828 #endif
 829
 830 static void vmx_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
 831 {
 832         vcpu->cr4 &= KVM_GUEST_CR4_MASK;
 833         vcpu->cr4 |= vmcs_readl(GUEST_CR4) & ~KVM_GUEST_CR4_MASK;
 834 }
 835
 836 static void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
 837 {
 838         if (vcpu->rmode.active && (cr0 & CR0_PE_MASK))
 839                 enter_pmode(vcpu);
 840
 841         if (!vcpu->rmode.active && !(cr0 & CR0_PE_MASK))
 842                 enter_rmode(vcpu);
 843
 844 #ifdef CONFIG_X86_64
 845         if (vcpu->shadow_efer & EFER_LME) {
 846                 if (!is_paging(vcpu) && (cr0 & CR0_PG_MASK))
 847                         enter_lmode(vcpu);
 848                 if (is_paging(vcpu) && !(cr0 & CR0_PG_MASK))
 849                         exit_lmode(vcpu);
 850         }
 851 #endif
 852
 853         if (!(cr0 & CR0_TS_MASK)) {
 854                 vcpu->fpu_active = 1;
 855                 vmcs_clear_bits(EXCEPTION_BITMAP, CR0_TS_MASK);
 856         }
 857
 858         vmcs_writel(CR0_READ_SHADOW, cr0);
 859         vmcs_writel(GUEST_CR0,
 860                     (cr0 & ~KVM_GUEST_CR0_MASK) | KVM_VM_CR0_ALWAYS_ON);
 861         vcpu->cr0 = cr0;
 862 }
 863
 864 static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
 865 {
 866         vmcs_writel(GUEST_CR3, cr3);
 867
 868         if (!(vcpu->cr0 & CR0_TS_MASK)) {
 869                 vcpu->fpu_active = 0;
 870                 vmcs_set_bits(GUEST_CR0, CR0_TS_MASK);
 871                 vmcs_set_bits(EXCEPTION_BITMAP, 1 << NM_VECTOR);
 872         }
 873 }
 874
 875 static void vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
 876 {
 877         vmcs_writel(CR4_READ_SHADOW, cr4);
 878         vmcs_writel(GUEST_CR4, cr4 | (vcpu->rmode.active ?
 879                     KVM_RMODE_VM_CR4_ALWAYS_ON : KVM_PMODE_VM_CR4_ALWAYS_ON));
 880         vcpu->cr4 = cr4;
 881 }
 882
 883 #ifdef CONFIG_X86_64
 884
 885 static void vmx_set_efer(struct kvm_vcpu *vcpu, u64 efer)
 886 {
 887         struct vmx_msr_entry *msr = find_msr_entry(vcpu, MSR_EFER);
 888
 889         vcpu->shadow_efer = efer;
 890         if (efer & EFER_LMA) {
 891                 vmcs_write32(VM_ENTRY_CONTROLS,
 892                                      vmcs_read32(VM_ENTRY_CONTROLS) |
 893                                      VM_ENTRY_CONTROLS_IA32E_MASK);
 894                 msr->data = efer;
 895
 896         } else {
 897                 vmcs_write32(VM_ENTRY_CONTROLS,
 898                                      vmcs_read32(VM_ENTRY_CONTROLS) &
 899                                      ~VM_ENTRY_CONTROLS_IA32E_MASK);
 900
 901                 msr->data = efer & ~EFER_LME;
 902         }
 903         setup_msrs(vcpu);
 904 }
 905
 906 #endif
 907
 908 static u64 vmx_get_segment_base(struct kvm_vcpu *vcpu, int seg)
 909 {
 910         struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
 911
 912         return vmcs_readl(sf->base);
 913 }
 914
 915 static void vmx_get_segment(struct kvm_vcpu *vcpu,
 916                             struct kvm_segment *var, int seg)
 917 {
 918         struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
 919         u32 ar;
 920
 921         var->base = vmcs_readl(sf->base);
 922         var->limit = vmcs_read32(sf->limit);
 923         var->selector = vmcs_read16(sf->selector);
 924         ar = vmcs_read32(sf->ar_bytes);
 925         if (ar & AR_UNUSABLE_MASK)
 926                 ar = 0;
 927         var->type = ar & 15;
 928         var->s = (ar >> 4) & 1;
 929         var->dpl = (ar >> 5) & 3;
 930         var->present = (ar >> 7) & 1;
 931         var->avl = (ar >> 12) & 1;
 932         var->l = (ar >> 13) & 1;
 933         var->db = (ar >> 14) & 1;
 934         var->g = (ar >> 15) & 1;
 935         var->unusable = (ar >> 16) & 1;
 936 }
 937
 938 static void vmx_set_segment(struct kvm_vcpu *vcpu,
 939                             struct kvm_segment *var, int seg)
 940 {
 941         struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
 942         u32 ar;
 943
 944         vmcs_writel(sf->base, var->base);
 945         vmcs_write32(sf->limit, var->limit);
 946         vmcs_write16(sf->selector, var->selector);
 947         if (vcpu->rmode.active && var->s) {
 948                 /*
 949                  * Hack real-mode segments into vm86 compatibility.
 950                  */
 951                 if (var->base == 0xffff0000 && var->selector == 0xf000)
 952                         vmcs_writel(sf->base, 0xf0000);
 953                 ar = 0xf3;
 954         } else if (var->unusable)
 955                 ar = 1 << 16;
 956         else {
 957                 ar = var->type & 15;
 958                 ar |= (var->s & 1) << 4;
 959                 ar |= (var->dpl & 3) << 5;
 960                 ar |= (var->present & 1) << 7;
 961                 ar |= (var->avl & 1) << 12;
 962                 ar |= (var->l & 1) << 13;
 963                 ar |= (var->db & 1) << 14;
 964                 ar |= (var->g & 1) << 15;
 965         }
 966         if (ar == 0) /* a 0 value means unusable */
 967                 ar = AR_UNUSABLE_MASK;
 968         vmcs_write32(sf->ar_bytes, ar);
 969 }
 970
 971 static void vmx_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
 972 {
 973         u32 ar = vmcs_read32(GUEST_CS_AR_BYTES);
 974
 975         *db = (ar >> 14) & 1;
 976         *l = (ar >> 13) & 1;
 977 }
 978
 979 static void vmx_get_idt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
 980 {
 981         dt->limit = vmcs_read32(GUEST_IDTR_LIMIT);
 982         dt->base = vmcs_readl(GUEST_IDTR_BASE);
 983 }
 984
 985 static void vmx_set_idt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
 986 {
 987         vmcs_write32(GUEST_IDTR_LIMIT, dt->limit);
 988         vmcs_writel(GUEST_IDTR_BASE, dt->base);
 989 }
 990
 991 static void vmx_get_gdt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
 992 {
 993         dt->limit = vmcs_read32(GUEST_GDTR_LIMIT);
 994         dt->base = vmcs_readl(GUEST_GDTR_BASE);
 995 }
 996
 997 static void vmx_set_gdt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
 998 {
 999         vmcs_write32(GUEST_GDTR_LIMIT, dt->limit);
1000         vmcs_writel(GUEST_GDTR_BASE, dt->base);
1001 }
1002
1003 static int init_rmode_tss(struct kvm* kvm)
1004 {
1005         struct page *p1, *p2, *p3;
1006         gfn_t fn = rmode_tss_base(kvm) >> PAGE_SHIFT;
1007         char *page;
1008
1009         p1 = gfn_to_page(kvm, fn++);
1010         p2 = gfn_to_page(kvm, fn++);
1011         p3 = gfn_to_page(kvm, fn);
1012
1013         if (!p1 || !p2 || !p3) {
1014                 kvm_printf(kvm,"%s: gfn_to_page failed\n", __FUNCTION__);
1015                 return 0;
1016         }
1017
1018         page = kmap_atomic(p1, KM_USER0);
1019         memset(page, 0, PAGE_SIZE);
1020         *(u16*)(page + 0x66) = TSS_BASE_SIZE + TSS_REDIRECTION_SIZE;
1021         kunmap_atomic(page, KM_USER0);
1022
1023         page = kmap_atomic(p2, KM_USER0);
1024         memset(page, 0, PAGE_SIZE);
1025         kunmap_atomic(page, KM_USER0);
1026
1027         page = kmap_atomic(p3, KM_USER0);
1028         memset(page, 0, PAGE_SIZE);
1029         *(page + RMODE_TSS_SIZE - 2 * PAGE_SIZE - 1) = ~0;
1030         kunmap_atomic(page, KM_USER0);
1031
1032         return 1;
1033 }
1034
1035 static void vmcs_write32_fixedbits(u32 msr, u32 vmcs_field, u32 val)
1036 {
1037         u32 msr_high, msr_low;
1038
1039         rdmsr(msr, msr_low, msr_high);
1040
1041         val &= msr_high;
1042         val |= msr_low;
1043         vmcs_write32(vmcs_field, val);
1044 }
1045
1046 static void seg_setup(int seg)
1047 {
1048         struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
1049
1050         vmcs_write16(sf->selector, 0);
1051         vmcs_writel(sf->base, 0);
1052         vmcs_write32(sf->limit, 0xffff);
1053         vmcs_write32(sf->ar_bytes, 0x93);
1054 }
1055
1056 /*
1057  * Sets up the vmcs for emulated real mode.
1058  */
1059 static int vmx_vcpu_setup(struct kvm_vcpu *vcpu)
1060 {
1061         u32 host_sysenter_cs;
1062         u32 junk;
1063         unsigned long a;
1064         struct descriptor_table dt;
1065         int i;
1066         int ret = 0;
1067         extern asmlinkage void kvm_vmx_return(void);
1068
1069         if (!init_rmode_tss(vcpu->kvm)) {
1070                 ret = -ENOMEM;
1071                 goto out;
1072         }
1073
1074         memset(vcpu->regs, 0, sizeof(vcpu->regs));
1075         vcpu->regs[VCPU_REGS_RDX] = get_rdx_init_val();
1076         vcpu->cr8 = 0;
1077         vcpu->apic_base = 0xfee00000 |
1078                         /*for vcpu 0*/ MSR_IA32_APICBASE_BSP |
1079                         MSR_IA32_APICBASE_ENABLE;
1080
1081         fx_init(vcpu);
1082
1083         /*
1084          * GUEST_CS_BASE should really be 0xffff0000, but VT vm86 mode
1085          * insists on having GUEST_CS_BASE == GUEST_CS_SELECTOR << 4.  Sigh.
1086          */
1087         vmcs_write16(GUEST_CS_SELECTOR, 0xf000);
1088         vmcs_writel(GUEST_CS_BASE, 0x000f0000);
1089         vmcs_write32(GUEST_CS_LIMIT, 0xffff);
1090         vmcs_write32(GUEST_CS_AR_BYTES, 0x9b);
1091
1092         seg_setup(VCPU_SREG_DS);
1093         seg_setup(VCPU_SREG_ES);
1094         seg_setup(VCPU_SREG_FS);
1095         seg_setup(VCPU_SREG_GS);
1096         seg_setup(VCPU_SREG_SS);
1097
1098         vmcs_write16(GUEST_TR_SELECTOR, 0);
1099         vmcs_writel(GUEST_TR_BASE, 0);
1100         vmcs_write32(GUEST_TR_LIMIT, 0xffff);
1101         vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);
1102
1103         vmcs_write16(GUEST_LDTR_SELECTOR, 0);
1104         vmcs_writel(GUEST_LDTR_BASE, 0);
1105         vmcs_write32(GUEST_LDTR_LIMIT, 0xffff);
1106         vmcs_write32(GUEST_LDTR_AR_BYTES, 0x00082);
1107
1108         vmcs_write32(GUEST_SYSENTER_CS, 0);
1109         vmcs_writel(GUEST_SYSENTER_ESP, 0);
1110         vmcs_writel(GUEST_SYSENTER_EIP, 0);
1111
1112         vmcs_writel(GUEST_RFLAGS, 0x02);
1113         vmcs_writel(GUEST_RIP, 0xfff0);
1114         vmcs_writel(GUEST_RSP, 0);
1115
1116         //todo: dr0 = dr1 = dr2 = dr3 = 0; dr6 = 0xffff0ff0
1117         vmcs_writel(GUEST_DR7, 0x400);
1118
1119         vmcs_writel(GUEST_GDTR_BASE, 0);
1120         vmcs_write32(GUEST_GDTR_LIMIT, 0xffff);
1121
1122         vmcs_writel(GUEST_IDTR_BASE, 0);
1123         vmcs_write32(GUEST_IDTR_LIMIT, 0xffff);
1124
1125         vmcs_write32(GUEST_ACTIVITY_STATE, 0);
1126         vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0);
1127         vmcs_write32(GUEST_PENDING_DBG_EXCEPTIONS, 0);
1128
1129         /* I/O */
1130         vmcs_write64(IO_BITMAP_A, 0);
1131         vmcs_write64(IO_BITMAP_B, 0);
1132
1133         guest_write_tsc(0);
1134
1135         vmcs_write64(VMCS_LINK_POINTER, -1ull); /* 22.3.1.5 */
1136
1137         /* Special registers */
1138         vmcs_write64(GUEST_IA32_DEBUGCTL, 0);
1139
1140         /* Control */
1141         vmcs_write32_fixedbits(MSR_IA32_VMX_PINBASED_CTLS,
1142                                PIN_BASED_VM_EXEC_CONTROL,
1143                                PIN_BASED_EXT_INTR_MASK   /* 20.6.1 */
1144                                | PIN_BASED_NMI_EXITING   /* 20.6.1 */
1145                         );
1146         vmcs_write32_fixedbits(MSR_IA32_VMX_PROCBASED_CTLS,
1147                                CPU_BASED_VM_EXEC_CONTROL,
1148                                CPU_BASED_HLT_EXITING         /* 20.6.2 */
1149                                | CPU_BASED_CR8_LOAD_EXITING    /* 20.6.2 */
1150                                | CPU_BASED_CR8_STORE_EXITING   /* 20.6.2 */
1151                                | CPU_BASED_UNCOND_IO_EXITING   /* 20.6.2 */
1152                                | CPU_BASED_MOV_DR_EXITING
1153                                | CPU_BASED_USE_TSC_OFFSETING   /* 21.3 */
1154                         );
1155
1156         vmcs_write32(EXCEPTION_BITMAP, 1 << PF_VECTOR);
1157         vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, 0);
1158         vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, 0);
1159         vmcs_write32(CR3_TARGET_COUNT, 0);           /* 22.2.1 */
1160
1161         vmcs_writel(HOST_CR0, read_cr0());  /* 22.2.3 */
1162         vmcs_writel(HOST_CR4, read_cr4());  /* 22.2.3, 22.2.5 */
1163         vmcs_writel(HOST_CR3, read_cr3());  /* 22.2.3  FIXME: shadow tables */
1164
1165         vmcs_write16(HOST_CS_SELECTOR, __KERNEL_CS);  /* 22.2.4 */
1166         vmcs_write16(HOST_DS_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
1167         vmcs_write16(HOST_ES_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
1168         vmcs_write16(HOST_FS_SELECTOR, read_fs());    /* 22.2.4 */
1169         vmcs_write16(HOST_GS_SELECTOR, read_gs());    /* 22.2.4 */
1170         vmcs_write16(HOST_SS_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
1171 #ifdef CONFIG_X86_64
1172         rdmsrl(MSR_FS_BASE, a);
1173         vmcs_writel(HOST_FS_BASE, a); /* 22.2.4 */
1174         rdmsrl(MSR_GS_BASE, a);
1175         vmcs_writel(HOST_GS_BASE, a); /* 22.2.4 */
1176 #else
1177         vmcs_writel(HOST_FS_BASE, 0); /* 22.2.4 */
1178         vmcs_writel(HOST_GS_BASE, 0); /* 22.2.4 */
1179 #endif
1180
1181         vmcs_write16(HOST_TR_SELECTOR, GDT_ENTRY_TSS*8);  /* 22.2.4 */
1182
1183         get_idt(&dt);
1184         vmcs_writel(HOST_IDTR_BASE, dt.base);   /* 22.2.4 */
1185
1186
1187         vmcs_writel(HOST_RIP, (unsigned long)kvm_vmx_return); /* 22.2.5 */
1188
1189         rdmsr(MSR_IA32_SYSENTER_CS, host_sysenter_cs, junk);
1190         vmcs_write32(HOST_IA32_SYSENTER_CS, host_sysenter_cs);
1191         rdmsrl(MSR_IA32_SYSENTER_ESP, a);
1192         vmcs_writel(HOST_IA32_SYSENTER_ESP, a);   /* 22.2.3 */
1193         rdmsrl(MSR_IA32_SYSENTER_EIP, a);
1194         vmcs_writel(HOST_IA32_SYSENTER_EIP, a);   /* 22.2.3 */
1195
1196         for (i = 0; i < NR_VMX_MSR; ++i) {
1197                 u32 index = vmx_msr_index[i];
1198                 u32 data_low, data_high;
1199                 u64 data;
1200                 int j = vcpu->nmsrs;
1201
1202                 if (rdmsr_safe(index, &data_low, &data_high) < 0)
1203                         continue;
1204                 if (wrmsr_safe(index, data_low, data_high) < 0)
1205                         continue;
1206                 data = data_low | ((u64)data_high << 32);
1207                 vcpu->host_msrs[j].index = index;
1208                 vcpu->host_msrs[j].reserved = 0;
1209                 vcpu->host_msrs[j].data = data;
1210                 vcpu->guest_msrs[j] = vcpu->host_msrs[j];
1211 #ifdef CONFIG_X86_64
1212                 if (index == MSR_KERNEL_GS_BASE)
1213                         msr_offset_kernel_gs_base = j;
1214 #endif
1215                 ++vcpu->nmsrs;
1216         }
1217
1218         setup_msrs(vcpu);
1219
1220         vmcs_write32_fixedbits(MSR_IA32_VMX_EXIT_CTLS, VM_EXIT_CONTROLS,
1221                                (HOST_IS_64 << 9));  /* 22.2,1, 20.7.1 */
1222
1223         /* 22.2.1, 20.8.1 */
1224         vmcs_write32_fixedbits(MSR_IA32_VMX_ENTRY_CTLS,
1225                                VM_ENTRY_CONTROLS, 0);
1226         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0);  /* 22.2.1 */
1227
1228 #ifdef CONFIG_X86_64
1229         vmcs_writel(VIRTUAL_APIC_PAGE_ADDR, 0);
1230         vmcs_writel(TPR_THRESHOLD, 0);
1231 #endif
1232
1233         vmcs_writel(CR0_GUEST_HOST_MASK, ~0UL);
1234         vmcs_writel(CR4_GUEST_HOST_MASK, KVM_GUEST_CR4_MASK);
1235
1236         vcpu->cr0 = 0x60000010;
1237         vmx_set_cr0(vcpu, vcpu->cr0); // enter rmode
1238         vmx_set_cr4(vcpu, 0);
1239 #ifdef CONFIG_X86_64
1240         vmx_set_efer(vcpu, 0);
1241 #endif
1242
1243         return 0;
1244
1245 out:
1246         return ret;
1247 }
1248
1249 static void inject_rmode_irq(struct kvm_vcpu *vcpu, int irq)
1250 {
1251         u16 ent[2];
1252         u16 cs;
1253         u16 ip;
1254         unsigned long flags;
1255         unsigned long ss_base = vmcs_readl(GUEST_SS_BASE);
1256         u16 sp =  vmcs_readl(GUEST_RSP);
1257         u32 ss_limit = vmcs_read32(GUEST_SS_LIMIT);
1258
1259         if (sp > ss_limit || sp < 6 ) {
1260                 vcpu_printf(vcpu, "%s: #SS, rsp 0x%lx ss 0x%lx limit 0x%x\n",
1261                             __FUNCTION__,
1262                             vmcs_readl(GUEST_RSP),
1263                             vmcs_readl(GUEST_SS_BASE),
1264                             vmcs_read32(GUEST_SS_LIMIT));
1265                 return;
1266         }
1267
1268         if (kvm_read_guest(vcpu, irq * sizeof(ent), sizeof(ent), &ent) !=
1269                                                                 sizeof(ent)) {
1270                 vcpu_printf(vcpu, "%s: read guest err\n", __FUNCTION__);
1271                 return;
1272         }
1273
1274         flags =  vmcs_readl(GUEST_RFLAGS);
1275         cs =  vmcs_readl(GUEST_CS_BASE) >> 4;
1276         ip =  vmcs_readl(GUEST_RIP);
1277
1278
1279         if (kvm_write_guest(vcpu, ss_base + sp - 2, 2, &flags) != 2 ||
1280             kvm_write_guest(vcpu, ss_base + sp - 4, 2, &cs) != 2 ||
1281             kvm_write_guest(vcpu, ss_base + sp - 6, 2, &ip) != 2) {
1282                 vcpu_printf(vcpu, "%s: write guest err\n", __FUNCTION__);
1283                 return;
1284         }
1285
1286         vmcs_writel(GUEST_RFLAGS, flags &
1287                     ~( X86_EFLAGS_IF | X86_EFLAGS_AC | X86_EFLAGS_TF));
1288         vmcs_write16(GUEST_CS_SELECTOR, ent[1]) ;
1289         vmcs_writel(GUEST_CS_BASE, ent[1] << 4);
1290         vmcs_writel(GUEST_RIP, ent[0]);
1291         vmcs_writel(GUEST_RSP, (vmcs_readl(GUEST_RSP) & ~0xffff) | (sp - 6));
1292 }
1293
1294 static void kvm_do_inject_irq(struct kvm_vcpu *vcpu)
1295 {
1296         int word_index = __ffs(vcpu->irq_summary);
1297         int bit_index = __ffs(vcpu->irq_pending[word_index]);
1298         int irq = word_index * BITS_PER_LONG + bit_index;
1299
1300         clear_bit(bit_index, &vcpu->irq_pending[word_index]);
1301         if (!vcpu->irq_pending[word_index])
1302                 clear_bit(word_index, &vcpu->irq_summary);
1303
1304         if (vcpu->rmode.active) {
1305                 inject_rmode_irq(vcpu, irq);
1306                 return;
1307         }
1308         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
1309                         irq | INTR_TYPE_EXT_INTR | INTR_INFO_VALID_MASK);
1310 }
1311
1312
1313 static void do_interrupt_requests(struct kvm_vcpu *vcpu,
1314                                        struct kvm_run *kvm_run)
1315 {
1316         u32 cpu_based_vm_exec_control;
1317
1318         vcpu->interrupt_window_open =
1319                 ((vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_IF) &&
1320                  (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 3) == 0);
1321
1322         if (vcpu->interrupt_window_open &&
1323             vcpu->irq_summary &&
1324             !(vmcs_read32(VM_ENTRY_INTR_INFO_FIELD) & INTR_INFO_VALID_MASK))
1325                 /*
1326                  * If interrupts enabled, and not blocked by sti or mov ss. Good.
1327                  */
1328                 kvm_do_inject_irq(vcpu);
1329
1330         cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
1331         if (!vcpu->interrupt_window_open &&
1332             (vcpu->irq_summary || kvm_run->request_interrupt_window))
1333                 /*
1334                  * Interrupts blocked.  Wait for unblock.
1335                  */
1336                 cpu_based_vm_exec_control |= CPU_BASED_VIRTUAL_INTR_PENDING;
1337         else
1338                 cpu_based_vm_exec_control &= ~CPU_BASED_VIRTUAL_INTR_PENDING;
1339         vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
1340 }
1341
1342 static void kvm_guest_debug_pre(struct kvm_vcpu *vcpu)
1343 {
1344         struct kvm_guest_debug *dbg = &vcpu->guest_debug;
1345
1346         set_debugreg(dbg->bp[0], 0);
1347         set_debugreg(dbg->bp[1], 1);
1348         set_debugreg(dbg->bp[2], 2);
1349         set_debugreg(dbg->bp[3], 3);
1350
1351         if (dbg->singlestep) {
1352                 unsigned long flags;
1353
1354                 flags = vmcs_readl(GUEST_RFLAGS);
1355                 flags |= X86_EFLAGS_TF | X86_EFLAGS_RF;
1356                 vmcs_writel(GUEST_RFLAGS, flags);
1357         }
1358 }
1359
1360 static int handle_rmode_exception(struct kvm_vcpu *vcpu,
1361                                   int vec, u32 err_code)
1362 {
1363         if (!vcpu->rmode.active)
1364                 return 0;
1365
1366         if (vec == GP_VECTOR && err_code == 0)
1367                 if (emulate_instruction(vcpu, NULL, 0, 0) == EMULATE_DONE)
1368                         return 1;
1369         return 0;
1370 }
1371
1372 static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1373 {
1374         u32 intr_info, error_code;
1375         unsigned long cr2, rip;
1376         u32 vect_info;
1377         enum emulation_result er;
1378         int r;
1379
1380         vect_info = vmcs_read32(IDT_VECTORING_INFO_FIELD);
1381         intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
1382
1383         if ((vect_info & VECTORING_INFO_VALID_MASK) &&
1384                                                 !is_page_fault(intr_info)) {
1385                 printk(KERN_ERR "%s: unexpected, vectoring info 0x%x "
1386                        "intr info 0x%x\n", __FUNCTION__, vect_info, intr_info);
1387         }
1388
1389         if (is_external_interrupt(vect_info)) {
1390                 int irq = vect_info & VECTORING_INFO_VECTOR_MASK;
1391                 set_bit(irq, vcpu->irq_pending);
1392                 set_bit(irq / BITS_PER_LONG, &vcpu->irq_summary);
1393         }
1394
1395         if ((intr_info & INTR_INFO_INTR_TYPE_MASK) == 0x200) { /* nmi */
1396                 asm ("int $2");
1397                 return 1;
1398         }
1399
1400         if (is_no_device(intr_info)) {
1401                 vcpu->fpu_active = 1;
1402                 vmcs_clear_bits(EXCEPTION_BITMAP, 1 << NM_VECTOR);
1403                 if (!(vcpu->cr0 & CR0_TS_MASK))
1404                         vmcs_clear_bits(GUEST_CR0, CR0_TS_MASK);
1405                 return 1;
1406         }
1407
1408         error_code = 0;
1409         rip = vmcs_readl(GUEST_RIP);
1410         if (intr_info & INTR_INFO_DELIEVER_CODE_MASK)
1411                 error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE);
1412         if (is_page_fault(intr_info)) {
1413                 cr2 = vmcs_readl(EXIT_QUALIFICATION);
1414
1415                 spin_lock(&vcpu->kvm->lock);
1416                 r = kvm_mmu_page_fault(vcpu, cr2, error_code);
1417                 if (r < 0) {
1418                         spin_unlock(&vcpu->kvm->lock);
1419                         return r;
1420                 }
1421                 if (!r) {
1422                         spin_unlock(&vcpu->kvm->lock);
1423                         return 1;
1424                 }
1425
1426                 er = emulate_instruction(vcpu, kvm_run, cr2, error_code);
1427                 spin_unlock(&vcpu->kvm->lock);
1428
1429                 switch (er) {
1430                 case EMULATE_DONE:
1431                         return 1;
1432                 case EMULATE_DO_MMIO:
1433                         ++vcpu->stat.mmio_exits;
1434                         kvm_run->exit_reason = KVM_EXIT_MMIO;
1435                         return 0;
1436                  case EMULATE_FAIL:
1437                         vcpu_printf(vcpu, "%s: emulate fail\n", __FUNCTION__);
1438                         break;
1439                 default:
1440                         BUG();
1441                 }
1442         }
1443
1444         if (vcpu->rmode.active &&
1445             handle_rmode_exception(vcpu, intr_info & INTR_INFO_VECTOR_MASK,
1446                                                                 error_code))
1447                 return 1;
1448
1449         if ((intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK)) == (INTR_TYPE_EXCEPTION | 1)) {
1450                 kvm_run->exit_reason = KVM_EXIT_DEBUG;
1451                 return 0;
1452         }
1453         kvm_run->exit_reason = KVM_EXIT_EXCEPTION;
1454         kvm_run->ex.exception = intr_info & INTR_INFO_VECTOR_MASK;
1455         kvm_run->ex.error_code = error_code;
1456         return 0;
1457 }
1458
1459 static int handle_external_interrupt(struct kvm_vcpu *vcpu,
1460                                      struct kvm_run *kvm_run)
1461 {
1462         ++vcpu->stat.irq_exits;
1463         return 1;
1464 }
1465
1466 static int handle_triple_fault(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1467 {
1468         kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
1469         return 0;
1470 }
1471
1472 static int get_io_count(struct kvm_vcpu *vcpu, unsigned long *count)
1473 {
1474         u64 inst;
1475         gva_t rip;
1476         int countr_size;
1477         int i, n;
1478
1479         if ((vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_VM)) {
1480                 countr_size = 2;
1481         } else {
1482                 u32 cs_ar = vmcs_read32(GUEST_CS_AR_BYTES);
1483
1484                 countr_size = (cs_ar & AR_L_MASK) ? 8:
1485                               (cs_ar & AR_DB_MASK) ? 4: 2;
1486         }
1487
1488         rip =  vmcs_readl(GUEST_RIP);
1489         if (countr_size != 8)
1490                 rip += vmcs_readl(GUEST_CS_BASE);
1491
1492         n = kvm_read_guest(vcpu, rip, sizeof(inst), &inst);
1493
1494         for (i = 0; i < n; i++) {
1495                 switch (((u8*)&inst)[i]) {
1496                 case 0xf0:
1497                 case 0xf2:
1498                 case 0xf3:
1499                 case 0x2e:
1500                 case 0x36:
1501                 case 0x3e:
1502                 case 0x26:
1503                 case 0x64:
1504                 case 0x65:
1505                 case 0x66:
1506                         break;
1507                 case 0x67:
1508                         countr_size = (countr_size == 2) ? 4: (countr_size >> 1);
1509                 default:
1510                         goto done;
1511                 }
1512         }
1513         return 0;
1514 done:
1515         countr_size *= 8;
1516         *count = vcpu->regs[VCPU_REGS_RCX] & (~0ULL >> (64 - countr_size));
1517         //printk("cx: %lx\n", vcpu->regs[VCPU_REGS_RCX]);
1518         return 1;
1519 }
1520
1521 static int handle_io(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1522 {
1523         u64 exit_qualification;
1524         int size, down, in, string, rep;
1525         unsigned port;
1526         unsigned long count;
1527         gva_t address;
1528
1529         ++vcpu->stat.io_exits;
1530         exit_qualification = vmcs_read64(EXIT_QUALIFICATION);
1531         in = (exit_qualification & 8) != 0;
1532         size = (exit_qualification & 7) + 1;
1533         string = (exit_qualification & 16) != 0;
1534         down = (vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_DF) != 0;
1535         count = 1;
1536         rep = (exit_qualification & 32) != 0;
1537         port = exit_qualification >> 16;
1538         address = 0;
1539         if (string) {
1540                 if (rep && !get_io_count(vcpu, &count))
1541                         return 1;
1542                 address = vmcs_readl(GUEST_LINEAR_ADDRESS);
1543         }
1544         return kvm_setup_pio(vcpu, kvm_run, in, size, count, string, down,
1545                              address, rep, port);
1546 }
1547
1548 static void
1549 vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
1550 {
1551         /*
1552          * Patch in the VMCALL instruction:
1553          */
1554         hypercall[0] = 0x0f;
1555         hypercall[1] = 0x01;
1556         hypercall[2] = 0xc1;
1557         hypercall[3] = 0xc3;
1558 }
1559
1560 static int handle_cr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1561 {
1562         u64 exit_qualification;
1563         int cr;
1564         int reg;
1565
1566         exit_qualification = vmcs_read64(EXIT_QUALIFICATION);
1567         cr = exit_qualification & 15;
1568         reg = (exit_qualification >> 8) & 15;
1569         switch ((exit_qualification >> 4) & 3) {
1570         case 0: /* mov to cr */
1571                 switch (cr) {
1572                 case 0:
1573                         vcpu_load_rsp_rip(vcpu);
1574                         set_cr0(vcpu, vcpu->regs[reg]);
1575                         skip_emulated_instruction(vcpu);
1576                         return 1;
1577                 case 3:
1578                         vcpu_load_rsp_rip(vcpu);
1579                         set_cr3(vcpu, vcpu->regs[reg]);
1580                         skip_emulated_instruction(vcpu);
1581                         return 1;
1582                 case 4:
1583                         vcpu_load_rsp_rip(vcpu);
1584                         set_cr4(vcpu, vcpu->regs[reg]);
1585                         skip_emulated_instruction(vcpu);
1586                         return 1;
1587                 case 8:
1588                         vcpu_load_rsp_rip(vcpu);
1589                         set_cr8(vcpu, vcpu->regs[reg]);
1590                         skip_emulated_instruction(vcpu);
1591                         return 1;
1592                 };
1593                 break;
1594         case 2: /* clts */
1595                 vcpu_load_rsp_rip(vcpu);
1596                 vcpu->fpu_active = 1;
1597                 vmcs_clear_bits(EXCEPTION_BITMAP, 1 << NM_VECTOR);
1598                 vmcs_clear_bits(GUEST_CR0, CR0_TS_MASK);
1599                 vcpu->cr0 &= ~CR0_TS_MASK;
1600                 vmcs_writel(CR0_READ_SHADOW, vcpu->cr0);
1601                 skip_emulated_instruction(vcpu);
1602                 return 1;
1603         case 1: /*mov from cr*/
1604                 switch (cr) {
1605                 case 3:
1606                         vcpu_load_rsp_rip(vcpu);
1607                         vcpu->regs[reg] = vcpu->cr3;
1608                         vcpu_put_rsp_rip(vcpu);
1609                         skip_emulated_instruction(vcpu);
1610                         return 1;
1611                 case 8:
1612                         vcpu_load_rsp_rip(vcpu);
1613                         vcpu->regs[reg] = vcpu->cr8;
1614                         vcpu_put_rsp_rip(vcpu);
1615                         skip_emulated_instruction(vcpu);
1616                         return 1;
1617                 }
1618                 break;
1619         case 3: /* lmsw */
1620                 lmsw(vcpu, (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f);
1621
1622                 skip_emulated_instruction(vcpu);
1623                 return 1;
1624         default:
1625                 break;
1626         }
1627         kvm_run->exit_reason = 0;
1628         printk(KERN_ERR "kvm: unhandled control register: op %d cr %d\n",
1629                (int)(exit_qualification >> 4) & 3, cr);
1630         return 0;
1631 }
1632
1633 static int handle_dr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1634 {
1635         u64 exit_qualification;
1636         unsigned long val;
1637         int dr, reg;
1638
1639         /*
1640          * FIXME: this code assumes the host is debugging the guest.
1641          *        need to deal with guest debugging itself too.
1642          */
1643         exit_qualification = vmcs_read64(EXIT_QUALIFICATION);
1644         dr = exit_qualification & 7;
1645         reg = (exit_qualification >> 8) & 15;
1646         vcpu_load_rsp_rip(vcpu);
1647         if (exit_qualification & 16) {
1648                 /* mov from dr */
1649                 switch (dr) {
1650                 case 6:
1651                         val = 0xffff0ff0;
1652                         break;
1653                 case 7:
1654                         val = 0x400;
1655                         break;
1656                 default:
1657                         val = 0;
1658                 }
1659                 vcpu->regs[reg] = val;
1660         } else {
1661                 /* mov to dr */
1662         }
1663         vcpu_put_rsp_rip(vcpu);
1664         skip_emulated_instruction(vcpu);
1665         return 1;
1666 }
1667
1668 static int handle_cpuid(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1669 {
1670         kvm_emulate_cpuid(vcpu);
1671         return 1;
1672 }
1673
1674 static int handle_rdmsr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1675 {
1676         u32 ecx = vcpu->regs[VCPU_REGS_RCX];
1677         u64 data;
1678
1679         if (vmx_get_msr(vcpu, ecx, &data)) {
1680                 vmx_inject_gp(vcpu, 0);
1681                 return 1;
1682         }
1683
1684         /* FIXME: handling of bits 32:63 of rax, rdx */
1685         vcpu->regs[VCPU_REGS_RAX] = data & -1u;
1686         vcpu->regs[VCPU_REGS_RDX] = (data >> 32) & -1u;
1687         skip_emulated_instruction(vcpu);
1688         return 1;
1689 }
1690
1691 static int handle_wrmsr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1692 {
1693         u32 ecx = vcpu->regs[VCPU_REGS_RCX];
1694         u64 data = (vcpu->regs[VCPU_REGS_RAX] & -1u)
1695                 | ((u64)(vcpu->regs[VCPU_REGS_RDX] & -1u) << 32);
1696
1697         if (vmx_set_msr(vcpu, ecx, data) != 0) {
1698                 vmx_inject_gp(vcpu, 0);
1699                 return 1;
1700         }
1701
1702         skip_emulated_instruction(vcpu);
1703         return 1;
1704 }
1705
1706 static void post_kvm_run_save(struct kvm_vcpu *vcpu,
1707                               struct kvm_run *kvm_run)
1708 {
1709         kvm_run->if_flag = (vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_IF) != 0;
1710         kvm_run->cr8 = vcpu->cr8;
1711         kvm_run->apic_base = vcpu->apic_base;
1712         kvm_run->ready_for_interrupt_injection = (vcpu->interrupt_window_open &&
1713                                                   vcpu->irq_summary == 0);
1714 }
1715
1716 static int handle_interrupt_window(struct kvm_vcpu *vcpu,
1717                                    struct kvm_run *kvm_run)
1718 {
1719         /*
1720          * If the user space waits to inject interrupts, exit as soon as
1721          * possible
1722          */
1723         if (kvm_run->request_interrupt_window &&
1724             !vcpu->irq_summary) {
1725                 kvm_run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
1726                 ++vcpu->stat.irq_window_exits;
1727                 return 0;
1728         }
1729         return 1;
1730 }
1731
1732 static int handle_halt(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1733 {
1734         skip_emulated_instruction(vcpu);
1735         if (vcpu->irq_summary)
1736                 return 1;
1737
1738         kvm_run->exit_reason = KVM_EXIT_HLT;
1739         ++vcpu->stat.halt_exits;
1740         return 0;
1741 }
1742
1743 static int handle_vmcall(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1744 {
1745         skip_emulated_instruction(vcpu);
1746         return kvm_hypercall(vcpu, kvm_run);
1747 }
1748
1749 /*
1750  * The exit handlers return 1 if the exit was handled fully and guest execution
1751  * may resume.  Otherwise they set the kvm_run parameter to indicate what needs
1752  * to be done to userspace and return 0.
1753  */
1754 static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu,
1755                                       struct kvm_run *kvm_run) = {
1756         [EXIT_REASON_EXCEPTION_NMI]           = handle_exception,
1757         [EXIT_REASON_EXTERNAL_INTERRUPT]      = handle_external_interrupt,
1758         [EXIT_REASON_TRIPLE_FAULT]            = handle_triple_fault,
1759         [EXIT_REASON_IO_INSTRUCTION]          = handle_io,
1760         [EXIT_REASON_CR_ACCESS]               = handle_cr,
1761         [EXIT_REASON_DR_ACCESS]               = handle_dr,
1762         [EXIT_REASON_CPUID]                   = handle_cpuid,
1763         [EXIT_REASON_MSR_READ]                = handle_rdmsr,
1764         [EXIT_REASON_MSR_WRITE]               = handle_wrmsr,
1765         [EXIT_REASON_PENDING_INTERRUPT]       = handle_interrupt_window,
1766         [EXIT_REASON_HLT]                     = handle_halt,
1767         [EXIT_REASON_VMCALL]                  = handle_vmcall,
1768 };
1769
1770 static const int kvm_vmx_max_exit_handlers =
1771         sizeof(kvm_vmx_exit_handlers) / sizeof(*kvm_vmx_exit_handlers);
1772
1773 /*
1774  * The guest has exited.  See if we can fix it or if we need userspace
1775  * assistance.
1776  */
1777 static int kvm_handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
1778 {
1779         u32 vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD);
1780         u32 exit_reason = vmcs_read32(VM_EXIT_REASON);
1781
1782         if ( (vectoring_info & VECTORING_INFO_VALID_MASK) &&
1783                                 exit_reason != EXIT_REASON_EXCEPTION_NMI )
1784                 printk(KERN_WARNING "%s: unexpected, valid vectoring info and "
1785                        "exit reason is 0x%x\n", __FUNCTION__, exit_reason);
1786         if (exit_reason < kvm_vmx_max_exit_handlers
1787             && kvm_vmx_exit_handlers[exit_reason])
1788                 return kvm_vmx_exit_handlers[exit_reason](vcpu, kvm_run);
1789         else {
1790                 kvm_run->exit_reason = KVM_EXIT_UNKNOWN;
1791                 kvm_run->hw.hardware_exit_reason = exit_reason;
1792         }
1793         return 0;
1794 }
1795
1796 /*
1797  * Check if userspace requested an interrupt window, and that the
1798  * interrupt window is open.
1799  *
1800  * No need to exit to userspace if we already have an interrupt queued.
1801  */
1802 static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu,
1803                                           struct kvm_run *kvm_run)
1804 {
1805         return (!vcpu->irq_summary &&
1806                 kvm_run->request_interrupt_window &&
1807                 vcpu->interrupt_window_open &&
1808                 (vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_IF));
1809 }
1810
1811 static int vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1812 {
1813         u8 fail;
1814         u16 fs_sel, gs_sel, ldt_sel;
1815         int fs_gs_ldt_reload_needed;
1816         int r;
1817
1818 again:
1819         /*
1820          * Set host fs and gs selectors.  Unfortunately, 22.2.3 does not
1821          * allow segment selectors with cpl > 0 or ti == 1.
1822          */
1823         fs_sel = read_fs();
1824         gs_sel = read_gs();
1825         ldt_sel = read_ldt();
1826         fs_gs_ldt_reload_needed = (fs_sel & 7) | (gs_sel & 7) | ldt_sel;
1827         if (!fs_gs_ldt_reload_needed) {
1828                 vmcs_write16(HOST_FS_SELECTOR, fs_sel);
1829                 vmcs_write16(HOST_GS_SELECTOR, gs_sel);
1830         } else {
1831                 vmcs_write16(HOST_FS_SELECTOR, 0);
1832                 vmcs_write16(HOST_GS_SELECTOR, 0);
1833         }
1834
1835 #ifdef CONFIG_X86_64
1836         vmcs_writel(HOST_FS_BASE, read_msr(MSR_FS_BASE));
1837         vmcs_writel(HOST_GS_BASE, read_msr(MSR_GS_BASE));
1838 #else
1839         vmcs_writel(HOST_FS_BASE, segment_base(fs_sel));
1840         vmcs_writel(HOST_GS_BASE, segment_base(gs_sel));
1841 #endif
1842
1843         if (!vcpu->mmio_read_completed)
1844                 do_interrupt_requests(vcpu, kvm_run);
1845
1846         if (vcpu->guest_debug.enabled)
1847                 kvm_guest_debug_pre(vcpu);
1848
1849         if (vcpu->fpu_active) {
1850                 fx_save(vcpu->host_fx_image);
1851                 fx_restore(vcpu->guest_fx_image);
1852         }
1853         /*
1854          * Loading guest fpu may have cleared host cr0.ts
1855          */
1856         vmcs_writel(HOST_CR0, read_cr0());
1857
1858 #ifdef CONFIG_X86_64
1859         if (is_long_mode(vcpu)) {
1860                 save_msrs(vcpu->host_msrs + msr_offset_kernel_gs_base, 1);
1861                 load_msrs(vcpu->guest_msrs, NR_BAD_MSRS);
1862         }
1863 #endif
1864
1865         asm (
1866                 /* Store host registers */
1867                 "pushf \n\t"
1868 #ifdef CONFIG_X86_64
1869                 "push %%rax; push %%rbx; push %%rdx;"
1870                 "push %%rsi; push %%rdi; push %%rbp;"
1871                 "push %%r8;  push %%r9;  push %%r10; push %%r11;"
1872                 "push %%r12; push %%r13; push %%r14; push %%r15;"
1873                 "push %%rcx \n\t"
1874                 ASM_VMX_VMWRITE_RSP_RDX "\n\t"
1875 #else
1876                 "pusha; push %%ecx \n\t"
1877                 ASM_VMX_VMWRITE_RSP_RDX "\n\t"
1878 #endif
1879                 /* Check if vmlaunch of vmresume is needed */
1880                 "cmp $0, %1 \n\t"
1881                 /* Load guest registers.  Don't clobber flags. */
1882 #ifdef CONFIG_X86_64
1883                 "mov %c[cr2](%3), %%rax \n\t"
1884                 "mov %%rax, %%cr2 \n\t"
1885                 "mov %c[rax](%3), %%rax \n\t"
1886                 "mov %c[rbx](%3), %%rbx \n\t"
1887                 "mov %c[rdx](%3), %%rdx \n\t"
1888                 "mov %c[rsi](%3), %%rsi \n\t"
1889                 "mov %c[rdi](%3), %%rdi \n\t"
1890                 "mov %c[rbp](%3), %%rbp \n\t"
1891                 "mov %c[r8](%3),  %%r8  \n\t"
1892                 "mov %c[r9](%3),  %%r9  \n\t"
1893                 "mov %c[r10](%3), %%r10 \n\t"
1894                 "mov %c[r11](%3), %%r11 \n\t"
1895                 "mov %c[r12](%3), %%r12 \n\t"
1896                 "mov %c[r13](%3), %%r13 \n\t"
1897                 "mov %c[r14](%3), %%r14 \n\t"
1898                 "mov %c[r15](%3), %%r15 \n\t"
1899                 "mov %c[rcx](%3), %%rcx \n\t" /* kills %3 (rcx) */
1900 #else
1901                 "mov %c[cr2](%3), %%eax \n\t"
1902                 "mov %%eax,   %%cr2 \n\t"
1903                 "mov %c[rax](%3), %%eax \n\t"
1904                 "mov %c[rbx](%3), %%ebx \n\t"
1905                 "mov %c[rdx](%3), %%edx \n\t"
1906                 "mov %c[rsi](%3), %%esi \n\t"
1907                 "mov %c[rdi](%3), %%edi \n\t"
1908                 "mov %c[rbp](%3), %%ebp \n\t"
1909                 "mov %c[rcx](%3), %%ecx \n\t" /* kills %3 (ecx) */
1910 #endif
1911                 /* Enter guest mode */
1912                 "jne launched \n\t"
1913                 ASM_VMX_VMLAUNCH "\n\t"
1914                 "jmp kvm_vmx_return \n\t"
1915                 "launched: " ASM_VMX_VMRESUME "\n\t"
1916                 ".globl kvm_vmx_return \n\t"
1917                 "kvm_vmx_return: "
1918                 /* Save guest registers, load host registers, keep flags */
1919 #ifdef CONFIG_X86_64
1920                 "xchg %3,     (%%rsp) \n\t"
1921                 "mov %%rax, %c[rax](%3) \n\t"
1922                 "mov %%rbx, %c[rbx](%3) \n\t"
1923                 "pushq (%%rsp); popq %c[rcx](%3) \n\t"
1924                 "mov %%rdx, %c[rdx](%3) \n\t"
1925                 "mov %%rsi, %c[rsi](%3) \n\t"
1926                 "mov %%rdi, %c[rdi](%3) \n\t"
1927                 "mov %%rbp, %c[rbp](%3) \n\t"
1928                 "mov %%r8,  %c[r8](%3) \n\t"
1929                 "mov %%r9,  %c[r9](%3) \n\t"
1930                 "mov %%r10, %c[r10](%3) \n\t"
1931                 "mov %%r11, %c[r11](%3) \n\t"
1932                 "mov %%r12, %c[r12](%3) \n\t"
1933                 "mov %%r13, %c[r13](%3) \n\t"
1934                 "mov %%r14, %c[r14](%3) \n\t"
1935                 "mov %%r15, %c[r15](%3) \n\t"
1936                 "mov %%cr2, %%rax   \n\t"
1937                 "mov %%rax, %c[cr2](%3) \n\t"
1938                 "mov (%%rsp), %3 \n\t"
1939
1940                 "pop  %%rcx; pop  %%r15; pop  %%r14; pop  %%r13; pop  %%r12;"
1941                 "pop  %%r11; pop  %%r10; pop  %%r9;  pop  %%r8;"
1942                 "pop  %%rbp; pop  %%rdi; pop  %%rsi;"
1943                 "pop  %%rdx; pop  %%rbx; pop  %%rax \n\t"
1944 #else
1945                 "xchg %3, (%%esp) \n\t"
1946                 "mov %%eax, %c[rax](%3) \n\t"
1947                 "mov %%ebx, %c[rbx](%3) \n\t"
1948                 "pushl (%%esp); popl %c[rcx](%3) \n\t"
1949                 "mov %%edx, %c[rdx](%3) \n\t"
1950                 "mov %%esi, %c[rsi](%3) \n\t"
1951                 "mov %%edi, %c[rdi](%3) \n\t"
1952                 "mov %%ebp, %c[rbp](%3) \n\t"
1953                 "mov %%cr2, %%eax  \n\t"
1954                 "mov %%eax, %c[cr2](%3) \n\t"
1955                 "mov (%%esp), %3 \n\t"
1956
1957                 "pop %%ecx; popa \n\t"
1958 #endif
1959                 "setbe %0 \n\t"
1960                 "popf \n\t"
1961               : "=q" (fail)
1962               : "r"(vcpu->launched), "d"((unsigned long)HOST_RSP),
1963                 "c"(vcpu),
1964                 [rax]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RAX])),
1965                 [rbx]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RBX])),
1966                 [rcx]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RCX])),
1967                 [rdx]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RDX])),
1968                 [rsi]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RSI])),
1969                 [rdi]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RDI])),
1970                 [rbp]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RBP])),
1971 #ifdef CONFIG_X86_64
1972                 [r8 ]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R8 ])),
1973                 [r9 ]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R9 ])),
1974                 [r10]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R10])),
1975                 [r11]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R11])),
1976                 [r12]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R12])),
1977                 [r13]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R13])),
1978                 [r14]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R14])),
1979                 [r15]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R15])),
1980 #endif
1981                 [cr2]"i"(offsetof(struct kvm_vcpu, cr2))
1982               : "cc", "memory" );
1983
1984         /*
1985          * Reload segment selectors ASAP. (it's needed for a functional
1986          * kernel: x86 relies on having __KERNEL_PDA in %fs and x86_64
1987          * relies on having 0 in %gs for the CPU PDA to work.)
1988          */
1989         if (fs_gs_ldt_reload_needed) {
1990                 load_ldt(ldt_sel);
1991                 load_fs(fs_sel);
1992                 /*
1993                  * If we have to reload gs, we must take care to
1994                  * preserve our gs base.
1995                  */
1996                 local_irq_disable();
1997                 load_gs(gs_sel);
1998 #ifdef CONFIG_X86_64
1999                 wrmsrl(MSR_GS_BASE, vmcs_readl(HOST_GS_BASE));
2000 #endif
2001                 local_irq_enable();
2002
2003                 reload_tss();
2004         }
2005         ++vcpu->stat.exits;
2006
2007 #ifdef CONFIG_X86_64
2008         if (is_long_mode(vcpu)) {
2009                 save_msrs(vcpu->guest_msrs, NR_BAD_MSRS);
2010                 load_msrs(vcpu->host_msrs, NR_BAD_MSRS);
2011         }
2012 #endif
2013
2014         if (vcpu->fpu_active) {
2015                 fx_save(vcpu->guest_fx_image);
2016                 fx_restore(vcpu->host_fx_image);
2017         }
2018
2019         vcpu->interrupt_window_open = (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 3) == 0;
2020
2021         asm ("mov %0, %%ds; mov %0, %%es" : : "r"(__USER_DS));
2022
2023         if (fail) {
2024                 kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
2025                 kvm_run->fail_entry.hardware_entry_failure_reason
2026                         = vmcs_read32(VM_INSTRUCTION_ERROR);
2027                 r = 0;
2028         } else {
2029                 /*
2030                  * Profile KVM exit RIPs:
2031                  */
2032                 if (unlikely(prof_on == KVM_PROFILING))
2033                         profile_hit(KVM_PROFILING, (void *)vmcs_readl(GUEST_RIP));
2034
2035                 vcpu->launched = 1;
2036                 r = kvm_handle_exit(kvm_run, vcpu);
2037                 if (r > 0) {
2038                         /* Give scheduler a change to reschedule. */
2039                         if (signal_pending(current)) {
2040                                 ++vcpu->stat.signal_exits;
2041                                 post_kvm_run_save(vcpu, kvm_run);
2042                                 kvm_run->exit_reason = KVM_EXIT_INTR;
2043                                 return -EINTR;
2044                         }
2045
2046                         if (dm_request_for_irq_injection(vcpu, kvm_run)) {
2047                                 ++vcpu->stat.request_irq_exits;
2048                                 post_kvm_run_save(vcpu, kvm_run);
2049                                 kvm_run->exit_reason = KVM_EXIT_INTR;
2050                                 return -EINTR;
2051                         }
2052
2053                         kvm_resched(vcpu);
2054                         goto again;
2055                 }
2056         }
2057
2058         post_kvm_run_save(vcpu, kvm_run);
2059         return r;
2060 }
2061
2062 static void vmx_flush_tlb(struct kvm_vcpu *vcpu)
2063 {
2064         vmcs_writel(GUEST_CR3, vmcs_readl(GUEST_CR3));
2065 }
2066
2067 static void vmx_inject_page_fault(struct kvm_vcpu *vcpu,
2068                                   unsigned long addr,
2069                                   u32 err_code)
2070 {
2071         u32 vect_info = vmcs_read32(IDT_VECTORING_INFO_FIELD);
2072
2073         ++vcpu->stat.pf_guest;
2074
2075         if (is_page_fault(vect_info)) {
2076                 printk(KERN_DEBUG "inject_page_fault: "
2077                        "double fault 0x%lx @ 0x%lx\n",
2078                        addr, vmcs_readl(GUEST_RIP));
2079                 vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, 0);
2080                 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
2081                              DF_VECTOR |
2082                              INTR_TYPE_EXCEPTION |
2083                              INTR_INFO_DELIEVER_CODE_MASK |
2084                              INTR_INFO_VALID_MASK);
2085                 return;
2086         }
2087         vcpu->cr2 = addr;
2088         vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, err_code);
2089         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
2090                      PF_VECTOR |
2091                      INTR_TYPE_EXCEPTION |
2092                      INTR_INFO_DELIEVER_CODE_MASK |
2093                      INTR_INFO_VALID_MASK);
2094
2095 }
2096
2097 static void vmx_free_vmcs(struct kvm_vcpu *vcpu)
2098 {
2099         if (vcpu->vmcs) {
2100                 on_each_cpu(__vcpu_clear, vcpu, 0, 1);
2101                 free_vmcs(vcpu->vmcs);
2102                 vcpu->vmcs = NULL;
2103         }
2104 }
2105
2106 static void vmx_free_vcpu(struct kvm_vcpu *vcpu)
2107 {
2108         vmx_free_vmcs(vcpu);
2109 }
2110
2111 static int vmx_create_vcpu(struct kvm_vcpu *vcpu)
2112 {
2113         struct vmcs *vmcs;
2114
2115         vcpu->guest_msrs = kmalloc(PAGE_SIZE, GFP_KERNEL);
2116         if (!vcpu->guest_msrs)
2117                 return -ENOMEM;
2118
2119         vcpu->host_msrs = kmalloc(PAGE_SIZE, GFP_KERNEL);
2120         if (!vcpu->host_msrs)
2121                 goto out_free_guest_msrs;
2122
2123         vmcs = alloc_vmcs();
2124         if (!vmcs)
2125                 goto out_free_msrs;
2126
2127         vmcs_clear(vmcs);
2128         vcpu->vmcs = vmcs;
2129         vcpu->launched = 0;
2130         vcpu->fpu_active = 1;
2131
2132         return 0;
2133
2134 out_free_msrs:
2135         kfree(vcpu->host_msrs);
2136         vcpu->host_msrs = NULL;
2137
2138 out_free_guest_msrs:
2139         kfree(vcpu->guest_msrs);
2140         vcpu->guest_msrs = NULL;
2141
2142         return -ENOMEM;
2143 }
2144
2145 static struct kvm_arch_ops vmx_arch_ops = {
2146         .cpu_has_kvm_support = cpu_has_kvm_support,
2147         .disabled_by_bios = vmx_disabled_by_bios,
2148         .hardware_setup = hardware_setup,
2149         .hardware_unsetup = hardware_unsetup,
2150         .hardware_enable = hardware_enable,
2151         .hardware_disable = hardware_disable,
2152
2153         .vcpu_create = vmx_create_vcpu,
2154         .vcpu_free = vmx_free_vcpu,
2155
2156         .vcpu_load = vmx_vcpu_load,
2157         .vcpu_put = vmx_vcpu_put,
2158         .vcpu_decache = vmx_vcpu_decache,
2159
2160         .set_guest_debug = set_guest_debug,
2161         .get_msr = vmx_get_msr,
2162         .set_msr = vmx_set_msr,
2163         .get_segment_base = vmx_get_segment_base,
2164         .get_segment = vmx_get_segment,
2165         .set_segment = vmx_set_segment,
2166         .get_cs_db_l_bits = vmx_get_cs_db_l_bits,
2167         .decache_cr4_guest_bits = vmx_decache_cr4_guest_bits,
2168         .set_cr0 = vmx_set_cr0,
2169         .set_cr3 = vmx_set_cr3,
2170         .set_cr4 = vmx_set_cr4,
2171 #ifdef CONFIG_X86_64
2172         .set_efer = vmx_set_efer,
2173 #endif
2174         .get_idt = vmx_get_idt,
2175         .set_idt = vmx_set_idt,
2176         .get_gdt = vmx_get_gdt,
2177         .set_gdt = vmx_set_gdt,
2178         .cache_regs = vcpu_load_rsp_rip,
2179         .decache_regs = vcpu_put_rsp_rip,
2180         .get_rflags = vmx_get_rflags,
2181         .set_rflags = vmx_set_rflags,
2182
2183         .tlb_flush = vmx_flush_tlb,
2184         .inject_page_fault = vmx_inject_page_fault,
2185
2186         .inject_gp = vmx_inject_gp,
2187
2188         .run = vmx_vcpu_run,
2189         .skip_emulated_instruction = skip_emulated_instruction,
2190         .vcpu_setup = vmx_vcpu_setup,
2191         .patch_hypercall = vmx_patch_hypercall,
2192 };
2193
2194 static int __init vmx_init(void)
2195 {
2196         return kvm_init_arch(&vmx_arch_ops, THIS_MODULE);
2197 }
2198
2199 static void __exit vmx_exit(void)
2200 {
2201         kvm_exit_arch();
2202 }
2203
2204 module_init(vmx_init)
2205 module_exit(vmx_exit)