arch/mips/kvm/tlb.c

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * KVM/MIPS TLB handling, this file is part of the Linux host kernel so that
   7  * TLB handlers run from KSEG0
   8  *
   9  * Copyright (C) 2012  MIPS Technologies, Inc.  All rights reserved.
  10  * Authors: Sanjay Lal <sanjayl@kymasys.com>
  11  */
  12
  13 #include <linux/sched.h>
  14 #include <linux/smp.h>
  15 #include <linux/mm.h>
  16 #include <linux/delay.h>
  17 #include <linux/module.h>
  18 #include <linux/kvm_host.h>
  19 #include <linux/srcu.h>
  20
  21 #include <asm/cpu.h>
  22 #include <asm/bootinfo.h>
  23 #include <asm/mmu_context.h>
  24 #include <asm/pgtable.h>
  25 #include <asm/cacheflush.h>
  26 #include <asm/tlb.h>
  27
  28 #undef CONFIG_MIPS_MT
  29 #include <asm/r4kcache.h>
  30 #define CONFIG_MIPS_MT
  31
  32 #define KVM_GUEST_PC_TLB    0
  33 #define KVM_GUEST_SP_TLB    1
  34
  35 #define PRIx64 "llx"
  36
  37 atomic_t kvm_mips_instance;
  38 EXPORT_SYMBOL_GPL(kvm_mips_instance);
  39
  40 /* These function pointers are initialized once the KVM module is loaded */
  41 kvm_pfn_t (*kvm_mips_gfn_to_pfn)(struct kvm *kvm, gfn_t gfn);
  42 EXPORT_SYMBOL_GPL(kvm_mips_gfn_to_pfn);
  43
  44 void (*kvm_mips_release_pfn_clean)(kvm_pfn_t pfn);
  45 EXPORT_SYMBOL_GPL(kvm_mips_release_pfn_clean);
  46
  47 bool (*kvm_mips_is_error_pfn)(kvm_pfn_t pfn);
  48 EXPORT_SYMBOL_GPL(kvm_mips_is_error_pfn);
  49
  50 uint32_t kvm_mips_get_kernel_asid(struct kvm_vcpu *vcpu)
  51 {
  52         int cpu = smp_processor_id();
  53
  54         return vcpu->arch.guest_kernel_asid[cpu] &
  55                         cpu_asid_mask(&cpu_data[cpu]);
  56 }
  57
  58 uint32_t kvm_mips_get_user_asid(struct kvm_vcpu *vcpu)
  59 {
  60         int cpu = smp_processor_id();
  61
  62         return vcpu->arch.guest_user_asid[cpu] &
  63                         cpu_asid_mask(&cpu_data[cpu]);
  64 }
  65
  66 inline uint32_t kvm_mips_get_commpage_asid(struct kvm_vcpu *vcpu)
  67 {
  68         return vcpu->kvm->arch.commpage_tlb;
  69 }
  70
  71 /* Structure defining an tlb entry data set. */
  72
  73 void kvm_mips_dump_host_tlbs(void)
  74 {
  75         unsigned long old_entryhi;
  76         unsigned long old_pagemask;
  77         struct kvm_mips_tlb tlb;
  78         unsigned long flags;
  79         int i;
  80
  81         local_irq_save(flags);
  82
  83         old_entryhi = read_c0_entryhi();
  84         old_pagemask = read_c0_pagemask();
  85
  86         kvm_info("HOST TLBs:\n");
  87         kvm_info("ASID: %#lx\n", read_c0_entryhi() &
  88                  cpu_asid_mask(&current_cpu_data));
  89
  90         for (i = 0; i < current_cpu_data.tlbsize; i++) {
  91                 write_c0_index(i);
  92                 mtc0_tlbw_hazard();
  93
  94                 tlb_read();
  95                 tlbw_use_hazard();
  96
  97                 tlb.tlb_hi = read_c0_entryhi();
  98                 tlb.tlb_lo0 = read_c0_entrylo0();
  99                 tlb.tlb_lo1 = read_c0_entrylo1();
 100                 tlb.tlb_mask = read_c0_pagemask();
 101
 102                 kvm_info("TLB%c%3d Hi 0x%08lx ",
 103                          (tlb.tlb_lo0 | tlb.tlb_lo1) & MIPS3_PG_V ? ' ' : '*',
 104                          i, tlb.tlb_hi);
 105                 kvm_info("Lo0=0x%09" PRIx64 " %c%c attr %lx ",
 106                          (uint64_t) mips3_tlbpfn_to_paddr(tlb.tlb_lo0),
 107                          (tlb.tlb_lo0 & MIPS3_PG_D) ? 'D' : ' ',
 108                          (tlb.tlb_lo0 & MIPS3_PG_G) ? 'G' : ' ',
 109                          (tlb.tlb_lo0 >> 3) & 7);
 110                 kvm_info("Lo1=0x%09" PRIx64 " %c%c attr %lx sz=%lx\n",
 111                          (uint64_t) mips3_tlbpfn_to_paddr(tlb.tlb_lo1),
 112                          (tlb.tlb_lo1 & MIPS3_PG_D) ? 'D' : ' ',
 113                          (tlb.tlb_lo1 & MIPS3_PG_G) ? 'G' : ' ',
 114                          (tlb.tlb_lo1 >> 3) & 7, tlb.tlb_mask);
 115         }
 116         write_c0_entryhi(old_entryhi);
 117         write_c0_pagemask(old_pagemask);
 118         mtc0_tlbw_hazard();
 119         local_irq_restore(flags);
 120 }
 121 EXPORT_SYMBOL_GPL(kvm_mips_dump_host_tlbs);
 122
 123 void kvm_mips_dump_guest_tlbs(struct kvm_vcpu *vcpu)
 124 {
 125         struct mips_coproc *cop0 = vcpu->arch.cop0;
 126         struct kvm_mips_tlb tlb;
 127         int i;
 128
 129         kvm_info("Guest TLBs:\n");
 130         kvm_info("Guest EntryHi: %#lx\n", kvm_read_c0_guest_entryhi(cop0));
 131
 132         for (i = 0; i < KVM_MIPS_GUEST_TLB_SIZE; i++) {
 133                 tlb = vcpu->arch.guest_tlb[i];
 134                 kvm_info("TLB%c%3d Hi 0x%08lx ",
 135                          (tlb.tlb_lo0 | tlb.tlb_lo1) & MIPS3_PG_V ? ' ' : '*',
 136                          i, tlb.tlb_hi);
 137                 kvm_info("Lo0=0x%09" PRIx64 " %c%c attr %lx ",
 138                          (uint64_t) mips3_tlbpfn_to_paddr(tlb.tlb_lo0),
 139                          (tlb.tlb_lo0 & MIPS3_PG_D) ? 'D' : ' ',
 140                          (tlb.tlb_lo0 & MIPS3_PG_G) ? 'G' : ' ',
 141                          (tlb.tlb_lo0 >> 3) & 7);
 142                 kvm_info("Lo1=0x%09" PRIx64 " %c%c attr %lx sz=%lx\n",
 143                          (uint64_t) mips3_tlbpfn_to_paddr(tlb.tlb_lo1),
 144                          (tlb.tlb_lo1 & MIPS3_PG_D) ? 'D' : ' ',
 145                          (tlb.tlb_lo1 & MIPS3_PG_G) ? 'G' : ' ',
 146                          (tlb.tlb_lo1 >> 3) & 7, tlb.tlb_mask);
 147         }
 148 }
 149 EXPORT_SYMBOL_GPL(kvm_mips_dump_guest_tlbs);
 150
 151 static int kvm_mips_map_page(struct kvm *kvm, gfn_t gfn)
 152 {
 153         int srcu_idx, err = 0;
 154         kvm_pfn_t pfn;
 155
 156         if (kvm->arch.guest_pmap[gfn] != KVM_INVALID_PAGE)
 157                 return 0;
 158
 159         srcu_idx = srcu_read_lock(&kvm->srcu);
 160         pfn = kvm_mips_gfn_to_pfn(kvm, gfn);
 161
 162         if (kvm_mips_is_error_pfn(pfn)) {
 163                 kvm_err("Couldn't get pfn for gfn %#" PRIx64 "!\n", gfn);
 164                 err = -EFAULT;
 165                 goto out;
 166         }
 167
 168         kvm->arch.guest_pmap[gfn] = pfn;
 169 out:
 170         srcu_read_unlock(&kvm->srcu, srcu_idx);
 171         return err;
 172 }
 173
 174 /* Translate guest KSEG0 addresses to Host PA */
 175 unsigned long kvm_mips_translate_guest_kseg0_to_hpa(struct kvm_vcpu *vcpu,
 176                                                     unsigned long gva)
 177 {
 178         gfn_t gfn;
 179         uint32_t offset = gva & ~PAGE_MASK;
 180         struct kvm *kvm = vcpu->kvm;
 181
 182         if (KVM_GUEST_KSEGX(gva) != KVM_GUEST_KSEG0) {
 183                 kvm_err("%s/%p: Invalid gva: %#lx\n", __func__,
 184                         __builtin_return_address(0), gva);
 185                 return KVM_INVALID_PAGE;
 186         }
 187
 188         gfn = (KVM_GUEST_CPHYSADDR(gva) >> PAGE_SHIFT);
 189
 190         if (gfn >= kvm->arch.guest_pmap_npages) {
 191                 kvm_err("%s: Invalid gfn: %#llx, GVA: %#lx\n", __func__, gfn,
 192                         gva);
 193                 return KVM_INVALID_PAGE;
 194         }
 195
 196         if (kvm_mips_map_page(vcpu->kvm, gfn) < 0)
 197                 return KVM_INVALID_ADDR;
 198
 199         return (kvm->arch.guest_pmap[gfn] << PAGE_SHIFT) + offset;
 200 }
 201 EXPORT_SYMBOL_GPL(kvm_mips_translate_guest_kseg0_to_hpa);
 202
 203 /* XXXKYMA: Must be called with interrupts disabled */
 204 /* set flush_dcache_mask == 0 if no dcache flush required */
 205 int kvm_mips_host_tlb_write(struct kvm_vcpu *vcpu, unsigned long entryhi,
 206                             unsigned long entrylo0, unsigned long entrylo1,
 207                             int flush_dcache_mask)
 208 {
 209         unsigned long flags;
 210         unsigned long old_entryhi;
 211         int idx;
 212
 213         local_irq_save(flags);
 214
 215         old_entryhi = read_c0_entryhi();
 216         write_c0_entryhi(entryhi);
 217         mtc0_tlbw_hazard();
 218
 219         tlb_probe();
 220         tlb_probe_hazard();
 221         idx = read_c0_index();
 222
 223         if (idx > current_cpu_data.tlbsize) {
 224                 kvm_err("%s: Invalid Index: %d\n", __func__, idx);
 225                 kvm_mips_dump_host_tlbs();
 226                 local_irq_restore(flags);
 227                 return -1;
 228         }
 229
 230         write_c0_entrylo0(entrylo0);
 231         write_c0_entrylo1(entrylo1);
 232         mtc0_tlbw_hazard();
 233
 234         if (idx < 0)
 235                 tlb_write_random();
 236         else
 237                 tlb_write_indexed();
 238         tlbw_use_hazard();
 239
 240         kvm_debug("@ %#lx idx: %2d [entryhi(R): %#lx] entrylo0(R): 0x%08lx, entrylo1(R): 0x%08lx\n",
 241                   vcpu->arch.pc, idx, read_c0_entryhi(),
 242                   read_c0_entrylo0(), read_c0_entrylo1());
 243
 244         /* Flush D-cache */
 245         if (flush_dcache_mask) {
 246                 if (entrylo0 & MIPS3_PG_V) {
 247                         ++vcpu->stat.flush_dcache_exits;
 248                         flush_data_cache_page((entryhi & VPN2_MASK) &
 249                                               ~flush_dcache_mask);
 250                 }
 251                 if (entrylo1 & MIPS3_PG_V) {
 252                         ++vcpu->stat.flush_dcache_exits;
 253                         flush_data_cache_page(((entryhi & VPN2_MASK) &
 254                                                ~flush_dcache_mask) |
 255                                               (0x1 << PAGE_SHIFT));
 256                 }
 257         }
 258
 259         /* Restore old ASID */
 260         write_c0_entryhi(old_entryhi);
 261         mtc0_tlbw_hazard();
 262         tlbw_use_hazard();
 263         local_irq_restore(flags);
 264         return 0;
 265 }
 266
 267 /* XXXKYMA: Must be called with interrupts disabled */
 268 int kvm_mips_handle_kseg0_tlb_fault(unsigned long badvaddr,
 269                                     struct kvm_vcpu *vcpu)
 270 {
 271         gfn_t gfn;
 272         kvm_pfn_t pfn0, pfn1;
 273         unsigned long vaddr = 0;
 274         unsigned long entryhi = 0, entrylo0 = 0, entrylo1 = 0;
 275         int even;
 276         struct kvm *kvm = vcpu->kvm;
 277         const int flush_dcache_mask = 0;
 278         int ret;
 279
 280         if (KVM_GUEST_KSEGX(badvaddr) != KVM_GUEST_KSEG0) {
 281                 kvm_err("%s: Invalid BadVaddr: %#lx\n", __func__, badvaddr);
 282                 kvm_mips_dump_host_tlbs();
 283                 return -1;
 284         }
 285
 286         gfn = (KVM_GUEST_CPHYSADDR(badvaddr) >> PAGE_SHIFT);
 287         if (gfn >= kvm->arch.guest_pmap_npages) {
 288                 kvm_err("%s: Invalid gfn: %#llx, BadVaddr: %#lx\n", __func__,
 289                         gfn, badvaddr);
 290                 kvm_mips_dump_host_tlbs();
 291                 return -1;
 292         }
 293         even = !(gfn & 0x1);
 294         vaddr = badvaddr & (PAGE_MASK << 1);
 295
 296         if (kvm_mips_map_page(vcpu->kvm, gfn) < 0)
 297                 return -1;
 298
 299         if (kvm_mips_map_page(vcpu->kvm, gfn ^ 0x1) < 0)
 300                 return -1;
 301
 302         if (even) {
 303                 pfn0 = kvm->arch.guest_pmap[gfn];
 304                 pfn1 = kvm->arch.guest_pmap[gfn ^ 0x1];
 305         } else {
 306                 pfn0 = kvm->arch.guest_pmap[gfn ^ 0x1];
 307                 pfn1 = kvm->arch.guest_pmap[gfn];
 308         }
 309
 310         entrylo0 = mips3_paddr_to_tlbpfn(pfn0 << PAGE_SHIFT) | (0x3 << 3) |
 311                    (1 << 2) | (0x1 << 1);
 312         entrylo1 = mips3_paddr_to_tlbpfn(pfn1 << PAGE_SHIFT) | (0x3 << 3) |
 313                    (1 << 2) | (0x1 << 1);
 314
 315         preempt_disable();
 316         entryhi = (vaddr | kvm_mips_get_kernel_asid(vcpu));
 317         ret = kvm_mips_host_tlb_write(vcpu, entryhi, entrylo0, entrylo1,
 318                                       flush_dcache_mask);
 319         preempt_enable();
 320
 321         return ret;
 322 }
 323 EXPORT_SYMBOL_GPL(kvm_mips_handle_kseg0_tlb_fault);
 324
 325 int kvm_mips_handle_commpage_tlb_fault(unsigned long badvaddr,
 326         struct kvm_vcpu *vcpu)
 327 {
 328         kvm_pfn_t pfn0, pfn1;
 329         unsigned long flags, old_entryhi = 0, vaddr = 0;
 330         unsigned long entrylo0 = 0, entrylo1 = 0;
 331
 332         pfn0 = CPHYSADDR(vcpu->arch.kseg0_commpage) >> PAGE_SHIFT;
 333         pfn1 = 0;
 334         entrylo0 = mips3_paddr_to_tlbpfn(pfn0 << PAGE_SHIFT) | (0x3 << 3) |
 335                    (1 << 2) | (0x1 << 1);
 336         entrylo1 = 0;
 337
 338         local_irq_save(flags);
 339
 340         old_entryhi = read_c0_entryhi();
 341         vaddr = badvaddr & (PAGE_MASK << 1);
 342         write_c0_entryhi(vaddr | kvm_mips_get_kernel_asid(vcpu));
 343         mtc0_tlbw_hazard();
 344         write_c0_entrylo0(entrylo0);
 345         mtc0_tlbw_hazard();
 346         write_c0_entrylo1(entrylo1);
 347         mtc0_tlbw_hazard();
 348         write_c0_index(kvm_mips_get_commpage_asid(vcpu));
 349         mtc0_tlbw_hazard();
 350         tlb_write_indexed();
 351         mtc0_tlbw_hazard();
 352         tlbw_use_hazard();
 353
 354         kvm_debug("@ %#lx idx: %2d [entryhi(R): %#lx] entrylo0 (R): 0x%08lx, entrylo1(R): 0x%08lx\n",
 355                   vcpu->arch.pc, read_c0_index(), read_c0_entryhi(),
 356                   read_c0_entrylo0(), read_c0_entrylo1());
 357
 358         /* Restore old ASID */
 359         write_c0_entryhi(old_entryhi);
 360         mtc0_tlbw_hazard();
 361         tlbw_use_hazard();
 362         local_irq_restore(flags);
 363
 364         return 0;
 365 }
 366 EXPORT_SYMBOL_GPL(kvm_mips_handle_commpage_tlb_fault);
 367
 368 int kvm_mips_handle_mapped_seg_tlb_fault(struct kvm_vcpu *vcpu,
 369                                          struct kvm_mips_tlb *tlb,
 370                                          unsigned long *hpa0,
 371                                          unsigned long *hpa1)
 372 {
 373         unsigned long entryhi = 0, entrylo0 = 0, entrylo1 = 0;
 374         struct kvm *kvm = vcpu->kvm;
 375         kvm_pfn_t pfn0, pfn1;
 376         int ret;
 377
 378         if ((tlb->tlb_hi & VPN2_MASK) == 0) {
 379                 pfn0 = 0;
 380                 pfn1 = 0;
 381         } else {
 382                 if (kvm_mips_map_page(kvm, mips3_tlbpfn_to_paddr(tlb->tlb_lo0)
 383                                            >> PAGE_SHIFT) < 0)
 384                         return -1;
 385
 386                 if (kvm_mips_map_page(kvm, mips3_tlbpfn_to_paddr(tlb->tlb_lo1)
 387                                            >> PAGE_SHIFT) < 0)
 388                         return -1;
 389
 390                 pfn0 = kvm->arch.guest_pmap[mips3_tlbpfn_to_paddr(tlb->tlb_lo0)
 391                                             >> PAGE_SHIFT];
 392                 pfn1 = kvm->arch.guest_pmap[mips3_tlbpfn_to_paddr(tlb->tlb_lo1)
 393                                             >> PAGE_SHIFT];
 394         }
 395
 396         if (hpa0)
 397                 *hpa0 = pfn0 << PAGE_SHIFT;
 398
 399         if (hpa1)
 400                 *hpa1 = pfn1 << PAGE_SHIFT;
 401
 402         /* Get attributes from the Guest TLB */
 403         entrylo0 = mips3_paddr_to_tlbpfn(pfn0 << PAGE_SHIFT) | (0x3 << 3) |
 404                    (tlb->tlb_lo0 & MIPS3_PG_D) | (tlb->tlb_lo0 & MIPS3_PG_V);
 405         entrylo1 = mips3_paddr_to_tlbpfn(pfn1 << PAGE_SHIFT) | (0x3 << 3) |
 406                    (tlb->tlb_lo1 & MIPS3_PG_D) | (tlb->tlb_lo1 & MIPS3_PG_V);
 407
 408         kvm_debug("@ %#lx tlb_lo0: 0x%08lx tlb_lo1: 0x%08lx\n", vcpu->arch.pc,
 409                   tlb->tlb_lo0, tlb->tlb_lo1);
 410
 411         preempt_disable();
 412         entryhi = (tlb->tlb_hi & VPN2_MASK) | (KVM_GUEST_KERNEL_MODE(vcpu) ?
 413                                                kvm_mips_get_kernel_asid(vcpu) :
 414                                                kvm_mips_get_user_asid(vcpu));
 415         ret = kvm_mips_host_tlb_write(vcpu, entryhi, entrylo0, entrylo1,
 416                                       tlb->tlb_mask);
 417         preempt_enable();
 418
 419         return ret;
 420 }
 421 EXPORT_SYMBOL_GPL(kvm_mips_handle_mapped_seg_tlb_fault);
 422
 423 int kvm_mips_guest_tlb_lookup(struct kvm_vcpu *vcpu, unsigned long entryhi)
 424 {
 425         int i;
 426         int index = -1;
 427         struct kvm_mips_tlb *tlb = vcpu->arch.guest_tlb;
 428
 429         for (i = 0; i < KVM_MIPS_GUEST_TLB_SIZE; i++) {
 430                 if (TLB_HI_VPN2_HIT(tlb[i], entryhi) &&
 431                     TLB_HI_ASID_HIT(tlb[i], entryhi)) {
 432                         index = i;
 433                         break;
 434                 }
 435         }
 436
 437         kvm_debug("%s: entryhi: %#lx, index: %d lo0: %#lx, lo1: %#lx\n",
 438                   __func__, entryhi, index, tlb[i].tlb_lo0, tlb[i].tlb_lo1);
 439
 440         return index;
 441 }
 442 EXPORT_SYMBOL_GPL(kvm_mips_guest_tlb_lookup);
 443
 444 int kvm_mips_host_tlb_lookup(struct kvm_vcpu *vcpu, unsigned long vaddr)
 445 {
 446         unsigned long old_entryhi, flags;
 447         int idx;
 448
 449         local_irq_save(flags);
 450
 451         old_entryhi = read_c0_entryhi();
 452
 453         if (KVM_GUEST_KERNEL_MODE(vcpu))
 454                 write_c0_entryhi((vaddr & VPN2_MASK) |
 455                                  kvm_mips_get_kernel_asid(vcpu));
 456         else {
 457                 write_c0_entryhi((vaddr & VPN2_MASK) |
 458                                  kvm_mips_get_user_asid(vcpu));
 459         }
 460
 461         mtc0_tlbw_hazard();
 462
 463         tlb_probe();
 464         tlb_probe_hazard();
 465         idx = read_c0_index();
 466
 467         /* Restore old ASID */
 468         write_c0_entryhi(old_entryhi);
 469         mtc0_tlbw_hazard();
 470         tlbw_use_hazard();
 471
 472         local_irq_restore(flags);
 473
 474         kvm_debug("Host TLB lookup, %#lx, idx: %2d\n", vaddr, idx);
 475
 476         return idx;
 477 }
 478 EXPORT_SYMBOL_GPL(kvm_mips_host_tlb_lookup);
 479
 480 int kvm_mips_host_tlb_inv(struct kvm_vcpu *vcpu, unsigned long va)
 481 {
 482         int idx;
 483         unsigned long flags, old_entryhi;
 484
 485         local_irq_save(flags);
 486
 487         old_entryhi = read_c0_entryhi();
 488
 489         write_c0_entryhi((va & VPN2_MASK) | kvm_mips_get_user_asid(vcpu));
 490         mtc0_tlbw_hazard();
 491
 492         tlb_probe();
 493         tlb_probe_hazard();
 494         idx = read_c0_index();
 495
 496         if (idx >= current_cpu_data.tlbsize)
 497                 BUG();
 498
 499         if (idx > 0) {
 500                 write_c0_entryhi(UNIQUE_ENTRYHI(idx));
 501                 mtc0_tlbw_hazard();
 502
 503                 write_c0_entrylo0(0);
 504                 mtc0_tlbw_hazard();
 505
 506                 write_c0_entrylo1(0);
 507                 mtc0_tlbw_hazard();
 508
 509                 tlb_write_indexed();
 510                 mtc0_tlbw_hazard();
 511         }
 512
 513         write_c0_entryhi(old_entryhi);
 514         mtc0_tlbw_hazard();
 515         tlbw_use_hazard();
 516
 517         local_irq_restore(flags);
 518
 519         if (idx > 0)
 520                 kvm_debug("%s: Invalidated entryhi %#lx @ idx %d\n", __func__,
 521                           (va & VPN2_MASK) | kvm_mips_get_user_asid(vcpu), idx);
 522
 523         return 0;
 524 }
 525 EXPORT_SYMBOL_GPL(kvm_mips_host_tlb_inv);
 526
 527 void kvm_mips_flush_host_tlb(int skip_kseg0)
 528 {
 529         unsigned long flags;
 530         unsigned long old_entryhi, entryhi;
 531         unsigned long old_pagemask;
 532         int entry = 0;
 533         int maxentry = current_cpu_data.tlbsize;
 534
 535         local_irq_save(flags);
 536
 537         old_entryhi = read_c0_entryhi();
 538         old_pagemask = read_c0_pagemask();
 539
 540         /* Blast 'em all away. */
 541         for (entry = 0; entry < maxentry; entry++) {
 542                 write_c0_index(entry);
 543                 mtc0_tlbw_hazard();
 544
 545                 if (skip_kseg0) {
 546                         tlb_read();
 547                         tlbw_use_hazard();
 548
 549                         entryhi = read_c0_entryhi();
 550
 551                         /* Don't blow away guest kernel entries */
 552                         if (KVM_GUEST_KSEGX(entryhi) == KVM_GUEST_KSEG0)
 553                                 continue;
 554                 }
 555
 556                 /* Make sure all entries differ. */
 557                 write_c0_entryhi(UNIQUE_ENTRYHI(entry));
 558                 mtc0_tlbw_hazard();
 559                 write_c0_entrylo0(0);
 560                 mtc0_tlbw_hazard();
 561                 write_c0_entrylo1(0);
 562                 mtc0_tlbw_hazard();
 563
 564                 tlb_write_indexed();
 565                 mtc0_tlbw_hazard();
 566         }
 567
 568         tlbw_use_hazard();
 569
 570         write_c0_entryhi(old_entryhi);
 571         write_c0_pagemask(old_pagemask);
 572         mtc0_tlbw_hazard();
 573         tlbw_use_hazard();
 574
 575         local_irq_restore(flags);
 576 }
 577 EXPORT_SYMBOL_GPL(kvm_mips_flush_host_tlb);
 578
 579 void kvm_get_new_mmu_context(struct mm_struct *mm, unsigned long cpu,
 580                              struct kvm_vcpu *vcpu)
 581 {
 582         unsigned long asid = asid_cache(cpu);
 583
 584         asid += cpu_asid_inc();
 585         if (!(asid & cpu_asid_mask(&cpu_data[cpu]))) {
 586                 if (cpu_has_vtag_icache)
 587                         flush_icache_all();
 588
 589                 kvm_local_flush_tlb_all();      /* start new asid cycle */
 590
 591                 if (!asid)      /* fix version if needed */
 592                         asid = asid_first_version(cpu);
 593         }
 594
 595         cpu_context(cpu, mm) = asid_cache(cpu) = asid;
 596 }
 597
 598 void kvm_local_flush_tlb_all(void)
 599 {
 600         unsigned long flags;
 601         unsigned long old_ctx;
 602         int entry = 0;
 603
 604         local_irq_save(flags);
 605         /* Save old context and create impossible VPN2 value */
 606         old_ctx = read_c0_entryhi();
 607         write_c0_entrylo0(0);
 608         write_c0_entrylo1(0);
 609
 610         /* Blast 'em all away. */
 611         while (entry < current_cpu_data.tlbsize) {
 612                 /* Make sure all entries differ. */
 613                 write_c0_entryhi(UNIQUE_ENTRYHI(entry));
 614                 write_c0_index(entry);
 615                 mtc0_tlbw_hazard();
 616                 tlb_write_indexed();
 617                 entry++;
 618         }
 619         tlbw_use_hazard();
 620         write_c0_entryhi(old_ctx);
 621         mtc0_tlbw_hazard();
 622
 623         local_irq_restore(flags);
 624 }
 625 EXPORT_SYMBOL_GPL(kvm_local_flush_tlb_all);
 626
 627 /**
 628  * kvm_mips_migrate_count() - Migrate timer.
 629  * @vcpu:       Virtual CPU.
 630  *
 631  * Migrate CP0_Count hrtimer to the current CPU by cancelling and restarting it
 632  * if it was running prior to being cancelled.
 633  *
 634  * Must be called when the VCPU is migrated to a different CPU to ensure that
 635  * timer expiry during guest execution interrupts the guest and causes the
 636  * interrupt to be delivered in a timely manner.
 637  */
 638 static void kvm_mips_migrate_count(struct kvm_vcpu *vcpu)
 639 {
 640         if (hrtimer_cancel(&vcpu->arch.comparecount_timer))
 641                 hrtimer_restart(&vcpu->arch.comparecount_timer);
 642 }
 643
 644 /* Restore ASID once we are scheduled back after preemption */
 645 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 646 {
 647         unsigned long asid_mask = cpu_asid_mask(&cpu_data[cpu]);
 648         unsigned long flags;
 649         int newasid = 0;
 650
 651         kvm_debug("%s: vcpu %p, cpu: %d\n", __func__, vcpu, cpu);
 652
 653         /* Allocate new kernel and user ASIDs if needed */
 654
 655         local_irq_save(flags);
 656
 657         if ((vcpu->arch.guest_kernel_asid[cpu] ^ asid_cache(cpu)) &
 658                                                 asid_version_mask(cpu)) {
 659                 kvm_get_new_mmu_context(&vcpu->arch.guest_kernel_mm, cpu, vcpu);
 660                 vcpu->arch.guest_kernel_asid[cpu] =
 661                     vcpu->arch.guest_kernel_mm.context.asid[cpu];
 662                 kvm_get_new_mmu_context(&vcpu->arch.guest_user_mm, cpu, vcpu);
 663                 vcpu->arch.guest_user_asid[cpu] =
 664                     vcpu->arch.guest_user_mm.context.asid[cpu];
 665                 newasid++;
 666
 667                 kvm_debug("[%d]: cpu_context: %#lx\n", cpu,
 668                           cpu_context(cpu, current->mm));
 669                 kvm_debug("[%d]: Allocated new ASID for Guest Kernel: %#x\n",
 670                           cpu, vcpu->arch.guest_kernel_asid[cpu]);
 671                 kvm_debug("[%d]: Allocated new ASID for Guest User: %#x\n", cpu,
 672                           vcpu->arch.guest_user_asid[cpu]);
 673         }
 674
 675         if (vcpu->arch.last_sched_cpu != cpu) {
 676                 kvm_debug("[%d->%d]KVM VCPU[%d] switch\n",
 677                           vcpu->arch.last_sched_cpu, cpu, vcpu->vcpu_id);
 678                 /*
 679                  * Migrate the timer interrupt to the current CPU so that it
 680                  * always interrupts the guest and synchronously triggers a
 681                  * guest timer interrupt.
 682                  */
 683                 kvm_mips_migrate_count(vcpu);
 684         }
 685
 686         if (!newasid) {
 687                 /*
 688                  * If we preempted while the guest was executing, then reload
 689                  * the pre-empted ASID
 690                  */
 691                 if (current->flags & PF_VCPU) {
 692                         write_c0_entryhi(vcpu->arch.
 693                                          preempt_entryhi & asid_mask);
 694                         ehb();
 695                 }
 696         } else {
 697                 /* New ASIDs were allocated for the VM */
 698
 699                 /*
 700                  * Were we in guest context? If so then the pre-empted ASID is
 701                  * no longer valid, we need to set it to what it should be based
 702                  * on the mode of the Guest (Kernel/User)
 703                  */
 704                 if (current->flags & PF_VCPU) {
 705                         if (KVM_GUEST_KERNEL_MODE(vcpu))
 706                                 write_c0_entryhi(vcpu->arch.
 707                                                  guest_kernel_asid[cpu] &
 708                                                  asid_mask);
 709                         else
 710                                 write_c0_entryhi(vcpu->arch.
 711                                                  guest_user_asid[cpu] &
 712                                                  asid_mask);
 713                         ehb();
 714                 }
 715         }
 716
 717         /* restore guest state to registers */
 718         kvm_mips_callbacks->vcpu_set_regs(vcpu);
 719
 720         local_irq_restore(flags);
 721
 722 }
 723 EXPORT_SYMBOL_GPL(kvm_arch_vcpu_load);
 724
 725 /* ASID can change if another task is scheduled during preemption */
 726 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
 727 {
 728         unsigned long flags;
 729         uint32_t cpu;
 730
 731         local_irq_save(flags);
 732
 733         cpu = smp_processor_id();
 734
 735         vcpu->arch.preempt_entryhi = read_c0_entryhi();
 736         vcpu->arch.last_sched_cpu = cpu;
 737
 738         /* save guest state in registers */
 739         kvm_mips_callbacks->vcpu_get_regs(vcpu);
 740
 741         if (((cpu_context(cpu, current->mm) ^ asid_cache(cpu)) &
 742              asid_version_mask(cpu))) {
 743                 kvm_debug("%s: Dropping MMU Context:  %#lx\n", __func__,
 744                           cpu_context(cpu, current->mm));
 745                 drop_mmu_context(current->mm, cpu);
 746         }
 747         write_c0_entryhi(cpu_asid(cpu, current->mm));
 748         ehb();
 749
 750         local_irq_restore(flags);
 751 }
 752 EXPORT_SYMBOL_GPL(kvm_arch_vcpu_put);
 753
 754 uint32_t kvm_get_inst(uint32_t *opc, struct kvm_vcpu *vcpu)
 755 {
 756         struct mips_coproc *cop0 = vcpu->arch.cop0;
 757         unsigned long paddr, flags, vpn2, asid;
 758         uint32_t inst;
 759         int index;
 760
 761         if (KVM_GUEST_KSEGX((unsigned long) opc) < KVM_GUEST_KSEG0 ||
 762             KVM_GUEST_KSEGX((unsigned long) opc) == KVM_GUEST_KSEG23) {
 763                 local_irq_save(flags);
 764                 index = kvm_mips_host_tlb_lookup(vcpu, (unsigned long) opc);
 765                 if (index >= 0) {
 766                         inst = *(opc);
 767                 } else {
 768                         vpn2 = (unsigned long) opc & VPN2_MASK;
 769                         asid = kvm_read_c0_guest_entryhi(cop0) &
 770                                                 KVM_ENTRYHI_ASID;
 771                         index = kvm_mips_guest_tlb_lookup(vcpu, vpn2 | asid);
 772                         if (index < 0) {
 773                                 kvm_err("%s: get_user_failed for %p, vcpu: %p, ASID: %#lx\n",
 774                                         __func__, opc, vcpu, read_c0_entryhi());
 775                                 kvm_mips_dump_host_tlbs();
 776                                 local_irq_restore(flags);
 777                                 return KVM_INVALID_INST;
 778                         }
 779                         kvm_mips_handle_mapped_seg_tlb_fault(vcpu,
 780                                                              &vcpu->arch.
 781                                                              guest_tlb[index],
 782                                                              NULL, NULL);
 783                         inst = *(opc);
 784                 }
 785                 local_irq_restore(flags);
 786         } else if (KVM_GUEST_KSEGX(opc) == KVM_GUEST_KSEG0) {
 787                 paddr =
 788                     kvm_mips_translate_guest_kseg0_to_hpa(vcpu,
 789                                                           (unsigned long) opc);
 790                 inst = *(uint32_t *) CKSEG0ADDR(paddr);
 791         } else {
 792                 kvm_err("%s: illegal address: %p\n", __func__, opc);
 793                 return KVM_INVALID_INST;
 794         }
 795
 796         return inst;
 797 }
 798 EXPORT_SYMBOL_GPL(kvm_get_inst);