arch/mips/kvm/emulate.c

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * KVM/MIPS: Instruction/Exception emulation
   7  *
   8  * Copyright (C) 2012  MIPS Technologies, Inc.  All rights reserved.
   9  * Authors: Sanjay Lal <sanjayl@kymasys.com>
  10  */
  11
  12 #include <linux/errno.h>
  13 #include <linux/err.h>
  14 #include <linux/ktime.h>
  15 #include <linux/kvm_host.h>
  16 #include <linux/module.h>
  17 #include <linux/vmalloc.h>
  18 #include <linux/fs.h>
  19 #include <linux/bootmem.h>
  20 #include <linux/random.h>
  21 #include <asm/page.h>
  22 #include <asm/cacheflush.h>
  23 #include <asm/cacheops.h>
  24 #include <asm/cpu-info.h>
  25 #include <asm/mmu_context.h>
  26 #include <asm/tlbflush.h>
  27 #include <asm/inst.h>
  28
  29 #undef CONFIG_MIPS_MT
  30 #include <asm/r4kcache.h>
  31 #define CONFIG_MIPS_MT
  32
  33 #include "interrupt.h"
  34 #include "commpage.h"
  35
  36 #include "trace.h"
  37
  38 /*
  39  * Compute the return address and do emulate branch simulation, if required.
  40  * This function should be called only in branch delay slot active.
  41  */
  42 unsigned long kvm_compute_return_epc(struct kvm_vcpu *vcpu,
  43         unsigned long instpc)
  44 {
  45         unsigned int dspcontrol;
  46         union mips_instruction insn;
  47         struct kvm_vcpu_arch *arch = &vcpu->arch;
  48         long epc = instpc;
  49         long nextpc = KVM_INVALID_INST;
  50
  51         if (epc & 3)
  52                 goto unaligned;
  53
  54         /* Read the instruction */
  55         insn.word = kvm_get_inst((u32 *) epc, vcpu);
  56
  57         if (insn.word == KVM_INVALID_INST)
  58                 return KVM_INVALID_INST;
  59
  60         switch (insn.i_format.opcode) {
  61                 /* jr and jalr are in r_format format. */
  62         case spec_op:
  63                 switch (insn.r_format.func) {
  64                 case jalr_op:
  65                         arch->gprs[insn.r_format.rd] = epc + 8;
  66                         /* Fall through */
  67                 case jr_op:
  68                         nextpc = arch->gprs[insn.r_format.rs];
  69                         break;
  70                 }
  71                 break;
  72
  73                 /*
  74                  * This group contains:
  75                  * bltz_op, bgez_op, bltzl_op, bgezl_op,
  76                  * bltzal_op, bgezal_op, bltzall_op, bgezall_op.
  77                  */
  78         case bcond_op:
  79                 switch (insn.i_format.rt) {
  80                 case bltz_op:
  81                 case bltzl_op:
  82                         if ((long)arch->gprs[insn.i_format.rs] < 0)
  83                                 epc = epc + 4 + (insn.i_format.simmediate << 2);
  84                         else
  85                                 epc += 8;
  86                         nextpc = epc;
  87                         break;
  88
  89                 case bgez_op:
  90                 case bgezl_op:
  91                         if ((long)arch->gprs[insn.i_format.rs] >= 0)
  92                                 epc = epc + 4 + (insn.i_format.simmediate << 2);
  93                         else
  94                                 epc += 8;
  95                         nextpc = epc;
  96                         break;
  97
  98                 case bltzal_op:
  99                 case bltzall_op:
 100                         arch->gprs[31] = epc + 8;
 101                         if ((long)arch->gprs[insn.i_format.rs] < 0)
 102                                 epc = epc + 4 + (insn.i_format.simmediate << 2);
 103                         else
 104                                 epc += 8;
 105                         nextpc = epc;
 106                         break;
 107
 108                 case bgezal_op:
 109                 case bgezall_op:
 110                         arch->gprs[31] = epc + 8;
 111                         if ((long)arch->gprs[insn.i_format.rs] >= 0)
 112                                 epc = epc + 4 + (insn.i_format.simmediate << 2);
 113                         else
 114                                 epc += 8;
 115                         nextpc = epc;
 116                         break;
 117                 case bposge32_op:
 118                         if (!cpu_has_dsp)
 119                                 goto sigill;
 120
 121                         dspcontrol = rddsp(0x01);
 122
 123                         if (dspcontrol >= 32)
 124                                 epc = epc + 4 + (insn.i_format.simmediate << 2);
 125                         else
 126                                 epc += 8;
 127                         nextpc = epc;
 128                         break;
 129                 }
 130                 break;
 131
 132                 /* These are unconditional and in j_format. */
 133         case jal_op:
 134                 arch->gprs[31] = instpc + 8;
 135         case j_op:
 136                 epc += 4;
 137                 epc >>= 28;
 138                 epc <<= 28;
 139                 epc |= (insn.j_format.target << 2);
 140                 nextpc = epc;
 141                 break;
 142
 143                 /* These are conditional and in i_format. */
 144         case beq_op:
 145         case beql_op:
 146                 if (arch->gprs[insn.i_format.rs] ==
 147                     arch->gprs[insn.i_format.rt])
 148                         epc = epc + 4 + (insn.i_format.simmediate << 2);
 149                 else
 150                         epc += 8;
 151                 nextpc = epc;
 152                 break;
 153
 154         case bne_op:
 155         case bnel_op:
 156                 if (arch->gprs[insn.i_format.rs] !=
 157                     arch->gprs[insn.i_format.rt])
 158                         epc = epc + 4 + (insn.i_format.simmediate << 2);
 159                 else
 160                         epc += 8;
 161                 nextpc = epc;
 162                 break;
 163
 164         case blez_op:           /* not really i_format */
 165         case blezl_op:
 166                 /* rt field assumed to be zero */
 167                 if ((long)arch->gprs[insn.i_format.rs] <= 0)
 168                         epc = epc + 4 + (insn.i_format.simmediate << 2);
 169                 else
 170                         epc += 8;
 171                 nextpc = epc;
 172                 break;
 173
 174         case bgtz_op:
 175         case bgtzl_op:
 176                 /* rt field assumed to be zero */
 177                 if ((long)arch->gprs[insn.i_format.rs] > 0)
 178                         epc = epc + 4 + (insn.i_format.simmediate << 2);
 179                 else
 180                         epc += 8;
 181                 nextpc = epc;
 182                 break;
 183
 184                 /* And now the FPA/cp1 branch instructions. */
 185         case cop1_op:
 186                 kvm_err("%s: unsupported cop1_op\n", __func__);
 187                 break;
 188         }
 189
 190         return nextpc;
 191
 192 unaligned:
 193         kvm_err("%s: unaligned epc\n", __func__);
 194         return nextpc;
 195
 196 sigill:
 197         kvm_err("%s: DSP branch but not DSP ASE\n", __func__);
 198         return nextpc;
 199 }
 200
 201 enum emulation_result update_pc(struct kvm_vcpu *vcpu, u32 cause)
 202 {
 203         unsigned long branch_pc;
 204         enum emulation_result er = EMULATE_DONE;
 205
 206         if (cause & CAUSEF_BD) {
 207                 branch_pc = kvm_compute_return_epc(vcpu, vcpu->arch.pc);
 208                 if (branch_pc == KVM_INVALID_INST) {
 209                         er = EMULATE_FAIL;
 210                 } else {
 211                         vcpu->arch.pc = branch_pc;
 212                         kvm_debug("BD update_pc(): New PC: %#lx\n",
 213                                   vcpu->arch.pc);
 214                 }
 215         } else
 216                 vcpu->arch.pc += 4;
 217
 218         kvm_debug("update_pc(): New PC: %#lx\n", vcpu->arch.pc);
 219
 220         return er;
 221 }
 222
 223 /**
 224  * kvm_mips_count_disabled() - Find whether the CP0_Count timer is disabled.
 225  * @vcpu:       Virtual CPU.
 226  *
 227  * Returns:     1 if the CP0_Count timer is disabled by either the guest
 228  *              CP0_Cause.DC bit or the count_ctl.DC bit.
 229  *              0 otherwise (in which case CP0_Count timer is running).
 230  */
 231 static inline int kvm_mips_count_disabled(struct kvm_vcpu *vcpu)
 232 {
 233         struct mips_coproc *cop0 = vcpu->arch.cop0;
 234
 235         return  (vcpu->arch.count_ctl & KVM_REG_MIPS_COUNT_CTL_DC) ||
 236                 (kvm_read_c0_guest_cause(cop0) & CAUSEF_DC);
 237 }
 238
 239 /**
 240  * kvm_mips_ktime_to_count() - Scale ktime_t to a 32-bit count.
 241  *
 242  * Caches the dynamic nanosecond bias in vcpu->arch.count_dyn_bias.
 243  *
 244  * Assumes !kvm_mips_count_disabled(@vcpu) (guest CP0_Count timer is running).
 245  */
 246 static u32 kvm_mips_ktime_to_count(struct kvm_vcpu *vcpu, ktime_t now)
 247 {
 248         s64 now_ns, periods;
 249         u64 delta;
 250
 251         now_ns = ktime_to_ns(now);
 252         delta = now_ns + vcpu->arch.count_dyn_bias;
 253
 254         if (delta >= vcpu->arch.count_period) {
 255                 /* If delta is out of safe range the bias needs adjusting */
 256                 periods = div64_s64(now_ns, vcpu->arch.count_period);
 257                 vcpu->arch.count_dyn_bias = -periods * vcpu->arch.count_period;
 258                 /* Recalculate delta with new bias */
 259                 delta = now_ns + vcpu->arch.count_dyn_bias;
 260         }
 261
 262         /*
 263          * We've ensured that:
 264          *   delta < count_period
 265          *
 266          * Therefore the intermediate delta*count_hz will never overflow since
 267          * at the boundary condition:
 268          *   delta = count_period
 269          *   delta = NSEC_PER_SEC * 2^32 / count_hz
 270          *   delta * count_hz = NSEC_PER_SEC * 2^32
 271          */
 272         return div_u64(delta * vcpu->arch.count_hz, NSEC_PER_SEC);
 273 }
 274
 275 /**
 276  * kvm_mips_count_time() - Get effective current time.
 277  * @vcpu:       Virtual CPU.
 278  *
 279  * Get effective monotonic ktime. This is usually a straightforward ktime_get(),
 280  * except when the master disable bit is set in count_ctl, in which case it is
 281  * count_resume, i.e. the time that the count was disabled.
 282  *
 283  * Returns:     Effective monotonic ktime for CP0_Count.
 284  */
 285 static inline ktime_t kvm_mips_count_time(struct kvm_vcpu *vcpu)
 286 {
 287         if (unlikely(vcpu->arch.count_ctl & KVM_REG_MIPS_COUNT_CTL_DC))
 288                 return vcpu->arch.count_resume;
 289
 290         return ktime_get();
 291 }
 292
 293 /**
 294  * kvm_mips_read_count_running() - Read the current count value as if running.
 295  * @vcpu:       Virtual CPU.
 296  * @now:        Kernel time to read CP0_Count at.
 297  *
 298  * Returns the current guest CP0_Count register at time @now and handles if the
 299  * timer interrupt is pending and hasn't been handled yet.
 300  *
 301  * Returns:     The current value of the guest CP0_Count register.
 302  */
 303 static u32 kvm_mips_read_count_running(struct kvm_vcpu *vcpu, ktime_t now)
 304 {
 305         struct mips_coproc *cop0 = vcpu->arch.cop0;
 306         ktime_t expires, threshold;
 307         u32 count, compare;
 308         int running;
 309
 310         /* Calculate the biased and scaled guest CP0_Count */
 311         count = vcpu->arch.count_bias + kvm_mips_ktime_to_count(vcpu, now);
 312         compare = kvm_read_c0_guest_compare(cop0);
 313
 314         /*
 315          * Find whether CP0_Count has reached the closest timer interrupt. If
 316          * not, we shouldn't inject it.
 317          */
 318         if ((s32)(count - compare) < 0)
 319                 return count;
 320
 321         /*
 322          * The CP0_Count we're going to return has already reached the closest
 323          * timer interrupt. Quickly check if it really is a new interrupt by
 324          * looking at whether the interval until the hrtimer expiry time is
 325          * less than 1/4 of the timer period.
 326          */
 327         expires = hrtimer_get_expires(&vcpu->arch.comparecount_timer);
 328         threshold = ktime_add_ns(now, vcpu->arch.count_period / 4);
 329         if (ktime_before(expires, threshold)) {
 330                 /*
 331                  * Cancel it while we handle it so there's no chance of
 332                  * interference with the timeout handler.
 333                  */
 334                 running = hrtimer_cancel(&vcpu->arch.comparecount_timer);
 335
 336                 /* Nothing should be waiting on the timeout */
 337                 kvm_mips_callbacks->queue_timer_int(vcpu);
 338
 339                 /*
 340                  * Restart the timer if it was running based on the expiry time
 341                  * we read, so that we don't push it back 2 periods.
 342                  */
 343                 if (running) {
 344                         expires = ktime_add_ns(expires,
 345                                                vcpu->arch.count_period);
 346                         hrtimer_start(&vcpu->arch.comparecount_timer, expires,
 347                                       HRTIMER_MODE_ABS);
 348                 }
 349         }
 350
 351         return count;
 352 }
 353
 354 /**
 355  * kvm_mips_read_count() - Read the current count value.
 356  * @vcpu:       Virtual CPU.
 357  *
 358  * Read the current guest CP0_Count value, taking into account whether the timer
 359  * is stopped.
 360  *
 361  * Returns:     The current guest CP0_Count value.
 362  */
 363 u32 kvm_mips_read_count(struct kvm_vcpu *vcpu)
 364 {
 365         struct mips_coproc *cop0 = vcpu->arch.cop0;
 366
 367         /* If count disabled just read static copy of count */
 368         if (kvm_mips_count_disabled(vcpu))
 369                 return kvm_read_c0_guest_count(cop0);
 370
 371         return kvm_mips_read_count_running(vcpu, ktime_get());
 372 }
 373
 374 /**
 375  * kvm_mips_freeze_hrtimer() - Safely stop the hrtimer.
 376  * @vcpu:       Virtual CPU.
 377  * @count:      Output pointer for CP0_Count value at point of freeze.
 378  *
 379  * Freeze the hrtimer safely and return both the ktime and the CP0_Count value
 380  * at the point it was frozen. It is guaranteed that any pending interrupts at
 381  * the point it was frozen are handled, and none after that point.
 382  *
 383  * This is useful where the time/CP0_Count is needed in the calculation of the
 384  * new parameters.
 385  *
 386  * Assumes !kvm_mips_count_disabled(@vcpu) (guest CP0_Count timer is running).
 387  *
 388  * Returns:     The ktime at the point of freeze.
 389  */
 390 static ktime_t kvm_mips_freeze_hrtimer(struct kvm_vcpu *vcpu, u32 *count)
 391 {
 392         ktime_t now;
 393
 394         /* stop hrtimer before finding time */
 395         hrtimer_cancel(&vcpu->arch.comparecount_timer);
 396         now = ktime_get();
 397
 398         /* find count at this point and handle pending hrtimer */
 399         *count = kvm_mips_read_count_running(vcpu, now);
 400
 401         return now;
 402 }
 403
 404 /**
 405  * kvm_mips_resume_hrtimer() - Resume hrtimer, updating expiry.
 406  * @vcpu:       Virtual CPU.
 407  * @now:        ktime at point of resume.
 408  * @count:      CP0_Count at point of resume.
 409  *
 410  * Resumes the timer and updates the timer expiry based on @now and @count.
 411  * This can be used in conjunction with kvm_mips_freeze_timer() when timer
 412  * parameters need to be changed.
 413  *
 414  * It is guaranteed that a timer interrupt immediately after resume will be
 415  * handled, but not if CP_Compare is exactly at @count. That case is already
 416  * handled by kvm_mips_freeze_timer().
 417  *
 418  * Assumes !kvm_mips_count_disabled(@vcpu) (guest CP0_Count timer is running).
 419  */
 420 static void kvm_mips_resume_hrtimer(struct kvm_vcpu *vcpu,
 421                                     ktime_t now, u32 count)
 422 {
 423         struct mips_coproc *cop0 = vcpu->arch.cop0;
 424         u32 compare;
 425         u64 delta;
 426         ktime_t expire;
 427
 428         /* Calculate timeout (wrap 0 to 2^32) */
 429         compare = kvm_read_c0_guest_compare(cop0);
 430         delta = (u64)(u32)(compare - count - 1) + 1;
 431         delta = div_u64(delta * NSEC_PER_SEC, vcpu->arch.count_hz);
 432         expire = ktime_add_ns(now, delta);
 433
 434         /* Update hrtimer to use new timeout */
 435         hrtimer_cancel(&vcpu->arch.comparecount_timer);
 436         hrtimer_start(&vcpu->arch.comparecount_timer, expire, HRTIMER_MODE_ABS);
 437 }
 438
 439 /**
 440  * kvm_mips_write_count() - Modify the count and update timer.
 441  * @vcpu:       Virtual CPU.
 442  * @count:      Guest CP0_Count value to set.
 443  *
 444  * Sets the CP0_Count value and updates the timer accordingly.
 445  */
 446 void kvm_mips_write_count(struct kvm_vcpu *vcpu, u32 count)
 447 {
 448         struct mips_coproc *cop0 = vcpu->arch.cop0;
 449         ktime_t now;
 450
 451         /* Calculate bias */
 452         now = kvm_mips_count_time(vcpu);
 453         vcpu->arch.count_bias = count - kvm_mips_ktime_to_count(vcpu, now);
 454
 455         if (kvm_mips_count_disabled(vcpu))
 456                 /* The timer's disabled, adjust the static count */
 457                 kvm_write_c0_guest_count(cop0, count);
 458         else
 459                 /* Update timeout */
 460                 kvm_mips_resume_hrtimer(vcpu, now, count);
 461 }
 462
 463 /**
 464  * kvm_mips_init_count() - Initialise timer.
 465  * @vcpu:       Virtual CPU.
 466  *
 467  * Initialise the timer to a sensible frequency, namely 100MHz, zero it, and set
 468  * it going if it's enabled.
 469  */
 470 void kvm_mips_init_count(struct kvm_vcpu *vcpu)
 471 {
 472         /* 100 MHz */
 473         vcpu->arch.count_hz = 100*1000*1000;
 474         vcpu->arch.count_period = div_u64((u64)NSEC_PER_SEC << 32,
 475                                           vcpu->arch.count_hz);
 476         vcpu->arch.count_dyn_bias = 0;
 477
 478         /* Starting at 0 */
 479         kvm_mips_write_count(vcpu, 0);
 480 }
 481
 482 /**
 483  * kvm_mips_set_count_hz() - Update the frequency of the timer.
 484  * @vcpu:       Virtual CPU.
 485  * @count_hz:   Frequency of CP0_Count timer in Hz.
 486  *
 487  * Change the frequency of the CP0_Count timer. This is done atomically so that
 488  * CP0_Count is continuous and no timer interrupt is lost.
 489  *
 490  * Returns:     -EINVAL if @count_hz is out of range.
 491  *              0 on success.
 492  */
 493 int kvm_mips_set_count_hz(struct kvm_vcpu *vcpu, s64 count_hz)
 494 {
 495         struct mips_coproc *cop0 = vcpu->arch.cop0;
 496         int dc;
 497         ktime_t now;
 498         u32 count;
 499
 500         /* ensure the frequency is in a sensible range... */
 501         if (count_hz <= 0 || count_hz > NSEC_PER_SEC)
 502                 return -EINVAL;
 503         /* ... and has actually changed */
 504         if (vcpu->arch.count_hz == count_hz)
 505                 return 0;
 506
 507         /* Safely freeze timer so we can keep it continuous */
 508         dc = kvm_mips_count_disabled(vcpu);
 509         if (dc) {
 510                 now = kvm_mips_count_time(vcpu);
 511                 count = kvm_read_c0_guest_count(cop0);
 512         } else {
 513                 now = kvm_mips_freeze_hrtimer(vcpu, &count);
 514         }
 515
 516         /* Update the frequency */
 517         vcpu->arch.count_hz = count_hz;
 518         vcpu->arch.count_period = div_u64((u64)NSEC_PER_SEC << 32, count_hz);
 519         vcpu->arch.count_dyn_bias = 0;
 520
 521         /* Calculate adjusted bias so dynamic count is unchanged */
 522         vcpu->arch.count_bias = count - kvm_mips_ktime_to_count(vcpu, now);
 523
 524         /* Update and resume hrtimer */
 525         if (!dc)
 526                 kvm_mips_resume_hrtimer(vcpu, now, count);
 527         return 0;
 528 }
 529
 530 /**
 531  * kvm_mips_write_compare() - Modify compare and update timer.
 532  * @vcpu:       Virtual CPU.
 533  * @compare:    New CP0_Compare value.
 534  * @ack:        Whether to acknowledge timer interrupt.
 535  *
 536  * Update CP0_Compare to a new value and update the timeout.
 537  * If @ack, atomically acknowledge any pending timer interrupt, otherwise ensure
 538  * any pending timer interrupt is preserved.
 539  */
 540 void kvm_mips_write_compare(struct kvm_vcpu *vcpu, u32 compare, bool ack)
 541 {
 542         struct mips_coproc *cop0 = vcpu->arch.cop0;
 543         int dc;
 544         u32 old_compare = kvm_read_c0_guest_compare(cop0);
 545         ktime_t now;
 546         u32 count;
 547
 548         /* if unchanged, must just be an ack */
 549         if (old_compare == compare) {
 550                 if (!ack)
 551                         return;
 552                 kvm_mips_callbacks->dequeue_timer_int(vcpu);
 553                 kvm_write_c0_guest_compare(cop0, compare);
 554                 return;
 555         }
 556
 557         /* freeze_hrtimer() takes care of timer interrupts <= count */
 558         dc = kvm_mips_count_disabled(vcpu);
 559         if (!dc)
 560                 now = kvm_mips_freeze_hrtimer(vcpu, &count);
 561
 562         if (ack)
 563                 kvm_mips_callbacks->dequeue_timer_int(vcpu);
 564
 565         kvm_write_c0_guest_compare(cop0, compare);
 566
 567         /* resume_hrtimer() takes care of timer interrupts > count */
 568         if (!dc)
 569                 kvm_mips_resume_hrtimer(vcpu, now, count);
 570 }
 571
 572 /**
 573  * kvm_mips_count_disable() - Disable count.
 574  * @vcpu:       Virtual CPU.
 575  *
 576  * Disable the CP0_Count timer. A timer interrupt on or before the final stop
 577  * time will be handled but not after.
 578  *
 579  * Assumes CP0_Count was previously enabled but now Guest.CP0_Cause.DC or
 580  * count_ctl.DC has been set (count disabled).
 581  *
 582  * Returns:     The time that the timer was stopped.
 583  */
 584 static ktime_t kvm_mips_count_disable(struct kvm_vcpu *vcpu)
 585 {
 586         struct mips_coproc *cop0 = vcpu->arch.cop0;
 587         u32 count;
 588         ktime_t now;
 589
 590         /* Stop hrtimer */
 591         hrtimer_cancel(&vcpu->arch.comparecount_timer);
 592
 593         /* Set the static count from the dynamic count, handling pending TI */
 594         now = ktime_get();
 595         count = kvm_mips_read_count_running(vcpu, now);
 596         kvm_write_c0_guest_count(cop0, count);
 597
 598         return now;
 599 }
 600
 601 /**
 602  * kvm_mips_count_disable_cause() - Disable count using CP0_Cause.DC.
 603  * @vcpu:       Virtual CPU.
 604  *
 605  * Disable the CP0_Count timer and set CP0_Cause.DC. A timer interrupt on or
 606  * before the final stop time will be handled if the timer isn't disabled by
 607  * count_ctl.DC, but not after.
 608  *
 609  * Assumes CP0_Cause.DC is clear (count enabled).
 610  */
 611 void kvm_mips_count_disable_cause(struct kvm_vcpu *vcpu)
 612 {
 613         struct mips_coproc *cop0 = vcpu->arch.cop0;
 614
 615         kvm_set_c0_guest_cause(cop0, CAUSEF_DC);
 616         if (!(vcpu->arch.count_ctl & KVM_REG_MIPS_COUNT_CTL_DC))
 617                 kvm_mips_count_disable(vcpu);
 618 }
 619
 620 /**
 621  * kvm_mips_count_enable_cause() - Enable count using CP0_Cause.DC.
 622  * @vcpu:       Virtual CPU.
 623  *
 624  * Enable the CP0_Count timer and clear CP0_Cause.DC. A timer interrupt after
 625  * the start time will be handled if the timer isn't disabled by count_ctl.DC,
 626  * potentially before even returning, so the caller should be careful with
 627  * ordering of CP0_Cause modifications so as not to lose it.
 628  *
 629  * Assumes CP0_Cause.DC is set (count disabled).
 630  */
 631 void kvm_mips_count_enable_cause(struct kvm_vcpu *vcpu)
 632 {
 633         struct mips_coproc *cop0 = vcpu->arch.cop0;
 634         u32 count;
 635
 636         kvm_clear_c0_guest_cause(cop0, CAUSEF_DC);
 637
 638         /*
 639          * Set the dynamic count to match the static count.
 640          * This starts the hrtimer if count_ctl.DC allows it.
 641          * Otherwise it conveniently updates the biases.
 642          */
 643         count = kvm_read_c0_guest_count(cop0);
 644         kvm_mips_write_count(vcpu, count);
 645 }
 646
 647 /**
 648  * kvm_mips_set_count_ctl() - Update the count control KVM register.
 649  * @vcpu:       Virtual CPU.
 650  * @count_ctl:  Count control register new value.
 651  *
 652  * Set the count control KVM register. The timer is updated accordingly.
 653  *
 654  * Returns:     -EINVAL if reserved bits are set.
 655  *              0 on success.
 656  */
 657 int kvm_mips_set_count_ctl(struct kvm_vcpu *vcpu, s64 count_ctl)
 658 {
 659         struct mips_coproc *cop0 = vcpu->arch.cop0;
 660         s64 changed = count_ctl ^ vcpu->arch.count_ctl;
 661         s64 delta;
 662         ktime_t expire, now;
 663         u32 count, compare;
 664
 665         /* Only allow defined bits to be changed */
 666         if (changed & ~(s64)(KVM_REG_MIPS_COUNT_CTL_DC))
 667                 return -EINVAL;
 668
 669         /* Apply new value */
 670         vcpu->arch.count_ctl = count_ctl;
 671
 672         /* Master CP0_Count disable */
 673         if (changed & KVM_REG_MIPS_COUNT_CTL_DC) {
 674                 /* Is CP0_Cause.DC already disabling CP0_Count? */
 675                 if (kvm_read_c0_guest_cause(cop0) & CAUSEF_DC) {
 676                         if (count_ctl & KVM_REG_MIPS_COUNT_CTL_DC)
 677                                 /* Just record the current time */
 678                                 vcpu->arch.count_resume = ktime_get();
 679                 } else if (count_ctl & KVM_REG_MIPS_COUNT_CTL_DC) {
 680                         /* disable timer and record current time */
 681                         vcpu->arch.count_resume = kvm_mips_count_disable(vcpu);
 682                 } else {
 683                         /*
 684                          * Calculate timeout relative to static count at resume
 685                          * time (wrap 0 to 2^32).
 686                          */
 687                         count = kvm_read_c0_guest_count(cop0);
 688                         compare = kvm_read_c0_guest_compare(cop0);
 689                         delta = (u64)(u32)(compare - count - 1) + 1;
 690                         delta = div_u64(delta * NSEC_PER_SEC,
 691                                         vcpu->arch.count_hz);
 692                         expire = ktime_add_ns(vcpu->arch.count_resume, delta);
 693
 694                         /* Handle pending interrupt */
 695                         now = ktime_get();
 696                         if (ktime_compare(now, expire) >= 0)
 697                                 /* Nothing should be waiting on the timeout */
 698                                 kvm_mips_callbacks->queue_timer_int(vcpu);
 699
 700                         /* Resume hrtimer without changing bias */
 701                         count = kvm_mips_read_count_running(vcpu, now);
 702                         kvm_mips_resume_hrtimer(vcpu, now, count);
 703                 }
 704         }
 705
 706         return 0;
 707 }
 708
 709 /**
 710  * kvm_mips_set_count_resume() - Update the count resume KVM register.
 711  * @vcpu:               Virtual CPU.
 712  * @count_resume:       Count resume register new value.
 713  *
 714  * Set the count resume KVM register.
 715  *
 716  * Returns:     -EINVAL if out of valid range (0..now).
 717  *              0 on success.
 718  */
 719 int kvm_mips_set_count_resume(struct kvm_vcpu *vcpu, s64 count_resume)
 720 {
 721         /*
 722          * It doesn't make sense for the resume time to be in the future, as it
 723          * would be possible for the next interrupt to be more than a full
 724          * period in the future.
 725          */
 726         if (count_resume < 0 || count_resume > ktime_to_ns(ktime_get()))
 727                 return -EINVAL;
 728
 729         vcpu->arch.count_resume = ns_to_ktime(count_resume);
 730         return 0;
 731 }
 732
 733 /**
 734  * kvm_mips_count_timeout() - Push timer forward on timeout.
 735  * @vcpu:       Virtual CPU.
 736  *
 737  * Handle an hrtimer event by push the hrtimer forward a period.
 738  *
 739  * Returns:     The hrtimer_restart value to return to the hrtimer subsystem.
 740  */
 741 enum hrtimer_restart kvm_mips_count_timeout(struct kvm_vcpu *vcpu)
 742 {
 743         /* Add the Count period to the current expiry time */
 744         hrtimer_add_expires_ns(&vcpu->arch.comparecount_timer,
 745                                vcpu->arch.count_period);
 746         return HRTIMER_RESTART;
 747 }
 748
 749 enum emulation_result kvm_mips_emul_eret(struct kvm_vcpu *vcpu)
 750 {
 751         struct mips_coproc *cop0 = vcpu->arch.cop0;
 752         enum emulation_result er = EMULATE_DONE;
 753
 754         if (kvm_read_c0_guest_status(cop0) & ST0_EXL) {
 755                 kvm_debug("[%#lx] ERET to %#lx\n", vcpu->arch.pc,
 756                           kvm_read_c0_guest_epc(cop0));
 757                 kvm_clear_c0_guest_status(cop0, ST0_EXL);
 758                 vcpu->arch.pc = kvm_read_c0_guest_epc(cop0);
 759
 760         } else if (kvm_read_c0_guest_status(cop0) & ST0_ERL) {
 761                 kvm_clear_c0_guest_status(cop0, ST0_ERL);
 762                 vcpu->arch.pc = kvm_read_c0_guest_errorepc(cop0);
 763         } else {
 764                 kvm_err("[%#lx] ERET when MIPS_SR_EXL|MIPS_SR_ERL == 0\n",
 765                         vcpu->arch.pc);
 766                 er = EMULATE_FAIL;
 767         }
 768
 769         return er;
 770 }
 771
 772 enum emulation_result kvm_mips_emul_wait(struct kvm_vcpu *vcpu)
 773 {
 774         kvm_debug("[%#lx] !!!WAIT!!! (%#lx)\n", vcpu->arch.pc,
 775                   vcpu->arch.pending_exceptions);
 776
 777         ++vcpu->stat.wait_exits;
 778         trace_kvm_exit(vcpu, KVM_TRACE_EXIT_WAIT);
 779         if (!vcpu->arch.pending_exceptions) {
 780                 vcpu->arch.wait = 1;
 781                 kvm_vcpu_block(vcpu);
 782
 783                 /*
 784                  * We we are runnable, then definitely go off to user space to
 785                  * check if any I/O interrupts are pending.
 786                  */
 787                 if (kvm_check_request(KVM_REQ_UNHALT, vcpu)) {
 788                         clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
 789                         vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
 790                 }
 791         }
 792
 793         return EMULATE_DONE;
 794 }
 795
 796 /*
 797  * XXXKYMA: Linux doesn't seem to use TLBR, return EMULATE_FAIL for now so that
 798  * we can catch this, if things ever change
 799  */
 800 enum emulation_result kvm_mips_emul_tlbr(struct kvm_vcpu *vcpu)
 801 {
 802         struct mips_coproc *cop0 = vcpu->arch.cop0;
 803         unsigned long pc = vcpu->arch.pc;
 804
 805         kvm_err("[%#lx] COP0_TLBR [%ld]\n", pc, kvm_read_c0_guest_index(cop0));
 806         return EMULATE_FAIL;
 807 }
 808
 809 /* Write Guest TLB Entry @ Index */
 810 enum emulation_result kvm_mips_emul_tlbwi(struct kvm_vcpu *vcpu)
 811 {
 812         struct mips_coproc *cop0 = vcpu->arch.cop0;
 813         int index = kvm_read_c0_guest_index(cop0);
 814         struct kvm_mips_tlb *tlb = NULL;
 815         unsigned long pc = vcpu->arch.pc;
 816
 817         if (index < 0 || index >= KVM_MIPS_GUEST_TLB_SIZE) {
 818                 kvm_debug("%s: illegal index: %d\n", __func__, index);
 819                 kvm_debug("[%#lx] COP0_TLBWI [%d] (entryhi: %#lx, entrylo0: %#lx entrylo1: %#lx, mask: %#lx)\n",
 820                           pc, index, kvm_read_c0_guest_entryhi(cop0),
 821                           kvm_read_c0_guest_entrylo0(cop0),
 822                           kvm_read_c0_guest_entrylo1(cop0),
 823                           kvm_read_c0_guest_pagemask(cop0));
 824                 index = (index & ~0x80000000) % KVM_MIPS_GUEST_TLB_SIZE;
 825         }
 826
 827         tlb = &vcpu->arch.guest_tlb[index];
 828         /*
 829          * Probe the shadow host TLB for the entry being overwritten, if one
 830          * matches, invalidate it
 831          */
 832         kvm_mips_host_tlb_inv(vcpu, tlb->tlb_hi);
 833
 834         tlb->tlb_mask = kvm_read_c0_guest_pagemask(cop0);
 835         tlb->tlb_hi = kvm_read_c0_guest_entryhi(cop0);
 836         tlb->tlb_lo[0] = kvm_read_c0_guest_entrylo0(cop0);
 837         tlb->tlb_lo[1] = kvm_read_c0_guest_entrylo1(cop0);
 838
 839         kvm_debug("[%#lx] COP0_TLBWI [%d] (entryhi: %#lx, entrylo0: %#lx entrylo1: %#lx, mask: %#lx)\n",
 840                   pc, index, kvm_read_c0_guest_entryhi(cop0),
 841                   kvm_read_c0_guest_entrylo0(cop0),
 842                   kvm_read_c0_guest_entrylo1(cop0),
 843                   kvm_read_c0_guest_pagemask(cop0));
 844
 845         return EMULATE_DONE;
 846 }
 847
 848 /* Write Guest TLB Entry @ Random Index */
 849 enum emulation_result kvm_mips_emul_tlbwr(struct kvm_vcpu *vcpu)
 850 {
 851         struct mips_coproc *cop0 = vcpu->arch.cop0;
 852         struct kvm_mips_tlb *tlb = NULL;
 853         unsigned long pc = vcpu->arch.pc;
 854         int index;
 855
 856         get_random_bytes(&index, sizeof(index));
 857         index &= (KVM_MIPS_GUEST_TLB_SIZE - 1);
 858
 859         tlb = &vcpu->arch.guest_tlb[index];
 860
 861         /*
 862          * Probe the shadow host TLB for the entry being overwritten, if one
 863          * matches, invalidate it
 864          */
 865         kvm_mips_host_tlb_inv(vcpu, tlb->tlb_hi);
 866
 867         tlb->tlb_mask = kvm_read_c0_guest_pagemask(cop0);
 868         tlb->tlb_hi = kvm_read_c0_guest_entryhi(cop0);
 869         tlb->tlb_lo[0] = kvm_read_c0_guest_entrylo0(cop0);
 870         tlb->tlb_lo[1] = kvm_read_c0_guest_entrylo1(cop0);
 871
 872         kvm_debug("[%#lx] COP0_TLBWR[%d] (entryhi: %#lx, entrylo0: %#lx entrylo1: %#lx)\n",
 873                   pc, index, kvm_read_c0_guest_entryhi(cop0),
 874                   kvm_read_c0_guest_entrylo0(cop0),
 875                   kvm_read_c0_guest_entrylo1(cop0));
 876
 877         return EMULATE_DONE;
 878 }
 879
 880 enum emulation_result kvm_mips_emul_tlbp(struct kvm_vcpu *vcpu)
 881 {
 882         struct mips_coproc *cop0 = vcpu->arch.cop0;
 883         long entryhi = kvm_read_c0_guest_entryhi(cop0);
 884         unsigned long pc = vcpu->arch.pc;
 885         int index = -1;
 886
 887         index = kvm_mips_guest_tlb_lookup(vcpu, entryhi);
 888
 889         kvm_write_c0_guest_index(cop0, index);
 890
 891         kvm_debug("[%#lx] COP0_TLBP (entryhi: %#lx), index: %d\n", pc, entryhi,
 892                   index);
 893
 894         return EMULATE_DONE;
 895 }
 896
 897 /**
 898  * kvm_mips_config1_wrmask() - Find mask of writable bits in guest Config1
 899  * @vcpu:       Virtual CPU.
 900  *
 901  * Finds the mask of bits which are writable in the guest's Config1 CP0
 902  * register, by userland (currently read-only to the guest).
 903  */
 904 unsigned int kvm_mips_config1_wrmask(struct kvm_vcpu *vcpu)
 905 {
 906         unsigned int mask = 0;
 907
 908         /* Permit FPU to be present if FPU is supported */
 909         if (kvm_mips_guest_can_have_fpu(&vcpu->arch))
 910                 mask |= MIPS_CONF1_FP;
 911
 912         return mask;
 913 }
 914
 915 /**
 916  * kvm_mips_config3_wrmask() - Find mask of writable bits in guest Config3
 917  * @vcpu:       Virtual CPU.
 918  *
 919  * Finds the mask of bits which are writable in the guest's Config3 CP0
 920  * register, by userland (currently read-only to the guest).
 921  */
 922 unsigned int kvm_mips_config3_wrmask(struct kvm_vcpu *vcpu)
 923 {
 924         /* Config4 is optional */
 925         unsigned int mask = MIPS_CONF_M;
 926
 927         /* Permit MSA to be present if MSA is supported */
 928         if (kvm_mips_guest_can_have_msa(&vcpu->arch))
 929                 mask |= MIPS_CONF3_MSA;
 930
 931         return mask;
 932 }
 933
 934 /**
 935  * kvm_mips_config4_wrmask() - Find mask of writable bits in guest Config4
 936  * @vcpu:       Virtual CPU.
 937  *
 938  * Finds the mask of bits which are writable in the guest's Config4 CP0
 939  * register, by userland (currently read-only to the guest).
 940  */
 941 unsigned int kvm_mips_config4_wrmask(struct kvm_vcpu *vcpu)
 942 {
 943         /* Config5 is optional */
 944         return MIPS_CONF_M;
 945 }
 946
 947 /**
 948  * kvm_mips_config5_wrmask() - Find mask of writable bits in guest Config5
 949  * @vcpu:       Virtual CPU.
 950  *
 951  * Finds the mask of bits which are writable in the guest's Config5 CP0
 952  * register, by the guest itself.
 953  */
 954 unsigned int kvm_mips_config5_wrmask(struct kvm_vcpu *vcpu)
 955 {
 956         unsigned int mask = 0;
 957
 958         /* Permit MSAEn changes if MSA supported and enabled */
 959         if (kvm_mips_guest_has_msa(&vcpu->arch))
 960                 mask |= MIPS_CONF5_MSAEN;
 961
 962         /*
 963          * Permit guest FPU mode changes if FPU is enabled and the relevant
 964          * feature exists according to FIR register.
 965          */
 966         if (kvm_mips_guest_has_fpu(&vcpu->arch)) {
 967                 if (cpu_has_fre)
 968                         mask |= MIPS_CONF5_FRE;
 969                 /* We don't support UFR or UFE */
 970         }
 971
 972         return mask;
 973 }
 974
 975 enum emulation_result kvm_mips_emulate_CP0(u32 inst, u32 *opc, u32 cause,
 976                                            struct kvm_run *run,
 977                                            struct kvm_vcpu *vcpu)
 978 {
 979         struct mips_coproc *cop0 = vcpu->arch.cop0;
 980         enum emulation_result er = EMULATE_DONE;
 981         u32 rt, rd, copz, sel, co_bit, op;
 982         unsigned long curr_pc;
 983
 984         /*
 985          * Update PC and hold onto current PC in case there is
 986          * an error and we want to rollback the PC
 987          */
 988         curr_pc = vcpu->arch.pc;
 989         er = update_pc(vcpu, cause);
 990         if (er == EMULATE_FAIL)
 991                 return er;
 992
 993         copz = (inst >> 21) & 0x1f;
 994         rt = (inst >> 16) & 0x1f;
 995         rd = (inst >> 11) & 0x1f;
 996         sel = inst & 0x7;
 997         co_bit = (inst >> 25) & 1;
 998
 999         if (co_bit) {
1000                 op = (inst) & 0xff;
1001
1002                 switch (op) {
1003                 case tlbr_op:   /*  Read indexed TLB entry  */
1004                         er = kvm_mips_emul_tlbr(vcpu);
1005                         break;
1006                 case tlbwi_op:  /*  Write indexed  */
1007                         er = kvm_mips_emul_tlbwi(vcpu);
1008                         break;
1009                 case tlbwr_op:  /*  Write random  */
1010                         er = kvm_mips_emul_tlbwr(vcpu);
1011                         break;
1012                 case tlbp_op:   /* TLB Probe */
1013                         er = kvm_mips_emul_tlbp(vcpu);
1014                         break;
1015                 case rfe_op:
1016                         kvm_err("!!!COP0_RFE!!!\n");
1017                         break;
1018                 case eret_op:
1019                         er = kvm_mips_emul_eret(vcpu);
1020                         goto dont_update_pc;
1021                         break;
1022                 case wait_op:
1023                         er = kvm_mips_emul_wait(vcpu);
1024                         break;
1025                 }
1026         } else {
1027                 switch (copz) {
1028                 case mfc_op:
1029 #ifdef CONFIG_KVM_MIPS_DEBUG_COP0_COUNTERS
1030                         cop0->stat[rd][sel]++;
1031 #endif
1032                         /* Get reg */
1033                         if ((rd == MIPS_CP0_COUNT) && (sel == 0)) {
1034                                 vcpu->arch.gprs[rt] = kvm_mips_read_count(vcpu);
1035                         } else if ((rd == MIPS_CP0_ERRCTL) && (sel == 0)) {
1036                                 vcpu->arch.gprs[rt] = 0x0;
1037 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1038                                 kvm_mips_trans_mfc0(inst, opc, vcpu);
1039 #endif
1040                         } else {
1041                                 vcpu->arch.gprs[rt] = cop0->reg[rd][sel];
1042
1043 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1044                                 kvm_mips_trans_mfc0(inst, opc, vcpu);
1045 #endif
1046                         }
1047
1048                         trace_kvm_hwr(vcpu, KVM_TRACE_MFC0,
1049                                       KVM_TRACE_COP0(rd, sel),
1050                                       vcpu->arch.gprs[rt]);
1051                         break;
1052
1053                 case dmfc_op:
1054                         vcpu->arch.gprs[rt] = cop0->reg[rd][sel];
1055
1056                         trace_kvm_hwr(vcpu, KVM_TRACE_DMFC0,
1057                                       KVM_TRACE_COP0(rd, sel),
1058                                       vcpu->arch.gprs[rt]);
1059                         break;
1060
1061                 case mtc_op:
1062 #ifdef CONFIG_KVM_MIPS_DEBUG_COP0_COUNTERS
1063                         cop0->stat[rd][sel]++;
1064 #endif
1065                         trace_kvm_hwr(vcpu, KVM_TRACE_MTC0,
1066                                       KVM_TRACE_COP0(rd, sel),
1067                                       vcpu->arch.gprs[rt]);
1068
1069                         if ((rd == MIPS_CP0_TLB_INDEX)
1070                             && (vcpu->arch.gprs[rt] >=
1071                                 KVM_MIPS_GUEST_TLB_SIZE)) {
1072                                 kvm_err("Invalid TLB Index: %ld",
1073                                         vcpu->arch.gprs[rt]);
1074                                 er = EMULATE_FAIL;
1075                                 break;
1076                         }
1077 #define C0_EBASE_CORE_MASK 0xff
1078                         if ((rd == MIPS_CP0_PRID) && (sel == 1)) {
1079                                 /* Preserve CORE number */
1080                                 kvm_change_c0_guest_ebase(cop0,
1081                                                           ~(C0_EBASE_CORE_MASK),
1082                                                           vcpu->arch.gprs[rt]);
1083                                 kvm_err("MTCz, cop0->reg[EBASE]: %#lx\n",
1084                                         kvm_read_c0_guest_ebase(cop0));
1085                         } else if (rd == MIPS_CP0_TLB_HI && sel == 0) {
1086                                 u32 nasid =
1087                                         vcpu->arch.gprs[rt] & KVM_ENTRYHI_ASID;
1088                                 if ((KSEGX(vcpu->arch.gprs[rt]) != CKSEG0) &&
1089                                     ((kvm_read_c0_guest_entryhi(cop0) &
1090                                       KVM_ENTRYHI_ASID) != nasid)) {
1091                                         trace_kvm_asid_change(vcpu,
1092                                                 kvm_read_c0_guest_entryhi(cop0)
1093                                                         & KVM_ENTRYHI_ASID,
1094                                                 nasid);
1095
1096                                         /* Blow away the shadow host TLBs */
1097                                         kvm_mips_flush_host_tlb(1);
1098                                 }
1099                                 kvm_write_c0_guest_entryhi(cop0,
1100                                                            vcpu->arch.gprs[rt]);
1101                         }
1102                         /* Are we writing to COUNT */
1103                         else if ((rd == MIPS_CP0_COUNT) && (sel == 0)) {
1104                                 kvm_mips_write_count(vcpu, vcpu->arch.gprs[rt]);
1105                                 goto done;
1106                         } else if ((rd == MIPS_CP0_COMPARE) && (sel == 0)) {
1107                                 /* If we are writing to COMPARE */
1108                                 /* Clear pending timer interrupt, if any */
1109                                 kvm_mips_write_compare(vcpu,
1110                                                        vcpu->arch.gprs[rt],
1111                                                        true);
1112                         } else if ((rd == MIPS_CP0_STATUS) && (sel == 0)) {
1113                                 unsigned int old_val, val, change;
1114
1115                                 old_val = kvm_read_c0_guest_status(cop0);
1116                                 val = vcpu->arch.gprs[rt];
1117                                 change = val ^ old_val;
1118
1119                                 /* Make sure that the NMI bit is never set */
1120                                 val &= ~ST0_NMI;
1121
1122                                 /*
1123                                  * Don't allow CU1 or FR to be set unless FPU
1124                                  * capability enabled and exists in guest
1125                                  * configuration.
1126                                  */
1127                                 if (!kvm_mips_guest_has_fpu(&vcpu->arch))
1128                                         val &= ~(ST0_CU1 | ST0_FR);
1129
1130                                 /*
1131                                  * Also don't allow FR to be set if host doesn't
1132                                  * support it.
1133                                  */
1134                                 if (!(current_cpu_data.fpu_id & MIPS_FPIR_F64))
1135                                         val &= ~ST0_FR;
1136
1137
1138                                 /* Handle changes in FPU mode */
1139                                 preempt_disable();
1140
1141                                 /*
1142                                  * FPU and Vector register state is made
1143                                  * UNPREDICTABLE by a change of FR, so don't
1144                                  * even bother saving it.
1145                                  */
1146                                 if (change & ST0_FR)
1147                                         kvm_drop_fpu(vcpu);
1148
1149                                 /*
1150                                  * If MSA state is already live, it is undefined
1151                                  * how it interacts with FR=0 FPU state, and we
1152                                  * don't want to hit reserved instruction
1153                                  * exceptions trying to save the MSA state later
1154                                  * when CU=1 && FR=1, so play it safe and save
1155                                  * it first.
1156                                  */
1157                                 if (change & ST0_CU1 && !(val & ST0_FR) &&
1158                                     vcpu->arch.aux_inuse & KVM_MIPS_AUX_MSA)
1159                                         kvm_lose_fpu(vcpu);
1160
1161                                 /*
1162                                  * Propagate CU1 (FPU enable) changes
1163                                  * immediately if the FPU context is already
1164                                  * loaded. When disabling we leave the context
1165                                  * loaded so it can be quickly enabled again in
1166                                  * the near future.
1167                                  */
1168                                 if (change & ST0_CU1 &&
1169                                     vcpu->arch.aux_inuse & KVM_MIPS_AUX_FPU)
1170                                         change_c0_status(ST0_CU1, val);
1171
1172                                 preempt_enable();
1173
1174                                 kvm_write_c0_guest_status(cop0, val);
1175
1176 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1177                                 /*
1178                                  * If FPU present, we need CU1/FR bits to take
1179                                  * effect fairly soon.
1180                                  */
1181                                 if (!kvm_mips_guest_has_fpu(&vcpu->arch))
1182                                         kvm_mips_trans_mtc0(inst, opc, vcpu);
1183 #endif
1184                         } else if ((rd == MIPS_CP0_CONFIG) && (sel == 5)) {
1185                                 unsigned int old_val, val, change, wrmask;
1186
1187                                 old_val = kvm_read_c0_guest_config5(cop0);
1188                                 val = vcpu->arch.gprs[rt];
1189
1190                                 /* Only a few bits are writable in Config5 */
1191                                 wrmask = kvm_mips_config5_wrmask(vcpu);
1192                                 change = (val ^ old_val) & wrmask;
1193                                 val = old_val ^ change;
1194
1195
1196                                 /* Handle changes in FPU/MSA modes */
1197                                 preempt_disable();
1198
1199                                 /*
1200                                  * Propagate FRE changes immediately if the FPU
1201                                  * context is already loaded.
1202                                  */
1203                                 if (change & MIPS_CONF5_FRE &&
1204                                     vcpu->arch.aux_inuse & KVM_MIPS_AUX_FPU)
1205                                         change_c0_config5(MIPS_CONF5_FRE, val);
1206
1207                                 /*
1208                                  * Propagate MSAEn changes immediately if the
1209                                  * MSA context is already loaded. When disabling
1210                                  * we leave the context loaded so it can be
1211                                  * quickly enabled again in the near future.
1212                                  */
1213                                 if (change & MIPS_CONF5_MSAEN &&
1214                                     vcpu->arch.aux_inuse & KVM_MIPS_AUX_MSA)
1215                                         change_c0_config5(MIPS_CONF5_MSAEN,
1216                                                           val);
1217
1218                                 preempt_enable();
1219
1220                                 kvm_write_c0_guest_config5(cop0, val);
1221                         } else if ((rd == MIPS_CP0_CAUSE) && (sel == 0)) {
1222                                 u32 old_cause, new_cause;
1223
1224                                 old_cause = kvm_read_c0_guest_cause(cop0);
1225                                 new_cause = vcpu->arch.gprs[rt];
1226                                 /* Update R/W bits */
1227                                 kvm_change_c0_guest_cause(cop0, 0x08800300,
1228                                                           new_cause);
1229                                 /* DC bit enabling/disabling timer? */
1230                                 if ((old_cause ^ new_cause) & CAUSEF_DC) {
1231                                         if (new_cause & CAUSEF_DC)
1232                                                 kvm_mips_count_disable_cause(vcpu);
1233                                         else
1234                                                 kvm_mips_count_enable_cause(vcpu);
1235                                 }
1236                         } else {
1237                                 cop0->reg[rd][sel] = vcpu->arch.gprs[rt];
1238 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1239                                 kvm_mips_trans_mtc0(inst, opc, vcpu);
1240 #endif
1241                         }
1242                         break;
1243
1244                 case dmtc_op:
1245                         kvm_err("!!!!!!![%#lx]dmtc_op: rt: %d, rd: %d, sel: %d!!!!!!\n",
1246                                 vcpu->arch.pc, rt, rd, sel);
1247                         trace_kvm_hwr(vcpu, KVM_TRACE_DMTC0,
1248                                       KVM_TRACE_COP0(rd, sel),
1249                                       vcpu->arch.gprs[rt]);
1250                         er = EMULATE_FAIL;
1251                         break;
1252
1253                 case mfmc0_op:
1254 #ifdef KVM_MIPS_DEBUG_COP0_COUNTERS
1255                         cop0->stat[MIPS_CP0_STATUS][0]++;
1256 #endif
1257                         if (rt != 0)
1258                                 vcpu->arch.gprs[rt] =
1259                                     kvm_read_c0_guest_status(cop0);
1260                         /* EI */
1261                         if (inst & 0x20) {
1262                                 kvm_debug("[%#lx] mfmc0_op: EI\n",
1263                                           vcpu->arch.pc);
1264                                 kvm_set_c0_guest_status(cop0, ST0_IE);
1265                         } else {
1266                                 kvm_debug("[%#lx] mfmc0_op: DI\n",
1267                                           vcpu->arch.pc);
1268                                 kvm_clear_c0_guest_status(cop0, ST0_IE);
1269                         }
1270
1271                         break;
1272
1273                 case wrpgpr_op:
1274                         {
1275                                 u32 css = cop0->reg[MIPS_CP0_STATUS][2] & 0xf;
1276                                 u32 pss =
1277                                     (cop0->reg[MIPS_CP0_STATUS][2] >> 6) & 0xf;
1278                                 /*
1279                                  * We don't support any shadow register sets, so
1280                                  * SRSCtl[PSS] == SRSCtl[CSS] = 0
1281                                  */
1282                                 if (css || pss) {
1283                                         er = EMULATE_FAIL;
1284                                         break;
1285                                 }
1286                                 kvm_debug("WRPGPR[%d][%d] = %#lx\n", pss, rd,
1287                                           vcpu->arch.gprs[rt]);
1288                                 vcpu->arch.gprs[rd] = vcpu->arch.gprs[rt];
1289                         }
1290                         break;
1291                 default:
1292                         kvm_err("[%#lx]MachEmulateCP0: unsupported COP0, copz: 0x%x\n",
1293                                 vcpu->arch.pc, copz);
1294                         er = EMULATE_FAIL;
1295                         break;
1296                 }
1297         }
1298
1299 done:
1300         /* Rollback PC only if emulation was unsuccessful */
1301         if (er == EMULATE_FAIL)
1302                 vcpu->arch.pc = curr_pc;
1303
1304 dont_update_pc:
1305         /*
1306          * This is for special instructions whose emulation
1307          * updates the PC, so do not overwrite the PC under
1308          * any circumstances
1309          */
1310
1311         return er;
1312 }
1313
1314 enum emulation_result kvm_mips_emulate_store(u32 inst, u32 cause,
1315                                              struct kvm_run *run,
1316                                              struct kvm_vcpu *vcpu)
1317 {
1318         enum emulation_result er = EMULATE_DO_MMIO;
1319         u32 op, base, rt;
1320         s16 offset;
1321         u32 bytes;
1322         void *data = run->mmio.data;
1323         unsigned long curr_pc;
1324
1325         /*
1326          * Update PC and hold onto current PC in case there is
1327          * an error and we want to rollback the PC
1328          */
1329         curr_pc = vcpu->arch.pc;
1330         er = update_pc(vcpu, cause);
1331         if (er == EMULATE_FAIL)
1332                 return er;
1333
1334         rt = (inst >> 16) & 0x1f;
1335         base = (inst >> 21) & 0x1f;
1336         offset = (s16)inst;
1337         op = (inst >> 26) & 0x3f;
1338
1339         switch (op) {
1340         case sb_op:
1341                 bytes = 1;
1342                 if (bytes > sizeof(run->mmio.data)) {
1343                         kvm_err("%s: bad MMIO length: %d\n", __func__,
1344                                run->mmio.len);
1345                 }
1346                 run->mmio.phys_addr =
1347                     kvm_mips_callbacks->gva_to_gpa(vcpu->arch.
1348                                                    host_cp0_badvaddr);
1349                 if (run->mmio.phys_addr == KVM_INVALID_ADDR) {
1350                         er = EMULATE_FAIL;
1351                         break;
1352                 }
1353                 run->mmio.len = bytes;
1354                 run->mmio.is_write = 1;
1355                 vcpu->mmio_needed = 1;
1356                 vcpu->mmio_is_write = 1;
1357                 *(u8 *) data = vcpu->arch.gprs[rt];
1358                 kvm_debug("OP_SB: eaddr: %#lx, gpr: %#lx, data: %#x\n",
1359                           vcpu->arch.host_cp0_badvaddr, vcpu->arch.gprs[rt],
1360                           *(u8 *) data);
1361
1362                 break;
1363
1364         case sw_op:
1365                 bytes = 4;
1366                 if (bytes > sizeof(run->mmio.data)) {
1367                         kvm_err("%s: bad MMIO length: %d\n", __func__,
1368                                run->mmio.len);
1369                 }
1370                 run->mmio.phys_addr =
1371                     kvm_mips_callbacks->gva_to_gpa(vcpu->arch.
1372                                                    host_cp0_badvaddr);
1373                 if (run->mmio.phys_addr == KVM_INVALID_ADDR) {
1374                         er = EMULATE_FAIL;
1375                         break;
1376                 }
1377
1378                 run->mmio.len = bytes;
1379                 run->mmio.is_write = 1;
1380                 vcpu->mmio_needed = 1;
1381                 vcpu->mmio_is_write = 1;
1382                 *(u32 *) data = vcpu->arch.gprs[rt];
1383
1384                 kvm_debug("[%#lx] OP_SW: eaddr: %#lx, gpr: %#lx, data: %#x\n",
1385                           vcpu->arch.pc, vcpu->arch.host_cp0_badvaddr,
1386                           vcpu->arch.gprs[rt], *(u32 *) data);
1387                 break;
1388
1389         case sh_op:
1390                 bytes = 2;
1391                 if (bytes > sizeof(run->mmio.data)) {
1392                         kvm_err("%s: bad MMIO length: %d\n", __func__,
1393                                run->mmio.len);
1394                 }
1395                 run->mmio.phys_addr =
1396                     kvm_mips_callbacks->gva_to_gpa(vcpu->arch.
1397                                                    host_cp0_badvaddr);
1398                 if (run->mmio.phys_addr == KVM_INVALID_ADDR) {
1399                         er = EMULATE_FAIL;
1400                         break;
1401                 }
1402
1403                 run->mmio.len = bytes;
1404                 run->mmio.is_write = 1;
1405                 vcpu->mmio_needed = 1;
1406                 vcpu->mmio_is_write = 1;
1407                 *(u16 *) data = vcpu->arch.gprs[rt];
1408
1409                 kvm_debug("[%#lx] OP_SH: eaddr: %#lx, gpr: %#lx, data: %#x\n",
1410                           vcpu->arch.pc, vcpu->arch.host_cp0_badvaddr,
1411                           vcpu->arch.gprs[rt], *(u32 *) data);
1412                 break;
1413
1414         default:
1415                 kvm_err("Store not yet supported");
1416                 er = EMULATE_FAIL;
1417                 break;
1418         }
1419
1420         /* Rollback PC if emulation was unsuccessful */
1421         if (er == EMULATE_FAIL)
1422                 vcpu->arch.pc = curr_pc;
1423
1424         return er;
1425 }
1426
1427 enum emulation_result kvm_mips_emulate_load(u32 inst, u32 cause,
1428                                             struct kvm_run *run,
1429                                             struct kvm_vcpu *vcpu)
1430 {
1431         enum emulation_result er = EMULATE_DO_MMIO;
1432         u32 op, base, rt;
1433         s16 offset;
1434         u32 bytes;
1435
1436         rt = (inst >> 16) & 0x1f;
1437         base = (inst >> 21) & 0x1f;
1438         offset = (s16)inst;
1439         op = (inst >> 26) & 0x3f;
1440
1441         vcpu->arch.pending_load_cause = cause;
1442         vcpu->arch.io_gpr = rt;
1443
1444         switch (op) {
1445         case lw_op:
1446                 bytes = 4;
1447                 if (bytes > sizeof(run->mmio.data)) {
1448                         kvm_err("%s: bad MMIO length: %d\n", __func__,
1449                                run->mmio.len);
1450                         er = EMULATE_FAIL;
1451                         break;
1452                 }
1453                 run->mmio.phys_addr =
1454                     kvm_mips_callbacks->gva_to_gpa(vcpu->arch.
1455                                                    host_cp0_badvaddr);
1456                 if (run->mmio.phys_addr == KVM_INVALID_ADDR) {
1457                         er = EMULATE_FAIL;
1458                         break;
1459                 }
1460
1461                 run->mmio.len = bytes;
1462                 run->mmio.is_write = 0;
1463                 vcpu->mmio_needed = 1;
1464                 vcpu->mmio_is_write = 0;
1465                 break;
1466
1467         case lh_op:
1468         case lhu_op:
1469                 bytes = 2;
1470                 if (bytes > sizeof(run->mmio.data)) {
1471                         kvm_err("%s: bad MMIO length: %d\n", __func__,
1472                                run->mmio.len);
1473                         er = EMULATE_FAIL;
1474                         break;
1475                 }
1476                 run->mmio.phys_addr =
1477                     kvm_mips_callbacks->gva_to_gpa(vcpu->arch.
1478                                                    host_cp0_badvaddr);
1479                 if (run->mmio.phys_addr == KVM_INVALID_ADDR) {
1480                         er = EMULATE_FAIL;
1481                         break;
1482                 }
1483
1484                 run->mmio.len = bytes;
1485                 run->mmio.is_write = 0;
1486                 vcpu->mmio_needed = 1;
1487                 vcpu->mmio_is_write = 0;
1488
1489                 if (op == lh_op)
1490                         vcpu->mmio_needed = 2;
1491                 else
1492                         vcpu->mmio_needed = 1;
1493
1494                 break;
1495
1496         case lbu_op:
1497         case lb_op:
1498                 bytes = 1;
1499                 if (bytes > sizeof(run->mmio.data)) {
1500                         kvm_err("%s: bad MMIO length: %d\n", __func__,
1501                                run->mmio.len);
1502                         er = EMULATE_FAIL;
1503                         break;
1504                 }
1505                 run->mmio.phys_addr =
1506                     kvm_mips_callbacks->gva_to_gpa(vcpu->arch.
1507                                                    host_cp0_badvaddr);
1508                 if (run->mmio.phys_addr == KVM_INVALID_ADDR) {
1509                         er = EMULATE_FAIL;
1510                         break;
1511                 }
1512
1513                 run->mmio.len = bytes;
1514                 run->mmio.is_write = 0;
1515                 vcpu->mmio_is_write = 0;
1516
1517                 if (op == lb_op)
1518                         vcpu->mmio_needed = 2;
1519                 else
1520                         vcpu->mmio_needed = 1;
1521
1522                 break;
1523
1524         default:
1525                 kvm_err("Load not yet supported");
1526                 er = EMULATE_FAIL;
1527                 break;
1528         }
1529
1530         return er;
1531 }
1532
1533 enum emulation_result kvm_mips_emulate_cache(u32 inst, u32 *opc,
1534                                              u32 cause,
1535                                              struct kvm_run *run,
1536                                              struct kvm_vcpu *vcpu)
1537 {
1538         struct mips_coproc *cop0 = vcpu->arch.cop0;
1539         enum emulation_result er = EMULATE_DONE;
1540         u32 cache, op_inst, op, base;
1541         s16 offset;
1542         struct kvm_vcpu_arch *arch = &vcpu->arch;
1543         unsigned long va;
1544         unsigned long curr_pc;
1545
1546         /*
1547          * Update PC and hold onto current PC in case there is
1548          * an error and we want to rollback the PC
1549          */
1550         curr_pc = vcpu->arch.pc;
1551         er = update_pc(vcpu, cause);
1552         if (er == EMULATE_FAIL)
1553                 return er;
1554
1555         base = (inst >> 21) & 0x1f;
1556         op_inst = (inst >> 16) & 0x1f;
1557         offset = (s16)inst;
1558         cache = op_inst & CacheOp_Cache;
1559         op = op_inst & CacheOp_Op;
1560
1561         va = arch->gprs[base] + offset;
1562
1563         kvm_debug("CACHE (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n",
1564                   cache, op, base, arch->gprs[base], offset);
1565
1566         /*
1567          * Treat INDEX_INV as a nop, basically issued by Linux on startup to
1568          * invalidate the caches entirely by stepping through all the
1569          * ways/indexes
1570          */
1571         if (op == Index_Writeback_Inv) {
1572                 kvm_debug("@ %#lx/%#lx CACHE (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n",
1573                           vcpu->arch.pc, vcpu->arch.gprs[31], cache, op, base,
1574                           arch->gprs[base], offset);
1575
1576                 if (cache == Cache_D)
1577                         r4k_blast_dcache();
1578                 else if (cache == Cache_I)
1579                         r4k_blast_icache();
1580                 else {
1581                         kvm_err("%s: unsupported CACHE INDEX operation\n",
1582                                 __func__);
1583                         return EMULATE_FAIL;
1584                 }
1585
1586 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1587                 kvm_mips_trans_cache_index(inst, opc, vcpu);
1588 #endif
1589                 goto done;
1590         }
1591
1592         preempt_disable();
1593         if (KVM_GUEST_KSEGX(va) == KVM_GUEST_KSEG0) {
1594                 if (kvm_mips_host_tlb_lookup(vcpu, va) < 0)
1595                         kvm_mips_handle_kseg0_tlb_fault(va, vcpu);
1596         } else if ((KVM_GUEST_KSEGX(va) < KVM_GUEST_KSEG0) ||
1597                    KVM_GUEST_KSEGX(va) == KVM_GUEST_KSEG23) {
1598                 int index;
1599
1600                 /* If an entry already exists then skip */
1601                 if (kvm_mips_host_tlb_lookup(vcpu, va) >= 0)
1602                         goto skip_fault;
1603
1604                 /*
1605                  * If address not in the guest TLB, then give the guest a fault,
1606                  * the resulting handler will do the right thing
1607                  */
1608                 index = kvm_mips_guest_tlb_lookup(vcpu, (va & VPN2_MASK) |
1609                                                   (kvm_read_c0_guest_entryhi
1610                                                    (cop0) & KVM_ENTRYHI_ASID));
1611
1612                 if (index < 0) {
1613                         vcpu->arch.host_cp0_badvaddr = va;
1614                         vcpu->arch.pc = curr_pc;
1615                         er = kvm_mips_emulate_tlbmiss_ld(cause, NULL, run,
1616                                                          vcpu);
1617                         preempt_enable();
1618                         goto dont_update_pc;
1619                 } else {
1620                         struct kvm_mips_tlb *tlb = &vcpu->arch.guest_tlb[index];
1621                         /*
1622                          * Check if the entry is valid, if not then setup a TLB
1623                          * invalid exception to the guest
1624                          */
1625                         if (!TLB_IS_VALID(*tlb, va)) {
1626                                 vcpu->arch.host_cp0_badvaddr = va;
1627                                 vcpu->arch.pc = curr_pc;
1628                                 er = kvm_mips_emulate_tlbinv_ld(cause, NULL,
1629                                                                 run, vcpu);
1630                                 preempt_enable();
1631                                 goto dont_update_pc;
1632                         } else {
1633                                 /*
1634                                  * We fault an entry from the guest tlb to the
1635                                  * shadow host TLB
1636                                  */
1637                                 kvm_mips_handle_mapped_seg_tlb_fault(vcpu, tlb);
1638                         }
1639                 }
1640         } else {
1641                 kvm_err("INVALID CACHE INDEX/ADDRESS (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n",
1642                         cache, op, base, arch->gprs[base], offset);
1643                 er = EMULATE_FAIL;
1644                 preempt_enable();
1645                 goto done;
1646
1647         }
1648
1649 skip_fault:
1650         /* XXXKYMA: Only a subset of cache ops are supported, used by Linux */
1651         if (op_inst == Hit_Writeback_Inv_D || op_inst == Hit_Invalidate_D) {
1652                 flush_dcache_line(va);
1653
1654 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1655                 /*
1656                  * Replace the CACHE instruction, with a SYNCI, not the same,
1657                  * but avoids a trap
1658                  */
1659                 kvm_mips_trans_cache_va(inst, opc, vcpu);
1660 #endif
1661         } else if (op_inst == Hit_Invalidate_I) {
1662                 flush_dcache_line(va);
1663                 flush_icache_line(va);
1664
1665 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1666                 /* Replace the CACHE instruction, with a SYNCI */
1667                 kvm_mips_trans_cache_va(inst, opc, vcpu);
1668 #endif
1669         } else {
1670                 kvm_err("NO-OP CACHE (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n",
1671                         cache, op, base, arch->gprs[base], offset);
1672                 er = EMULATE_FAIL;
1673         }
1674
1675         preempt_enable();
1676 done:
1677         /* Rollback PC only if emulation was unsuccessful */
1678         if (er == EMULATE_FAIL)
1679                 vcpu->arch.pc = curr_pc;
1680
1681 dont_update_pc:
1682         /*
1683          * This is for exceptions whose emulation updates the PC, so do not
1684          * overwrite the PC under any circumstances
1685          */
1686
1687         return er;
1688 }
1689
1690 enum emulation_result kvm_mips_emulate_inst(u32 cause, u32 *opc,
1691                                             struct kvm_run *run,
1692                                             struct kvm_vcpu *vcpu)
1693 {
1694         enum emulation_result er = EMULATE_DONE;
1695         u32 inst;
1696
1697         /* Fetch the instruction. */
1698         if (cause & CAUSEF_BD)
1699                 opc += 1;
1700
1701         inst = kvm_get_inst(opc, vcpu);
1702
1703         switch (((union mips_instruction)inst).r_format.opcode) {
1704         case cop0_op:
1705                 er = kvm_mips_emulate_CP0(inst, opc, cause, run, vcpu);
1706                 break;
1707         case sb_op:
1708         case sh_op:
1709         case sw_op:
1710                 er = kvm_mips_emulate_store(inst, cause, run, vcpu);
1711                 break;
1712         case lb_op:
1713         case lbu_op:
1714         case lhu_op:
1715         case lh_op:
1716         case lw_op:
1717                 er = kvm_mips_emulate_load(inst, cause, run, vcpu);
1718                 break;
1719
1720         case cache_op:
1721                 ++vcpu->stat.cache_exits;
1722                 trace_kvm_exit(vcpu, KVM_TRACE_EXIT_CACHE);
1723                 er = kvm_mips_emulate_cache(inst, opc, cause, run, vcpu);
1724                 break;
1725
1726         default:
1727                 kvm_err("Instruction emulation not supported (%p/%#x)\n", opc,
1728                         inst);
1729                 kvm_arch_vcpu_dump_regs(vcpu);
1730                 er = EMULATE_FAIL;
1731                 break;
1732         }
1733
1734         return er;
1735 }
1736
1737 enum emulation_result kvm_mips_emulate_syscall(u32 cause,
1738                                                u32 *opc,
1739                                                struct kvm_run *run,
1740                                                struct kvm_vcpu *vcpu)
1741 {
1742         struct mips_coproc *cop0 = vcpu->arch.cop0;
1743         struct kvm_vcpu_arch *arch = &vcpu->arch;
1744         enum emulation_result er = EMULATE_DONE;
1745
1746         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
1747                 /* save old pc */
1748                 kvm_write_c0_guest_epc(cop0, arch->pc);
1749                 kvm_set_c0_guest_status(cop0, ST0_EXL);
1750
1751                 if (cause & CAUSEF_BD)
1752                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
1753                 else
1754                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
1755
1756                 kvm_debug("Delivering SYSCALL @ pc %#lx\n", arch->pc);
1757
1758                 kvm_change_c0_guest_cause(cop0, (0xff),
1759                                           (EXCCODE_SYS << CAUSEB_EXCCODE));
1760
1761                 /* Set PC to the exception entry point */
1762                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
1763
1764         } else {
1765                 kvm_err("Trying to deliver SYSCALL when EXL is already set\n");
1766                 er = EMULATE_FAIL;
1767         }
1768
1769         return er;
1770 }
1771
1772 enum emulation_result kvm_mips_emulate_tlbmiss_ld(u32 cause,
1773                                                   u32 *opc,
1774                                                   struct kvm_run *run,
1775                                                   struct kvm_vcpu *vcpu)
1776 {
1777         struct mips_coproc *cop0 = vcpu->arch.cop0;
1778         struct kvm_vcpu_arch *arch = &vcpu->arch;
1779         unsigned long entryhi = (vcpu->arch.  host_cp0_badvaddr & VPN2_MASK) |
1780                         (kvm_read_c0_guest_entryhi(cop0) & KVM_ENTRYHI_ASID);
1781
1782         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
1783                 /* save old pc */
1784                 kvm_write_c0_guest_epc(cop0, arch->pc);
1785                 kvm_set_c0_guest_status(cop0, ST0_EXL);
1786
1787                 if (cause & CAUSEF_BD)
1788                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
1789                 else
1790                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
1791
1792                 kvm_debug("[EXL == 0] delivering TLB MISS @ pc %#lx\n",
1793                           arch->pc);
1794
1795                 /* set pc to the exception entry point */
1796                 arch->pc = KVM_GUEST_KSEG0 + 0x0;
1797
1798         } else {
1799                 kvm_debug("[EXL == 1] delivering TLB MISS @ pc %#lx\n",
1800                           arch->pc);
1801
1802                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
1803         }
1804
1805         kvm_change_c0_guest_cause(cop0, (0xff),
1806                                   (EXCCODE_TLBL << CAUSEB_EXCCODE));
1807
1808         /* setup badvaddr, context and entryhi registers for the guest */
1809         kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr);
1810         /* XXXKYMA: is the context register used by linux??? */
1811         kvm_write_c0_guest_entryhi(cop0, entryhi);
1812         /* Blow away the shadow host TLBs */
1813         kvm_mips_flush_host_tlb(1);
1814
1815         return EMULATE_DONE;
1816 }
1817
1818 enum emulation_result kvm_mips_emulate_tlbinv_ld(u32 cause,
1819                                                  u32 *opc,
1820                                                  struct kvm_run *run,
1821                                                  struct kvm_vcpu *vcpu)
1822 {
1823         struct mips_coproc *cop0 = vcpu->arch.cop0;
1824         struct kvm_vcpu_arch *arch = &vcpu->arch;
1825         unsigned long entryhi =
1826                 (vcpu->arch.host_cp0_badvaddr & VPN2_MASK) |
1827                 (kvm_read_c0_guest_entryhi(cop0) & KVM_ENTRYHI_ASID);
1828
1829         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
1830                 /* save old pc */
1831                 kvm_write_c0_guest_epc(cop0, arch->pc);
1832                 kvm_set_c0_guest_status(cop0, ST0_EXL);
1833
1834                 if (cause & CAUSEF_BD)
1835                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
1836                 else
1837                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
1838
1839                 kvm_debug("[EXL == 0] delivering TLB INV @ pc %#lx\n",
1840                           arch->pc);
1841
1842                 /* set pc to the exception entry point */
1843                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
1844
1845         } else {
1846                 kvm_debug("[EXL == 1] delivering TLB MISS @ pc %#lx\n",
1847                           arch->pc);
1848                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
1849         }
1850
1851         kvm_change_c0_guest_cause(cop0, (0xff),
1852                                   (EXCCODE_TLBL << CAUSEB_EXCCODE));
1853
1854         /* setup badvaddr, context and entryhi registers for the guest */
1855         kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr);
1856         /* XXXKYMA: is the context register used by linux??? */
1857         kvm_write_c0_guest_entryhi(cop0, entryhi);
1858         /* Blow away the shadow host TLBs */
1859         kvm_mips_flush_host_tlb(1);
1860
1861         return EMULATE_DONE;
1862 }
1863
1864 enum emulation_result kvm_mips_emulate_tlbmiss_st(u32 cause,
1865                                                   u32 *opc,
1866                                                   struct kvm_run *run,
1867                                                   struct kvm_vcpu *vcpu)
1868 {
1869         struct mips_coproc *cop0 = vcpu->arch.cop0;
1870         struct kvm_vcpu_arch *arch = &vcpu->arch;
1871         unsigned long entryhi = (vcpu->arch.host_cp0_badvaddr & VPN2_MASK) |
1872                         (kvm_read_c0_guest_entryhi(cop0) & KVM_ENTRYHI_ASID);
1873
1874         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
1875                 /* save old pc */
1876                 kvm_write_c0_guest_epc(cop0, arch->pc);
1877                 kvm_set_c0_guest_status(cop0, ST0_EXL);
1878
1879                 if (cause & CAUSEF_BD)
1880                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
1881                 else
1882                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
1883
1884                 kvm_debug("[EXL == 0] Delivering TLB MISS @ pc %#lx\n",
1885                           arch->pc);
1886
1887                 /* Set PC to the exception entry point */
1888                 arch->pc = KVM_GUEST_KSEG0 + 0x0;
1889         } else {
1890                 kvm_debug("[EXL == 1] Delivering TLB MISS @ pc %#lx\n",
1891                           arch->pc);
1892                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
1893         }
1894
1895         kvm_change_c0_guest_cause(cop0, (0xff),
1896                                   (EXCCODE_TLBS << CAUSEB_EXCCODE));
1897
1898         /* setup badvaddr, context and entryhi registers for the guest */
1899         kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr);
1900         /* XXXKYMA: is the context register used by linux??? */
1901         kvm_write_c0_guest_entryhi(cop0, entryhi);
1902         /* Blow away the shadow host TLBs */
1903         kvm_mips_flush_host_tlb(1);
1904
1905         return EMULATE_DONE;
1906 }
1907
1908 enum emulation_result kvm_mips_emulate_tlbinv_st(u32 cause,
1909                                                  u32 *opc,
1910                                                  struct kvm_run *run,
1911                                                  struct kvm_vcpu *vcpu)
1912 {
1913         struct mips_coproc *cop0 = vcpu->arch.cop0;
1914         struct kvm_vcpu_arch *arch = &vcpu->arch;
1915         unsigned long entryhi = (vcpu->arch.host_cp0_badvaddr & VPN2_MASK) |
1916                 (kvm_read_c0_guest_entryhi(cop0) & KVM_ENTRYHI_ASID);
1917
1918         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
1919                 /* save old pc */
1920                 kvm_write_c0_guest_epc(cop0, arch->pc);
1921                 kvm_set_c0_guest_status(cop0, ST0_EXL);
1922
1923                 if (cause & CAUSEF_BD)
1924                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
1925                 else
1926                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
1927
1928                 kvm_debug("[EXL == 0] Delivering TLB MISS @ pc %#lx\n",
1929                           arch->pc);
1930
1931                 /* Set PC to the exception entry point */
1932                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
1933         } else {
1934                 kvm_debug("[EXL == 1] Delivering TLB MISS @ pc %#lx\n",
1935                           arch->pc);
1936                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
1937         }
1938
1939         kvm_change_c0_guest_cause(cop0, (0xff),
1940                                   (EXCCODE_TLBS << CAUSEB_EXCCODE));
1941
1942         /* setup badvaddr, context and entryhi registers for the guest */
1943         kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr);
1944         /* XXXKYMA: is the context register used by linux??? */
1945         kvm_write_c0_guest_entryhi(cop0, entryhi);
1946         /* Blow away the shadow host TLBs */
1947         kvm_mips_flush_host_tlb(1);
1948
1949         return EMULATE_DONE;
1950 }
1951
1952 /* TLBMOD: store into address matching TLB with Dirty bit off */
1953 enum emulation_result kvm_mips_handle_tlbmod(u32 cause, u32 *opc,
1954                                              struct kvm_run *run,
1955                                              struct kvm_vcpu *vcpu)
1956 {
1957         enum emulation_result er = EMULATE_DONE;
1958 #ifdef DEBUG
1959         struct mips_coproc *cop0 = vcpu->arch.cop0;
1960         unsigned long entryhi = (vcpu->arch.host_cp0_badvaddr & VPN2_MASK) |
1961                         (kvm_read_c0_guest_entryhi(cop0) & KVM_ENTRYHI_ASID);
1962         int index;
1963
1964         /* If address not in the guest TLB, then we are in trouble */
1965         index = kvm_mips_guest_tlb_lookup(vcpu, entryhi);
1966         if (index < 0) {
1967                 /* XXXKYMA Invalidate and retry */
1968                 kvm_mips_host_tlb_inv(vcpu, vcpu->arch.host_cp0_badvaddr);
1969                 kvm_err("%s: host got TLBMOD for %#lx but entry not present in Guest TLB\n",
1970                      __func__, entryhi);
1971                 kvm_mips_dump_guest_tlbs(vcpu);
1972                 kvm_mips_dump_host_tlbs();
1973                 return EMULATE_FAIL;
1974         }
1975 #endif
1976
1977         er = kvm_mips_emulate_tlbmod(cause, opc, run, vcpu);
1978         return er;
1979 }
1980
1981 enum emulation_result kvm_mips_emulate_tlbmod(u32 cause,
1982                                               u32 *opc,
1983                                               struct kvm_run *run,
1984                                               struct kvm_vcpu *vcpu)
1985 {
1986         struct mips_coproc *cop0 = vcpu->arch.cop0;
1987         unsigned long entryhi = (vcpu->arch.host_cp0_badvaddr & VPN2_MASK) |
1988                         (kvm_read_c0_guest_entryhi(cop0) & KVM_ENTRYHI_ASID);
1989         struct kvm_vcpu_arch *arch = &vcpu->arch;
1990
1991         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
1992                 /* save old pc */
1993                 kvm_write_c0_guest_epc(cop0, arch->pc);
1994                 kvm_set_c0_guest_status(cop0, ST0_EXL);
1995
1996                 if (cause & CAUSEF_BD)
1997                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
1998                 else
1999                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2000
2001                 kvm_debug("[EXL == 0] Delivering TLB MOD @ pc %#lx\n",
2002                           arch->pc);
2003
2004                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2005         } else {
2006                 kvm_debug("[EXL == 1] Delivering TLB MOD @ pc %#lx\n",
2007                           arch->pc);
2008                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2009         }
2010
2011         kvm_change_c0_guest_cause(cop0, (0xff),
2012                                   (EXCCODE_MOD << CAUSEB_EXCCODE));
2013
2014         /* setup badvaddr, context and entryhi registers for the guest */
2015         kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr);
2016         /* XXXKYMA: is the context register used by linux??? */
2017         kvm_write_c0_guest_entryhi(cop0, entryhi);
2018         /* Blow away the shadow host TLBs */
2019         kvm_mips_flush_host_tlb(1);
2020
2021         return EMULATE_DONE;
2022 }
2023
2024 enum emulation_result kvm_mips_emulate_fpu_exc(u32 cause,
2025                                                u32 *opc,
2026                                                struct kvm_run *run,
2027                                                struct kvm_vcpu *vcpu)
2028 {
2029         struct mips_coproc *cop0 = vcpu->arch.cop0;
2030         struct kvm_vcpu_arch *arch = &vcpu->arch;
2031
2032         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2033                 /* save old pc */
2034                 kvm_write_c0_guest_epc(cop0, arch->pc);
2035                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2036
2037                 if (cause & CAUSEF_BD)
2038                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2039                 else
2040                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2041
2042         }
2043
2044         arch->pc = KVM_GUEST_KSEG0 + 0x180;
2045
2046         kvm_change_c0_guest_cause(cop0, (0xff),
2047                                   (EXCCODE_CPU << CAUSEB_EXCCODE));
2048         kvm_change_c0_guest_cause(cop0, (CAUSEF_CE), (0x1 << CAUSEB_CE));
2049
2050         return EMULATE_DONE;
2051 }
2052
2053 enum emulation_result kvm_mips_emulate_ri_exc(u32 cause,
2054                                               u32 *opc,
2055                                               struct kvm_run *run,
2056                                               struct kvm_vcpu *vcpu)
2057 {
2058         struct mips_coproc *cop0 = vcpu->arch.cop0;
2059         struct kvm_vcpu_arch *arch = &vcpu->arch;
2060         enum emulation_result er = EMULATE_DONE;
2061
2062         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2063                 /* save old pc */
2064                 kvm_write_c0_guest_epc(cop0, arch->pc);
2065                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2066
2067                 if (cause & CAUSEF_BD)
2068                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2069                 else
2070                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2071
2072                 kvm_debug("Delivering RI @ pc %#lx\n", arch->pc);
2073
2074                 kvm_change_c0_guest_cause(cop0, (0xff),
2075                                           (EXCCODE_RI << CAUSEB_EXCCODE));
2076
2077                 /* Set PC to the exception entry point */
2078                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2079
2080         } else {
2081                 kvm_err("Trying to deliver RI when EXL is already set\n");
2082                 er = EMULATE_FAIL;
2083         }
2084
2085         return er;
2086 }
2087
2088 enum emulation_result kvm_mips_emulate_bp_exc(u32 cause,
2089                                               u32 *opc,
2090                                               struct kvm_run *run,
2091                                               struct kvm_vcpu *vcpu)
2092 {
2093         struct mips_coproc *cop0 = vcpu->arch.cop0;
2094         struct kvm_vcpu_arch *arch = &vcpu->arch;
2095         enum emulation_result er = EMULATE_DONE;
2096
2097         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2098                 /* save old pc */
2099                 kvm_write_c0_guest_epc(cop0, arch->pc);
2100                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2101
2102                 if (cause & CAUSEF_BD)
2103                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2104                 else
2105                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2106
2107                 kvm_debug("Delivering BP @ pc %#lx\n", arch->pc);
2108
2109                 kvm_change_c0_guest_cause(cop0, (0xff),
2110                                           (EXCCODE_BP << CAUSEB_EXCCODE));
2111
2112                 /* Set PC to the exception entry point */
2113                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2114
2115         } else {
2116                 kvm_err("Trying to deliver BP when EXL is already set\n");
2117                 er = EMULATE_FAIL;
2118         }
2119
2120         return er;
2121 }
2122
2123 enum emulation_result kvm_mips_emulate_trap_exc(u32 cause,
2124                                                 u32 *opc,
2125                                                 struct kvm_run *run,
2126                                                 struct kvm_vcpu *vcpu)
2127 {
2128         struct mips_coproc *cop0 = vcpu->arch.cop0;
2129         struct kvm_vcpu_arch *arch = &vcpu->arch;
2130         enum emulation_result er = EMULATE_DONE;
2131
2132         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2133                 /* save old pc */
2134                 kvm_write_c0_guest_epc(cop0, arch->pc);
2135                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2136
2137                 if (cause & CAUSEF_BD)
2138                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2139                 else
2140                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2141
2142                 kvm_debug("Delivering TRAP @ pc %#lx\n", arch->pc);
2143
2144                 kvm_change_c0_guest_cause(cop0, (0xff),
2145                                           (EXCCODE_TR << CAUSEB_EXCCODE));
2146
2147                 /* Set PC to the exception entry point */
2148                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2149
2150         } else {
2151                 kvm_err("Trying to deliver TRAP when EXL is already set\n");
2152                 er = EMULATE_FAIL;
2153         }
2154
2155         return er;
2156 }
2157
2158 enum emulation_result kvm_mips_emulate_msafpe_exc(u32 cause,
2159                                                   u32 *opc,
2160                                                   struct kvm_run *run,
2161                                                   struct kvm_vcpu *vcpu)
2162 {
2163         struct mips_coproc *cop0 = vcpu->arch.cop0;
2164         struct kvm_vcpu_arch *arch = &vcpu->arch;
2165         enum emulation_result er = EMULATE_DONE;
2166
2167         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2168                 /* save old pc */
2169                 kvm_write_c0_guest_epc(cop0, arch->pc);
2170                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2171
2172                 if (cause & CAUSEF_BD)
2173                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2174                 else
2175                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2176
2177                 kvm_debug("Delivering MSAFPE @ pc %#lx\n", arch->pc);
2178
2179                 kvm_change_c0_guest_cause(cop0, (0xff),
2180                                           (EXCCODE_MSAFPE << CAUSEB_EXCCODE));
2181
2182                 /* Set PC to the exception entry point */
2183                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2184
2185         } else {
2186                 kvm_err("Trying to deliver MSAFPE when EXL is already set\n");
2187                 er = EMULATE_FAIL;
2188         }
2189
2190         return er;
2191 }
2192
2193 enum emulation_result kvm_mips_emulate_fpe_exc(u32 cause,
2194                                                u32 *opc,
2195                                                struct kvm_run *run,
2196                                                struct kvm_vcpu *vcpu)
2197 {
2198         struct mips_coproc *cop0 = vcpu->arch.cop0;
2199         struct kvm_vcpu_arch *arch = &vcpu->arch;
2200         enum emulation_result er = EMULATE_DONE;
2201
2202         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2203                 /* save old pc */
2204                 kvm_write_c0_guest_epc(cop0, arch->pc);
2205                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2206
2207                 if (cause & CAUSEF_BD)
2208                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2209                 else
2210                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2211
2212                 kvm_debug("Delivering FPE @ pc %#lx\n", arch->pc);
2213
2214                 kvm_change_c0_guest_cause(cop0, (0xff),
2215                                           (EXCCODE_FPE << CAUSEB_EXCCODE));
2216
2217                 /* Set PC to the exception entry point */
2218                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2219
2220         } else {
2221                 kvm_err("Trying to deliver FPE when EXL is already set\n");
2222                 er = EMULATE_FAIL;
2223         }
2224
2225         return er;
2226 }
2227
2228 enum emulation_result kvm_mips_emulate_msadis_exc(u32 cause,
2229                                                   u32 *opc,
2230                                                   struct kvm_run *run,
2231                                                   struct kvm_vcpu *vcpu)
2232 {
2233         struct mips_coproc *cop0 = vcpu->arch.cop0;
2234         struct kvm_vcpu_arch *arch = &vcpu->arch;
2235         enum emulation_result er = EMULATE_DONE;
2236
2237         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2238                 /* save old pc */
2239                 kvm_write_c0_guest_epc(cop0, arch->pc);
2240                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2241
2242                 if (cause & CAUSEF_BD)
2243                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2244                 else
2245                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2246
2247                 kvm_debug("Delivering MSADIS @ pc %#lx\n", arch->pc);
2248
2249                 kvm_change_c0_guest_cause(cop0, (0xff),
2250                                           (EXCCODE_MSADIS << CAUSEB_EXCCODE));
2251
2252                 /* Set PC to the exception entry point */
2253                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2254
2255         } else {
2256                 kvm_err("Trying to deliver MSADIS when EXL is already set\n");
2257                 er = EMULATE_FAIL;
2258         }
2259
2260         return er;
2261 }
2262
2263 /* ll/sc, rdhwr, sync emulation */
2264
2265 #define OPCODE 0xfc000000
2266 #define BASE   0x03e00000
2267 #define RT     0x001f0000
2268 #define OFFSET 0x0000ffff
2269 #define LL     0xc0000000
2270 #define SC     0xe0000000
2271 #define SPEC0  0x00000000
2272 #define SPEC3  0x7c000000
2273 #define RD     0x0000f800
2274 #define FUNC   0x0000003f
2275 #define SYNC   0x0000000f
2276 #define RDHWR  0x0000003b
2277
2278 enum emulation_result kvm_mips_handle_ri(u32 cause, u32 *opc,
2279                                          struct kvm_run *run,
2280                                          struct kvm_vcpu *vcpu)
2281 {
2282         struct mips_coproc *cop0 = vcpu->arch.cop0;
2283         struct kvm_vcpu_arch *arch = &vcpu->arch;
2284         enum emulation_result er = EMULATE_DONE;
2285         unsigned long curr_pc;
2286         u32 inst;
2287
2288         /*
2289          * Update PC and hold onto current PC in case there is
2290          * an error and we want to rollback the PC
2291          */
2292         curr_pc = vcpu->arch.pc;
2293         er = update_pc(vcpu, cause);
2294         if (er == EMULATE_FAIL)
2295                 return er;
2296
2297         /* Fetch the instruction. */
2298         if (cause & CAUSEF_BD)
2299                 opc += 1;
2300
2301         inst = kvm_get_inst(opc, vcpu);
2302
2303         if (inst == KVM_INVALID_INST) {
2304                 kvm_err("%s: Cannot get inst @ %p\n", __func__, opc);
2305                 return EMULATE_FAIL;
2306         }
2307
2308         if ((inst & OPCODE) == SPEC3 && (inst & FUNC) == RDHWR) {
2309                 int usermode = !KVM_GUEST_KERNEL_MODE(vcpu);
2310                 int rd = (inst & RD) >> 11;
2311                 int rt = (inst & RT) >> 16;
2312                 int sel = (inst >> 6) & 0x7;
2313
2314                 /* If usermode, check RDHWR rd is allowed by guest HWREna */
2315                 if (usermode && !(kvm_read_c0_guest_hwrena(cop0) & BIT(rd))) {
2316                         kvm_debug("RDHWR %#x disallowed by HWREna @ %p\n",
2317                                   rd, opc);
2318                         goto emulate_ri;
2319                 }
2320                 switch (rd) {
2321                 case 0: /* CPU number */
2322                         arch->gprs[rt] = 0;
2323                         break;
2324                 case 1: /* SYNCI length */
2325                         arch->gprs[rt] = min(current_cpu_data.dcache.linesz,
2326                                              current_cpu_data.icache.linesz);
2327                         break;
2328                 case 2: /* Read count register */
2329                         arch->gprs[rt] = kvm_mips_read_count(vcpu);
2330                         break;
2331                 case 3: /* Count register resolution */
2332                         switch (current_cpu_data.cputype) {
2333                         case CPU_20KC:
2334                         case CPU_25KF:
2335                                 arch->gprs[rt] = 1;
2336                                 break;
2337                         default:
2338                                 arch->gprs[rt] = 2;
2339                         }
2340                         break;
2341                 case 29:
2342                         arch->gprs[rt] = kvm_read_c0_guest_userlocal(cop0);
2343                         break;
2344
2345                 default:
2346                         kvm_debug("RDHWR %#x not supported @ %p\n", rd, opc);
2347                         goto emulate_ri;
2348                 }
2349
2350                 trace_kvm_hwr(vcpu, KVM_TRACE_RDHWR, KVM_TRACE_HWR(rd, sel),
2351                               vcpu->arch.gprs[rt]);
2352         } else {
2353                 kvm_debug("Emulate RI not supported @ %p: %#x\n", opc, inst);
2354                 goto emulate_ri;
2355         }
2356
2357         return EMULATE_DONE;
2358
2359 emulate_ri:
2360         /*
2361          * Rollback PC (if in branch delay slot then the PC already points to
2362          * branch target), and pass the RI exception to the guest OS.
2363          */
2364         vcpu->arch.pc = curr_pc;
2365         return kvm_mips_emulate_ri_exc(cause, opc, run, vcpu);
2366 }
2367
2368 enum emulation_result kvm_mips_complete_mmio_load(struct kvm_vcpu *vcpu,
2369                                                   struct kvm_run *run)
2370 {
2371         unsigned long *gpr = &vcpu->arch.gprs[vcpu->arch.io_gpr];
2372         enum emulation_result er = EMULATE_DONE;
2373
2374         if (run->mmio.len > sizeof(*gpr)) {
2375                 kvm_err("Bad MMIO length: %d", run->mmio.len);
2376                 er = EMULATE_FAIL;
2377                 goto done;
2378         }
2379
2380         er = update_pc(vcpu, vcpu->arch.pending_load_cause);
2381         if (er == EMULATE_FAIL)
2382                 return er;
2383
2384         switch (run->mmio.len) {
2385         case 4:
2386                 *gpr = *(s32 *) run->mmio.data;
2387                 break;
2388
2389         case 2:
2390                 if (vcpu->mmio_needed == 2)
2391                         *gpr = *(s16 *) run->mmio.data;
2392                 else
2393                         *gpr = *(u16 *)run->mmio.data;
2394
2395                 break;
2396         case 1:
2397                 if (vcpu->mmio_needed == 2)
2398                         *gpr = *(s8 *) run->mmio.data;
2399                 else
2400                         *gpr = *(u8 *) run->mmio.data;
2401                 break;
2402         }
2403
2404         if (vcpu->arch.pending_load_cause & CAUSEF_BD)
2405                 kvm_debug("[%#lx] Completing %d byte BD Load to gpr %d (0x%08lx) type %d\n",
2406                           vcpu->arch.pc, run->mmio.len, vcpu->arch.io_gpr, *gpr,
2407                           vcpu->mmio_needed);
2408
2409 done:
2410         return er;
2411 }
2412
2413 static enum emulation_result kvm_mips_emulate_exc(u32 cause,
2414                                                   u32 *opc,
2415                                                   struct kvm_run *run,
2416                                                   struct kvm_vcpu *vcpu)
2417 {
2418         u32 exccode = (cause >> CAUSEB_EXCCODE) & 0x1f;
2419         struct mips_coproc *cop0 = vcpu->arch.cop0;
2420         struct kvm_vcpu_arch *arch = &vcpu->arch;
2421         enum emulation_result er = EMULATE_DONE;
2422
2423         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2424                 /* save old pc */
2425                 kvm_write_c0_guest_epc(cop0, arch->pc);
2426                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2427
2428                 if (cause & CAUSEF_BD)
2429                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2430                 else
2431                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2432
2433                 kvm_change_c0_guest_cause(cop0, (0xff),
2434                                           (exccode << CAUSEB_EXCCODE));
2435
2436                 /* Set PC to the exception entry point */
2437                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2438                 kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr);
2439
2440                 kvm_debug("Delivering EXC %d @ pc %#lx, badVaddr: %#lx\n",
2441                           exccode, kvm_read_c0_guest_epc(cop0),
2442                           kvm_read_c0_guest_badvaddr(cop0));
2443         } else {
2444                 kvm_err("Trying to deliver EXC when EXL is already set\n");
2445                 er = EMULATE_FAIL;
2446         }
2447
2448         return er;
2449 }
2450
2451 enum emulation_result kvm_mips_check_privilege(u32 cause,
2452                                                u32 *opc,
2453                                                struct kvm_run *run,
2454                                                struct kvm_vcpu *vcpu)
2455 {
2456         enum emulation_result er = EMULATE_DONE;
2457         u32 exccode = (cause >> CAUSEB_EXCCODE) & 0x1f;
2458         unsigned long badvaddr = vcpu->arch.host_cp0_badvaddr;
2459
2460         int usermode = !KVM_GUEST_KERNEL_MODE(vcpu);
2461
2462         if (usermode) {
2463                 switch (exccode) {
2464                 case EXCCODE_INT:
2465                 case EXCCODE_SYS:
2466                 case EXCCODE_BP:
2467                 case EXCCODE_RI:
2468                 case EXCCODE_TR:
2469                 case EXCCODE_MSAFPE:
2470                 case EXCCODE_FPE:
2471                 case EXCCODE_MSADIS:
2472                         break;
2473
2474                 case EXCCODE_CPU:
2475                         if (((cause & CAUSEF_CE) >> CAUSEB_CE) == 0)
2476                                 er = EMULATE_PRIV_FAIL;
2477                         break;
2478
2479                 case EXCCODE_MOD:
2480                         break;
2481
2482                 case EXCCODE_TLBL:
2483                         /*
2484                          * We we are accessing Guest kernel space, then send an
2485                          * address error exception to the guest
2486                          */
2487                         if (badvaddr >= (unsigned long) KVM_GUEST_KSEG0) {
2488                                 kvm_debug("%s: LD MISS @ %#lx\n", __func__,
2489                                           badvaddr);
2490                                 cause &= ~0xff;
2491                                 cause |= (EXCCODE_ADEL << CAUSEB_EXCCODE);
2492                                 er = EMULATE_PRIV_FAIL;
2493                         }
2494                         break;
2495
2496                 case EXCCODE_TLBS:
2497                         /*
2498                          * We we are accessing Guest kernel space, then send an
2499                          * address error exception to the guest
2500                          */
2501                         if (badvaddr >= (unsigned long) KVM_GUEST_KSEG0) {
2502                                 kvm_debug("%s: ST MISS @ %#lx\n", __func__,
2503                                           badvaddr);
2504                                 cause &= ~0xff;
2505                                 cause |= (EXCCODE_ADES << CAUSEB_EXCCODE);
2506                                 er = EMULATE_PRIV_FAIL;
2507                         }
2508                         break;
2509
2510                 case EXCCODE_ADES:
2511                         kvm_debug("%s: address error ST @ %#lx\n", __func__,
2512                                   badvaddr);
2513                         if ((badvaddr & PAGE_MASK) == KVM_GUEST_COMMPAGE_ADDR) {
2514                                 cause &= ~0xff;
2515                                 cause |= (EXCCODE_TLBS << CAUSEB_EXCCODE);
2516                         }
2517                         er = EMULATE_PRIV_FAIL;
2518                         break;
2519                 case EXCCODE_ADEL:
2520                         kvm_debug("%s: address error LD @ %#lx\n", __func__,
2521                                   badvaddr);
2522                         if ((badvaddr & PAGE_MASK) == KVM_GUEST_COMMPAGE_ADDR) {
2523                                 cause &= ~0xff;
2524                                 cause |= (EXCCODE_TLBL << CAUSEB_EXCCODE);
2525                         }
2526                         er = EMULATE_PRIV_FAIL;
2527                         break;
2528                 default:
2529                         er = EMULATE_PRIV_FAIL;
2530                         break;
2531                 }
2532         }
2533
2534         if (er == EMULATE_PRIV_FAIL)
2535                 kvm_mips_emulate_exc(cause, opc, run, vcpu);
2536
2537         return er;
2538 }
2539
2540 /*
2541  * User Address (UA) fault, this could happen if
2542  * (1) TLB entry not present/valid in both Guest and shadow host TLBs, in this
2543  *     case we pass on the fault to the guest kernel and let it handle it.
2544  * (2) TLB entry is present in the Guest TLB but not in the shadow, in this
2545  *     case we inject the TLB from the Guest TLB into the shadow host TLB
2546  */
2547 enum emulation_result kvm_mips_handle_tlbmiss(u32 cause,
2548                                               u32 *opc,
2549                                               struct kvm_run *run,
2550                                               struct kvm_vcpu *vcpu)
2551 {
2552         enum emulation_result er = EMULATE_DONE;
2553         u32 exccode = (cause >> CAUSEB_EXCCODE) & 0x1f;
2554         unsigned long va = vcpu->arch.host_cp0_badvaddr;
2555         int index;
2556
2557         kvm_debug("kvm_mips_handle_tlbmiss: badvaddr: %#lx\n",
2558                   vcpu->arch.host_cp0_badvaddr);
2559
2560         /*
2561          * KVM would not have got the exception if this entry was valid in the
2562          * shadow host TLB. Check the Guest TLB, if the entry is not there then
2563          * send the guest an exception. The guest exc handler should then inject
2564          * an entry into the guest TLB.
2565          */
2566         index = kvm_mips_guest_tlb_lookup(vcpu,
2567                       (va & VPN2_MASK) |
2568                       (kvm_read_c0_guest_entryhi(vcpu->arch.cop0) &
2569                        KVM_ENTRYHI_ASID));
2570         if (index < 0) {
2571                 if (exccode == EXCCODE_TLBL) {
2572                         er = kvm_mips_emulate_tlbmiss_ld(cause, opc, run, vcpu);
2573                 } else if (exccode == EXCCODE_TLBS) {
2574                         er = kvm_mips_emulate_tlbmiss_st(cause, opc, run, vcpu);
2575                 } else {
2576                         kvm_err("%s: invalid exc code: %d\n", __func__,
2577                                 exccode);
2578                         er = EMULATE_FAIL;
2579                 }
2580         } else {
2581                 struct kvm_mips_tlb *tlb = &vcpu->arch.guest_tlb[index];
2582
2583                 /*
2584                  * Check if the entry is valid, if not then setup a TLB invalid
2585                  * exception to the guest
2586                  */
2587                 if (!TLB_IS_VALID(*tlb, va)) {
2588                         if (exccode == EXCCODE_TLBL) {
2589                                 er = kvm_mips_emulate_tlbinv_ld(cause, opc, run,
2590                                                                 vcpu);
2591                         } else if (exccode == EXCCODE_TLBS) {
2592                                 er = kvm_mips_emulate_tlbinv_st(cause, opc, run,
2593                                                                 vcpu);
2594                         } else {
2595                                 kvm_err("%s: invalid exc code: %d\n", __func__,
2596                                         exccode);
2597                                 er = EMULATE_FAIL;
2598                         }
2599                 } else {
2600                         kvm_debug("Injecting hi: %#lx, lo0: %#lx, lo1: %#lx into shadow host TLB\n",
2601                                   tlb->tlb_hi, tlb->tlb_lo[0], tlb->tlb_lo[1]);
2602                         /*
2603                          * OK we have a Guest TLB entry, now inject it into the
2604                          * shadow host TLB
2605                          */
2606                         kvm_mips_handle_mapped_seg_tlb_fault(vcpu, tlb);
2607                 }
2608         }
2609
2610         return er;
2611 }