virt/kvm/arm/arch_timer.c

   1 /*
   2  * Copyright (C) 2012 ARM Ltd.
   3  * Author: Marc Zyngier <marc.zyngier@arm.com>
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License version 2 as
   7  * published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write to the Free Software
  16  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  17  */
  18
  19 #include <linux/cpu.h>
  20 #include <linux/kvm.h>
  21 #include <linux/kvm_host.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/irq.h>
  24
  25 #include <clocksource/arm_arch_timer.h>
  26 #include <asm/arch_timer.h>
  27
  28 #include <kvm/arm_vgic.h>
  29 #include <kvm/arm_arch_timer.h>
  30
  31 #include "trace.h"
  32
  33 static struct timecounter *timecounter;
  34 static struct workqueue_struct *wqueue;
  35 static unsigned int host_vtimer_irq;
  36 static u32 host_vtimer_irq_flags;
  37
  38 void kvm_timer_vcpu_put(struct kvm_vcpu *vcpu)
  39 {
  40         vcpu->arch.timer_cpu.active_cleared_last = false;
  41 }
  42
  43 static cycle_t kvm_phys_timer_read(void)
  44 {
  45         return timecounter->cc->read(timecounter->cc);
  46 }
  47
  48 static bool timer_is_armed(struct arch_timer_cpu *timer)
  49 {
  50         return timer->armed;
  51 }
  52
  53 /* timer_arm: as in "arm the timer", not as in ARM the company */
  54 static void timer_arm(struct arch_timer_cpu *timer, u64 ns)
  55 {
  56         timer->armed = true;
  57         hrtimer_start(&timer->timer, ktime_add_ns(ktime_get(), ns),
  58                       HRTIMER_MODE_ABS);
  59 }
  60
  61 static void timer_disarm(struct arch_timer_cpu *timer)
  62 {
  63         if (timer_is_armed(timer)) {
  64                 hrtimer_cancel(&timer->timer);
  65                 cancel_work_sync(&timer->expired);
  66                 timer->armed = false;
  67         }
  68 }
  69
  70 static irqreturn_t kvm_arch_timer_handler(int irq, void *dev_id)
  71 {
  72         struct kvm_vcpu *vcpu = *(struct kvm_vcpu **)dev_id;
  73
  74         /*
  75          * We disable the timer in the world switch and let it be
  76          * handled by kvm_timer_sync_hwstate(). Getting a timer
  77          * interrupt at this point is a sure sign of some major
  78          * breakage.
  79          */
  80         pr_warn("Unexpected interrupt %d on vcpu %p\n", irq, vcpu);
  81         return IRQ_HANDLED;
  82 }
  83
  84 /*
  85  * Work function for handling the backup timer that we schedule when a vcpu is
  86  * no longer running, but had a timer programmed to fire in the future.
  87  */
  88 static void kvm_timer_inject_irq_work(struct work_struct *work)
  89 {
  90         struct kvm_vcpu *vcpu;
  91
  92         vcpu = container_of(work, struct kvm_vcpu, arch.timer_cpu.expired);
  93         vcpu->arch.timer_cpu.armed = false;
  94
  95         WARN_ON(!kvm_timer_should_fire(vcpu));
  96
  97         /*
  98          * If the vcpu is blocked we want to wake it up so that it will see
  99          * the timer has expired when entering the guest.
 100          */
 101         kvm_vcpu_kick(vcpu);
 102 }
 103
 104 static u64 kvm_timer_compute_delta(struct kvm_vcpu *vcpu)
 105 {
 106         cycle_t cval, now;
 107
 108         cval = vcpu->arch.timer_cpu.cntv_cval;
 109         now = kvm_phys_timer_read() - vcpu->kvm->arch.timer.cntvoff;
 110
 111         if (now < cval) {
 112                 u64 ns;
 113
 114                 ns = cyclecounter_cyc2ns(timecounter->cc,
 115                                          cval - now,
 116                                          timecounter->mask,
 117                                          &timecounter->frac);
 118                 return ns;
 119         }
 120
 121         return 0;
 122 }
 123
 124 static enum hrtimer_restart kvm_timer_expire(struct hrtimer *hrt)
 125 {
 126         struct arch_timer_cpu *timer;
 127         struct kvm_vcpu *vcpu;
 128         u64 ns;
 129
 130         timer = container_of(hrt, struct arch_timer_cpu, timer);
 131         vcpu = container_of(timer, struct kvm_vcpu, arch.timer_cpu);
 132
 133         /*
 134          * Check that the timer has really expired from the guest's
 135          * PoV (NTP on the host may have forced it to expire
 136          * early). If we should have slept longer, restart it.
 137          */
 138         ns = kvm_timer_compute_delta(vcpu);
 139         if (unlikely(ns)) {
 140                 hrtimer_forward_now(hrt, ns_to_ktime(ns));
 141                 return HRTIMER_RESTART;
 142         }
 143
 144         queue_work(wqueue, &timer->expired);
 145         return HRTIMER_NORESTART;
 146 }
 147
 148 static bool kvm_timer_irq_can_fire(struct kvm_vcpu *vcpu)
 149 {
 150         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 151
 152         return !(timer->cntv_ctl & ARCH_TIMER_CTRL_IT_MASK) &&
 153                 (timer->cntv_ctl & ARCH_TIMER_CTRL_ENABLE);
 154 }
 155
 156 bool kvm_timer_should_fire(struct kvm_vcpu *vcpu)
 157 {
 158         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 159         cycle_t cval, now;
 160
 161         if (!kvm_timer_irq_can_fire(vcpu))
 162                 return false;
 163
 164         cval = timer->cntv_cval;
 165         now = kvm_phys_timer_read() - vcpu->kvm->arch.timer.cntvoff;
 166
 167         return cval <= now;
 168 }
 169
 170 static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level)
 171 {
 172         int ret;
 173         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 174
 175         BUG_ON(!vgic_initialized(vcpu->kvm));
 176
 177         timer->active_cleared_last = false;
 178         timer->irq.level = new_level;
 179         trace_kvm_timer_update_irq(vcpu->vcpu_id, timer->irq.irq,
 180                                    timer->irq.level);
 181         ret = kvm_vgic_inject_mapped_irq(vcpu->kvm, vcpu->vcpu_id,
 182                                          timer->irq.irq,
 183                                          timer->irq.level);
 184         WARN_ON(ret);
 185 }
 186
 187 /*
 188  * Check if there was a change in the timer state (should we raise or lower
 189  * the line level to the GIC).
 190  */
 191 static int kvm_timer_update_state(struct kvm_vcpu *vcpu)
 192 {
 193         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 194
 195         /*
 196          * If userspace modified the timer registers via SET_ONE_REG before
 197          * the vgic was initialized, we mustn't set the timer->irq.level value
 198          * because the guest would never see the interrupt.  Instead wait
 199          * until we call this function from kvm_timer_flush_hwstate.
 200          */
 201         if (!vgic_initialized(vcpu->kvm) || !timer->enabled)
 202                 return -ENODEV;
 203
 204         if (kvm_timer_should_fire(vcpu) != timer->irq.level)
 205                 kvm_timer_update_irq(vcpu, !timer->irq.level);
 206
 207         return 0;
 208 }
 209
 210 /*
 211  * Schedule the background timer before calling kvm_vcpu_block, so that this
 212  * thread is removed from its waitqueue and made runnable when there's a timer
 213  * interrupt to handle.
 214  */
 215 void kvm_timer_schedule(struct kvm_vcpu *vcpu)
 216 {
 217         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 218
 219         BUG_ON(timer_is_armed(timer));
 220
 221         /*
 222          * No need to schedule a background timer if the guest timer has
 223          * already expired, because kvm_vcpu_block will return before putting
 224          * the thread to sleep.
 225          */
 226         if (kvm_timer_should_fire(vcpu))
 227                 return;
 228
 229         /*
 230          * If the timer is not capable of raising interrupts (disabled or
 231          * masked), then there's no more work for us to do.
 232          */
 233         if (!kvm_timer_irq_can_fire(vcpu))
 234                 return;
 235
 236         /*  The timer has not yet expired, schedule a background timer */
 237         timer_arm(timer, kvm_timer_compute_delta(vcpu));
 238 }
 239
 240 void kvm_timer_unschedule(struct kvm_vcpu *vcpu)
 241 {
 242         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 243         timer_disarm(timer);
 244 }
 245
 246 /**
 247  * kvm_timer_flush_hwstate - prepare to move the virt timer to the cpu
 248  * @vcpu: The vcpu pointer
 249  *
 250  * Check if the virtual timer has expired while we were running in the host,
 251  * and inject an interrupt if that was the case.
 252  */
 253 void kvm_timer_flush_hwstate(struct kvm_vcpu *vcpu)
 254 {
 255         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 256         bool phys_active;
 257         int ret;
 258
 259         if (kvm_timer_update_state(vcpu))
 260                 return;
 261
 262         /*
 263         * If we enter the guest with the virtual input level to the VGIC
 264         * asserted, then we have already told the VGIC what we need to, and
 265         * we don't need to exit from the guest until the guest deactivates
 266         * the already injected interrupt, so therefore we should set the
 267         * hardware active state to prevent unnecessary exits from the guest.
 268         *
 269         * Also, if we enter the guest with the virtual timer interrupt active,
 270         * then it must be active on the physical distributor, because we set
 271         * the HW bit and the guest must be able to deactivate the virtual and
 272         * physical interrupt at the same time.
 273         *
 274         * Conversely, if the virtual input level is deasserted and the virtual
 275         * interrupt is not active, then always clear the hardware active state
 276         * to ensure that hardware interrupts from the timer triggers a guest
 277         * exit.
 278         */
 279         phys_active = timer->irq.level ||
 280                         kvm_vgic_map_is_active(vcpu, timer->irq.irq);
 281
 282         /*
 283          * We want to avoid hitting the (re)distributor as much as
 284          * possible, as this is a potentially expensive MMIO access
 285          * (not to mention locks in the irq layer), and a solution for
 286          * this is to cache the "active" state in memory.
 287          *
 288          * Things to consider: we cannot cache an "active set" state,
 289          * because the HW can change this behind our back (it becomes
 290          * "clear" in the HW). We must then restrict the caching to
 291          * the "clear" state.
 292          *
 293          * The cache is invalidated on:
 294          * - vcpu put, indicating that the HW cannot be trusted to be
 295          *   in a sane state on the next vcpu load,
 296          * - any change in the interrupt state
 297          *
 298          * Usage conditions:
 299          * - cached value is "active clear"
 300          * - value to be programmed is "active clear"
 301          */
 302         if (timer->active_cleared_last && !phys_active)
 303                 return;
 304
 305         ret = irq_set_irqchip_state(host_vtimer_irq,
 306                                     IRQCHIP_STATE_ACTIVE,
 307                                     phys_active);
 308         WARN_ON(ret);
 309
 310         timer->active_cleared_last = !phys_active;
 311 }
 312
 313 /**
 314  * kvm_timer_sync_hwstate - sync timer state from cpu
 315  * @vcpu: The vcpu pointer
 316  *
 317  * Check if the virtual timer has expired while we were running in the guest,
 318  * and inject an interrupt if that was the case.
 319  */
 320 void kvm_timer_sync_hwstate(struct kvm_vcpu *vcpu)
 321 {
 322         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 323
 324         BUG_ON(timer_is_armed(timer));
 325
 326         /*
 327          * The guest could have modified the timer registers or the timer
 328          * could have expired, update the timer state.
 329          */
 330         kvm_timer_update_state(vcpu);
 331 }
 332
 333 int kvm_timer_vcpu_reset(struct kvm_vcpu *vcpu,
 334                          const struct kvm_irq_level *irq)
 335 {
 336         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 337
 338         /*
 339          * The vcpu timer irq number cannot be determined in
 340          * kvm_timer_vcpu_init() because it is called much before
 341          * kvm_vcpu_set_target(). To handle this, we determine
 342          * vcpu timer irq number when the vcpu is reset.
 343          */
 344         timer->irq.irq = irq->irq;
 345
 346         /*
 347          * The bits in CNTV_CTL are architecturally reset to UNKNOWN for ARMv8
 348          * and to 0 for ARMv7.  We provide an implementation that always
 349          * resets the timer to be disabled and unmasked and is compliant with
 350          * the ARMv7 architecture.
 351          */
 352         timer->cntv_ctl = 0;
 353         kvm_timer_update_state(vcpu);
 354
 355         return 0;
 356 }
 357
 358 void kvm_timer_vcpu_init(struct kvm_vcpu *vcpu)
 359 {
 360         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 361
 362         INIT_WORK(&timer->expired, kvm_timer_inject_irq_work);
 363         hrtimer_init(&timer->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
 364         timer->timer.function = kvm_timer_expire;
 365 }
 366
 367 static void kvm_timer_init_interrupt(void *info)
 368 {
 369         enable_percpu_irq(host_vtimer_irq, host_vtimer_irq_flags);
 370 }
 371
 372 int kvm_arm_timer_set_reg(struct kvm_vcpu *vcpu, u64 regid, u64 value)
 373 {
 374         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 375
 376         switch (regid) {
 377         case KVM_REG_ARM_TIMER_CTL:
 378                 timer->cntv_ctl = value;
 379                 break;
 380         case KVM_REG_ARM_TIMER_CNT:
 381                 vcpu->kvm->arch.timer.cntvoff = kvm_phys_timer_read() - value;
 382                 break;
 383         case KVM_REG_ARM_TIMER_CVAL:
 384                 timer->cntv_cval = value;
 385                 break;
 386         default:
 387                 return -1;
 388         }
 389
 390         kvm_timer_update_state(vcpu);
 391         return 0;
 392 }
 393
 394 u64 kvm_arm_timer_get_reg(struct kvm_vcpu *vcpu, u64 regid)
 395 {
 396         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 397
 398         switch (regid) {
 399         case KVM_REG_ARM_TIMER_CTL:
 400                 return timer->cntv_ctl;
 401         case KVM_REG_ARM_TIMER_CNT:
 402                 return kvm_phys_timer_read() - vcpu->kvm->arch.timer.cntvoff;
 403         case KVM_REG_ARM_TIMER_CVAL:
 404                 return timer->cntv_cval;
 405         }
 406         return (u64)-1;
 407 }
 408
 409 static int kvm_timer_starting_cpu(unsigned int cpu)
 410 {
 411         kvm_timer_init_interrupt(NULL);
 412         return 0;
 413 }
 414
 415 static int kvm_timer_dying_cpu(unsigned int cpu)
 416 {
 417         disable_percpu_irq(host_vtimer_irq);
 418         return 0;
 419 }
 420
 421 int kvm_timer_hyp_init(void)
 422 {
 423         struct arch_timer_kvm_info *info;
 424         int err;
 425
 426         info = arch_timer_get_kvm_info();
 427         timecounter = &info->timecounter;
 428
 429         if (info->virtual_irq <= 0) {
 430                 kvm_err("kvm_arch_timer: invalid virtual timer IRQ: %d\n",
 431                         info->virtual_irq);
 432                 return -ENODEV;
 433         }
 434         host_vtimer_irq = info->virtual_irq;
 435
 436         host_vtimer_irq_flags = irq_get_trigger_type(host_vtimer_irq);
 437         if (host_vtimer_irq_flags != IRQF_TRIGGER_HIGH &&
 438             host_vtimer_irq_flags != IRQF_TRIGGER_LOW) {
 439                 kvm_err("Invalid trigger for IRQ%d, assuming level low\n",
 440                         host_vtimer_irq);
 441                 host_vtimer_irq_flags = IRQF_TRIGGER_LOW;
 442         }
 443
 444         err = request_percpu_irq(host_vtimer_irq, kvm_arch_timer_handler,
 445                                  "kvm guest timer", kvm_get_running_vcpus());
 446         if (err) {
 447                 kvm_err("kvm_arch_timer: can't request interrupt %d (%d)\n",
 448                         host_vtimer_irq, err);
 449                 goto out;
 450         }
 451
 452         wqueue = create_singlethread_workqueue("kvm_arch_timer");
 453         if (!wqueue) {
 454                 err = -ENOMEM;
 455                 goto out_free;
 456         }
 457
 458         kvm_info("virtual timer IRQ%d\n", host_vtimer_irq);
 459
 460         cpuhp_setup_state(CPUHP_AP_KVM_ARM_TIMER_STARTING,
 461                           "AP_KVM_ARM_TIMER_STARTING", kvm_timer_starting_cpu,
 462                           kvm_timer_dying_cpu);
 463         goto out;
 464 out_free:
 465         free_percpu_irq(host_vtimer_irq, kvm_get_running_vcpus());
 466 out:
 467         return err;
 468 }
 469
 470 void kvm_timer_vcpu_terminate(struct kvm_vcpu *vcpu)
 471 {
 472         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 473
 474         timer_disarm(timer);
 475         kvm_vgic_unmap_phys_irq(vcpu, timer->irq.irq);
 476 }
 477
 478 int kvm_timer_enable(struct kvm_vcpu *vcpu)
 479 {
 480         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 481         struct irq_desc *desc;
 482         struct irq_data *data;
 483         int phys_irq;
 484         int ret;
 485
 486         if (timer->enabled)
 487                 return 0;
 488
 489         /*
 490          * Find the physical IRQ number corresponding to the host_vtimer_irq
 491          */
 492         desc = irq_to_desc(host_vtimer_irq);
 493         if (!desc) {
 494                 kvm_err("%s: no interrupt descriptor\n", __func__);
 495                 return -EINVAL;
 496         }
 497
 498         data = irq_desc_get_irq_data(desc);
 499         while (data->parent_data)
 500                 data = data->parent_data;
 501
 502         phys_irq = data->hwirq;
 503
 504         /*
 505          * Tell the VGIC that the virtual interrupt is tied to a
 506          * physical interrupt. We do that once per VCPU.
 507          */
 508         ret = kvm_vgic_map_phys_irq(vcpu, timer->irq.irq, phys_irq);
 509         if (ret)
 510                 return ret;
 511
 512
 513         /*
 514          * There is a potential race here between VCPUs starting for the first
 515          * time, which may be enabling the timer multiple times.  That doesn't
 516          * hurt though, because we're just setting a variable to the same
 517          * variable that it already was.  The important thing is that all
 518          * VCPUs have the enabled variable set, before entering the guest, if
 519          * the arch timers are enabled.
 520          */
 521         if (timecounter && wqueue)
 522                 timer->enabled = 1;
 523
 524         return 0;
 525 }
 526
 527 void kvm_timer_init(struct kvm *kvm)
 528 {
 529         kvm->arch.timer.cntvoff = kvm_phys_timer_read();
 530 }