virt/kvm/arm/arch_timer.c

   1 /*
   2  * Copyright (C) 2012 ARM Ltd.
   3  * Author: Marc Zyngier <marc.zyngier@arm.com>
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License version 2 as
   7  * published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write to the Free Software
  16  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  17  */
  18
  19 #include <linux/cpu.h>
  20 #include <linux/of_irq.h>
  21 #include <linux/kvm.h>
  22 #include <linux/kvm_host.h>
  23 #include <linux/interrupt.h>
  24
  25 #include <clocksource/arm_arch_timer.h>
  26 #include <asm/arch_timer.h>
  27
  28 #include <kvm/arm_vgic.h>
  29 #include <kvm/arm_arch_timer.h>
  30
  31 #include "trace.h"
  32
  33 static struct timecounter *timecounter;
  34 static struct workqueue_struct *wqueue;
  35 static unsigned int host_vtimer_irq;
  36
  37 void kvm_timer_vcpu_put(struct kvm_vcpu *vcpu)
  38 {
  39         vcpu->arch.timer_cpu.active_cleared_last = false;
  40 }
  41
  42 static cycle_t kvm_phys_timer_read(void)
  43 {
  44         return timecounter->cc->read(timecounter->cc);
  45 }
  46
  47 static bool timer_is_armed(struct arch_timer_cpu *timer)
  48 {
  49         return timer->armed;
  50 }
  51
  52 /* timer_arm: as in "arm the timer", not as in ARM the company */
  53 static void timer_arm(struct arch_timer_cpu *timer, u64 ns)
  54 {
  55         timer->armed = true;
  56         hrtimer_start(&timer->timer, ktime_add_ns(ktime_get(), ns),
  57                       HRTIMER_MODE_ABS);
  58 }
  59
  60 static void timer_disarm(struct arch_timer_cpu *timer)
  61 {
  62         if (timer_is_armed(timer)) {
  63                 hrtimer_cancel(&timer->timer);
  64                 cancel_work_sync(&timer->expired);
  65                 timer->armed = false;
  66         }
  67 }
  68
  69 static irqreturn_t kvm_arch_timer_handler(int irq, void *dev_id)
  70 {
  71         struct kvm_vcpu *vcpu = *(struct kvm_vcpu **)dev_id;
  72
  73         /*
  74          * We disable the timer in the world switch and let it be
  75          * handled by kvm_timer_sync_hwstate(). Getting a timer
  76          * interrupt at this point is a sure sign of some major
  77          * breakage.
  78          */
  79         pr_warn("Unexpected interrupt %d on vcpu %p\n", irq, vcpu);
  80         return IRQ_HANDLED;
  81 }
  82
  83 /*
  84  * Work function for handling the backup timer that we schedule when a vcpu is
  85  * no longer running, but had a timer programmed to fire in the future.
  86  */
  87 static void kvm_timer_inject_irq_work(struct work_struct *work)
  88 {
  89         struct kvm_vcpu *vcpu;
  90
  91         vcpu = container_of(work, struct kvm_vcpu, arch.timer_cpu.expired);
  92         vcpu->arch.timer_cpu.armed = false;
  93
  94         /*
  95          * If the vcpu is blocked we want to wake it up so that it will see
  96          * the timer has expired when entering the guest.
  97          */
  98         kvm_vcpu_kick(vcpu);
  99 }
 100
 101 static enum hrtimer_restart kvm_timer_expire(struct hrtimer *hrt)
 102 {
 103         struct arch_timer_cpu *timer;
 104         timer = container_of(hrt, struct arch_timer_cpu, timer);
 105         queue_work(wqueue, &timer->expired);
 106         return HRTIMER_NORESTART;
 107 }
 108
 109 static bool kvm_timer_irq_can_fire(struct kvm_vcpu *vcpu)
 110 {
 111         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 112
 113         return !(timer->cntv_ctl & ARCH_TIMER_CTRL_IT_MASK) &&
 114                 (timer->cntv_ctl & ARCH_TIMER_CTRL_ENABLE);
 115 }
 116
 117 bool kvm_timer_should_fire(struct kvm_vcpu *vcpu)
 118 {
 119         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 120         cycle_t cval, now;
 121
 122         if (!kvm_timer_irq_can_fire(vcpu))
 123                 return false;
 124
 125         cval = timer->cntv_cval;
 126         now = kvm_phys_timer_read() - vcpu->kvm->arch.timer.cntvoff;
 127
 128         return cval <= now;
 129 }
 130
 131 static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level)
 132 {
 133         int ret;
 134         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 135
 136         BUG_ON(!vgic_initialized(vcpu->kvm));
 137
 138         timer->active_cleared_last = false;
 139         timer->irq.level = new_level;
 140         trace_kvm_timer_update_irq(vcpu->vcpu_id, timer->map->virt_irq,
 141                                    timer->irq.level);
 142         ret = kvm_vgic_inject_mapped_irq(vcpu->kvm, vcpu->vcpu_id,
 143                                          timer->map,
 144                                          timer->irq.level);
 145         WARN_ON(ret);
 146 }
 147
 148 /*
 149  * Check if there was a change in the timer state (should we raise or lower
 150  * the line level to the GIC).
 151  */
 152 static int kvm_timer_update_state(struct kvm_vcpu *vcpu)
 153 {
 154         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 155
 156         /*
 157          * If userspace modified the timer registers via SET_ONE_REG before
 158          * the vgic was initialized, we mustn't set the timer->irq.level value
 159          * because the guest would never see the interrupt.  Instead wait
 160          * until we call this function from kvm_timer_flush_hwstate.
 161          */
 162         if (!vgic_initialized(vcpu->kvm))
 163                 return -ENODEV;
 164
 165         if (kvm_timer_should_fire(vcpu) != timer->irq.level)
 166                 kvm_timer_update_irq(vcpu, !timer->irq.level);
 167
 168         return 0;
 169 }
 170
 171 /*
 172  * Schedule the background timer before calling kvm_vcpu_block, so that this
 173  * thread is removed from its waitqueue and made runnable when there's a timer
 174  * interrupt to handle.
 175  */
 176 void kvm_timer_schedule(struct kvm_vcpu *vcpu)
 177 {
 178         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 179         u64 ns;
 180         cycle_t cval, now;
 181
 182         BUG_ON(timer_is_armed(timer));
 183
 184         /*
 185          * No need to schedule a background timer if the guest timer has
 186          * already expired, because kvm_vcpu_block will return before putting
 187          * the thread to sleep.
 188          */
 189         if (kvm_timer_should_fire(vcpu))
 190                 return;
 191
 192         /*
 193          * If the timer is not capable of raising interrupts (disabled or
 194          * masked), then there's no more work for us to do.
 195          */
 196         if (!kvm_timer_irq_can_fire(vcpu))
 197                 return;
 198
 199         /*  The timer has not yet expired, schedule a background timer */
 200         cval = timer->cntv_cval;
 201         now = kvm_phys_timer_read() - vcpu->kvm->arch.timer.cntvoff;
 202
 203         ns = cyclecounter_cyc2ns(timecounter->cc,
 204                                  cval - now,
 205                                  timecounter->mask,
 206                                  &timecounter->frac);
 207         timer_arm(timer, ns);
 208 }
 209
 210 void kvm_timer_unschedule(struct kvm_vcpu *vcpu)
 211 {
 212         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 213         timer_disarm(timer);
 214 }
 215
 216 /**
 217  * kvm_timer_flush_hwstate - prepare to move the virt timer to the cpu
 218  * @vcpu: The vcpu pointer
 219  *
 220  * Check if the virtual timer has expired while we were running in the host,
 221  * and inject an interrupt if that was the case.
 222  */
 223 void kvm_timer_flush_hwstate(struct kvm_vcpu *vcpu)
 224 {
 225         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 226         bool phys_active;
 227         int ret;
 228
 229         if (kvm_timer_update_state(vcpu))
 230                 return;
 231
 232         /*
 233         * If we enter the guest with the virtual input level to the VGIC
 234         * asserted, then we have already told the VGIC what we need to, and
 235         * we don't need to exit from the guest until the guest deactivates
 236         * the already injected interrupt, so therefore we should set the
 237         * hardware active state to prevent unnecessary exits from the guest.
 238         *
 239         * Also, if we enter the guest with the virtual timer interrupt active,
 240         * then it must be active on the physical distributor, because we set
 241         * the HW bit and the guest must be able to deactivate the virtual and
 242         * physical interrupt at the same time.
 243         *
 244         * Conversely, if the virtual input level is deasserted and the virtual
 245         * interrupt is not active, then always clear the hardware active state
 246         * to ensure that hardware interrupts from the timer triggers a guest
 247         * exit.
 248         */
 249         if (timer->irq.level || kvm_vgic_map_is_active(vcpu, timer->map))
 250                 phys_active = true;
 251         else
 252                 phys_active = false;
 253
 254         /*
 255          * We want to avoid hitting the (re)distributor as much as
 256          * possible, as this is a potentially expensive MMIO access
 257          * (not to mention locks in the irq layer), and a solution for
 258          * this is to cache the "active" state in memory.
 259          *
 260          * Things to consider: we cannot cache an "active set" state,
 261          * because the HW can change this behind our back (it becomes
 262          * "clear" in the HW). We must then restrict the caching to
 263          * the "clear" state.
 264          *
 265          * The cache is invalidated on:
 266          * - vcpu put, indicating that the HW cannot be trusted to be
 267          *   in a sane state on the next vcpu load,
 268          * - any change in the interrupt state
 269          *
 270          * Usage conditions:
 271          * - cached value is "active clear"
 272          * - value to be programmed is "active clear"
 273          */
 274         if (timer->active_cleared_last && !phys_active)
 275                 return;
 276
 277         ret = irq_set_irqchip_state(timer->map->irq,
 278                                     IRQCHIP_STATE_ACTIVE,
 279                                     phys_active);
 280         WARN_ON(ret);
 281
 282         timer->active_cleared_last = !phys_active;
 283 }
 284
 285 /**
 286  * kvm_timer_sync_hwstate - sync timer state from cpu
 287  * @vcpu: The vcpu pointer
 288  *
 289  * Check if the virtual timer has expired while we were running in the guest,
 290  * and inject an interrupt if that was the case.
 291  */
 292 void kvm_timer_sync_hwstate(struct kvm_vcpu *vcpu)
 293 {
 294         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 295
 296         BUG_ON(timer_is_armed(timer));
 297
 298         /*
 299          * The guest could have modified the timer registers or the timer
 300          * could have expired, update the timer state.
 301          */
 302         kvm_timer_update_state(vcpu);
 303 }
 304
 305 int kvm_timer_vcpu_reset(struct kvm_vcpu *vcpu,
 306                          const struct kvm_irq_level *irq)
 307 {
 308         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 309         struct irq_phys_map *map;
 310
 311         /*
 312          * The vcpu timer irq number cannot be determined in
 313          * kvm_timer_vcpu_init() because it is called much before
 314          * kvm_vcpu_set_target(). To handle this, we determine
 315          * vcpu timer irq number when the vcpu is reset.
 316          */
 317         timer->irq.irq = irq->irq;
 318
 319         /*
 320          * The bits in CNTV_CTL are architecturally reset to UNKNOWN for ARMv8
 321          * and to 0 for ARMv7.  We provide an implementation that always
 322          * resets the timer to be disabled and unmasked and is compliant with
 323          * the ARMv7 architecture.
 324          */
 325         timer->cntv_ctl = 0;
 326         kvm_timer_update_state(vcpu);
 327
 328         /*
 329          * Tell the VGIC that the virtual interrupt is tied to a
 330          * physical interrupt. We do that once per VCPU.
 331          */
 332         map = kvm_vgic_map_phys_irq(vcpu, irq->irq, host_vtimer_irq);
 333         if (WARN_ON(IS_ERR(map)))
 334                 return PTR_ERR(map);
 335
 336         timer->map = map;
 337         return 0;
 338 }
 339
 340 void kvm_timer_vcpu_init(struct kvm_vcpu *vcpu)
 341 {
 342         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 343
 344         INIT_WORK(&timer->expired, kvm_timer_inject_irq_work);
 345         hrtimer_init(&timer->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
 346         timer->timer.function = kvm_timer_expire;
 347 }
 348
 349 static void kvm_timer_init_interrupt(void *info)
 350 {
 351         enable_percpu_irq(host_vtimer_irq, 0);
 352 }
 353
 354 int kvm_arm_timer_set_reg(struct kvm_vcpu *vcpu, u64 regid, u64 value)
 355 {
 356         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 357
 358         switch (regid) {
 359         case KVM_REG_ARM_TIMER_CTL:
 360                 timer->cntv_ctl = value;
 361                 break;
 362         case KVM_REG_ARM_TIMER_CNT:
 363                 vcpu->kvm->arch.timer.cntvoff = kvm_phys_timer_read() - value;
 364                 break;
 365         case KVM_REG_ARM_TIMER_CVAL:
 366                 timer->cntv_cval = value;
 367                 break;
 368         default:
 369                 return -1;
 370         }
 371
 372         kvm_timer_update_state(vcpu);
 373         return 0;
 374 }
 375
 376 u64 kvm_arm_timer_get_reg(struct kvm_vcpu *vcpu, u64 regid)
 377 {
 378         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 379
 380         switch (regid) {
 381         case KVM_REG_ARM_TIMER_CTL:
 382                 return timer->cntv_ctl;
 383         case KVM_REG_ARM_TIMER_CNT:
 384                 return kvm_phys_timer_read() - vcpu->kvm->arch.timer.cntvoff;
 385         case KVM_REG_ARM_TIMER_CVAL:
 386                 return timer->cntv_cval;
 387         }
 388         return (u64)-1;
 389 }
 390
 391 static int kvm_timer_cpu_notify(struct notifier_block *self,
 392                                 unsigned long action, void *cpu)
 393 {
 394         switch (action) {
 395         case CPU_STARTING:
 396         case CPU_STARTING_FROZEN:
 397                 kvm_timer_init_interrupt(NULL);
 398                 break;
 399         case CPU_DYING:
 400         case CPU_DYING_FROZEN:
 401                 disable_percpu_irq(host_vtimer_irq);
 402                 break;
 403         }
 404
 405         return NOTIFY_OK;
 406 }
 407
 408 static struct notifier_block kvm_timer_cpu_nb = {
 409         .notifier_call = kvm_timer_cpu_notify,
 410 };
 411
 412 static const struct of_device_id arch_timer_of_match[] = {
 413         { .compatible   = "arm,armv7-timer",    },
 414         { .compatible   = "arm,armv8-timer",    },
 415         {},
 416 };
 417
 418 int kvm_timer_hyp_init(void)
 419 {
 420         struct device_node *np;
 421         unsigned int ppi;
 422         int err;
 423
 424         timecounter = arch_timer_get_timecounter();
 425         if (!timecounter)
 426                 return -ENODEV;
 427
 428         np = of_find_matching_node(NULL, arch_timer_of_match);
 429         if (!np) {
 430                 kvm_err("kvm_arch_timer: can't find DT node\n");
 431                 return -ENODEV;
 432         }
 433
 434         ppi = irq_of_parse_and_map(np, 2);
 435         if (!ppi) {
 436                 kvm_err("kvm_arch_timer: no virtual timer interrupt\n");
 437                 err = -EINVAL;
 438                 goto out;
 439         }
 440
 441         err = request_percpu_irq(ppi, kvm_arch_timer_handler,
 442                                  "kvm guest timer", kvm_get_running_vcpus());
 443         if (err) {
 444                 kvm_err("kvm_arch_timer: can't request interrupt %d (%d)\n",
 445                         ppi, err);
 446                 goto out;
 447         }
 448
 449         host_vtimer_irq = ppi;
 450
 451         err = __register_cpu_notifier(&kvm_timer_cpu_nb);
 452         if (err) {
 453                 kvm_err("Cannot register timer CPU notifier\n");
 454                 goto out_free;
 455         }
 456
 457         wqueue = create_singlethread_workqueue("kvm_arch_timer");
 458         if (!wqueue) {
 459                 err = -ENOMEM;
 460                 goto out_free;
 461         }
 462
 463         kvm_info("%s IRQ%d\n", np->name, ppi);
 464         on_each_cpu(kvm_timer_init_interrupt, NULL, 1);
 465
 466         goto out;
 467 out_free:
 468         free_percpu_irq(ppi, kvm_get_running_vcpus());
 469 out:
 470         of_node_put(np);
 471         return err;
 472 }
 473
 474 void kvm_timer_vcpu_terminate(struct kvm_vcpu *vcpu)
 475 {
 476         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 477
 478         timer_disarm(timer);
 479         if (timer->map)
 480                 kvm_vgic_unmap_phys_irq(vcpu, timer->map);
 481 }
 482
 483 void kvm_timer_enable(struct kvm *kvm)
 484 {
 485         if (kvm->arch.timer.enabled)
 486                 return;
 487
 488         /*
 489          * There is a potential race here between VCPUs starting for the first
 490          * time, which may be enabling the timer multiple times.  That doesn't
 491          * hurt though, because we're just setting a variable to the same
 492          * variable that it already was.  The important thing is that all
 493          * VCPUs have the enabled variable set, before entering the guest, if
 494          * the arch timers are enabled.
 495          */
 496         if (timecounter && wqueue)
 497                 kvm->arch.timer.enabled = 1;
 498 }
 499
 500 void kvm_timer_init(struct kvm *kvm)
 501 {
 502         kvm->arch.timer.cntvoff = kvm_phys_timer_read();
 503 }