virt/kvm/arm/vgic/vgic.c

   1 /*
   2  * Copyright (C) 2015, 2016 ARM Ltd.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11  * GNU General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License
  14  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  15  */
  16
  17 #include <linux/kvm.h>
  18 #include <linux/kvm_host.h>
  19 #include <linux/list_sort.h>
  20
  21 #include "vgic.h"
  22
  23 #define CREATE_TRACE_POINTS
  24 #include "../trace.h"
  25
  26 #ifdef CONFIG_DEBUG_SPINLOCK
  27 #define DEBUG_SPINLOCK_BUG_ON(p) BUG_ON(p)
  28 #else
  29 #define DEBUG_SPINLOCK_BUG_ON(p)
  30 #endif
  31
  32 struct vgic_global __section(.hyp.text) kvm_vgic_global_state;
  33
  34 /*
  35  * Locking order is always:
  36  * its->cmd_lock (mutex)
  37  *   its->its_lock (mutex)
  38  *     vgic_cpu->ap_list_lock
  39  *       kvm->lpi_list_lock
  40  *         vgic_irq->irq_lock
  41  *
  42  * If you need to take multiple locks, always take the upper lock first,
  43  * then the lower ones, e.g. first take the its_lock, then the irq_lock.
  44  * If you are already holding a lock and need to take a higher one, you
  45  * have to drop the lower ranking lock first and re-aquire it after having
  46  * taken the upper one.
  47  *
  48  * When taking more than one ap_list_lock at the same time, always take the
  49  * lowest numbered VCPU's ap_list_lock first, so:
  50  *   vcpuX->vcpu_id < vcpuY->vcpu_id:
  51  *     spin_lock(vcpuX->arch.vgic_cpu.ap_list_lock);
  52  *     spin_lock(vcpuY->arch.vgic_cpu.ap_list_lock);
  53  */
  54
  55 /*
  56  * Iterate over the VM's list of mapped LPIs to find the one with a
  57  * matching interrupt ID and return a reference to the IRQ structure.
  58  */
  59 static struct vgic_irq *vgic_get_lpi(struct kvm *kvm, u32 intid)
  60 {
  61         struct vgic_dist *dist = &kvm->arch.vgic;
  62         struct vgic_irq *irq = NULL;
  63
  64         spin_lock(&dist->lpi_list_lock);
  65
  66         list_for_each_entry(irq, &dist->lpi_list_head, lpi_list) {
  67                 if (irq->intid != intid)
  68                         continue;
  69
  70                 /*
  71                  * This increases the refcount, the caller is expected to
  72                  * call vgic_put_irq() later once it's finished with the IRQ.
  73                  */
  74                 kref_get(&irq->refcount);
  75                 goto out_unlock;
  76         }
  77         irq = NULL;
  78
  79 out_unlock:
  80         spin_unlock(&dist->lpi_list_lock);
  81
  82         return irq;
  83 }
  84
  85 /*
  86  * This looks up the virtual interrupt ID to get the corresponding
  87  * struct vgic_irq. It also increases the refcount, so any caller is expected
  88  * to call vgic_put_irq() once it's finished with this IRQ.
  89  */
  90 struct vgic_irq *vgic_get_irq(struct kvm *kvm, struct kvm_vcpu *vcpu,
  91                               u32 intid)
  92 {
  93         /* SGIs and PPIs */
  94         if (intid <= VGIC_MAX_PRIVATE)
  95                 return &vcpu->arch.vgic_cpu.private_irqs[intid];
  96
  97         /* SPIs */
  98         if (intid <= VGIC_MAX_SPI)
  99                 return &kvm->arch.vgic.spis[intid - VGIC_NR_PRIVATE_IRQS];
 100
 101         /* LPIs */
 102         if (intid >= VGIC_MIN_LPI)
 103                 return vgic_get_lpi(kvm, intid);
 104
 105         WARN(1, "Looking up struct vgic_irq for reserved INTID");
 106         return NULL;
 107 }
 108
 109 static void vgic_get_irq_kref(struct vgic_irq *irq)
 110 {
 111         if (irq->intid < VGIC_MIN_LPI)
 112                 return;
 113
 114         kref_get(&irq->refcount);
 115 }
 116
 117 /*
 118  * We can't do anything in here, because we lack the kvm pointer to
 119  * lock and remove the item from the lpi_list. So we keep this function
 120  * empty and use the return value of kref_put() to trigger the freeing.
 121  */
 122 static void vgic_irq_release(struct kref *ref)
 123 {
 124 }
 125
 126 void vgic_put_irq(struct kvm *kvm, struct vgic_irq *irq)
 127 {
 128         struct vgic_dist *dist;
 129
 130         if (irq->intid < VGIC_MIN_LPI)
 131                 return;
 132
 133         if (!kref_put(&irq->refcount, vgic_irq_release))
 134                 return;
 135
 136         dist = &kvm->arch.vgic;
 137
 138         spin_lock(&dist->lpi_list_lock);
 139         list_del(&irq->lpi_list);
 140         dist->lpi_list_count--;
 141         spin_unlock(&dist->lpi_list_lock);
 142
 143         kfree(irq);
 144 }
 145
 146 /**
 147  * kvm_vgic_target_oracle - compute the target vcpu for an irq
 148  *
 149  * @irq:        The irq to route. Must be already locked.
 150  *
 151  * Based on the current state of the interrupt (enabled, pending,
 152  * active, vcpu and target_vcpu), compute the next vcpu this should be
 153  * given to. Return NULL if this shouldn't be injected at all.
 154  *
 155  * Requires the IRQ lock to be held.
 156  */
 157 static struct kvm_vcpu *vgic_target_oracle(struct vgic_irq *irq)
 158 {
 159         DEBUG_SPINLOCK_BUG_ON(!spin_is_locked(&irq->irq_lock));
 160
 161         /* If the interrupt is active, it must stay on the current vcpu */
 162         if (irq->active)
 163                 return irq->vcpu ? : irq->target_vcpu;
 164
 165         /*
 166          * If the IRQ is not active but enabled and pending, we should direct
 167          * it to its configured target VCPU.
 168          * If the distributor is disabled, pending interrupts shouldn't be
 169          * forwarded.
 170          */
 171         if (irq->enabled && irq->pending) {
 172                 if (unlikely(irq->target_vcpu &&
 173                              !irq->target_vcpu->kvm->arch.vgic.enabled))
 174                         return NULL;
 175
 176                 return irq->target_vcpu;
 177         }
 178
 179         /* If neither active nor pending and enabled, then this IRQ should not
 180          * be queued to any VCPU.
 181          */
 182         return NULL;
 183 }
 184
 185 /*
 186  * The order of items in the ap_lists defines how we'll pack things in LRs as
 187  * well, the first items in the list being the first things populated in the
 188  * LRs.
 189  *
 190  * A hard rule is that active interrupts can never be pushed out of the LRs
 191  * (and therefore take priority) since we cannot reliably trap on deactivation
 192  * of IRQs and therefore they have to be present in the LRs.
 193  *
 194  * Otherwise things should be sorted by the priority field and the GIC
 195  * hardware support will take care of preemption of priority groups etc.
 196  *
 197  * Return negative if "a" sorts before "b", 0 to preserve order, and positive
 198  * to sort "b" before "a".
 199  */
 200 static int vgic_irq_cmp(void *priv, struct list_head *a, struct list_head *b)
 201 {
 202         struct vgic_irq *irqa = container_of(a, struct vgic_irq, ap_list);
 203         struct vgic_irq *irqb = container_of(b, struct vgic_irq, ap_list);
 204         bool penda, pendb;
 205         int ret;
 206
 207         spin_lock(&irqa->irq_lock);
 208         spin_lock_nested(&irqb->irq_lock, SINGLE_DEPTH_NESTING);
 209
 210         if (irqa->active || irqb->active) {
 211                 ret = (int)irqb->active - (int)irqa->active;
 212                 goto out;
 213         }
 214
 215         penda = irqa->enabled && irqa->pending;
 216         pendb = irqb->enabled && irqb->pending;
 217
 218         if (!penda || !pendb) {
 219                 ret = (int)pendb - (int)penda;
 220                 goto out;
 221         }
 222
 223         /* Both pending and enabled, sort by priority */
 224         ret = irqa->priority - irqb->priority;
 225 out:
 226         spin_unlock(&irqb->irq_lock);
 227         spin_unlock(&irqa->irq_lock);
 228         return ret;
 229 }
 230
 231 /* Must be called with the ap_list_lock held */
 232 static void vgic_sort_ap_list(struct kvm_vcpu *vcpu)
 233 {
 234         struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
 235
 236         DEBUG_SPINLOCK_BUG_ON(!spin_is_locked(&vgic_cpu->ap_list_lock));
 237
 238         list_sort(NULL, &vgic_cpu->ap_list_head, vgic_irq_cmp);
 239 }
 240
 241 /*
 242  * Only valid injection if changing level for level-triggered IRQs or for a
 243  * rising edge.
 244  */
 245 static bool vgic_validate_injection(struct vgic_irq *irq, bool level)
 246 {
 247         switch (irq->config) {
 248         case VGIC_CONFIG_LEVEL:
 249                 return irq->line_level != level;
 250         case VGIC_CONFIG_EDGE:
 251                 return level;
 252         }
 253
 254         return false;
 255 }
 256
 257 /*
 258  * Check whether an IRQ needs to (and can) be queued to a VCPU's ap list.
 259  * Do the queuing if necessary, taking the right locks in the right order.
 260  * Returns true when the IRQ was queued, false otherwise.
 261  *
 262  * Needs to be entered with the IRQ lock already held, but will return
 263  * with all locks dropped.
 264  */
 265 bool vgic_queue_irq_unlock(struct kvm *kvm, struct vgic_irq *irq)
 266 {
 267         struct kvm_vcpu *vcpu;
 268
 269         DEBUG_SPINLOCK_BUG_ON(!spin_is_locked(&irq->irq_lock));
 270
 271 retry:
 272         vcpu = vgic_target_oracle(irq);
 273         if (irq->vcpu || !vcpu) {
 274                 /*
 275                  * If this IRQ is already on a VCPU's ap_list, then it
 276                  * cannot be moved or modified and there is no more work for
 277                  * us to do.
 278                  *
 279                  * Otherwise, if the irq is not pending and enabled, it does
 280                  * not need to be inserted into an ap_list and there is also
 281                  * no more work for us to do.
 282                  */
 283                 spin_unlock(&irq->irq_lock);
 284                 return false;
 285         }
 286
 287         /*
 288          * We must unlock the irq lock to take the ap_list_lock where
 289          * we are going to insert this new pending interrupt.
 290          */
 291         spin_unlock(&irq->irq_lock);
 292
 293         /* someone can do stuff here, which we re-check below */
 294
 295         spin_lock(&vcpu->arch.vgic_cpu.ap_list_lock);
 296         spin_lock(&irq->irq_lock);
 297
 298         /*
 299          * Did something change behind our backs?
 300          *
 301          * There are two cases:
 302          * 1) The irq lost its pending state or was disabled behind our
 303          *    backs and/or it was queued to another VCPU's ap_list.
 304          * 2) Someone changed the affinity on this irq behind our
 305          *    backs and we are now holding the wrong ap_list_lock.
 306          *
 307          * In both cases, drop the locks and retry.
 308          */
 309
 310         if (unlikely(irq->vcpu || vcpu != vgic_target_oracle(irq))) {
 311                 spin_unlock(&irq->irq_lock);
 312                 spin_unlock(&vcpu->arch.vgic_cpu.ap_list_lock);
 313
 314                 spin_lock(&irq->irq_lock);
 315                 goto retry;
 316         }
 317
 318         /*
 319          * Grab a reference to the irq to reflect the fact that it is
 320          * now in the ap_list.
 321          */
 322         vgic_get_irq_kref(irq);
 323         list_add_tail(&irq->ap_list, &vcpu->arch.vgic_cpu.ap_list_head);
 324         irq->vcpu = vcpu;
 325
 326         spin_unlock(&irq->irq_lock);
 327         spin_unlock(&vcpu->arch.vgic_cpu.ap_list_lock);
 328
 329         kvm_vcpu_kick(vcpu);
 330
 331         return true;
 332 }
 333
 334 static int vgic_update_irq_pending(struct kvm *kvm, int cpuid,
 335                                    unsigned int intid, bool level,
 336                                    bool mapped_irq)
 337 {
 338         struct kvm_vcpu *vcpu;
 339         struct vgic_irq *irq;
 340         int ret;
 341
 342         trace_vgic_update_irq_pending(cpuid, intid, level);
 343
 344         ret = vgic_lazy_init(kvm);
 345         if (ret)
 346                 return ret;
 347
 348         vcpu = kvm_get_vcpu(kvm, cpuid);
 349         if (!vcpu && intid < VGIC_NR_PRIVATE_IRQS)
 350                 return -EINVAL;
 351
 352         irq = vgic_get_irq(kvm, vcpu, intid);
 353         if (!irq)
 354                 return -EINVAL;
 355
 356         if (irq->hw != mapped_irq) {
 357                 vgic_put_irq(kvm, irq);
 358                 return -EINVAL;
 359         }
 360
 361         spin_lock(&irq->irq_lock);
 362
 363         if (!vgic_validate_injection(irq, level)) {
 364                 /* Nothing to see here, move along... */
 365                 spin_unlock(&irq->irq_lock);
 366                 vgic_put_irq(kvm, irq);
 367                 return 0;
 368         }
 369
 370         if (irq->config == VGIC_CONFIG_LEVEL) {
 371                 irq->line_level = level;
 372                 irq->pending = level || irq->soft_pending;
 373         } else {
 374                 irq->pending = true;
 375         }
 376
 377         vgic_queue_irq_unlock(kvm, irq);
 378         vgic_put_irq(kvm, irq);
 379
 380         return 0;
 381 }
 382
 383 /**
 384  * kvm_vgic_inject_irq - Inject an IRQ from a device to the vgic
 385  * @kvm:     The VM structure pointer
 386  * @cpuid:   The CPU for PPIs
 387  * @intid:   The INTID to inject a new state to.
 388  * @level:   Edge-triggered:  true:  to trigger the interrupt
 389  *                            false: to ignore the call
 390  *           Level-sensitive  true:  raise the input signal
 391  *                            false: lower the input signal
 392  *
 393  * The VGIC is not concerned with devices being active-LOW or active-HIGH for
 394  * level-sensitive interrupts.  You can think of the level parameter as 1
 395  * being HIGH and 0 being LOW and all devices being active-HIGH.
 396  */
 397 int kvm_vgic_inject_irq(struct kvm *kvm, int cpuid, unsigned int intid,
 398                         bool level)
 399 {
 400         return vgic_update_irq_pending(kvm, cpuid, intid, level, false);
 401 }
 402
 403 int kvm_vgic_inject_mapped_irq(struct kvm *kvm, int cpuid, unsigned int intid,
 404                                bool level)
 405 {
 406         return vgic_update_irq_pending(kvm, cpuid, intid, level, true);
 407 }
 408
 409 int kvm_vgic_map_phys_irq(struct kvm_vcpu *vcpu, u32 virt_irq, u32 phys_irq)
 410 {
 411         struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, virt_irq);
 412
 413         BUG_ON(!irq);
 414
 415         spin_lock(&irq->irq_lock);
 416
 417         irq->hw = true;
 418         irq->hwintid = phys_irq;
 419
 420         spin_unlock(&irq->irq_lock);
 421         vgic_put_irq(vcpu->kvm, irq);
 422
 423         return 0;
 424 }
 425
 426 int kvm_vgic_unmap_phys_irq(struct kvm_vcpu *vcpu, unsigned int virt_irq)
 427 {
 428         struct vgic_irq *irq;
 429
 430         if (!vgic_initialized(vcpu->kvm))
 431                 return -EAGAIN;
 432
 433         irq = vgic_get_irq(vcpu->kvm, vcpu, virt_irq);
 434         BUG_ON(!irq);
 435
 436         spin_lock(&irq->irq_lock);
 437
 438         irq->hw = false;
 439         irq->hwintid = 0;
 440
 441         spin_unlock(&irq->irq_lock);
 442         vgic_put_irq(vcpu->kvm, irq);
 443
 444         return 0;
 445 }
 446
 447 /**
 448  * vgic_prune_ap_list - Remove non-relevant interrupts from the list
 449  *
 450  * @vcpu: The VCPU pointer
 451  *
 452  * Go over the list of "interesting" interrupts, and prune those that we
 453  * won't have to consider in the near future.
 454  */
 455 static void vgic_prune_ap_list(struct kvm_vcpu *vcpu)
 456 {
 457         struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
 458         struct vgic_irq *irq, *tmp;
 459
 460 retry:
 461         spin_lock(&vgic_cpu->ap_list_lock);
 462
 463         list_for_each_entry_safe(irq, tmp, &vgic_cpu->ap_list_head, ap_list) {
 464                 struct kvm_vcpu *target_vcpu, *vcpuA, *vcpuB;
 465
 466                 spin_lock(&irq->irq_lock);
 467
 468                 BUG_ON(vcpu != irq->vcpu);
 469
 470                 target_vcpu = vgic_target_oracle(irq);
 471
 472                 if (!target_vcpu) {
 473                         /*
 474                          * We don't need to process this interrupt any
 475                          * further, move it off the list.
 476                          */
 477                         list_del(&irq->ap_list);
 478                         irq->vcpu = NULL;
 479                         spin_unlock(&irq->irq_lock);
 480
 481                         /*
 482                          * This vgic_put_irq call matches the
 483                          * vgic_get_irq_kref in vgic_queue_irq_unlock,
 484                          * where we added the LPI to the ap_list. As
 485                          * we remove the irq from the list, we drop
 486                          * also drop the refcount.
 487                          */
 488                         vgic_put_irq(vcpu->kvm, irq);
 489                         continue;
 490                 }
 491
 492                 if (target_vcpu == vcpu) {
 493                         /* We're on the right CPU */
 494                         spin_unlock(&irq->irq_lock);
 495                         continue;
 496                 }
 497
 498                 /* This interrupt looks like it has to be migrated. */
 499
 500                 spin_unlock(&irq->irq_lock);
 501                 spin_unlock(&vgic_cpu->ap_list_lock);
 502
 503                 /*
 504                  * Ensure locking order by always locking the smallest
 505                  * ID first.
 506                  */
 507                 if (vcpu->vcpu_id < target_vcpu->vcpu_id) {
 508                         vcpuA = vcpu;
 509                         vcpuB = target_vcpu;
 510                 } else {
 511                         vcpuA = target_vcpu;
 512                         vcpuB = vcpu;
 513                 }
 514
 515                 spin_lock(&vcpuA->arch.vgic_cpu.ap_list_lock);
 516                 spin_lock_nested(&vcpuB->arch.vgic_cpu.ap_list_lock,
 517                                  SINGLE_DEPTH_NESTING);
 518                 spin_lock(&irq->irq_lock);
 519
 520                 /*
 521                  * If the affinity has been preserved, move the
 522                  * interrupt around. Otherwise, it means things have
 523                  * changed while the interrupt was unlocked, and we
 524                  * need to replay this.
 525                  *
 526                  * In all cases, we cannot trust the list not to have
 527                  * changed, so we restart from the beginning.
 528                  */
 529                 if (target_vcpu == vgic_target_oracle(irq)) {
 530                         struct vgic_cpu *new_cpu = &target_vcpu->arch.vgic_cpu;
 531
 532                         list_del(&irq->ap_list);
 533                         irq->vcpu = target_vcpu;
 534                         list_add_tail(&irq->ap_list, &new_cpu->ap_list_head);
 535                 }
 536
 537                 spin_unlock(&irq->irq_lock);
 538                 spin_unlock(&vcpuB->arch.vgic_cpu.ap_list_lock);
 539                 spin_unlock(&vcpuA->arch.vgic_cpu.ap_list_lock);
 540                 goto retry;
 541         }
 542
 543         spin_unlock(&vgic_cpu->ap_list_lock);
 544 }
 545
 546 static inline void vgic_process_maintenance_interrupt(struct kvm_vcpu *vcpu)
 547 {
 548         if (kvm_vgic_global_state.type == VGIC_V2)
 549                 vgic_v2_process_maintenance(vcpu);
 550         else
 551                 vgic_v3_process_maintenance(vcpu);
 552 }
 553
 554 static inline void vgic_fold_lr_state(struct kvm_vcpu *vcpu)
 555 {
 556         if (kvm_vgic_global_state.type == VGIC_V2)
 557                 vgic_v2_fold_lr_state(vcpu);
 558         else
 559                 vgic_v3_fold_lr_state(vcpu);
 560 }
 561
 562 /* Requires the irq_lock to be held. */
 563 static inline void vgic_populate_lr(struct kvm_vcpu *vcpu,
 564                                     struct vgic_irq *irq, int lr)
 565 {
 566         DEBUG_SPINLOCK_BUG_ON(!spin_is_locked(&irq->irq_lock));
 567
 568         if (kvm_vgic_global_state.type == VGIC_V2)
 569                 vgic_v2_populate_lr(vcpu, irq, lr);
 570         else
 571                 vgic_v3_populate_lr(vcpu, irq, lr);
 572 }
 573
 574 static inline void vgic_clear_lr(struct kvm_vcpu *vcpu, int lr)
 575 {
 576         if (kvm_vgic_global_state.type == VGIC_V2)
 577                 vgic_v2_clear_lr(vcpu, lr);
 578         else
 579                 vgic_v3_clear_lr(vcpu, lr);
 580 }
 581
 582 static inline void vgic_set_underflow(struct kvm_vcpu *vcpu)
 583 {
 584         if (kvm_vgic_global_state.type == VGIC_V2)
 585                 vgic_v2_set_underflow(vcpu);
 586         else
 587                 vgic_v3_set_underflow(vcpu);
 588 }
 589
 590 /* Requires the ap_list_lock to be held. */
 591 static int compute_ap_list_depth(struct kvm_vcpu *vcpu)
 592 {
 593         struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
 594         struct vgic_irq *irq;
 595         int count = 0;
 596
 597         DEBUG_SPINLOCK_BUG_ON(!spin_is_locked(&vgic_cpu->ap_list_lock));
 598
 599         list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) {
 600                 spin_lock(&irq->irq_lock);
 601                 /* GICv2 SGIs can count for more than one... */
 602                 if (vgic_irq_is_sgi(irq->intid) && irq->source)
 603                         count += hweight8(irq->source);
 604                 else
 605                         count++;
 606                 spin_unlock(&irq->irq_lock);
 607         }
 608         return count;
 609 }
 610
 611 /* Requires the VCPU's ap_list_lock to be held. */
 612 static void vgic_flush_lr_state(struct kvm_vcpu *vcpu)
 613 {
 614         struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
 615         struct vgic_irq *irq;
 616         int count = 0;
 617
 618         DEBUG_SPINLOCK_BUG_ON(!spin_is_locked(&vgic_cpu->ap_list_lock));
 619
 620         if (compute_ap_list_depth(vcpu) > kvm_vgic_global_state.nr_lr) {
 621                 vgic_set_underflow(vcpu);
 622                 vgic_sort_ap_list(vcpu);
 623         }
 624
 625         list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) {
 626                 spin_lock(&irq->irq_lock);
 627
 628                 if (unlikely(vgic_target_oracle(irq) != vcpu))
 629                         goto next;
 630
 631                 /*
 632                  * If we get an SGI with multiple sources, try to get
 633                  * them in all at once.
 634                  */
 635                 do {
 636                         vgic_populate_lr(vcpu, irq, count++);
 637                 } while (irq->source && count < kvm_vgic_global_state.nr_lr);
 638
 639 next:
 640                 spin_unlock(&irq->irq_lock);
 641
 642                 if (count == kvm_vgic_global_state.nr_lr)
 643                         break;
 644         }
 645
 646         vcpu->arch.vgic_cpu.used_lrs = count;
 647
 648         /* Nuke remaining LRs */
 649         for ( ; count < kvm_vgic_global_state.nr_lr; count++)
 650                 vgic_clear_lr(vcpu, count);
 651 }
 652
 653 /* Sync back the hardware VGIC state into our emulation after a guest's run. */
 654 void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu)
 655 {
 656         vgic_process_maintenance_interrupt(vcpu);
 657         vgic_fold_lr_state(vcpu);
 658         vgic_prune_ap_list(vcpu);
 659 }
 660
 661 /* Flush our emulation state into the GIC hardware before entering the guest. */
 662 void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu)
 663 {
 664         spin_lock(&vcpu->arch.vgic_cpu.ap_list_lock);
 665         vgic_flush_lr_state(vcpu);
 666         spin_unlock(&vcpu->arch.vgic_cpu.ap_list_lock);
 667 }
 668
 669 int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu)
 670 {
 671         struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
 672         struct vgic_irq *irq;
 673         bool pending = false;
 674
 675         if (!vcpu->kvm->arch.vgic.enabled)
 676                 return false;
 677
 678         spin_lock(&vgic_cpu->ap_list_lock);
 679
 680         list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) {
 681                 spin_lock(&irq->irq_lock);
 682                 pending = irq->pending && irq->enabled;
 683                 spin_unlock(&irq->irq_lock);
 684
 685                 if (pending)
 686                         break;
 687         }
 688
 689         spin_unlock(&vgic_cpu->ap_list_lock);
 690
 691         return pending;
 692 }
 693
 694 void vgic_kick_vcpus(struct kvm *kvm)
 695 {
 696         struct kvm_vcpu *vcpu;
 697         int c;
 698
 699         /*
 700          * We've injected an interrupt, time to find out who deserves
 701          * a good kick...
 702          */
 703         kvm_for_each_vcpu(c, vcpu, kvm) {
 704                 if (kvm_vgic_vcpu_pending_irq(vcpu))
 705                         kvm_vcpu_kick(vcpu);
 706         }
 707 }
 708
 709 bool kvm_vgic_map_is_active(struct kvm_vcpu *vcpu, unsigned int virt_irq)
 710 {
 711         struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, virt_irq);
 712         bool map_is_active;
 713
 714         spin_lock(&irq->irq_lock);
 715         map_is_active = irq->hw && irq->active;
 716         spin_unlock(&irq->irq_lock);
 717         vgic_put_irq(vcpu->kvm, irq);
 718
 719         return map_is_active;
 720 }
 721
 722 int kvm_send_userspace_msi(struct kvm *kvm, struct kvm_msi *msi)
 723 {
 724         if (vgic_has_its(kvm))
 725                 return vgic_its_inject_msi(kvm, msi);
 726         else
 727                 return -ENODEV;
 728 }