arch/s390/kvm/interrupt.c

   1 /*
   2  * handling kvm guest interrupts
   3  *
   4  * Copyright IBM Corp. 2008, 2015
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License (version 2 only)
   8  * as published by the Free Software Foundation.
   9  *
  10  *    Author(s): Carsten Otte <cotte@de.ibm.com>
  11  */
  12
  13 #include <linux/interrupt.h>
  14 #include <linux/kvm_host.h>
  15 #include <linux/hrtimer.h>
  16 #include <linux/mmu_context.h>
  17 #include <linux/signal.h>
  18 #include <linux/slab.h>
  19 #include <linux/bitmap.h>
  20 #include <linux/vmalloc.h>
  21 #include <asm/asm-offsets.h>
  22 #include <asm/dis.h>
  23 #include <asm/uaccess.h>
  24 #include <asm/sclp.h>
  25 #include <asm/isc.h>
  26 #include "kvm-s390.h"
  27 #include "gaccess.h"
  28 #include "trace-s390.h"
  29
  30 #define IOINT_SCHID_MASK 0x0000ffff
  31 #define IOINT_SSID_MASK 0x00030000
  32 #define IOINT_CSSID_MASK 0x03fc0000
  33 #define PFAULT_INIT 0x0600
  34 #define PFAULT_DONE 0x0680
  35 #define VIRTIO_PARAM 0x0d00
  36
  37 int psw_extint_disabled(struct kvm_vcpu *vcpu)
  38 {
  39         return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_EXT);
  40 }
  41
  42 static int psw_ioint_disabled(struct kvm_vcpu *vcpu)
  43 {
  44         return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_IO);
  45 }
  46
  47 static int psw_mchk_disabled(struct kvm_vcpu *vcpu)
  48 {
  49         return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_MCHECK);
  50 }
  51
  52 static int psw_interrupts_disabled(struct kvm_vcpu *vcpu)
  53 {
  54         if ((vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PER) ||
  55             (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_IO) ||
  56             (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_EXT))
  57                 return 0;
  58         return 1;
  59 }
  60
  61 static int ckc_interrupts_enabled(struct kvm_vcpu *vcpu)
  62 {
  63         if (psw_extint_disabled(vcpu) ||
  64             !(vcpu->arch.sie_block->gcr[0] & 0x800ul))
  65                 return 0;
  66         if (guestdbg_enabled(vcpu) && guestdbg_sstep_enabled(vcpu))
  67                 /* No timer interrupts when single stepping */
  68                 return 0;
  69         return 1;
  70 }
  71
  72 static int ckc_irq_pending(struct kvm_vcpu *vcpu)
  73 {
  74         preempt_disable();
  75         if (!(vcpu->arch.sie_block->ckc <
  76               get_tod_clock_fast() + vcpu->arch.sie_block->epoch)) {
  77                 preempt_enable();
  78                 return 0;
  79         }
  80         preempt_enable();
  81         return ckc_interrupts_enabled(vcpu);
  82 }
  83
  84 static int cpu_timer_interrupts_enabled(struct kvm_vcpu *vcpu)
  85 {
  86         return !psw_extint_disabled(vcpu) &&
  87                (vcpu->arch.sie_block->gcr[0] & 0x400ul);
  88 }
  89
  90 static int cpu_timer_irq_pending(struct kvm_vcpu *vcpu)
  91 {
  92         return (vcpu->arch.sie_block->cputm >> 63) &&
  93                cpu_timer_interrupts_enabled(vcpu);
  94 }
  95
  96 static inline int is_ioirq(unsigned long irq_type)
  97 {
  98         return ((irq_type >= IRQ_PEND_IO_ISC_0) &&
  99                 (irq_type <= IRQ_PEND_IO_ISC_7));
 100 }
 101
 102 static uint64_t isc_to_isc_bits(int isc)
 103 {
 104         return (0x80 >> isc) << 24;
 105 }
 106
 107 static inline u8 int_word_to_isc(u32 int_word)
 108 {
 109         return (int_word & 0x38000000) >> 27;
 110 }
 111
 112 static inline unsigned long pending_floating_irqs(struct kvm_vcpu *vcpu)
 113 {
 114         return vcpu->kvm->arch.float_int.pending_irqs;
 115 }
 116
 117 static inline unsigned long pending_local_irqs(struct kvm_vcpu *vcpu)
 118 {
 119         return vcpu->arch.local_int.pending_irqs;
 120 }
 121
 122 static unsigned long disable_iscs(struct kvm_vcpu *vcpu,
 123                                    unsigned long active_mask)
 124 {
 125         int i;
 126
 127         for (i = 0; i <= MAX_ISC; i++)
 128                 if (!(vcpu->arch.sie_block->gcr[6] & isc_to_isc_bits(i)))
 129                         active_mask &= ~(1UL << (IRQ_PEND_IO_ISC_0 + i));
 130
 131         return active_mask;
 132 }
 133
 134 static unsigned long deliverable_irqs(struct kvm_vcpu *vcpu)
 135 {
 136         unsigned long active_mask;
 137
 138         active_mask = pending_local_irqs(vcpu);
 139         active_mask |= pending_floating_irqs(vcpu);
 140         if (!active_mask)
 141                 return 0;
 142
 143         if (psw_extint_disabled(vcpu))
 144                 active_mask &= ~IRQ_PEND_EXT_MASK;
 145         if (psw_ioint_disabled(vcpu))
 146                 active_mask &= ~IRQ_PEND_IO_MASK;
 147         else
 148                 active_mask = disable_iscs(vcpu, active_mask);
 149         if (!(vcpu->arch.sie_block->gcr[0] & 0x2000ul))
 150                 __clear_bit(IRQ_PEND_EXT_EXTERNAL, &active_mask);
 151         if (!(vcpu->arch.sie_block->gcr[0] & 0x4000ul))
 152                 __clear_bit(IRQ_PEND_EXT_EMERGENCY, &active_mask);
 153         if (!(vcpu->arch.sie_block->gcr[0] & 0x800ul))
 154                 __clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &active_mask);
 155         if (!(vcpu->arch.sie_block->gcr[0] & 0x400ul))
 156                 __clear_bit(IRQ_PEND_EXT_CPU_TIMER, &active_mask);
 157         if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
 158                 __clear_bit(IRQ_PEND_EXT_SERVICE, &active_mask);
 159         if (psw_mchk_disabled(vcpu))
 160                 active_mask &= ~IRQ_PEND_MCHK_MASK;
 161         if (!(vcpu->arch.sie_block->gcr[14] &
 162               vcpu->kvm->arch.float_int.mchk.cr14))
 163                 __clear_bit(IRQ_PEND_MCHK_REP, &active_mask);
 164
 165         /*
 166          * STOP irqs will never be actively delivered. They are triggered via
 167          * intercept requests and cleared when the stop intercept is performed.
 168          */
 169         __clear_bit(IRQ_PEND_SIGP_STOP, &active_mask);
 170
 171         return active_mask;
 172 }
 173
 174 static void __set_cpu_idle(struct kvm_vcpu *vcpu)
 175 {
 176         atomic_or(CPUSTAT_WAIT, &vcpu->arch.sie_block->cpuflags);
 177         set_bit(vcpu->vcpu_id, vcpu->arch.local_int.float_int->idle_mask);
 178 }
 179
 180 static void __unset_cpu_idle(struct kvm_vcpu *vcpu)
 181 {
 182         atomic_andnot(CPUSTAT_WAIT, &vcpu->arch.sie_block->cpuflags);
 183         clear_bit(vcpu->vcpu_id, vcpu->arch.local_int.float_int->idle_mask);
 184 }
 185
 186 static void __reset_intercept_indicators(struct kvm_vcpu *vcpu)
 187 {
 188         atomic_andnot(CPUSTAT_IO_INT | CPUSTAT_EXT_INT | CPUSTAT_STOP_INT,
 189                     &vcpu->arch.sie_block->cpuflags);
 190         vcpu->arch.sie_block->lctl = 0x0000;
 191         vcpu->arch.sie_block->ictl &= ~(ICTL_LPSW | ICTL_STCTL | ICTL_PINT);
 192
 193         if (guestdbg_enabled(vcpu)) {
 194                 vcpu->arch.sie_block->lctl |= (LCTL_CR0 | LCTL_CR9 |
 195                                                LCTL_CR10 | LCTL_CR11);
 196                 vcpu->arch.sie_block->ictl |= (ICTL_STCTL | ICTL_PINT);
 197         }
 198 }
 199
 200 static void __set_cpuflag(struct kvm_vcpu *vcpu, u32 flag)
 201 {
 202         atomic_or(flag, &vcpu->arch.sie_block->cpuflags);
 203 }
 204
 205 static void set_intercept_indicators_io(struct kvm_vcpu *vcpu)
 206 {
 207         if (!(pending_floating_irqs(vcpu) & IRQ_PEND_IO_MASK))
 208                 return;
 209         else if (psw_ioint_disabled(vcpu))
 210                 __set_cpuflag(vcpu, CPUSTAT_IO_INT);
 211         else
 212                 vcpu->arch.sie_block->lctl |= LCTL_CR6;
 213 }
 214
 215 static void set_intercept_indicators_ext(struct kvm_vcpu *vcpu)
 216 {
 217         if (!(pending_local_irqs(vcpu) & IRQ_PEND_EXT_MASK))
 218                 return;
 219         if (psw_extint_disabled(vcpu))
 220                 __set_cpuflag(vcpu, CPUSTAT_EXT_INT);
 221         else
 222                 vcpu->arch.sie_block->lctl |= LCTL_CR0;
 223 }
 224
 225 static void set_intercept_indicators_mchk(struct kvm_vcpu *vcpu)
 226 {
 227         if (!(pending_local_irqs(vcpu) & IRQ_PEND_MCHK_MASK))
 228                 return;
 229         if (psw_mchk_disabled(vcpu))
 230                 vcpu->arch.sie_block->ictl |= ICTL_LPSW;
 231         else
 232                 vcpu->arch.sie_block->lctl |= LCTL_CR14;
 233 }
 234
 235 static void set_intercept_indicators_stop(struct kvm_vcpu *vcpu)
 236 {
 237         if (kvm_s390_is_stop_irq_pending(vcpu))
 238                 __set_cpuflag(vcpu, CPUSTAT_STOP_INT);
 239 }
 240
 241 /* Set interception request for non-deliverable interrupts */
 242 static void set_intercept_indicators(struct kvm_vcpu *vcpu)
 243 {
 244         set_intercept_indicators_io(vcpu);
 245         set_intercept_indicators_ext(vcpu);
 246         set_intercept_indicators_mchk(vcpu);
 247         set_intercept_indicators_stop(vcpu);
 248 }
 249
 250 static u16 get_ilc(struct kvm_vcpu *vcpu)
 251 {
 252         switch (vcpu->arch.sie_block->icptcode) {
 253         case ICPT_INST:
 254         case ICPT_INSTPROGI:
 255         case ICPT_OPEREXC:
 256         case ICPT_PARTEXEC:
 257         case ICPT_IOINST:
 258                 /* last instruction only stored for these icptcodes */
 259                 return insn_length(vcpu->arch.sie_block->ipa >> 8);
 260         case ICPT_PROGI:
 261                 return vcpu->arch.sie_block->pgmilc;
 262         default:
 263                 return 0;
 264         }
 265 }
 266
 267 static int __must_check __deliver_cpu_timer(struct kvm_vcpu *vcpu)
 268 {
 269         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 270         int rc;
 271
 272         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_CPU_TIMER,
 273                                          0, 0);
 274
 275         rc  = put_guest_lc(vcpu, EXT_IRQ_CPU_TIMER,
 276                            (u16 *)__LC_EXT_INT_CODE);
 277         rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR);
 278         rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
 279                              &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 280         rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
 281                             &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 282         clear_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
 283         return rc ? -EFAULT : 0;
 284 }
 285
 286 static int __must_check __deliver_ckc(struct kvm_vcpu *vcpu)
 287 {
 288         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 289         int rc;
 290
 291         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_CLOCK_COMP,
 292                                          0, 0);
 293
 294         rc  = put_guest_lc(vcpu, EXT_IRQ_CLK_COMP,
 295                            (u16 __user *)__LC_EXT_INT_CODE);
 296         rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR);
 297         rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
 298                              &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 299         rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
 300                             &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 301         clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
 302         return rc ? -EFAULT : 0;
 303 }
 304
 305 static int __must_check __deliver_pfault_init(struct kvm_vcpu *vcpu)
 306 {
 307         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 308         struct kvm_s390_ext_info ext;
 309         int rc;
 310
 311         spin_lock(&li->lock);
 312         ext = li->irq.ext;
 313         clear_bit(IRQ_PEND_PFAULT_INIT, &li->pending_irqs);
 314         li->irq.ext.ext_params2 = 0;
 315         spin_unlock(&li->lock);
 316
 317         VCPU_EVENT(vcpu, 4, "deliver: pfault init token 0x%llx",
 318                    ext.ext_params2);
 319         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
 320                                          KVM_S390_INT_PFAULT_INIT,
 321                                          0, ext.ext_params2);
 322
 323         rc  = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE, (u16 *) __LC_EXT_INT_CODE);
 324         rc |= put_guest_lc(vcpu, PFAULT_INIT, (u16 *) __LC_EXT_CPU_ADDR);
 325         rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
 326                              &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 327         rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
 328                             &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 329         rc |= put_guest_lc(vcpu, ext.ext_params2, (u64 *) __LC_EXT_PARAMS2);
 330         return rc ? -EFAULT : 0;
 331 }
 332
 333 static int __must_check __deliver_machine_check(struct kvm_vcpu *vcpu)
 334 {
 335         struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
 336         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 337         struct kvm_s390_mchk_info mchk = {};
 338         unsigned long adtl_status_addr;
 339         int deliver = 0;
 340         int rc = 0;
 341
 342         spin_lock(&fi->lock);
 343         spin_lock(&li->lock);
 344         if (test_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs) ||
 345             test_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs)) {
 346                 /*
 347                  * If there was an exigent machine check pending, then any
 348                  * repressible machine checks that might have been pending
 349                  * are indicated along with it, so always clear bits for
 350                  * repressible and exigent interrupts
 351                  */
 352                 mchk = li->irq.mchk;
 353                 clear_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs);
 354                 clear_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs);
 355                 memset(&li->irq.mchk, 0, sizeof(mchk));
 356                 deliver = 1;
 357         }
 358         /*
 359          * We indicate floating repressible conditions along with
 360          * other pending conditions. Channel Report Pending and Channel
 361          * Subsystem damage are the only two and and are indicated by
 362          * bits in mcic and masked in cr14.
 363          */
 364         if (test_and_clear_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs)) {
 365                 mchk.mcic |= fi->mchk.mcic;
 366                 mchk.cr14 |= fi->mchk.cr14;
 367                 memset(&fi->mchk, 0, sizeof(mchk));
 368                 deliver = 1;
 369         }
 370         spin_unlock(&li->lock);
 371         spin_unlock(&fi->lock);
 372
 373         if (deliver) {
 374                 VCPU_EVENT(vcpu, 3, "deliver: machine check mcic 0x%llx",
 375                            mchk.mcic);
 376                 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
 377                                                  KVM_S390_MCHK,
 378                                                  mchk.cr14, mchk.mcic);
 379
 380                 rc  = kvm_s390_vcpu_store_status(vcpu,
 381                                                  KVM_S390_STORE_STATUS_PREFIXED);
 382                 rc |= read_guest_lc(vcpu, __LC_VX_SAVE_AREA_ADDR,
 383                                     &adtl_status_addr,
 384                                     sizeof(unsigned long));
 385                 rc |= kvm_s390_vcpu_store_adtl_status(vcpu,
 386                                                       adtl_status_addr);
 387                 rc |= put_guest_lc(vcpu, mchk.mcic,
 388                                    (u64 __user *) __LC_MCCK_CODE);
 389                 rc |= put_guest_lc(vcpu, mchk.failing_storage_address,
 390                                    (u64 __user *) __LC_MCCK_FAIL_STOR_ADDR);
 391                 rc |= write_guest_lc(vcpu, __LC_PSW_SAVE_AREA,
 392                                      &mchk.fixed_logout,
 393                                      sizeof(mchk.fixed_logout));
 394                 rc |= write_guest_lc(vcpu, __LC_MCK_OLD_PSW,
 395                                      &vcpu->arch.sie_block->gpsw,
 396                                      sizeof(psw_t));
 397                 rc |= read_guest_lc(vcpu, __LC_MCK_NEW_PSW,
 398                                     &vcpu->arch.sie_block->gpsw,
 399                                     sizeof(psw_t));
 400         }
 401         return rc ? -EFAULT : 0;
 402 }
 403
 404 static int __must_check __deliver_restart(struct kvm_vcpu *vcpu)
 405 {
 406         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 407         int rc;
 408
 409         VCPU_EVENT(vcpu, 3, "%s", "deliver: cpu restart");
 410         vcpu->stat.deliver_restart_signal++;
 411         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_RESTART, 0, 0);
 412
 413         rc  = write_guest_lc(vcpu,
 414                              offsetof(struct _lowcore, restart_old_psw),
 415                              &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 416         rc |= read_guest_lc(vcpu, offsetof(struct _lowcore, restart_psw),
 417                             &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 418         clear_bit(IRQ_PEND_RESTART, &li->pending_irqs);
 419         return rc ? -EFAULT : 0;
 420 }
 421
 422 static int __must_check __deliver_set_prefix(struct kvm_vcpu *vcpu)
 423 {
 424         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 425         struct kvm_s390_prefix_info prefix;
 426
 427         spin_lock(&li->lock);
 428         prefix = li->irq.prefix;
 429         li->irq.prefix.address = 0;
 430         clear_bit(IRQ_PEND_SET_PREFIX, &li->pending_irqs);
 431         spin_unlock(&li->lock);
 432
 433         vcpu->stat.deliver_prefix_signal++;
 434         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
 435                                          KVM_S390_SIGP_SET_PREFIX,
 436                                          prefix.address, 0);
 437
 438         kvm_s390_set_prefix(vcpu, prefix.address);
 439         return 0;
 440 }
 441
 442 static int __must_check __deliver_emergency_signal(struct kvm_vcpu *vcpu)
 443 {
 444         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 445         int rc;
 446         int cpu_addr;
 447
 448         spin_lock(&li->lock);
 449         cpu_addr = find_first_bit(li->sigp_emerg_pending, KVM_MAX_VCPUS);
 450         clear_bit(cpu_addr, li->sigp_emerg_pending);
 451         if (bitmap_empty(li->sigp_emerg_pending, KVM_MAX_VCPUS))
 452                 clear_bit(IRQ_PEND_EXT_EMERGENCY, &li->pending_irqs);
 453         spin_unlock(&li->lock);
 454
 455         VCPU_EVENT(vcpu, 4, "%s", "deliver: sigp emerg");
 456         vcpu->stat.deliver_emergency_signal++;
 457         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_EMERGENCY,
 458                                          cpu_addr, 0);
 459
 460         rc  = put_guest_lc(vcpu, EXT_IRQ_EMERGENCY_SIG,
 461                            (u16 *)__LC_EXT_INT_CODE);
 462         rc |= put_guest_lc(vcpu, cpu_addr, (u16 *)__LC_EXT_CPU_ADDR);
 463         rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
 464                              &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 465         rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
 466                             &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 467         return rc ? -EFAULT : 0;
 468 }
 469
 470 static int __must_check __deliver_external_call(struct kvm_vcpu *vcpu)
 471 {
 472         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 473         struct kvm_s390_extcall_info extcall;
 474         int rc;
 475
 476         spin_lock(&li->lock);
 477         extcall = li->irq.extcall;
 478         li->irq.extcall.code = 0;
 479         clear_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs);
 480         spin_unlock(&li->lock);
 481
 482         VCPU_EVENT(vcpu, 4, "%s", "deliver: sigp ext call");
 483         vcpu->stat.deliver_external_call++;
 484         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
 485                                          KVM_S390_INT_EXTERNAL_CALL,
 486                                          extcall.code, 0);
 487
 488         rc  = put_guest_lc(vcpu, EXT_IRQ_EXTERNAL_CALL,
 489                            (u16 *)__LC_EXT_INT_CODE);
 490         rc |= put_guest_lc(vcpu, extcall.code, (u16 *)__LC_EXT_CPU_ADDR);
 491         rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
 492                              &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 493         rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, &vcpu->arch.sie_block->gpsw,
 494                             sizeof(psw_t));
 495         return rc ? -EFAULT : 0;
 496 }
 497
 498 static int __must_check __deliver_prog(struct kvm_vcpu *vcpu)
 499 {
 500         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 501         struct kvm_s390_pgm_info pgm_info;
 502         int rc = 0, nullifying = false;
 503         u16 ilc = get_ilc(vcpu);
 504
 505         spin_lock(&li->lock);
 506         pgm_info = li->irq.pgm;
 507         clear_bit(IRQ_PEND_PROG, &li->pending_irqs);
 508         memset(&li->irq.pgm, 0, sizeof(pgm_info));
 509         spin_unlock(&li->lock);
 510
 511         VCPU_EVENT(vcpu, 3, "deliver: program irq code 0x%x, ilc:%d",
 512                    pgm_info.code, ilc);
 513         vcpu->stat.deliver_program_int++;
 514         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_PROGRAM_INT,
 515                                          pgm_info.code, 0);
 516
 517         switch (pgm_info.code & ~PGM_PER) {
 518         case PGM_AFX_TRANSLATION:
 519         case PGM_ASX_TRANSLATION:
 520         case PGM_EX_TRANSLATION:
 521         case PGM_LFX_TRANSLATION:
 522         case PGM_LSTE_SEQUENCE:
 523         case PGM_LSX_TRANSLATION:
 524         case PGM_LX_TRANSLATION:
 525         case PGM_PRIMARY_AUTHORITY:
 526         case PGM_SECONDARY_AUTHORITY:
 527                 nullifying = true;
 528                 /* fall through */
 529         case PGM_SPACE_SWITCH:
 530                 rc = put_guest_lc(vcpu, pgm_info.trans_exc_code,
 531                                   (u64 *)__LC_TRANS_EXC_CODE);
 532                 break;
 533         case PGM_ALEN_TRANSLATION:
 534         case PGM_ALE_SEQUENCE:
 535         case PGM_ASTE_INSTANCE:
 536         case PGM_ASTE_SEQUENCE:
 537         case PGM_ASTE_VALIDITY:
 538         case PGM_EXTENDED_AUTHORITY:
 539                 rc = put_guest_lc(vcpu, pgm_info.exc_access_id,
 540                                   (u8 *)__LC_EXC_ACCESS_ID);
 541                 nullifying = true;
 542                 break;
 543         case PGM_ASCE_TYPE:
 544         case PGM_PAGE_TRANSLATION:
 545         case PGM_REGION_FIRST_TRANS:
 546         case PGM_REGION_SECOND_TRANS:
 547         case PGM_REGION_THIRD_TRANS:
 548         case PGM_SEGMENT_TRANSLATION:
 549                 rc = put_guest_lc(vcpu, pgm_info.trans_exc_code,
 550                                   (u64 *)__LC_TRANS_EXC_CODE);
 551                 rc |= put_guest_lc(vcpu, pgm_info.exc_access_id,
 552                                    (u8 *)__LC_EXC_ACCESS_ID);
 553                 rc |= put_guest_lc(vcpu, pgm_info.op_access_id,
 554                                    (u8 *)__LC_OP_ACCESS_ID);
 555                 nullifying = true;
 556                 break;
 557         case PGM_MONITOR:
 558                 rc = put_guest_lc(vcpu, pgm_info.mon_class_nr,
 559                                   (u16 *)__LC_MON_CLASS_NR);
 560                 rc |= put_guest_lc(vcpu, pgm_info.mon_code,
 561                                    (u64 *)__LC_MON_CODE);
 562                 break;
 563         case PGM_VECTOR_PROCESSING:
 564         case PGM_DATA:
 565                 rc = put_guest_lc(vcpu, pgm_info.data_exc_code,
 566                                   (u32 *)__LC_DATA_EXC_CODE);
 567                 break;
 568         case PGM_PROTECTION:
 569                 rc = put_guest_lc(vcpu, pgm_info.trans_exc_code,
 570                                   (u64 *)__LC_TRANS_EXC_CODE);
 571                 rc |= put_guest_lc(vcpu, pgm_info.exc_access_id,
 572                                    (u8 *)__LC_EXC_ACCESS_ID);
 573                 break;
 574         case PGM_STACK_FULL:
 575         case PGM_STACK_EMPTY:
 576         case PGM_STACK_SPECIFICATION:
 577         case PGM_STACK_TYPE:
 578         case PGM_STACK_OPERATION:
 579         case PGM_TRACE_TABEL:
 580         case PGM_CRYPTO_OPERATION:
 581                 nullifying = true;
 582                 break;
 583         }
 584
 585         if (pgm_info.code & PGM_PER) {
 586                 rc |= put_guest_lc(vcpu, pgm_info.per_code,
 587                                    (u8 *) __LC_PER_CODE);
 588                 rc |= put_guest_lc(vcpu, pgm_info.per_atmid,
 589                                    (u8 *)__LC_PER_ATMID);
 590                 rc |= put_guest_lc(vcpu, pgm_info.per_address,
 591                                    (u64 *) __LC_PER_ADDRESS);
 592                 rc |= put_guest_lc(vcpu, pgm_info.per_access_id,
 593                                    (u8 *) __LC_PER_ACCESS_ID);
 594         }
 595
 596         if (nullifying && vcpu->arch.sie_block->icptcode == ICPT_INST)
 597                 kvm_s390_rewind_psw(vcpu, ilc);
 598
 599         rc |= put_guest_lc(vcpu, ilc, (u16 *) __LC_PGM_ILC);
 600         rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->gbea,
 601                                  (u64 *) __LC_LAST_BREAK);
 602         rc |= put_guest_lc(vcpu, pgm_info.code,
 603                            (u16 *)__LC_PGM_INT_CODE);
 604         rc |= write_guest_lc(vcpu, __LC_PGM_OLD_PSW,
 605                              &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 606         rc |= read_guest_lc(vcpu, __LC_PGM_NEW_PSW,
 607                             &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 608         return rc ? -EFAULT : 0;
 609 }
 610
 611 static int __must_check __deliver_service(struct kvm_vcpu *vcpu)
 612 {
 613         struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
 614         struct kvm_s390_ext_info ext;
 615         int rc = 0;
 616
 617         spin_lock(&fi->lock);
 618         if (!(test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs))) {
 619                 spin_unlock(&fi->lock);
 620                 return 0;
 621         }
 622         ext = fi->srv_signal;
 623         memset(&fi->srv_signal, 0, sizeof(ext));
 624         clear_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs);
 625         spin_unlock(&fi->lock);
 626
 627         VCPU_EVENT(vcpu, 4, "deliver: sclp parameter 0x%x",
 628                    ext.ext_params);
 629         vcpu->stat.deliver_service_signal++;
 630         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_SERVICE,
 631                                          ext.ext_params, 0);
 632
 633         rc  = put_guest_lc(vcpu, EXT_IRQ_SERVICE_SIG, (u16 *)__LC_EXT_INT_CODE);
 634         rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR);
 635         rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
 636                              &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 637         rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
 638                             &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 639         rc |= put_guest_lc(vcpu, ext.ext_params,
 640                            (u32 *)__LC_EXT_PARAMS);
 641
 642         return rc ? -EFAULT : 0;
 643 }
 644
 645 static int __must_check __deliver_pfault_done(struct kvm_vcpu *vcpu)
 646 {
 647         struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
 648         struct kvm_s390_interrupt_info *inti;
 649         int rc = 0;
 650
 651         spin_lock(&fi->lock);
 652         inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_PFAULT],
 653                                         struct kvm_s390_interrupt_info,
 654                                         list);
 655         if (inti) {
 656                 list_del(&inti->list);
 657                 fi->counters[FIRQ_CNTR_PFAULT] -= 1;
 658         }
 659         if (list_empty(&fi->lists[FIRQ_LIST_PFAULT]))
 660                 clear_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs);
 661         spin_unlock(&fi->lock);
 662
 663         if (inti) {
 664                 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
 665                                                  KVM_S390_INT_PFAULT_DONE, 0,
 666                                                  inti->ext.ext_params2);
 667                 VCPU_EVENT(vcpu, 4, "deliver: pfault done token 0x%llx",
 668                            inti->ext.ext_params2);
 669
 670                 rc  = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE,
 671                                 (u16 *)__LC_EXT_INT_CODE);
 672                 rc |= put_guest_lc(vcpu, PFAULT_DONE,
 673                                 (u16 *)__LC_EXT_CPU_ADDR);
 674                 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
 675                                 &vcpu->arch.sie_block->gpsw,
 676                                 sizeof(psw_t));
 677                 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
 678                                 &vcpu->arch.sie_block->gpsw,
 679                                 sizeof(psw_t));
 680                 rc |= put_guest_lc(vcpu, inti->ext.ext_params2,
 681                                 (u64 *)__LC_EXT_PARAMS2);
 682                 kfree(inti);
 683         }
 684         return rc ? -EFAULT : 0;
 685 }
 686
 687 static int __must_check __deliver_virtio(struct kvm_vcpu *vcpu)
 688 {
 689         struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
 690         struct kvm_s390_interrupt_info *inti;
 691         int rc = 0;
 692
 693         spin_lock(&fi->lock);
 694         inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_VIRTIO],
 695                                         struct kvm_s390_interrupt_info,
 696                                         list);
 697         if (inti) {
 698                 VCPU_EVENT(vcpu, 4,
 699                            "deliver: virtio parm: 0x%x,parm64: 0x%llx",
 700                            inti->ext.ext_params, inti->ext.ext_params2);
 701                 vcpu->stat.deliver_virtio_interrupt++;
 702                 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
 703                                 inti->type,
 704                                 inti->ext.ext_params,
 705                                 inti->ext.ext_params2);
 706                 list_del(&inti->list);
 707                 fi->counters[FIRQ_CNTR_VIRTIO] -= 1;
 708         }
 709         if (list_empty(&fi->lists[FIRQ_LIST_VIRTIO]))
 710                 clear_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs);
 711         spin_unlock(&fi->lock);
 712
 713         if (inti) {
 714                 rc  = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE,
 715                                 (u16 *)__LC_EXT_INT_CODE);
 716                 rc |= put_guest_lc(vcpu, VIRTIO_PARAM,
 717                                 (u16 *)__LC_EXT_CPU_ADDR);
 718                 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
 719                                 &vcpu->arch.sie_block->gpsw,
 720                                 sizeof(psw_t));
 721                 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
 722                                 &vcpu->arch.sie_block->gpsw,
 723                                 sizeof(psw_t));
 724                 rc |= put_guest_lc(vcpu, inti->ext.ext_params,
 725                                 (u32 *)__LC_EXT_PARAMS);
 726                 rc |= put_guest_lc(vcpu, inti->ext.ext_params2,
 727                                 (u64 *)__LC_EXT_PARAMS2);
 728                 kfree(inti);
 729         }
 730         return rc ? -EFAULT : 0;
 731 }
 732
 733 static int __must_check __deliver_io(struct kvm_vcpu *vcpu,
 734                                      unsigned long irq_type)
 735 {
 736         struct list_head *isc_list;
 737         struct kvm_s390_float_interrupt *fi;
 738         struct kvm_s390_interrupt_info *inti = NULL;
 739         int rc = 0;
 740
 741         fi = &vcpu->kvm->arch.float_int;
 742
 743         spin_lock(&fi->lock);
 744         isc_list = &fi->lists[irq_type - IRQ_PEND_IO_ISC_0];
 745         inti = list_first_entry_or_null(isc_list,
 746                                         struct kvm_s390_interrupt_info,
 747                                         list);
 748         if (inti) {
 749                 VCPU_EVENT(vcpu, 4, "deliver: I/O 0x%llx", inti->type);
 750                 vcpu->stat.deliver_io_int++;
 751                 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
 752                                 inti->type,
 753                                 ((__u32)inti->io.subchannel_id << 16) |
 754                                 inti->io.subchannel_nr,
 755                                 ((__u64)inti->io.io_int_parm << 32) |
 756                                 inti->io.io_int_word);
 757                 list_del(&inti->list);
 758                 fi->counters[FIRQ_CNTR_IO] -= 1;
 759         }
 760         if (list_empty(isc_list))
 761                 clear_bit(irq_type, &fi->pending_irqs);
 762         spin_unlock(&fi->lock);
 763
 764         if (inti) {
 765                 rc  = put_guest_lc(vcpu, inti->io.subchannel_id,
 766                                 (u16 *)__LC_SUBCHANNEL_ID);
 767                 rc |= put_guest_lc(vcpu, inti->io.subchannel_nr,
 768                                 (u16 *)__LC_SUBCHANNEL_NR);
 769                 rc |= put_guest_lc(vcpu, inti->io.io_int_parm,
 770                                 (u32 *)__LC_IO_INT_PARM);
 771                 rc |= put_guest_lc(vcpu, inti->io.io_int_word,
 772                                 (u32 *)__LC_IO_INT_WORD);
 773                 rc |= write_guest_lc(vcpu, __LC_IO_OLD_PSW,
 774                                 &vcpu->arch.sie_block->gpsw,
 775                                 sizeof(psw_t));
 776                 rc |= read_guest_lc(vcpu, __LC_IO_NEW_PSW,
 777                                 &vcpu->arch.sie_block->gpsw,
 778                                 sizeof(psw_t));
 779                 kfree(inti);
 780         }
 781
 782         return rc ? -EFAULT : 0;
 783 }
 784
 785 typedef int (*deliver_irq_t)(struct kvm_vcpu *vcpu);
 786
 787 static const deliver_irq_t deliver_irq_funcs[] = {
 788         [IRQ_PEND_MCHK_EX]        = __deliver_machine_check,
 789         [IRQ_PEND_MCHK_REP]       = __deliver_machine_check,
 790         [IRQ_PEND_PROG]           = __deliver_prog,
 791         [IRQ_PEND_EXT_EMERGENCY]  = __deliver_emergency_signal,
 792         [IRQ_PEND_EXT_EXTERNAL]   = __deliver_external_call,
 793         [IRQ_PEND_EXT_CLOCK_COMP] = __deliver_ckc,
 794         [IRQ_PEND_EXT_CPU_TIMER]  = __deliver_cpu_timer,
 795         [IRQ_PEND_RESTART]        = __deliver_restart,
 796         [IRQ_PEND_SET_PREFIX]     = __deliver_set_prefix,
 797         [IRQ_PEND_PFAULT_INIT]    = __deliver_pfault_init,
 798         [IRQ_PEND_EXT_SERVICE]    = __deliver_service,
 799         [IRQ_PEND_PFAULT_DONE]    = __deliver_pfault_done,
 800         [IRQ_PEND_VIRTIO]         = __deliver_virtio,
 801 };
 802
 803 /* Check whether an external call is pending (deliverable or not) */
 804 int kvm_s390_ext_call_pending(struct kvm_vcpu *vcpu)
 805 {
 806         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 807         uint8_t sigp_ctrl = vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sigp_ctrl;
 808
 809         if (!sclp.has_sigpif)
 810                 return test_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs);
 811
 812         return (sigp_ctrl & SIGP_CTRL_C) &&
 813                (atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_ECALL_PEND);
 814 }
 815
 816 int kvm_s390_vcpu_has_irq(struct kvm_vcpu *vcpu, int exclude_stop)
 817 {
 818         int rc;
 819
 820         rc = !!deliverable_irqs(vcpu);
 821
 822         if (!rc && kvm_cpu_has_pending_timer(vcpu))
 823                 rc = 1;
 824
 825         /* external call pending and deliverable */
 826         if (!rc && kvm_s390_ext_call_pending(vcpu) &&
 827             !psw_extint_disabled(vcpu) &&
 828             (vcpu->arch.sie_block->gcr[0] & 0x2000ul))
 829                 rc = 1;
 830
 831         if (!rc && !exclude_stop && kvm_s390_is_stop_irq_pending(vcpu))
 832                 rc = 1;
 833
 834         return rc;
 835 }
 836
 837 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
 838 {
 839         return ckc_irq_pending(vcpu) || cpu_timer_irq_pending(vcpu);
 840 }
 841
 842 int kvm_s390_handle_wait(struct kvm_vcpu *vcpu)
 843 {
 844         u64 now, sltime;
 845
 846         vcpu->stat.exit_wait_state++;
 847
 848         /* fast path */
 849         if (kvm_cpu_has_pending_timer(vcpu) || kvm_arch_vcpu_runnable(vcpu))
 850                 return 0;
 851
 852         if (psw_interrupts_disabled(vcpu)) {
 853                 VCPU_EVENT(vcpu, 3, "%s", "disabled wait");
 854                 return -EOPNOTSUPP; /* disabled wait */
 855         }
 856
 857         if (!ckc_interrupts_enabled(vcpu)) {
 858                 VCPU_EVENT(vcpu, 3, "%s", "enabled wait w/o timer");
 859                 __set_cpu_idle(vcpu);
 860                 goto no_timer;
 861         }
 862
 863         preempt_disable();
 864         now = get_tod_clock_fast() + vcpu->arch.sie_block->epoch;
 865         preempt_enable();
 866         sltime = tod_to_ns(vcpu->arch.sie_block->ckc - now);
 867
 868         /* underflow */
 869         if (vcpu->arch.sie_block->ckc < now)
 870                 return 0;
 871
 872         __set_cpu_idle(vcpu);
 873         hrtimer_start(&vcpu->arch.ckc_timer, ktime_set (0, sltime) , HRTIMER_MODE_REL);
 874         VCPU_EVENT(vcpu, 4, "enabled wait via clock comparator: %llu ns", sltime);
 875 no_timer:
 876         srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
 877         kvm_vcpu_block(vcpu);
 878         __unset_cpu_idle(vcpu);
 879         vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
 880
 881         hrtimer_cancel(&vcpu->arch.ckc_timer);
 882         return 0;
 883 }
 884
 885 void kvm_s390_vcpu_wakeup(struct kvm_vcpu *vcpu)
 886 {
 887         if (waitqueue_active(&vcpu->wq)) {
 888                 /*
 889                  * The vcpu gave up the cpu voluntarily, mark it as a good
 890                  * yield-candidate.
 891                  */
 892                 vcpu->preempted = true;
 893                 wake_up_interruptible(&vcpu->wq);
 894                 vcpu->stat.halt_wakeup++;
 895         }
 896 }
 897
 898 enum hrtimer_restart kvm_s390_idle_wakeup(struct hrtimer *timer)
 899 {
 900         struct kvm_vcpu *vcpu;
 901         u64 now, sltime;
 902
 903         vcpu = container_of(timer, struct kvm_vcpu, arch.ckc_timer);
 904         preempt_disable();
 905         now = get_tod_clock_fast() + vcpu->arch.sie_block->epoch;
 906         preempt_enable();
 907         sltime = tod_to_ns(vcpu->arch.sie_block->ckc - now);
 908
 909         /*
 910          * If the monotonic clock runs faster than the tod clock we might be
 911          * woken up too early and have to go back to sleep to avoid deadlocks.
 912          */
 913         if (vcpu->arch.sie_block->ckc > now &&
 914             hrtimer_forward_now(timer, ns_to_ktime(sltime)))
 915                 return HRTIMER_RESTART;
 916         kvm_s390_vcpu_wakeup(vcpu);
 917         return HRTIMER_NORESTART;
 918 }
 919
 920 void kvm_s390_clear_local_irqs(struct kvm_vcpu *vcpu)
 921 {
 922         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 923
 924         spin_lock(&li->lock);
 925         li->pending_irqs = 0;
 926         bitmap_zero(li->sigp_emerg_pending, KVM_MAX_VCPUS);
 927         memset(&li->irq, 0, sizeof(li->irq));
 928         spin_unlock(&li->lock);
 929
 930         /* clear pending external calls set by sigp interpretation facility */
 931         atomic_andnot(CPUSTAT_ECALL_PEND, li->cpuflags);
 932         vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sigp_ctrl = 0;
 933 }
 934
 935 int __must_check kvm_s390_deliver_pending_interrupts(struct kvm_vcpu *vcpu)
 936 {
 937         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 938         deliver_irq_t func;
 939         int rc = 0;
 940         unsigned long irq_type;
 941         unsigned long irqs;
 942
 943         __reset_intercept_indicators(vcpu);
 944
 945         /* pending ckc conditions might have been invalidated */
 946         clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
 947         if (ckc_irq_pending(vcpu))
 948                 set_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
 949
 950         /* pending cpu timer conditions might have been invalidated */
 951         clear_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
 952         if (cpu_timer_irq_pending(vcpu))
 953                 set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
 954
 955         while ((irqs = deliverable_irqs(vcpu)) && !rc) {
 956                 /* bits are in the order of interrupt priority */
 957                 irq_type = find_first_bit(&irqs, IRQ_PEND_COUNT);
 958                 if (is_ioirq(irq_type)) {
 959                         rc = __deliver_io(vcpu, irq_type);
 960                 } else {
 961                         func = deliver_irq_funcs[irq_type];
 962                         if (!func) {
 963                                 WARN_ON_ONCE(func == NULL);
 964                                 clear_bit(irq_type, &li->pending_irqs);
 965                                 continue;
 966                         }
 967                         rc = func(vcpu);
 968                 }
 969         }
 970
 971         set_intercept_indicators(vcpu);
 972
 973         return rc;
 974 }
 975
 976 static int __inject_prog(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
 977 {
 978         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 979
 980         VCPU_EVENT(vcpu, 3, "inject: program irq code 0x%x", irq->u.pgm.code);
 981         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_PROGRAM_INT,
 982                                    irq->u.pgm.code, 0);
 983
 984         li->irq.pgm = irq->u.pgm;
 985         set_bit(IRQ_PEND_PROG, &li->pending_irqs);
 986         return 0;
 987 }
 988
 989 int kvm_s390_inject_program_int(struct kvm_vcpu *vcpu, u16 code)
 990 {
 991         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 992         struct kvm_s390_irq irq;
 993
 994         spin_lock(&li->lock);
 995         irq.u.pgm.code = code;
 996         __inject_prog(vcpu, &irq);
 997         BUG_ON(waitqueue_active(li->wq));
 998         spin_unlock(&li->lock);
 999         return 0;
1000 }
1001
1002 int kvm_s390_inject_prog_irq(struct kvm_vcpu *vcpu,
1003                              struct kvm_s390_pgm_info *pgm_info)
1004 {
1005         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1006         struct kvm_s390_irq irq;
1007         int rc;
1008
1009         spin_lock(&li->lock);
1010         irq.u.pgm = *pgm_info;
1011         rc = __inject_prog(vcpu, &irq);
1012         BUG_ON(waitqueue_active(li->wq));
1013         spin_unlock(&li->lock);
1014         return rc;
1015 }
1016
1017 static int __inject_pfault_init(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1018 {
1019         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1020
1021         VCPU_EVENT(vcpu, 4, "inject: pfault init parameter block at 0x%llx",
1022                    irq->u.ext.ext_params2);
1023         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_PFAULT_INIT,
1024                                    irq->u.ext.ext_params,
1025                                    irq->u.ext.ext_params2);
1026
1027         li->irq.ext = irq->u.ext;
1028         set_bit(IRQ_PEND_PFAULT_INIT, &li->pending_irqs);
1029         atomic_or(CPUSTAT_EXT_INT, li->cpuflags);
1030         return 0;
1031 }
1032
1033 static int __inject_extcall_sigpif(struct kvm_vcpu *vcpu, uint16_t src_id)
1034 {
1035         unsigned char new_val, old_val;
1036         uint8_t *sigp_ctrl = &vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sigp_ctrl;
1037
1038         new_val = SIGP_CTRL_C | (src_id & SIGP_CTRL_SCN_MASK);
1039         old_val = *sigp_ctrl & ~SIGP_CTRL_C;
1040         if (cmpxchg(sigp_ctrl, old_val, new_val) != old_val) {
1041                 /* another external call is pending */
1042                 return -EBUSY;
1043         }
1044         atomic_or(CPUSTAT_ECALL_PEND, &vcpu->arch.sie_block->cpuflags);
1045         return 0;
1046 }
1047
1048 static int __inject_extcall(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1049 {
1050         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1051         struct kvm_s390_extcall_info *extcall = &li->irq.extcall;
1052         uint16_t src_id = irq->u.extcall.code;
1053
1054         VCPU_EVENT(vcpu, 4, "inject: external call source-cpu:%u",
1055                    src_id);
1056         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_EXTERNAL_CALL,
1057                                    src_id, 0);
1058
1059         /* sending vcpu invalid */
1060         if (src_id >= KVM_MAX_VCPUS ||
1061             kvm_get_vcpu(vcpu->kvm, src_id) == NULL)
1062                 return -EINVAL;
1063
1064         if (sclp.has_sigpif)
1065                 return __inject_extcall_sigpif(vcpu, src_id);
1066
1067         if (test_and_set_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs))
1068                 return -EBUSY;
1069         *extcall = irq->u.extcall;
1070         atomic_or(CPUSTAT_EXT_INT, li->cpuflags);
1071         return 0;
1072 }
1073
1074 static int __inject_set_prefix(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1075 {
1076         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1077         struct kvm_s390_prefix_info *prefix = &li->irq.prefix;
1078
1079         VCPU_EVENT(vcpu, 3, "inject: set prefix to %x",
1080                    irq->u.prefix.address);
1081         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_SIGP_SET_PREFIX,
1082                                    irq->u.prefix.address, 0);
1083
1084         if (!is_vcpu_stopped(vcpu))
1085                 return -EBUSY;
1086
1087         *prefix = irq->u.prefix;
1088         set_bit(IRQ_PEND_SET_PREFIX, &li->pending_irqs);
1089         return 0;
1090 }
1091
1092 #define KVM_S390_STOP_SUPP_FLAGS (KVM_S390_STOP_FLAG_STORE_STATUS)
1093 static int __inject_sigp_stop(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1094 {
1095         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1096         struct kvm_s390_stop_info *stop = &li->irq.stop;
1097         int rc = 0;
1098
1099         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_SIGP_STOP, 0, 0);
1100
1101         if (irq->u.stop.flags & ~KVM_S390_STOP_SUPP_FLAGS)
1102                 return -EINVAL;
1103
1104         if (is_vcpu_stopped(vcpu)) {
1105                 if (irq->u.stop.flags & KVM_S390_STOP_FLAG_STORE_STATUS)
1106                         rc = kvm_s390_store_status_unloaded(vcpu,
1107                                                 KVM_S390_STORE_STATUS_NOADDR);
1108                 return rc;
1109         }
1110
1111         if (test_and_set_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs))
1112                 return -EBUSY;
1113         stop->flags = irq->u.stop.flags;
1114         __set_cpuflag(vcpu, CPUSTAT_STOP_INT);
1115         return 0;
1116 }
1117
1118 static int __inject_sigp_restart(struct kvm_vcpu *vcpu,
1119                                  struct kvm_s390_irq *irq)
1120 {
1121         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1122
1123         VCPU_EVENT(vcpu, 3, "%s", "inject: restart int");
1124         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_RESTART, 0, 0);
1125
1126         set_bit(IRQ_PEND_RESTART, &li->pending_irqs);
1127         return 0;
1128 }
1129
1130 static int __inject_sigp_emergency(struct kvm_vcpu *vcpu,
1131                                    struct kvm_s390_irq *irq)
1132 {
1133         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1134
1135         VCPU_EVENT(vcpu, 4, "inject: emergency from cpu %u",
1136                    irq->u.emerg.code);
1137         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_EMERGENCY,
1138                                    irq->u.emerg.code, 0);
1139
1140         set_bit(irq->u.emerg.code, li->sigp_emerg_pending);
1141         set_bit(IRQ_PEND_EXT_EMERGENCY, &li->pending_irqs);
1142         atomic_or(CPUSTAT_EXT_INT, li->cpuflags);
1143         return 0;
1144 }
1145
1146 static int __inject_mchk(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1147 {
1148         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1149         struct kvm_s390_mchk_info *mchk = &li->irq.mchk;
1150
1151         VCPU_EVENT(vcpu, 3, "inject: machine check mcic 0x%llx",
1152                    irq->u.mchk.mcic);
1153         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_MCHK, 0,
1154                                    irq->u.mchk.mcic);
1155
1156         /*
1157          * Because repressible machine checks can be indicated along with
1158          * exigent machine checks (PoP, Chapter 11, Interruption action)
1159          * we need to combine cr14, mcic and external damage code.
1160          * Failing storage address and the logout area should not be or'ed
1161          * together, we just indicate the last occurrence of the corresponding
1162          * machine check
1163          */
1164         mchk->cr14 |= irq->u.mchk.cr14;
1165         mchk->mcic |= irq->u.mchk.mcic;
1166         mchk->ext_damage_code |= irq->u.mchk.ext_damage_code;
1167         mchk->failing_storage_address = irq->u.mchk.failing_storage_address;
1168         memcpy(&mchk->fixed_logout, &irq->u.mchk.fixed_logout,
1169                sizeof(mchk->fixed_logout));
1170         if (mchk->mcic & MCHK_EX_MASK)
1171                 set_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs);
1172         else if (mchk->mcic & MCHK_REP_MASK)
1173                 set_bit(IRQ_PEND_MCHK_REP,  &li->pending_irqs);
1174         return 0;
1175 }
1176
1177 static int __inject_ckc(struct kvm_vcpu *vcpu)
1178 {
1179         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1180
1181         VCPU_EVENT(vcpu, 3, "%s", "inject: clock comparator external");
1182         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_CLOCK_COMP,
1183                                    0, 0);
1184
1185         set_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
1186         atomic_or(CPUSTAT_EXT_INT, li->cpuflags);
1187         return 0;
1188 }
1189
1190 static int __inject_cpu_timer(struct kvm_vcpu *vcpu)
1191 {
1192         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1193
1194         VCPU_EVENT(vcpu, 3, "%s", "inject: cpu timer external");
1195         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_CPU_TIMER,
1196                                    0, 0);
1197
1198         set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
1199         atomic_or(CPUSTAT_EXT_INT, li->cpuflags);
1200         return 0;
1201 }
1202
1203 static struct kvm_s390_interrupt_info *get_io_int(struct kvm *kvm,
1204                                                   int isc, u32 schid)
1205 {
1206         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1207         struct list_head *isc_list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc];
1208         struct kvm_s390_interrupt_info *iter;
1209         u16 id = (schid & 0xffff0000U) >> 16;
1210         u16 nr = schid & 0x0000ffffU;
1211
1212         spin_lock(&fi->lock);
1213         list_for_each_entry(iter, isc_list, list) {
1214                 if (schid && (id != iter->io.subchannel_id ||
1215                               nr != iter->io.subchannel_nr))
1216                         continue;
1217                 /* found an appropriate entry */
1218                 list_del_init(&iter->list);
1219                 fi->counters[FIRQ_CNTR_IO] -= 1;
1220                 if (list_empty(isc_list))
1221                         clear_bit(IRQ_PEND_IO_ISC_0 + isc, &fi->pending_irqs);
1222                 spin_unlock(&fi->lock);
1223                 return iter;
1224         }
1225         spin_unlock(&fi->lock);
1226         return NULL;
1227 }
1228
1229 /*
1230  * Dequeue and return an I/O interrupt matching any of the interruption
1231  * subclasses as designated by the isc mask in cr6 and the schid (if != 0).
1232  */
1233 struct kvm_s390_interrupt_info *kvm_s390_get_io_int(struct kvm *kvm,
1234                                                     u64 isc_mask, u32 schid)
1235 {
1236         struct kvm_s390_interrupt_info *inti = NULL;
1237         int isc;
1238
1239         for (isc = 0; isc <= MAX_ISC && !inti; isc++) {
1240                 if (isc_mask & isc_to_isc_bits(isc))
1241                         inti = get_io_int(kvm, isc, schid);
1242         }
1243         return inti;
1244 }
1245
1246 #define SCCB_MASK 0xFFFFFFF8
1247 #define SCCB_EVENT_PENDING 0x3
1248
1249 static int __inject_service(struct kvm *kvm,
1250                              struct kvm_s390_interrupt_info *inti)
1251 {
1252         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1253
1254         spin_lock(&fi->lock);
1255         fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_EVENT_PENDING;
1256         /*
1257          * Early versions of the QEMU s390 bios will inject several
1258          * service interrupts after another without handling a
1259          * condition code indicating busy.
1260          * We will silently ignore those superfluous sccb values.
1261          * A future version of QEMU will take care of serialization
1262          * of servc requests
1263          */
1264         if (fi->srv_signal.ext_params & SCCB_MASK)
1265                 goto out;
1266         fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_MASK;
1267         set_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs);
1268 out:
1269         spin_unlock(&fi->lock);
1270         kfree(inti);
1271         return 0;
1272 }
1273
1274 static int __inject_virtio(struct kvm *kvm,
1275                             struct kvm_s390_interrupt_info *inti)
1276 {
1277         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1278
1279         spin_lock(&fi->lock);
1280         if (fi->counters[FIRQ_CNTR_VIRTIO] >= KVM_S390_MAX_VIRTIO_IRQS) {
1281                 spin_unlock(&fi->lock);
1282                 return -EBUSY;
1283         }
1284         fi->counters[FIRQ_CNTR_VIRTIO] += 1;
1285         list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_VIRTIO]);
1286         set_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs);
1287         spin_unlock(&fi->lock);
1288         return 0;
1289 }
1290
1291 static int __inject_pfault_done(struct kvm *kvm,
1292                                  struct kvm_s390_interrupt_info *inti)
1293 {
1294         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1295
1296         spin_lock(&fi->lock);
1297         if (fi->counters[FIRQ_CNTR_PFAULT] >=
1298                 (ASYNC_PF_PER_VCPU * KVM_MAX_VCPUS)) {
1299                 spin_unlock(&fi->lock);
1300                 return -EBUSY;
1301         }
1302         fi->counters[FIRQ_CNTR_PFAULT] += 1;
1303         list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_PFAULT]);
1304         set_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs);
1305         spin_unlock(&fi->lock);
1306         return 0;
1307 }
1308
1309 #define CR_PENDING_SUBCLASS 28
1310 static int __inject_float_mchk(struct kvm *kvm,
1311                                 struct kvm_s390_interrupt_info *inti)
1312 {
1313         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1314
1315         spin_lock(&fi->lock);
1316         fi->mchk.cr14 |= inti->mchk.cr14 & (1UL << CR_PENDING_SUBCLASS);
1317         fi->mchk.mcic |= inti->mchk.mcic;
1318         set_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs);
1319         spin_unlock(&fi->lock);
1320         kfree(inti);
1321         return 0;
1322 }
1323
1324 static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
1325 {
1326         struct kvm_s390_float_interrupt *fi;
1327         struct list_head *list;
1328         int isc;
1329
1330         fi = &kvm->arch.float_int;
1331         spin_lock(&fi->lock);
1332         if (fi->counters[FIRQ_CNTR_IO] >= KVM_S390_MAX_FLOAT_IRQS) {
1333                 spin_unlock(&fi->lock);
1334                 return -EBUSY;
1335         }
1336         fi->counters[FIRQ_CNTR_IO] += 1;
1337
1338         isc = int_word_to_isc(inti->io.io_int_word);
1339         list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc];
1340         list_add_tail(&inti->list, list);
1341         set_bit(IRQ_PEND_IO_ISC_0 + isc, &fi->pending_irqs);
1342         spin_unlock(&fi->lock);
1343         return 0;
1344 }
1345
1346 /*
1347  * Find a destination VCPU for a floating irq and kick it.
1348  */
1349 static void __floating_irq_kick(struct kvm *kvm, u64 type)
1350 {
1351         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1352         struct kvm_s390_local_interrupt *li;
1353         struct kvm_vcpu *dst_vcpu;
1354         int sigcpu, online_vcpus, nr_tries = 0;
1355
1356         online_vcpus = atomic_read(&kvm->online_vcpus);
1357         if (!online_vcpus)
1358                 return;
1359
1360         /* find idle VCPUs first, then round robin */
1361         sigcpu = find_first_bit(fi->idle_mask, online_vcpus);
1362         if (sigcpu == online_vcpus) {
1363                 do {
1364                         sigcpu = fi->next_rr_cpu;
1365                         fi->next_rr_cpu = (fi->next_rr_cpu + 1) % online_vcpus;
1366                         /* avoid endless loops if all vcpus are stopped */
1367                         if (nr_tries++ >= online_vcpus)
1368                                 return;
1369                 } while (is_vcpu_stopped(kvm_get_vcpu(kvm, sigcpu)));
1370         }
1371         dst_vcpu = kvm_get_vcpu(kvm, sigcpu);
1372
1373         /* make the VCPU drop out of the SIE, or wake it up if sleeping */
1374         li = &dst_vcpu->arch.local_int;
1375         spin_lock(&li->lock);
1376         switch (type) {
1377         case KVM_S390_MCHK:
1378                 atomic_or(CPUSTAT_STOP_INT, li->cpuflags);
1379                 break;
1380         case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
1381                 atomic_or(CPUSTAT_IO_INT, li->cpuflags);
1382                 break;
1383         default:
1384                 atomic_or(CPUSTAT_EXT_INT, li->cpuflags);
1385                 break;
1386         }
1387         spin_unlock(&li->lock);
1388         kvm_s390_vcpu_wakeup(dst_vcpu);
1389 }
1390
1391 static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
1392 {
1393         struct kvm_s390_float_interrupt *fi;
1394         u64 type = READ_ONCE(inti->type);
1395         int rc;
1396
1397         fi = &kvm->arch.float_int;
1398
1399         switch (type) {
1400         case KVM_S390_MCHK:
1401                 rc = __inject_float_mchk(kvm, inti);
1402                 break;
1403         case KVM_S390_INT_VIRTIO:
1404                 rc = __inject_virtio(kvm, inti);
1405                 break;
1406         case KVM_S390_INT_SERVICE:
1407                 rc = __inject_service(kvm, inti);
1408                 break;
1409         case KVM_S390_INT_PFAULT_DONE:
1410                 rc = __inject_pfault_done(kvm, inti);
1411                 break;
1412         case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
1413                 rc = __inject_io(kvm, inti);
1414                 break;
1415         default:
1416                 rc = -EINVAL;
1417         }
1418         if (rc)
1419                 return rc;
1420
1421         __floating_irq_kick(kvm, type);
1422         return 0;
1423 }
1424
1425 int kvm_s390_inject_vm(struct kvm *kvm,
1426                        struct kvm_s390_interrupt *s390int)
1427 {
1428         struct kvm_s390_interrupt_info *inti;
1429         int rc;
1430
1431         inti = kzalloc(sizeof(*inti), GFP_KERNEL);
1432         if (!inti)
1433                 return -ENOMEM;
1434
1435         inti->type = s390int->type;
1436         switch (inti->type) {
1437         case KVM_S390_INT_VIRTIO:
1438                 VM_EVENT(kvm, 5, "inject: virtio parm:%x,parm64:%llx",
1439                          s390int->parm, s390int->parm64);
1440                 inti->ext.ext_params = s390int->parm;
1441                 inti->ext.ext_params2 = s390int->parm64;
1442                 break;
1443         case KVM_S390_INT_SERVICE:
1444                 VM_EVENT(kvm, 4, "inject: sclp parm:%x", s390int->parm);
1445                 inti->ext.ext_params = s390int->parm;
1446                 break;
1447         case KVM_S390_INT_PFAULT_DONE:
1448                 inti->ext.ext_params2 = s390int->parm64;
1449                 break;
1450         case KVM_S390_MCHK:
1451                 VM_EVENT(kvm, 3, "inject: machine check mcic 0x%llx",
1452                          s390int->parm64);
1453                 inti->mchk.cr14 = s390int->parm; /* upper bits are not used */
1454                 inti->mchk.mcic = s390int->parm64;
1455                 break;
1456         case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
1457                 if (inti->type & KVM_S390_INT_IO_AI_MASK)
1458                         VM_EVENT(kvm, 5, "%s", "inject: I/O (AI)");
1459                 else
1460                         VM_EVENT(kvm, 5, "inject: I/O css %x ss %x schid %04x",
1461                                  s390int->type & IOINT_CSSID_MASK,
1462                                  s390int->type & IOINT_SSID_MASK,
1463                                  s390int->type & IOINT_SCHID_MASK);
1464                 inti->io.subchannel_id = s390int->parm >> 16;
1465                 inti->io.subchannel_nr = s390int->parm & 0x0000ffffu;
1466                 inti->io.io_int_parm = s390int->parm64 >> 32;
1467                 inti->io.io_int_word = s390int->parm64 & 0x00000000ffffffffull;
1468                 break;
1469         default:
1470                 kfree(inti);
1471                 return -EINVAL;
1472         }
1473         trace_kvm_s390_inject_vm(s390int->type, s390int->parm, s390int->parm64,
1474                                  2);
1475
1476         rc = __inject_vm(kvm, inti);
1477         if (rc)
1478                 kfree(inti);
1479         return rc;
1480 }
1481
1482 int kvm_s390_reinject_io_int(struct kvm *kvm,
1483                               struct kvm_s390_interrupt_info *inti)
1484 {
1485         return __inject_vm(kvm, inti);
1486 }
1487
1488 int s390int_to_s390irq(struct kvm_s390_interrupt *s390int,
1489                        struct kvm_s390_irq *irq)
1490 {
1491         irq->type = s390int->type;
1492         switch (irq->type) {
1493         case KVM_S390_PROGRAM_INT:
1494                 if (s390int->parm & 0xffff0000)
1495                         return -EINVAL;
1496                 irq->u.pgm.code = s390int->parm;
1497                 break;
1498         case KVM_S390_SIGP_SET_PREFIX:
1499                 irq->u.prefix.address = s390int->parm;
1500                 break;
1501         case KVM_S390_SIGP_STOP:
1502                 irq->u.stop.flags = s390int->parm;
1503                 break;
1504         case KVM_S390_INT_EXTERNAL_CALL:
1505                 if (s390int->parm & 0xffff0000)
1506                         return -EINVAL;
1507                 irq->u.extcall.code = s390int->parm;
1508                 break;
1509         case KVM_S390_INT_EMERGENCY:
1510                 if (s390int->parm & 0xffff0000)
1511                         return -EINVAL;
1512                 irq->u.emerg.code = s390int->parm;
1513                 break;
1514         case KVM_S390_MCHK:
1515                 irq->u.mchk.mcic = s390int->parm64;
1516                 break;
1517         }
1518         return 0;
1519 }
1520
1521 int kvm_s390_is_stop_irq_pending(struct kvm_vcpu *vcpu)
1522 {
1523         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1524
1525         return test_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs);
1526 }
1527
1528 void kvm_s390_clear_stop_irq(struct kvm_vcpu *vcpu)
1529 {
1530         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1531
1532         spin_lock(&li->lock);
1533         li->irq.stop.flags = 0;
1534         clear_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs);
1535         spin_unlock(&li->lock);
1536 }
1537
1538 static int do_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1539 {
1540         int rc;
1541
1542         switch (irq->type) {
1543         case KVM_S390_PROGRAM_INT:
1544                 rc = __inject_prog(vcpu, irq);
1545                 break;
1546         case KVM_S390_SIGP_SET_PREFIX:
1547                 rc = __inject_set_prefix(vcpu, irq);
1548                 break;
1549         case KVM_S390_SIGP_STOP:
1550                 rc = __inject_sigp_stop(vcpu, irq);
1551                 break;
1552         case KVM_S390_RESTART:
1553                 rc = __inject_sigp_restart(vcpu, irq);
1554                 break;
1555         case KVM_S390_INT_CLOCK_COMP:
1556                 rc = __inject_ckc(vcpu);
1557                 break;
1558         case KVM_S390_INT_CPU_TIMER:
1559                 rc = __inject_cpu_timer(vcpu);
1560                 break;
1561         case KVM_S390_INT_EXTERNAL_CALL:
1562                 rc = __inject_extcall(vcpu, irq);
1563                 break;
1564         case KVM_S390_INT_EMERGENCY:
1565                 rc = __inject_sigp_emergency(vcpu, irq);
1566                 break;
1567         case KVM_S390_MCHK:
1568                 rc = __inject_mchk(vcpu, irq);
1569                 break;
1570         case KVM_S390_INT_PFAULT_INIT:
1571                 rc = __inject_pfault_init(vcpu, irq);
1572                 break;
1573         case KVM_S390_INT_VIRTIO:
1574         case KVM_S390_INT_SERVICE:
1575         case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
1576         default:
1577                 rc = -EINVAL;
1578         }
1579
1580         return rc;
1581 }
1582
1583 int kvm_s390_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1584 {
1585         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1586         int rc;
1587
1588         spin_lock(&li->lock);
1589         rc = do_inject_vcpu(vcpu, irq);
1590         spin_unlock(&li->lock);
1591         if (!rc)
1592                 kvm_s390_vcpu_wakeup(vcpu);
1593         return rc;
1594 }
1595
1596 static inline void clear_irq_list(struct list_head *_list)
1597 {
1598         struct kvm_s390_interrupt_info *inti, *n;
1599
1600         list_for_each_entry_safe(inti, n, _list, list) {
1601                 list_del(&inti->list);
1602                 kfree(inti);
1603         }
1604 }
1605
1606 static void inti_to_irq(struct kvm_s390_interrupt_info *inti,
1607                        struct kvm_s390_irq *irq)
1608 {
1609         irq->type = inti->type;
1610         switch (inti->type) {
1611         case KVM_S390_INT_PFAULT_INIT:
1612         case KVM_S390_INT_PFAULT_DONE:
1613         case KVM_S390_INT_VIRTIO:
1614                 irq->u.ext = inti->ext;
1615                 break;
1616         case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
1617                 irq->u.io = inti->io;
1618                 break;
1619         }
1620 }
1621
1622 void kvm_s390_clear_float_irqs(struct kvm *kvm)
1623 {
1624         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1625         int i;
1626
1627         spin_lock(&fi->lock);
1628         fi->pending_irqs = 0;
1629         memset(&fi->srv_signal, 0, sizeof(fi->srv_signal));
1630         memset(&fi->mchk, 0, sizeof(fi->mchk));
1631         for (i = 0; i < FIRQ_LIST_COUNT; i++)
1632                 clear_irq_list(&fi->lists[i]);
1633         for (i = 0; i < FIRQ_MAX_COUNT; i++)
1634                 fi->counters[i] = 0;
1635         spin_unlock(&fi->lock);
1636 };
1637
1638 static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len)
1639 {
1640         struct kvm_s390_interrupt_info *inti;
1641         struct kvm_s390_float_interrupt *fi;
1642         struct kvm_s390_irq *buf;
1643         struct kvm_s390_irq *irq;
1644         int max_irqs;
1645         int ret = 0;
1646         int n = 0;
1647         int i;
1648
1649         if (len > KVM_S390_FLIC_MAX_BUFFER || len == 0)
1650                 return -EINVAL;
1651
1652         /*
1653          * We are already using -ENOMEM to signal
1654          * userspace it may retry with a bigger buffer,
1655          * so we need to use something else for this case
1656          */
1657         buf = vzalloc(len);
1658         if (!buf)
1659                 return -ENOBUFS;
1660
1661         max_irqs = len / sizeof(struct kvm_s390_irq);
1662
1663         fi = &kvm->arch.float_int;
1664         spin_lock(&fi->lock);
1665         for (i = 0; i < FIRQ_LIST_COUNT; i++) {
1666                 list_for_each_entry(inti, &fi->lists[i], list) {
1667                         if (n == max_irqs) {
1668                                 /* signal userspace to try again */
1669                                 ret = -ENOMEM;
1670                                 goto out;
1671                         }
1672                         inti_to_irq(inti, &buf[n]);
1673                         n++;
1674                 }
1675         }
1676         if (test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs)) {
1677                 if (n == max_irqs) {
1678                         /* signal userspace to try again */
1679                         ret = -ENOMEM;
1680                         goto out;
1681                 }
1682                 irq = (struct kvm_s390_irq *) &buf[n];
1683                 irq->type = KVM_S390_INT_SERVICE;
1684                 irq->u.ext = fi->srv_signal;
1685                 n++;
1686         }
1687         if (test_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs)) {
1688                 if (n == max_irqs) {
1689                                 /* signal userspace to try again */
1690                                 ret = -ENOMEM;
1691                                 goto out;
1692                 }
1693                 irq = (struct kvm_s390_irq *) &buf[n];
1694                 irq->type = KVM_S390_MCHK;
1695                 irq->u.mchk = fi->mchk;
1696                 n++;
1697 }
1698
1699 out:
1700         spin_unlock(&fi->lock);
1701         if (!ret && n > 0) {
1702                 if (copy_to_user(usrbuf, buf, sizeof(struct kvm_s390_irq) * n))
1703                         ret = -EFAULT;
1704         }
1705         vfree(buf);
1706
1707         return ret < 0 ? ret : n;
1708 }
1709
1710 static int flic_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
1711 {
1712         int r;
1713
1714         switch (attr->group) {
1715         case KVM_DEV_FLIC_GET_ALL_IRQS:
1716                 r = get_all_floating_irqs(dev->kvm, (u8 __user *) attr->addr,
1717                                           attr->attr);
1718                 break;
1719         default:
1720                 r = -EINVAL;
1721         }
1722
1723         return r;
1724 }
1725
1726 static inline int copy_irq_from_user(struct kvm_s390_interrupt_info *inti,
1727                                      u64 addr)
1728 {
1729         struct kvm_s390_irq __user *uptr = (struct kvm_s390_irq __user *) addr;
1730         void *target = NULL;
1731         void __user *source;
1732         u64 size;
1733
1734         if (get_user(inti->type, (u64 __user *)addr))
1735                 return -EFAULT;
1736
1737         switch (inti->type) {
1738         case KVM_S390_INT_PFAULT_INIT:
1739         case KVM_S390_INT_PFAULT_DONE:
1740         case KVM_S390_INT_VIRTIO:
1741         case KVM_S390_INT_SERVICE:
1742                 target = (void *) &inti->ext;
1743                 source = &uptr->u.ext;
1744                 size = sizeof(inti->ext);
1745                 break;
1746         case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
1747                 target = (void *) &inti->io;
1748                 source = &uptr->u.io;
1749                 size = sizeof(inti->io);
1750                 break;
1751         case KVM_S390_MCHK:
1752                 target = (void *) &inti->mchk;
1753                 source = &uptr->u.mchk;
1754                 size = sizeof(inti->mchk);
1755                 break;
1756         default:
1757                 return -EINVAL;
1758         }
1759
1760         if (copy_from_user(target, source, size))
1761                 return -EFAULT;
1762
1763         return 0;
1764 }
1765
1766 static int enqueue_floating_irq(struct kvm_device *dev,
1767                                 struct kvm_device_attr *attr)
1768 {
1769         struct kvm_s390_interrupt_info *inti = NULL;
1770         int r = 0;
1771         int len = attr->attr;
1772
1773         if (len % sizeof(struct kvm_s390_irq) != 0)
1774                 return -EINVAL;
1775         else if (len > KVM_S390_FLIC_MAX_BUFFER)
1776                 return -EINVAL;
1777
1778         while (len >= sizeof(struct kvm_s390_irq)) {
1779                 inti = kzalloc(sizeof(*inti), GFP_KERNEL);
1780                 if (!inti)
1781                         return -ENOMEM;
1782
1783                 r = copy_irq_from_user(inti, attr->addr);
1784                 if (r) {
1785                         kfree(inti);
1786                         return r;
1787                 }
1788                 r = __inject_vm(dev->kvm, inti);
1789                 if (r) {
1790                         kfree(inti);
1791                         return r;
1792                 }
1793                 len -= sizeof(struct kvm_s390_irq);
1794                 attr->addr += sizeof(struct kvm_s390_irq);
1795         }
1796
1797         return r;
1798 }
1799
1800 static struct s390_io_adapter *get_io_adapter(struct kvm *kvm, unsigned int id)
1801 {
1802         if (id >= MAX_S390_IO_ADAPTERS)
1803                 return NULL;
1804         return kvm->arch.adapters[id];
1805 }
1806
1807 static int register_io_adapter(struct kvm_device *dev,
1808                                struct kvm_device_attr *attr)
1809 {
1810         struct s390_io_adapter *adapter;
1811         struct kvm_s390_io_adapter adapter_info;
1812
1813         if (copy_from_user(&adapter_info,
1814                            (void __user *)attr->addr, sizeof(adapter_info)))
1815                 return -EFAULT;
1816
1817         if ((adapter_info.id >= MAX_S390_IO_ADAPTERS) ||
1818             (dev->kvm->arch.adapters[adapter_info.id] != NULL))
1819                 return -EINVAL;
1820
1821         adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
1822         if (!adapter)
1823                 return -ENOMEM;
1824
1825         INIT_LIST_HEAD(&adapter->maps);
1826         init_rwsem(&adapter->maps_lock);
1827         atomic_set(&adapter->nr_maps, 0);
1828         adapter->id = adapter_info.id;
1829         adapter->isc = adapter_info.isc;
1830         adapter->maskable = adapter_info.maskable;
1831         adapter->masked = false;
1832         adapter->swap = adapter_info.swap;
1833         dev->kvm->arch.adapters[adapter->id] = adapter;
1834
1835         return 0;
1836 }
1837
1838 int kvm_s390_mask_adapter(struct kvm *kvm, unsigned int id, bool masked)
1839 {
1840         int ret;
1841         struct s390_io_adapter *adapter = get_io_adapter(kvm, id);
1842
1843         if (!adapter || !adapter->maskable)
1844                 return -EINVAL;
1845         ret = adapter->masked;
1846         adapter->masked = masked;
1847         return ret;
1848 }
1849
1850 static int kvm_s390_adapter_map(struct kvm *kvm, unsigned int id, __u64 addr)
1851 {
1852         struct s390_io_adapter *adapter = get_io_adapter(kvm, id);
1853         struct s390_map_info *map;
1854         int ret;
1855
1856         if (!adapter || !addr)
1857                 return -EINVAL;
1858
1859         map = kzalloc(sizeof(*map), GFP_KERNEL);
1860         if (!map) {
1861                 ret = -ENOMEM;
1862                 goto out;
1863         }
1864         INIT_LIST_HEAD(&map->list);
1865         map->guest_addr = addr;
1866         map->addr = gmap_translate(kvm->arch.gmap, addr);
1867         if (map->addr == -EFAULT) {
1868                 ret = -EFAULT;
1869                 goto out;
1870         }
1871         ret = get_user_pages_fast(map->addr, 1, 1, &map->page);
1872         if (ret < 0)
1873                 goto out;
1874         BUG_ON(ret != 1);
1875         down_write(&adapter->maps_lock);
1876         if (atomic_inc_return(&adapter->nr_maps) < MAX_S390_ADAPTER_MAPS) {
1877                 list_add_tail(&map->list, &adapter->maps);
1878                 ret = 0;
1879         } else {
1880                 put_page(map->page);
1881                 ret = -EINVAL;
1882         }
1883         up_write(&adapter->maps_lock);
1884 out:
1885         if (ret)
1886                 kfree(map);
1887         return ret;
1888 }
1889
1890 static int kvm_s390_adapter_unmap(struct kvm *kvm, unsigned int id, __u64 addr)
1891 {
1892         struct s390_io_adapter *adapter = get_io_adapter(kvm, id);
1893         struct s390_map_info *map, *tmp;
1894         int found = 0;
1895
1896         if (!adapter || !addr)
1897                 return -EINVAL;
1898
1899         down_write(&adapter->maps_lock);
1900         list_for_each_entry_safe(map, tmp, &adapter->maps, list) {
1901                 if (map->guest_addr == addr) {
1902                         found = 1;
1903                         atomic_dec(&adapter->nr_maps);
1904                         list_del(&map->list);
1905                         put_page(map->page);
1906                         kfree(map);
1907                         break;
1908                 }
1909         }
1910         up_write(&adapter->maps_lock);
1911
1912         return found ? 0 : -EINVAL;
1913 }
1914
1915 void kvm_s390_destroy_adapters(struct kvm *kvm)
1916 {
1917         int i;
1918         struct s390_map_info *map, *tmp;
1919
1920         for (i = 0; i < MAX_S390_IO_ADAPTERS; i++) {
1921                 if (!kvm->arch.adapters[i])
1922                         continue;
1923                 list_for_each_entry_safe(map, tmp,
1924                                          &kvm->arch.adapters[i]->maps, list) {
1925                         list_del(&map->list);
1926                         put_page(map->page);
1927                         kfree(map);
1928                 }
1929                 kfree(kvm->arch.adapters[i]);
1930         }
1931 }
1932
1933 static int modify_io_adapter(struct kvm_device *dev,
1934                              struct kvm_device_attr *attr)
1935 {
1936         struct kvm_s390_io_adapter_req req;
1937         struct s390_io_adapter *adapter;
1938         int ret;
1939
1940         if (copy_from_user(&req, (void __user *)attr->addr, sizeof(req)))
1941                 return -EFAULT;
1942
1943         adapter = get_io_adapter(dev->kvm, req.id);
1944         if (!adapter)
1945                 return -EINVAL;
1946         switch (req.type) {
1947         case KVM_S390_IO_ADAPTER_MASK:
1948                 ret = kvm_s390_mask_adapter(dev->kvm, req.id, req.mask);
1949                 if (ret > 0)
1950                         ret = 0;
1951                 break;
1952         case KVM_S390_IO_ADAPTER_MAP:
1953                 ret = kvm_s390_adapter_map(dev->kvm, req.id, req.addr);
1954                 break;
1955         case KVM_S390_IO_ADAPTER_UNMAP:
1956                 ret = kvm_s390_adapter_unmap(dev->kvm, req.id, req.addr);
1957                 break;
1958         default:
1959                 ret = -EINVAL;
1960         }
1961
1962         return ret;
1963 }
1964
1965 static int flic_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
1966 {
1967         int r = 0;
1968         unsigned int i;
1969         struct kvm_vcpu *vcpu;
1970
1971         switch (attr->group) {
1972         case KVM_DEV_FLIC_ENQUEUE:
1973                 r = enqueue_floating_irq(dev, attr);
1974                 break;
1975         case KVM_DEV_FLIC_CLEAR_IRQS:
1976                 kvm_s390_clear_float_irqs(dev->kvm);
1977                 break;
1978         case KVM_DEV_FLIC_APF_ENABLE:
1979                 dev->kvm->arch.gmap->pfault_enabled = 1;
1980                 break;
1981         case KVM_DEV_FLIC_APF_DISABLE_WAIT:
1982                 dev->kvm->arch.gmap->pfault_enabled = 0;
1983                 /*
1984                  * Make sure no async faults are in transition when
1985                  * clearing the queues. So we don't need to worry
1986                  * about late coming workers.
1987                  */
1988                 synchronize_srcu(&dev->kvm->srcu);
1989                 kvm_for_each_vcpu(i, vcpu, dev->kvm)
1990                         kvm_clear_async_pf_completion_queue(vcpu);
1991                 break;
1992         case KVM_DEV_FLIC_ADAPTER_REGISTER:
1993                 r = register_io_adapter(dev, attr);
1994                 break;
1995         case KVM_DEV_FLIC_ADAPTER_MODIFY:
1996                 r = modify_io_adapter(dev, attr);
1997                 break;
1998         default:
1999                 r = -EINVAL;
2000         }
2001
2002         return r;
2003 }
2004
2005 static int flic_create(struct kvm_device *dev, u32 type)
2006 {
2007         if (!dev)
2008                 return -EINVAL;
2009         if (dev->kvm->arch.flic)
2010                 return -EINVAL;
2011         dev->kvm->arch.flic = dev;
2012         return 0;
2013 }
2014
2015 static void flic_destroy(struct kvm_device *dev)
2016 {
2017         dev->kvm->arch.flic = NULL;
2018         kfree(dev);
2019 }
2020
2021 /* s390 floating irq controller (flic) */
2022 struct kvm_device_ops kvm_flic_ops = {
2023         .name = "kvm-flic",
2024         .get_attr = flic_get_attr,
2025         .set_attr = flic_set_attr,
2026         .create = flic_create,
2027         .destroy = flic_destroy,
2028 };
2029
2030 static unsigned long get_ind_bit(__u64 addr, unsigned long bit_nr, bool swap)
2031 {
2032         unsigned long bit;
2033
2034         bit = bit_nr + (addr % PAGE_SIZE) * 8;
2035
2036         return swap ? (bit ^ (BITS_PER_LONG - 1)) : bit;
2037 }
2038
2039 static struct s390_map_info *get_map_info(struct s390_io_adapter *adapter,
2040                                           u64 addr)
2041 {
2042         struct s390_map_info *map;
2043
2044         if (!adapter)
2045                 return NULL;
2046
2047         list_for_each_entry(map, &adapter->maps, list) {
2048                 if (map->guest_addr == addr)
2049                         return map;
2050         }
2051         return NULL;
2052 }
2053
2054 static int adapter_indicators_set(struct kvm *kvm,
2055                                   struct s390_io_adapter *adapter,
2056                                   struct kvm_s390_adapter_int *adapter_int)
2057 {
2058         unsigned long bit;
2059         int summary_set, idx;
2060         struct s390_map_info *info;
2061         void *map;
2062
2063         info = get_map_info(adapter, adapter_int->ind_addr);
2064         if (!info)
2065                 return -1;
2066         map = page_address(info->page);
2067         bit = get_ind_bit(info->addr, adapter_int->ind_offset, adapter->swap);
2068         set_bit(bit, map);
2069         idx = srcu_read_lock(&kvm->srcu);
2070         mark_page_dirty(kvm, info->guest_addr >> PAGE_SHIFT);
2071         set_page_dirty_lock(info->page);
2072         info = get_map_info(adapter, adapter_int->summary_addr);
2073         if (!info) {
2074                 srcu_read_unlock(&kvm->srcu, idx);
2075                 return -1;
2076         }
2077         map = page_address(info->page);
2078         bit = get_ind_bit(info->addr, adapter_int->summary_offset,
2079                           adapter->swap);
2080         summary_set = test_and_set_bit(bit, map);
2081         mark_page_dirty(kvm, info->guest_addr >> PAGE_SHIFT);
2082         set_page_dirty_lock(info->page);
2083         srcu_read_unlock(&kvm->srcu, idx);
2084         return summary_set ? 0 : 1;
2085 }
2086
2087 /*
2088  * < 0 - not injected due to error
2089  * = 0 - coalesced, summary indicator already active
2090  * > 0 - injected interrupt
2091  */
2092 static int set_adapter_int(struct kvm_kernel_irq_routing_entry *e,
2093                            struct kvm *kvm, int irq_source_id, int level,
2094                            bool line_status)
2095 {
2096         int ret;
2097         struct s390_io_adapter *adapter;
2098
2099         /* We're only interested in the 0->1 transition. */
2100         if (!level)
2101                 return 0;
2102         adapter = get_io_adapter(kvm, e->adapter.adapter_id);
2103         if (!adapter)
2104                 return -1;
2105         down_read(&adapter->maps_lock);
2106         ret = adapter_indicators_set(kvm, adapter, &e->adapter);
2107         up_read(&adapter->maps_lock);
2108         if ((ret > 0) && !adapter->masked) {
2109                 struct kvm_s390_interrupt s390int = {
2110                         .type = KVM_S390_INT_IO(1, 0, 0, 0),
2111                         .parm = 0,
2112                         .parm64 = (adapter->isc << 27) | 0x80000000,
2113                 };
2114                 ret = kvm_s390_inject_vm(kvm, &s390int);
2115                 if (ret == 0)
2116                         ret = 1;
2117         }
2118         return ret;
2119 }
2120
2121 int kvm_set_routing_entry(struct kvm_kernel_irq_routing_entry *e,
2122                           const struct kvm_irq_routing_entry *ue)
2123 {
2124         int ret;
2125
2126         switch (ue->type) {
2127         case KVM_IRQ_ROUTING_S390_ADAPTER:
2128                 e->set = set_adapter_int;
2129                 e->adapter.summary_addr = ue->u.adapter.summary_addr;
2130                 e->adapter.ind_addr = ue->u.adapter.ind_addr;
2131                 e->adapter.summary_offset = ue->u.adapter.summary_offset;
2132                 e->adapter.ind_offset = ue->u.adapter.ind_offset;
2133                 e->adapter.adapter_id = ue->u.adapter.adapter_id;
2134                 ret = 0;
2135                 break;
2136         default:
2137                 ret = -EINVAL;
2138         }
2139
2140         return ret;
2141 }
2142
2143 int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm,
2144                 int irq_source_id, int level, bool line_status)
2145 {
2146         return -EINVAL;
2147 }
2148
2149 int kvm_s390_set_irq_state(struct kvm_vcpu *vcpu, void __user *irqstate, int len)
2150 {
2151         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2152         struct kvm_s390_irq *buf;
2153         int r = 0;
2154         int n;
2155
2156         buf = vmalloc(len);
2157         if (!buf)
2158                 return -ENOMEM;
2159
2160         if (copy_from_user((void *) buf, irqstate, len)) {
2161                 r = -EFAULT;
2162                 goto out_free;
2163         }
2164
2165         /*
2166          * Don't allow setting the interrupt state
2167          * when there are already interrupts pending
2168          */
2169         spin_lock(&li->lock);
2170         if (li->pending_irqs) {
2171                 r = -EBUSY;
2172                 goto out_unlock;
2173         }
2174
2175         for (n = 0; n < len / sizeof(*buf); n++) {
2176                 r = do_inject_vcpu(vcpu, &buf[n]);
2177                 if (r)
2178                         break;
2179         }
2180
2181 out_unlock:
2182         spin_unlock(&li->lock);
2183 out_free:
2184         vfree(buf);
2185
2186         return r;
2187 }
2188
2189 static void store_local_irq(struct kvm_s390_local_interrupt *li,
2190                             struct kvm_s390_irq *irq,
2191                             unsigned long irq_type)
2192 {
2193         switch (irq_type) {
2194         case IRQ_PEND_MCHK_EX:
2195         case IRQ_PEND_MCHK_REP:
2196                 irq->type = KVM_S390_MCHK;
2197                 irq->u.mchk = li->irq.mchk;
2198                 break;
2199         case IRQ_PEND_PROG:
2200                 irq->type = KVM_S390_PROGRAM_INT;
2201                 irq->u.pgm = li->irq.pgm;
2202                 break;
2203         case IRQ_PEND_PFAULT_INIT:
2204                 irq->type = KVM_S390_INT_PFAULT_INIT;
2205                 irq->u.ext = li->irq.ext;
2206                 break;
2207         case IRQ_PEND_EXT_EXTERNAL:
2208                 irq->type = KVM_S390_INT_EXTERNAL_CALL;
2209                 irq->u.extcall = li->irq.extcall;
2210                 break;
2211         case IRQ_PEND_EXT_CLOCK_COMP:
2212                 irq->type = KVM_S390_INT_CLOCK_COMP;
2213                 break;
2214         case IRQ_PEND_EXT_CPU_TIMER:
2215                 irq->type = KVM_S390_INT_CPU_TIMER;
2216                 break;
2217         case IRQ_PEND_SIGP_STOP:
2218                 irq->type = KVM_S390_SIGP_STOP;
2219                 irq->u.stop = li->irq.stop;
2220                 break;
2221         case IRQ_PEND_RESTART:
2222                 irq->type = KVM_S390_RESTART;
2223                 break;
2224         case IRQ_PEND_SET_PREFIX:
2225                 irq->type = KVM_S390_SIGP_SET_PREFIX;
2226                 irq->u.prefix = li->irq.prefix;
2227                 break;
2228         }
2229 }
2230
2231 int kvm_s390_get_irq_state(struct kvm_vcpu *vcpu, __u8 __user *buf, int len)
2232 {
2233         uint8_t sigp_ctrl = vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sigp_ctrl;
2234         unsigned long sigp_emerg_pending[BITS_TO_LONGS(KVM_MAX_VCPUS)];
2235         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2236         unsigned long pending_irqs;
2237         struct kvm_s390_irq irq;
2238         unsigned long irq_type;
2239         int cpuaddr;
2240         int n = 0;
2241
2242         spin_lock(&li->lock);
2243         pending_irqs = li->pending_irqs;
2244         memcpy(&sigp_emerg_pending, &li->sigp_emerg_pending,
2245                sizeof(sigp_emerg_pending));
2246         spin_unlock(&li->lock);
2247
2248         for_each_set_bit(irq_type, &pending_irqs, IRQ_PEND_COUNT) {
2249                 memset(&irq, 0, sizeof(irq));
2250                 if (irq_type == IRQ_PEND_EXT_EMERGENCY)
2251                         continue;
2252                 if (n + sizeof(irq) > len)
2253                         return -ENOBUFS;
2254                 store_local_irq(&vcpu->arch.local_int, &irq, irq_type);
2255                 if (copy_to_user(&buf[n], &irq, sizeof(irq)))
2256                         return -EFAULT;
2257                 n += sizeof(irq);
2258         }
2259
2260         if (test_bit(IRQ_PEND_EXT_EMERGENCY, &pending_irqs)) {
2261                 for_each_set_bit(cpuaddr, sigp_emerg_pending, KVM_MAX_VCPUS) {
2262                         memset(&irq, 0, sizeof(irq));
2263                         if (n + sizeof(irq) > len)
2264                                 return -ENOBUFS;
2265                         irq.type = KVM_S390_INT_EMERGENCY;
2266                         irq.u.emerg.code = cpuaddr;
2267                         if (copy_to_user(&buf[n], &irq, sizeof(irq)))
2268                                 return -EFAULT;
2269                         n += sizeof(irq);
2270                 }
2271         }
2272
2273         if ((sigp_ctrl & SIGP_CTRL_C) &&
2274             (atomic_read(&vcpu->arch.sie_block->cpuflags) &
2275              CPUSTAT_ECALL_PEND)) {
2276                 if (n + sizeof(irq) > len)
2277                         return -ENOBUFS;
2278                 memset(&irq, 0, sizeof(irq));
2279                 irq.type = KVM_S390_INT_EXTERNAL_CALL;
2280                 irq.u.extcall.code = sigp_ctrl & SIGP_CTRL_SCN_MASK;
2281                 if (copy_to_user(&buf[n], &irq, sizeof(irq)))
2282                         return -EFAULT;
2283                 n += sizeof(irq);
2284         }
2285
2286         return n;
2287 }