arch/s390/kvm/kvm-s390.c

   1 /*
   2  * hosting zSeries kernel virtual machines
   3  *
   4  * Copyright IBM Corp. 2008, 2009
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License (version 2 only)
   8  * as published by the Free Software Foundation.
   9  *
  10  *    Author(s): Carsten Otte <cotte@de.ibm.com>
  11  *               Christian Borntraeger <borntraeger@de.ibm.com>
  12  *               Heiko Carstens <heiko.carstens@de.ibm.com>
  13  *               Christian Ehrhardt <ehrhardt@de.ibm.com>
  14  */
  15
  16 #include <linux/compiler.h>
  17 #include <linux/err.h>
  18 #include <linux/fs.h>
  19 #include <linux/hrtimer.h>
  20 #include <linux/init.h>
  21 #include <linux/kvm.h>
  22 #include <linux/kvm_host.h>
  23 #include <linux/module.h>
  24 #include <linux/slab.h>
  25 #include <linux/timer.h>
  26 #include <asm/asm-offsets.h>
  27 #include <asm/lowcore.h>
  28 #include <asm/pgtable.h>
  29 #include <asm/nmi.h>
  30 #include <asm/switch_to.h>
  31 #include <asm/facility.h>
  32 #include <asm/sclp.h>
  33 #include "kvm-s390.h"
  34 #include "gaccess.h"
  35
  36 #define CREATE_TRACE_POINTS
  37 #include "trace.h"
  38 #include "trace-s390.h"
  39
  40 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
  41
  42 struct kvm_stats_debugfs_item debugfs_entries[] = {
  43         { "userspace_handled", VCPU_STAT(exit_userspace) },
  44         { "exit_null", VCPU_STAT(exit_null) },
  45         { "exit_validity", VCPU_STAT(exit_validity) },
  46         { "exit_stop_request", VCPU_STAT(exit_stop_request) },
  47         { "exit_external_request", VCPU_STAT(exit_external_request) },
  48         { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
  49         { "exit_instruction", VCPU_STAT(exit_instruction) },
  50         { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
  51         { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
  52         { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
  53         { "instruction_lctl", VCPU_STAT(instruction_lctl) },
  54         { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
  55         { "deliver_external_call", VCPU_STAT(deliver_external_call) },
  56         { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
  57         { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
  58         { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
  59         { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
  60         { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
  61         { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
  62         { "exit_wait_state", VCPU_STAT(exit_wait_state) },
  63         { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
  64         { "instruction_stidp", VCPU_STAT(instruction_stidp) },
  65         { "instruction_spx", VCPU_STAT(instruction_spx) },
  66         { "instruction_stpx", VCPU_STAT(instruction_stpx) },
  67         { "instruction_stap", VCPU_STAT(instruction_stap) },
  68         { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
  69         { "instruction_stsch", VCPU_STAT(instruction_stsch) },
  70         { "instruction_chsc", VCPU_STAT(instruction_chsc) },
  71         { "instruction_essa", VCPU_STAT(instruction_essa) },
  72         { "instruction_stsi", VCPU_STAT(instruction_stsi) },
  73         { "instruction_stfl", VCPU_STAT(instruction_stfl) },
  74         { "instruction_tprot", VCPU_STAT(instruction_tprot) },
  75         { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
  76         { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
  77         { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
  78         { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
  79         { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
  80         { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
  81         { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
  82         { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
  83         { "diagnose_10", VCPU_STAT(diagnose_10) },
  84         { "diagnose_44", VCPU_STAT(diagnose_44) },
  85         { "diagnose_9c", VCPU_STAT(diagnose_9c) },
  86         { NULL }
  87 };
  88
  89 unsigned long *vfacilities;
  90 static struct gmap_notifier gmap_notifier;
  91
  92 /* test availability of vfacility */
  93 static inline int test_vfacility(unsigned long nr)
  94 {
  95         return __test_facility(nr, (void *) vfacilities);
  96 }
  97
  98 /* Section: not file related */
  99 int kvm_arch_hardware_enable(void *garbage)
 100 {
 101         /* every s390 is virtualization enabled ;-) */
 102         return 0;
 103 }
 104
 105 void kvm_arch_hardware_disable(void *garbage)
 106 {
 107 }
 108
 109 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
 110
 111 int kvm_arch_hardware_setup(void)
 112 {
 113         gmap_notifier.notifier_call = kvm_gmap_notifier;
 114         gmap_register_ipte_notifier(&gmap_notifier);
 115         return 0;
 116 }
 117
 118 void kvm_arch_hardware_unsetup(void)
 119 {
 120         gmap_unregister_ipte_notifier(&gmap_notifier);
 121 }
 122
 123 void kvm_arch_check_processor_compat(void *rtn)
 124 {
 125 }
 126
 127 int kvm_arch_init(void *opaque)
 128 {
 129         return 0;
 130 }
 131
 132 void kvm_arch_exit(void)
 133 {
 134 }
 135
 136 /* Section: device related */
 137 long kvm_arch_dev_ioctl(struct file *filp,
 138                         unsigned int ioctl, unsigned long arg)
 139 {
 140         if (ioctl == KVM_S390_ENABLE_SIE)
 141                 return s390_enable_sie();
 142         return -EINVAL;
 143 }
 144
 145 int kvm_dev_ioctl_check_extension(long ext)
 146 {
 147         int r;
 148
 149         switch (ext) {
 150         case KVM_CAP_S390_PSW:
 151         case KVM_CAP_S390_GMAP:
 152         case KVM_CAP_SYNC_MMU:
 153 #ifdef CONFIG_KVM_S390_UCONTROL
 154         case KVM_CAP_S390_UCONTROL:
 155 #endif
 156         case KVM_CAP_ASYNC_PF:
 157         case KVM_CAP_SYNC_REGS:
 158         case KVM_CAP_ONE_REG:
 159         case KVM_CAP_ENABLE_CAP:
 160         case KVM_CAP_S390_CSS_SUPPORT:
 161         case KVM_CAP_IRQFD:
 162         case KVM_CAP_IOEVENTFD:
 163         case KVM_CAP_DEVICE_CTRL:
 164         case KVM_CAP_ENABLE_CAP_VM:
 165                 r = 1;
 166                 break;
 167         case KVM_CAP_NR_VCPUS:
 168         case KVM_CAP_MAX_VCPUS:
 169                 r = KVM_MAX_VCPUS;
 170                 break;
 171         case KVM_CAP_NR_MEMSLOTS:
 172                 r = KVM_USER_MEM_SLOTS;
 173                 break;
 174         case KVM_CAP_S390_COW:
 175                 r = MACHINE_HAS_ESOP;
 176                 break;
 177         default:
 178                 r = 0;
 179         }
 180         return r;
 181 }
 182
 183 /* Section: vm related */
 184 /*
 185  * Get (and clear) the dirty memory log for a memory slot.
 186  */
 187 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
 188                                struct kvm_dirty_log *log)
 189 {
 190         return 0;
 191 }
 192
 193 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
 194 {
 195         int r;
 196
 197         if (cap->flags)
 198                 return -EINVAL;
 199
 200         switch (cap->cap) {
 201         case KVM_CAP_S390_IRQCHIP:
 202                 kvm->arch.use_irqchip = 1;
 203                 r = 0;
 204                 break;
 205         default:
 206                 r = -EINVAL;
 207                 break;
 208         }
 209         return r;
 210 }
 211
 212 long kvm_arch_vm_ioctl(struct file *filp,
 213                        unsigned int ioctl, unsigned long arg)
 214 {
 215         struct kvm *kvm = filp->private_data;
 216         void __user *argp = (void __user *)arg;
 217         int r;
 218
 219         switch (ioctl) {
 220         case KVM_S390_INTERRUPT: {
 221                 struct kvm_s390_interrupt s390int;
 222
 223                 r = -EFAULT;
 224                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
 225                         break;
 226                 r = kvm_s390_inject_vm(kvm, &s390int);
 227                 break;
 228         }
 229         case KVM_ENABLE_CAP: {
 230                 struct kvm_enable_cap cap;
 231                 r = -EFAULT;
 232                 if (copy_from_user(&cap, argp, sizeof(cap)))
 233                         break;
 234                 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
 235                 break;
 236         }
 237         case KVM_CREATE_IRQCHIP: {
 238                 struct kvm_irq_routing_entry routing;
 239
 240                 r = -EINVAL;
 241                 if (kvm->arch.use_irqchip) {
 242                         /* Set up dummy routing. */
 243                         memset(&routing, 0, sizeof(routing));
 244                         kvm_set_irq_routing(kvm, &routing, 0, 0);
 245                         r = 0;
 246                 }
 247                 break;
 248         }
 249         default:
 250                 r = -ENOTTY;
 251         }
 252
 253         return r;
 254 }
 255
 256 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
 257 {
 258         int rc;
 259         char debug_name[16];
 260         static unsigned long sca_offset;
 261
 262         rc = -EINVAL;
 263 #ifdef CONFIG_KVM_S390_UCONTROL
 264         if (type & ~KVM_VM_S390_UCONTROL)
 265                 goto out_err;
 266         if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
 267                 goto out_err;
 268 #else
 269         if (type)
 270                 goto out_err;
 271 #endif
 272
 273         rc = s390_enable_sie();
 274         if (rc)
 275                 goto out_err;
 276
 277         rc = -ENOMEM;
 278
 279         kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL);
 280         if (!kvm->arch.sca)
 281                 goto out_err;
 282         spin_lock(&kvm_lock);
 283         sca_offset = (sca_offset + 16) & 0x7f0;
 284         kvm->arch.sca = (struct sca_block *) ((char *) kvm->arch.sca + sca_offset);
 285         spin_unlock(&kvm_lock);
 286
 287         sprintf(debug_name, "kvm-%u", current->pid);
 288
 289         kvm->arch.dbf = debug_register(debug_name, 8, 2, 8 * sizeof(long));
 290         if (!kvm->arch.dbf)
 291                 goto out_nodbf;
 292
 293         spin_lock_init(&kvm->arch.float_int.lock);
 294         INIT_LIST_HEAD(&kvm->arch.float_int.list);
 295
 296         debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
 297         VM_EVENT(kvm, 3, "%s", "vm created");
 298
 299         if (type & KVM_VM_S390_UCONTROL) {
 300                 kvm->arch.gmap = NULL;
 301         } else {
 302                 kvm->arch.gmap = gmap_alloc(current->mm);
 303                 if (!kvm->arch.gmap)
 304                         goto out_nogmap;
 305                 kvm->arch.gmap->private = kvm;
 306                 kvm->arch.gmap->pfault_enabled = 0;
 307         }
 308
 309         kvm->arch.css_support = 0;
 310         kvm->arch.use_irqchip = 0;
 311
 312         return 0;
 313 out_nogmap:
 314         debug_unregister(kvm->arch.dbf);
 315 out_nodbf:
 316         free_page((unsigned long)(kvm->arch.sca));
 317 out_err:
 318         return rc;
 319 }
 320
 321 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
 322 {
 323         VCPU_EVENT(vcpu, 3, "%s", "free cpu");
 324         trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
 325         kvm_clear_async_pf_completion_queue(vcpu);
 326         if (!kvm_is_ucontrol(vcpu->kvm)) {
 327                 clear_bit(63 - vcpu->vcpu_id,
 328                           (unsigned long *) &vcpu->kvm->arch.sca->mcn);
 329                 if (vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda ==
 330                     (__u64) vcpu->arch.sie_block)
 331                         vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda = 0;
 332         }
 333         smp_mb();
 334
 335         if (kvm_is_ucontrol(vcpu->kvm))
 336                 gmap_free(vcpu->arch.gmap);
 337
 338         if (vcpu->arch.sie_block->cbrlo)
 339                 __free_page(__pfn_to_page(
 340                                 vcpu->arch.sie_block->cbrlo >> PAGE_SHIFT));
 341         free_page((unsigned long)(vcpu->arch.sie_block));
 342
 343         kvm_vcpu_uninit(vcpu);
 344         kmem_cache_free(kvm_vcpu_cache, vcpu);
 345 }
 346
 347 static void kvm_free_vcpus(struct kvm *kvm)
 348 {
 349         unsigned int i;
 350         struct kvm_vcpu *vcpu;
 351
 352         kvm_for_each_vcpu(i, vcpu, kvm)
 353                 kvm_arch_vcpu_destroy(vcpu);
 354
 355         mutex_lock(&kvm->lock);
 356         for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
 357                 kvm->vcpus[i] = NULL;
 358
 359         atomic_set(&kvm->online_vcpus, 0);
 360         mutex_unlock(&kvm->lock);
 361 }
 362
 363 void kvm_arch_sync_events(struct kvm *kvm)
 364 {
 365 }
 366
 367 void kvm_arch_destroy_vm(struct kvm *kvm)
 368 {
 369         kvm_free_vcpus(kvm);
 370         free_page((unsigned long)(kvm->arch.sca));
 371         debug_unregister(kvm->arch.dbf);
 372         if (!kvm_is_ucontrol(kvm))
 373                 gmap_free(kvm->arch.gmap);
 374         kvm_s390_destroy_adapters(kvm);
 375 }
 376
 377 /* Section: vcpu related */
 378 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
 379 {
 380         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
 381         kvm_clear_async_pf_completion_queue(vcpu);
 382         if (kvm_is_ucontrol(vcpu->kvm)) {
 383                 vcpu->arch.gmap = gmap_alloc(current->mm);
 384                 if (!vcpu->arch.gmap)
 385                         return -ENOMEM;
 386                 vcpu->arch.gmap->private = vcpu->kvm;
 387                 return 0;
 388         }
 389
 390         vcpu->arch.gmap = vcpu->kvm->arch.gmap;
 391         vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
 392                                     KVM_SYNC_GPRS |
 393                                     KVM_SYNC_ACRS |
 394                                     KVM_SYNC_CRS;
 395         return 0;
 396 }
 397
 398 void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu)
 399 {
 400         /* Nothing todo */
 401 }
 402
 403 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 404 {
 405         save_fp_ctl(&vcpu->arch.host_fpregs.fpc);
 406         save_fp_regs(vcpu->arch.host_fpregs.fprs);
 407         save_access_regs(vcpu->arch.host_acrs);
 408         restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
 409         restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
 410         restore_access_regs(vcpu->run->s.regs.acrs);
 411         gmap_enable(vcpu->arch.gmap);
 412         atomic_set_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
 413 }
 414
 415 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
 416 {
 417         atomic_clear_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
 418         gmap_disable(vcpu->arch.gmap);
 419         save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
 420         save_fp_regs(vcpu->arch.guest_fpregs.fprs);
 421         save_access_regs(vcpu->run->s.regs.acrs);
 422         restore_fp_ctl(&vcpu->arch.host_fpregs.fpc);
 423         restore_fp_regs(vcpu->arch.host_fpregs.fprs);
 424         restore_access_regs(vcpu->arch.host_acrs);
 425 }
 426
 427 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
 428 {
 429         /* this equals initial cpu reset in pop, but we don't switch to ESA */
 430         vcpu->arch.sie_block->gpsw.mask = 0UL;
 431         vcpu->arch.sie_block->gpsw.addr = 0UL;
 432         kvm_s390_set_prefix(vcpu, 0);
 433         vcpu->arch.sie_block->cputm     = 0UL;
 434         vcpu->arch.sie_block->ckc       = 0UL;
 435         vcpu->arch.sie_block->todpr     = 0;
 436         memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
 437         vcpu->arch.sie_block->gcr[0]  = 0xE0UL;
 438         vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
 439         vcpu->arch.guest_fpregs.fpc = 0;
 440         asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc));
 441         vcpu->arch.sie_block->gbea = 1;
 442         vcpu->arch.sie_block->pp = 0;
 443         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
 444         kvm_clear_async_pf_completion_queue(vcpu);
 445         atomic_set_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
 446         kvm_s390_clear_local_irqs(vcpu);
 447 }
 448
 449 int kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
 450 {
 451         return 0;
 452 }
 453
 454 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
 455 {
 456         struct page *cbrl;
 457
 458         atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
 459                                                     CPUSTAT_SM |
 460                                                     CPUSTAT_STOPPED |
 461                                                     CPUSTAT_GED);
 462         vcpu->arch.sie_block->ecb   = 6;
 463         if (test_vfacility(50) && test_vfacility(73))
 464                 vcpu->arch.sie_block->ecb |= 0x10;
 465
 466         vcpu->arch.sie_block->ecb2  = 8;
 467         vcpu->arch.sie_block->eca   = 0xC1002001U;
 468         vcpu->arch.sie_block->fac   = (int) (long) vfacilities;
 469         if (kvm_enabled_cmma()) {
 470                 cbrl = alloc_page(GFP_KERNEL | __GFP_ZERO);
 471                 if (cbrl) {
 472                         vcpu->arch.sie_block->ecb2 |= 0x80;
 473                         vcpu->arch.sie_block->ecb2 &= ~0x08;
 474                         vcpu->arch.sie_block->cbrlo = page_to_phys(cbrl);
 475                 }
 476         }
 477         hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_REALTIME, HRTIMER_MODE_ABS);
 478         tasklet_init(&vcpu->arch.tasklet, kvm_s390_tasklet,
 479                      (unsigned long) vcpu);
 480         vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
 481         get_cpu_id(&vcpu->arch.cpu_id);
 482         vcpu->arch.cpu_id.version = 0xff;
 483         return 0;
 484 }
 485
 486 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
 487                                       unsigned int id)
 488 {
 489         struct kvm_vcpu *vcpu;
 490         struct sie_page *sie_page;
 491         int rc = -EINVAL;
 492
 493         if (id >= KVM_MAX_VCPUS)
 494                 goto out;
 495
 496         rc = -ENOMEM;
 497
 498         vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
 499         if (!vcpu)
 500                 goto out;
 501
 502         sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
 503         if (!sie_page)
 504                 goto out_free_cpu;
 505
 506         vcpu->arch.sie_block = &sie_page->sie_block;
 507         vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
 508
 509         vcpu->arch.sie_block->icpua = id;
 510         if (!kvm_is_ucontrol(kvm)) {
 511                 if (!kvm->arch.sca) {
 512                         WARN_ON_ONCE(1);
 513                         goto out_free_cpu;
 514                 }
 515                 if (!kvm->arch.sca->cpu[id].sda)
 516                         kvm->arch.sca->cpu[id].sda =
 517                                 (__u64) vcpu->arch.sie_block;
 518                 vcpu->arch.sie_block->scaoh =
 519                         (__u32)(((__u64)kvm->arch.sca) >> 32);
 520                 vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca;
 521                 set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn);
 522         }
 523
 524         spin_lock_init(&vcpu->arch.local_int.lock);
 525         INIT_LIST_HEAD(&vcpu->arch.local_int.list);
 526         vcpu->arch.local_int.float_int = &kvm->arch.float_int;
 527         vcpu->arch.local_int.wq = &vcpu->wq;
 528         vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
 529
 530         rc = kvm_vcpu_init(vcpu, kvm, id);
 531         if (rc)
 532                 goto out_free_sie_block;
 533         VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu,
 534                  vcpu->arch.sie_block);
 535         trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
 536
 537         return vcpu;
 538 out_free_sie_block:
 539         free_page((unsigned long)(vcpu->arch.sie_block));
 540 out_free_cpu:
 541         kmem_cache_free(kvm_vcpu_cache, vcpu);
 542 out:
 543         return ERR_PTR(rc);
 544 }
 545
 546 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
 547 {
 548         return kvm_cpu_has_interrupt(vcpu);
 549 }
 550
 551 void s390_vcpu_block(struct kvm_vcpu *vcpu)
 552 {
 553         atomic_set_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
 554 }
 555
 556 void s390_vcpu_unblock(struct kvm_vcpu *vcpu)
 557 {
 558         atomic_clear_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
 559 }
 560
 561 /*
 562  * Kick a guest cpu out of SIE and wait until SIE is not running.
 563  * If the CPU is not running (e.g. waiting as idle) the function will
 564  * return immediately. */
 565 void exit_sie(struct kvm_vcpu *vcpu)
 566 {
 567         atomic_set_mask(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
 568         while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
 569                 cpu_relax();
 570 }
 571
 572 /* Kick a guest cpu out of SIE and prevent SIE-reentry */
 573 void exit_sie_sync(struct kvm_vcpu *vcpu)
 574 {
 575         s390_vcpu_block(vcpu);
 576         exit_sie(vcpu);
 577 }
 578
 579 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
 580 {
 581         int i;
 582         struct kvm *kvm = gmap->private;
 583         struct kvm_vcpu *vcpu;
 584
 585         kvm_for_each_vcpu(i, vcpu, kvm) {
 586                 /* match against both prefix pages */
 587                 if (vcpu->arch.sie_block->prefix == (address & ~0x1000UL)) {
 588                         VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
 589                         kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
 590                         exit_sie_sync(vcpu);
 591                 }
 592         }
 593 }
 594
 595 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
 596 {
 597         /* kvm common code refers to this, but never calls it */
 598         BUG();
 599         return 0;
 600 }
 601
 602 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
 603                                            struct kvm_one_reg *reg)
 604 {
 605         int r = -EINVAL;
 606
 607         switch (reg->id) {
 608         case KVM_REG_S390_TODPR:
 609                 r = put_user(vcpu->arch.sie_block->todpr,
 610                              (u32 __user *)reg->addr);
 611                 break;
 612         case KVM_REG_S390_EPOCHDIFF:
 613                 r = put_user(vcpu->arch.sie_block->epoch,
 614                              (u64 __user *)reg->addr);
 615                 break;
 616         case KVM_REG_S390_CPU_TIMER:
 617                 r = put_user(vcpu->arch.sie_block->cputm,
 618                              (u64 __user *)reg->addr);
 619                 break;
 620         case KVM_REG_S390_CLOCK_COMP:
 621                 r = put_user(vcpu->arch.sie_block->ckc,
 622                              (u64 __user *)reg->addr);
 623                 break;
 624         case KVM_REG_S390_PFTOKEN:
 625                 r = put_user(vcpu->arch.pfault_token,
 626                              (u64 __user *)reg->addr);
 627                 break;
 628         case KVM_REG_S390_PFCOMPARE:
 629                 r = put_user(vcpu->arch.pfault_compare,
 630                              (u64 __user *)reg->addr);
 631                 break;
 632         case KVM_REG_S390_PFSELECT:
 633                 r = put_user(vcpu->arch.pfault_select,
 634                              (u64 __user *)reg->addr);
 635                 break;
 636         case KVM_REG_S390_PP:
 637                 r = put_user(vcpu->arch.sie_block->pp,
 638                              (u64 __user *)reg->addr);
 639                 break;
 640         case KVM_REG_S390_GBEA:
 641                 r = put_user(vcpu->arch.sie_block->gbea,
 642                              (u64 __user *)reg->addr);
 643                 break;
 644         default:
 645                 break;
 646         }
 647
 648         return r;
 649 }
 650
 651 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
 652                                            struct kvm_one_reg *reg)
 653 {
 654         int r = -EINVAL;
 655
 656         switch (reg->id) {
 657         case KVM_REG_S390_TODPR:
 658                 r = get_user(vcpu->arch.sie_block->todpr,
 659                              (u32 __user *)reg->addr);
 660                 break;
 661         case KVM_REG_S390_EPOCHDIFF:
 662                 r = get_user(vcpu->arch.sie_block->epoch,
 663                              (u64 __user *)reg->addr);
 664                 break;
 665         case KVM_REG_S390_CPU_TIMER:
 666                 r = get_user(vcpu->arch.sie_block->cputm,
 667                              (u64 __user *)reg->addr);
 668                 break;
 669         case KVM_REG_S390_CLOCK_COMP:
 670                 r = get_user(vcpu->arch.sie_block->ckc,
 671                              (u64 __user *)reg->addr);
 672                 break;
 673         case KVM_REG_S390_PFTOKEN:
 674                 r = get_user(vcpu->arch.pfault_token,
 675                              (u64 __user *)reg->addr);
 676                 break;
 677         case KVM_REG_S390_PFCOMPARE:
 678                 r = get_user(vcpu->arch.pfault_compare,
 679                              (u64 __user *)reg->addr);
 680                 break;
 681         case KVM_REG_S390_PFSELECT:
 682                 r = get_user(vcpu->arch.pfault_select,
 683                              (u64 __user *)reg->addr);
 684                 break;
 685         case KVM_REG_S390_PP:
 686                 r = get_user(vcpu->arch.sie_block->pp,
 687                              (u64 __user *)reg->addr);
 688                 break;
 689         case KVM_REG_S390_GBEA:
 690                 r = get_user(vcpu->arch.sie_block->gbea,
 691                              (u64 __user *)reg->addr);
 692                 break;
 693         default:
 694                 break;
 695         }
 696
 697         return r;
 698 }
 699
 700 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
 701 {
 702         kvm_s390_vcpu_initial_reset(vcpu);
 703         return 0;
 704 }
 705
 706 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 707 {
 708         memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
 709         return 0;
 710 }
 711
 712 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 713 {
 714         memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
 715         return 0;
 716 }
 717
 718 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
 719                                   struct kvm_sregs *sregs)
 720 {
 721         memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
 722         memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
 723         restore_access_regs(vcpu->run->s.regs.acrs);
 724         return 0;
 725 }
 726
 727 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
 728                                   struct kvm_sregs *sregs)
 729 {
 730         memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
 731         memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
 732         return 0;
 733 }
 734
 735 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 736 {
 737         if (test_fp_ctl(fpu->fpc))
 738                 return -EINVAL;
 739         memcpy(&vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs));
 740         vcpu->arch.guest_fpregs.fpc = fpu->fpc;
 741         restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
 742         restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
 743         return 0;
 744 }
 745
 746 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 747 {
 748         memcpy(&fpu->fprs, &vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs));
 749         fpu->fpc = vcpu->arch.guest_fpregs.fpc;
 750         return 0;
 751 }
 752
 753 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
 754 {
 755         int rc = 0;
 756
 757         if (!(atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_STOPPED))
 758                 rc = -EBUSY;
 759         else {
 760                 vcpu->run->psw_mask = psw.mask;
 761                 vcpu->run->psw_addr = psw.addr;
 762         }
 763         return rc;
 764 }
 765
 766 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
 767                                   struct kvm_translation *tr)
 768 {
 769         return -EINVAL; /* not implemented yet */
 770 }
 771
 772 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
 773                                         struct kvm_guest_debug *dbg)
 774 {
 775         return -EINVAL; /* not implemented yet */
 776 }
 777
 778 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
 779                                     struct kvm_mp_state *mp_state)
 780 {
 781         return -EINVAL; /* not implemented yet */
 782 }
 783
 784 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
 785                                     struct kvm_mp_state *mp_state)
 786 {
 787         return -EINVAL; /* not implemented yet */
 788 }
 789
 790 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
 791 {
 792         /*
 793          * We use MMU_RELOAD just to re-arm the ipte notifier for the
 794          * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
 795          * This ensures that the ipte instruction for this request has
 796          * already finished. We might race against a second unmapper that
 797          * wants to set the blocking bit. Lets just retry the request loop.
 798          */
 799         while (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
 800                 int rc;
 801                 rc = gmap_ipte_notify(vcpu->arch.gmap,
 802                                       vcpu->arch.sie_block->prefix,
 803                                       PAGE_SIZE * 2);
 804                 if (rc)
 805                         return rc;
 806                 s390_vcpu_unblock(vcpu);
 807         }
 808         return 0;
 809 }
 810
 811 static long kvm_arch_fault_in_sync(struct kvm_vcpu *vcpu)
 812 {
 813         long rc;
 814         hva_t fault = gmap_fault(current->thread.gmap_addr, vcpu->arch.gmap);
 815         struct mm_struct *mm = current->mm;
 816         down_read(&mm->mmap_sem);
 817         rc = get_user_pages(current, mm, fault, 1, 1, 0, NULL, NULL);
 818         up_read(&mm->mmap_sem);
 819         return rc;
 820 }
 821
 822 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
 823                                       unsigned long token)
 824 {
 825         struct kvm_s390_interrupt inti;
 826         inti.parm64 = token;
 827
 828         if (start_token) {
 829                 inti.type = KVM_S390_INT_PFAULT_INIT;
 830                 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &inti));
 831         } else {
 832                 inti.type = KVM_S390_INT_PFAULT_DONE;
 833                 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
 834         }
 835 }
 836
 837 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
 838                                      struct kvm_async_pf *work)
 839 {
 840         trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
 841         __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
 842 }
 843
 844 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
 845                                  struct kvm_async_pf *work)
 846 {
 847         trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
 848         __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
 849 }
 850
 851 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
 852                                struct kvm_async_pf *work)
 853 {
 854         /* s390 will always inject the page directly */
 855 }
 856
 857 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
 858 {
 859         /*
 860          * s390 will always inject the page directly,
 861          * but we still want check_async_completion to cleanup
 862          */
 863         return true;
 864 }
 865
 866 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
 867 {
 868         hva_t hva;
 869         struct kvm_arch_async_pf arch;
 870         int rc;
 871
 872         if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
 873                 return 0;
 874         if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
 875             vcpu->arch.pfault_compare)
 876                 return 0;
 877         if (psw_extint_disabled(vcpu))
 878                 return 0;
 879         if (kvm_cpu_has_interrupt(vcpu))
 880                 return 0;
 881         if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
 882                 return 0;
 883         if (!vcpu->arch.gmap->pfault_enabled)
 884                 return 0;
 885
 886         hva = gmap_fault(current->thread.gmap_addr, vcpu->arch.gmap);
 887         if (copy_from_guest(vcpu, &arch.pfault_token, vcpu->arch.pfault_token, 8))
 888                 return 0;
 889
 890         rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
 891         return rc;
 892 }
 893
 894 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
 895 {
 896         int rc, cpuflags;
 897
 898         /*
 899          * On s390 notifications for arriving pages will be delivered directly
 900          * to the guest but the house keeping for completed pfaults is
 901          * handled outside the worker.
 902          */
 903         kvm_check_async_pf_completion(vcpu);
 904
 905         memcpy(&vcpu->arch.sie_block->gg14, &vcpu->run->s.regs.gprs[14], 16);
 906
 907         if (need_resched())
 908                 schedule();
 909
 910         if (test_thread_flag(TIF_MCCK_PENDING))
 911                 s390_handle_mcck();
 912
 913         if (!kvm_is_ucontrol(vcpu->kvm))
 914                 kvm_s390_deliver_pending_interrupts(vcpu);
 915
 916         rc = kvm_s390_handle_requests(vcpu);
 917         if (rc)
 918                 return rc;
 919
 920         vcpu->arch.sie_block->icptcode = 0;
 921         cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
 922         VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
 923         trace_kvm_s390_sie_enter(vcpu, cpuflags);
 924
 925         return 0;
 926 }
 927
 928 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
 929 {
 930         int rc = -1;
 931
 932         VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
 933                    vcpu->arch.sie_block->icptcode);
 934         trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
 935
 936         if (exit_reason >= 0) {
 937                 rc = 0;
 938         } else if (kvm_is_ucontrol(vcpu->kvm)) {
 939                 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
 940                 vcpu->run->s390_ucontrol.trans_exc_code =
 941                                                 current->thread.gmap_addr;
 942                 vcpu->run->s390_ucontrol.pgm_code = 0x10;
 943                 rc = -EREMOTE;
 944
 945         } else if (current->thread.gmap_pfault) {
 946                 trace_kvm_s390_major_guest_pfault(vcpu);
 947                 current->thread.gmap_pfault = 0;
 948                 if (kvm_arch_setup_async_pf(vcpu) ||
 949                     (kvm_arch_fault_in_sync(vcpu) >= 0))
 950                         rc = 0;
 951         }
 952
 953         if (rc == -1) {
 954                 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
 955                 trace_kvm_s390_sie_fault(vcpu);
 956                 rc = kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
 957         }
 958
 959         memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16);
 960
 961         if (rc == 0) {
 962                 if (kvm_is_ucontrol(vcpu->kvm))
 963                         /* Don't exit for host interrupts. */
 964                         rc = vcpu->arch.sie_block->icptcode ? -EOPNOTSUPP : 0;
 965                 else
 966                         rc = kvm_handle_sie_intercept(vcpu);
 967         }
 968
 969         return rc;
 970 }
 971
 972 bool kvm_enabled_cmma(void)
 973 {
 974         if (!MACHINE_IS_LPAR)
 975                 return false;
 976         /* only enable for z10 and later */
 977         if (!MACHINE_HAS_EDAT1)
 978                 return false;
 979         return true;
 980 }
 981
 982 static int __vcpu_run(struct kvm_vcpu *vcpu)
 983 {
 984         int rc, exit_reason;
 985
 986         /*
 987          * We try to hold kvm->srcu during most of vcpu_run (except when run-
 988          * ning the guest), so that memslots (and other stuff) are protected
 989          */
 990         vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
 991
 992         do {
 993                 rc = vcpu_pre_run(vcpu);
 994                 if (rc)
 995                         break;
 996
 997                 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
 998                 /*
 999                  * As PF_VCPU will be used in fault handler, between
1000                  * guest_enter and guest_exit should be no uaccess.
1001                  */
1002                 preempt_disable();
1003                 kvm_guest_enter();
1004                 preempt_enable();
1005                 exit_reason = sie64a(vcpu->arch.sie_block,
1006                                      vcpu->run->s.regs.gprs);
1007                 kvm_guest_exit();
1008                 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1009
1010                 rc = vcpu_post_run(vcpu, exit_reason);
1011         } while (!signal_pending(current) && !rc);
1012
1013         srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
1014         return rc;
1015 }
1016
1017 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1018 {
1019         int rc;
1020         sigset_t sigsaved;
1021
1022         if (vcpu->sigset_active)
1023                 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
1024
1025         atomic_clear_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
1026
1027         switch (kvm_run->exit_reason) {
1028         case KVM_EXIT_S390_SIEIC:
1029         case KVM_EXIT_UNKNOWN:
1030         case KVM_EXIT_INTR:
1031         case KVM_EXIT_S390_RESET:
1032         case KVM_EXIT_S390_UCONTROL:
1033         case KVM_EXIT_S390_TSCH:
1034                 break;
1035         default:
1036                 BUG();
1037         }
1038
1039         vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
1040         vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
1041         if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX) {
1042                 kvm_run->kvm_dirty_regs &= ~KVM_SYNC_PREFIX;
1043                 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
1044         }
1045         if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
1046                 kvm_run->kvm_dirty_regs &= ~KVM_SYNC_CRS;
1047                 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
1048                 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
1049         }
1050
1051         might_fault();
1052         rc = __vcpu_run(vcpu);
1053
1054         if (signal_pending(current) && !rc) {
1055                 kvm_run->exit_reason = KVM_EXIT_INTR;
1056                 rc = -EINTR;
1057         }
1058
1059         if (rc == -EOPNOTSUPP) {
1060                 /* intercept cannot be handled in-kernel, prepare kvm-run */
1061                 kvm_run->exit_reason         = KVM_EXIT_S390_SIEIC;
1062                 kvm_run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
1063                 kvm_run->s390_sieic.ipa      = vcpu->arch.sie_block->ipa;
1064                 kvm_run->s390_sieic.ipb      = vcpu->arch.sie_block->ipb;
1065                 rc = 0;
1066         }
1067
1068         if (rc == -EREMOTE) {
1069                 /* intercept was handled, but userspace support is needed
1070                  * kvm_run has been prepared by the handler */
1071                 rc = 0;
1072         }
1073
1074         kvm_run->psw_mask     = vcpu->arch.sie_block->gpsw.mask;
1075         kvm_run->psw_addr     = vcpu->arch.sie_block->gpsw.addr;
1076         kvm_run->s.regs.prefix = vcpu->arch.sie_block->prefix;
1077         memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
1078
1079         if (vcpu->sigset_active)
1080                 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
1081
1082         vcpu->stat.exit_userspace++;
1083         return rc;
1084 }
1085
1086 static int __guestcopy(struct kvm_vcpu *vcpu, u64 guestdest, void *from,
1087                        unsigned long n, int prefix)
1088 {
1089         if (prefix)
1090                 return copy_to_guest(vcpu, guestdest, from, n);
1091         else
1092                 return copy_to_guest_absolute(vcpu, guestdest, from, n);
1093 }
1094
1095 /*
1096  * store status at address
1097  * we use have two special cases:
1098  * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
1099  * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
1100  */
1101 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long addr)
1102 {
1103         unsigned char archmode = 1;
1104         int prefix;
1105         u64 clkcomp;
1106
1107         if (addr == KVM_S390_STORE_STATUS_NOADDR) {
1108                 if (copy_to_guest_absolute(vcpu, 163ul, &archmode, 1))
1109                         return -EFAULT;
1110                 addr = SAVE_AREA_BASE;
1111                 prefix = 0;
1112         } else if (addr == KVM_S390_STORE_STATUS_PREFIXED) {
1113                 if (copy_to_guest(vcpu, 163ul, &archmode, 1))
1114                         return -EFAULT;
1115                 addr = SAVE_AREA_BASE;
1116                 prefix = 1;
1117         } else
1118                 prefix = 0;
1119
1120         if (__guestcopy(vcpu, addr + offsetof(struct save_area, fp_regs),
1121                         vcpu->arch.guest_fpregs.fprs, 128, prefix))
1122                 return -EFAULT;
1123
1124         if (__guestcopy(vcpu, addr + offsetof(struct save_area, gp_regs),
1125                         vcpu->run->s.regs.gprs, 128, prefix))
1126                 return -EFAULT;
1127
1128         if (__guestcopy(vcpu, addr + offsetof(struct save_area, psw),
1129                         &vcpu->arch.sie_block->gpsw, 16, prefix))
1130                 return -EFAULT;
1131
1132         if (__guestcopy(vcpu, addr + offsetof(struct save_area, pref_reg),
1133                         &vcpu->arch.sie_block->prefix, 4, prefix))
1134                 return -EFAULT;
1135
1136         if (__guestcopy(vcpu,
1137                         addr + offsetof(struct save_area, fp_ctrl_reg),
1138                         &vcpu->arch.guest_fpregs.fpc, 4, prefix))
1139                 return -EFAULT;
1140
1141         if (__guestcopy(vcpu, addr + offsetof(struct save_area, tod_reg),
1142                         &vcpu->arch.sie_block->todpr, 4, prefix))
1143                 return -EFAULT;
1144
1145         if (__guestcopy(vcpu, addr + offsetof(struct save_area, timer),
1146                         &vcpu->arch.sie_block->cputm, 8, prefix))
1147                 return -EFAULT;
1148
1149         clkcomp = vcpu->arch.sie_block->ckc >> 8;
1150         if (__guestcopy(vcpu, addr + offsetof(struct save_area, clk_cmp),
1151                         &clkcomp, 8, prefix))
1152                 return -EFAULT;
1153
1154         if (__guestcopy(vcpu, addr + offsetof(struct save_area, acc_regs),
1155                         &vcpu->run->s.regs.acrs, 64, prefix))
1156                 return -EFAULT;
1157
1158         if (__guestcopy(vcpu,
1159                         addr + offsetof(struct save_area, ctrl_regs),
1160                         &vcpu->arch.sie_block->gcr, 128, prefix))
1161                 return -EFAULT;
1162         return 0;
1163 }
1164
1165 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
1166 {
1167         /*
1168          * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
1169          * copying in vcpu load/put. Lets update our copies before we save
1170          * it into the save area
1171          */
1172         save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1173         save_fp_regs(vcpu->arch.guest_fpregs.fprs);
1174         save_access_regs(vcpu->run->s.regs.acrs);
1175
1176         return kvm_s390_store_status_unloaded(vcpu, addr);
1177 }
1178
1179 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
1180                                      struct kvm_enable_cap *cap)
1181 {
1182         int r;
1183
1184         if (cap->flags)
1185                 return -EINVAL;
1186
1187         switch (cap->cap) {
1188         case KVM_CAP_S390_CSS_SUPPORT:
1189                 if (!vcpu->kvm->arch.css_support) {
1190                         vcpu->kvm->arch.css_support = 1;
1191                         trace_kvm_s390_enable_css(vcpu->kvm);
1192                 }
1193                 r = 0;
1194                 break;
1195         default:
1196                 r = -EINVAL;
1197                 break;
1198         }
1199         return r;
1200 }
1201
1202 long kvm_arch_vcpu_ioctl(struct file *filp,
1203                          unsigned int ioctl, unsigned long arg)
1204 {
1205         struct kvm_vcpu *vcpu = filp->private_data;
1206         void __user *argp = (void __user *)arg;
1207         int idx;
1208         long r;
1209
1210         switch (ioctl) {
1211         case KVM_S390_INTERRUPT: {
1212                 struct kvm_s390_interrupt s390int;
1213
1214                 r = -EFAULT;
1215                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
1216                         break;
1217                 r = kvm_s390_inject_vcpu(vcpu, &s390int);
1218                 break;
1219         }
1220         case KVM_S390_STORE_STATUS:
1221                 idx = srcu_read_lock(&vcpu->kvm->srcu);
1222                 r = kvm_s390_vcpu_store_status(vcpu, arg);
1223                 srcu_read_unlock(&vcpu->kvm->srcu, idx);
1224                 break;
1225         case KVM_S390_SET_INITIAL_PSW: {
1226                 psw_t psw;
1227
1228                 r = -EFAULT;
1229                 if (copy_from_user(&psw, argp, sizeof(psw)))
1230                         break;
1231                 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
1232                 break;
1233         }
1234         case KVM_S390_INITIAL_RESET:
1235                 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
1236                 break;
1237         case KVM_SET_ONE_REG:
1238         case KVM_GET_ONE_REG: {
1239                 struct kvm_one_reg reg;
1240                 r = -EFAULT;
1241                 if (copy_from_user(&reg, argp, sizeof(reg)))
1242                         break;
1243                 if (ioctl == KVM_SET_ONE_REG)
1244                         r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
1245                 else
1246                         r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
1247                 break;
1248         }
1249 #ifdef CONFIG_KVM_S390_UCONTROL
1250         case KVM_S390_UCAS_MAP: {
1251                 struct kvm_s390_ucas_mapping ucasmap;
1252
1253                 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
1254                         r = -EFAULT;
1255                         break;
1256                 }
1257
1258                 if (!kvm_is_ucontrol(vcpu->kvm)) {
1259                         r = -EINVAL;
1260                         break;
1261                 }
1262
1263                 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
1264                                      ucasmap.vcpu_addr, ucasmap.length);
1265                 break;
1266         }
1267         case KVM_S390_UCAS_UNMAP: {
1268                 struct kvm_s390_ucas_mapping ucasmap;
1269
1270                 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
1271                         r = -EFAULT;
1272                         break;
1273                 }
1274
1275                 if (!kvm_is_ucontrol(vcpu->kvm)) {
1276                         r = -EINVAL;
1277                         break;
1278                 }
1279
1280                 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
1281                         ucasmap.length);
1282                 break;
1283         }
1284 #endif
1285         case KVM_S390_VCPU_FAULT: {
1286                 r = gmap_fault(arg, vcpu->arch.gmap);
1287                 if (!IS_ERR_VALUE(r))
1288                         r = 0;
1289                 break;
1290         }
1291         case KVM_ENABLE_CAP:
1292         {
1293                 struct kvm_enable_cap cap;
1294                 r = -EFAULT;
1295                 if (copy_from_user(&cap, argp, sizeof(cap)))
1296                         break;
1297                 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
1298                 break;
1299         }
1300         default:
1301                 r = -ENOTTY;
1302         }
1303         return r;
1304 }
1305
1306 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
1307 {
1308 #ifdef CONFIG_KVM_S390_UCONTROL
1309         if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
1310                  && (kvm_is_ucontrol(vcpu->kvm))) {
1311                 vmf->page = virt_to_page(vcpu->arch.sie_block);
1312                 get_page(vmf->page);
1313                 return 0;
1314         }
1315 #endif
1316         return VM_FAULT_SIGBUS;
1317 }
1318
1319 void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free,
1320                            struct kvm_memory_slot *dont)
1321 {
1322 }
1323
1324 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
1325                             unsigned long npages)
1326 {
1327         return 0;
1328 }
1329
1330 void kvm_arch_memslots_updated(struct kvm *kvm)
1331 {
1332 }
1333
1334 /* Section: memory related */
1335 int kvm_arch_prepare_memory_region(struct kvm *kvm,
1336                                    struct kvm_memory_slot *memslot,
1337                                    struct kvm_userspace_memory_region *mem,
1338                                    enum kvm_mr_change change)
1339 {
1340         /* A few sanity checks. We can have memory slots which have to be
1341            located/ended at a segment boundary (1MB). The memory in userland is
1342            ok to be fragmented into various different vmas. It is okay to mmap()
1343            and munmap() stuff in this slot after doing this call at any time */
1344
1345         if (mem->userspace_addr & 0xffffful)
1346                 return -EINVAL;
1347
1348         if (mem->memory_size & 0xffffful)
1349                 return -EINVAL;
1350
1351         return 0;
1352 }
1353
1354 void kvm_arch_commit_memory_region(struct kvm *kvm,
1355                                 struct kvm_userspace_memory_region *mem,
1356                                 const struct kvm_memory_slot *old,
1357                                 enum kvm_mr_change change)
1358 {
1359         int rc;
1360
1361         /* If the basics of the memslot do not change, we do not want
1362          * to update the gmap. Every update causes several unnecessary
1363          * segment translation exceptions. This is usually handled just
1364          * fine by the normal fault handler + gmap, but it will also
1365          * cause faults on the prefix page of running guest CPUs.
1366          */
1367         if (old->userspace_addr == mem->userspace_addr &&
1368             old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
1369             old->npages * PAGE_SIZE == mem->memory_size)
1370                 return;
1371
1372         rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
1373                 mem->guest_phys_addr, mem->memory_size);
1374         if (rc)
1375                 printk(KERN_WARNING "kvm-s390: failed to commit memory region\n");
1376         return;
1377 }
1378
1379 void kvm_arch_flush_shadow_all(struct kvm *kvm)
1380 {
1381 }
1382
1383 void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
1384                                    struct kvm_memory_slot *slot)
1385 {
1386 }
1387
1388 static int __init kvm_s390_init(void)
1389 {
1390         int ret;
1391         ret = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
1392         if (ret)
1393                 return ret;
1394
1395         /*
1396          * guests can ask for up to 255+1 double words, we need a full page
1397          * to hold the maximum amount of facilities. On the other hand, we
1398          * only set facilities that are known to work in KVM.
1399          */
1400         vfacilities = (unsigned long *) get_zeroed_page(GFP_KERNEL|GFP_DMA);
1401         if (!vfacilities) {
1402                 kvm_exit();
1403                 return -ENOMEM;
1404         }
1405         memcpy(vfacilities, S390_lowcore.stfle_fac_list, 16);
1406         vfacilities[0] &= 0xff82fff3f4fc2000UL;
1407         vfacilities[1] &= 0x005c000000000000UL;
1408         return 0;
1409 }
1410
1411 static void __exit kvm_s390_exit(void)
1412 {
1413         free_page((unsigned long) vfacilities);
1414         kvm_exit();
1415 }
1416
1417 module_init(kvm_s390_init);
1418 module_exit(kvm_s390_exit);
1419
1420 /*
1421  * Enable autoloading of the kvm module.
1422  * Note that we add the module alias here instead of virt/kvm/kvm_main.c
1423  * since x86 takes a different approach.
1424  */
1425 #include <linux/miscdevice.h>
1426 MODULE_ALIAS_MISCDEV(KVM_MINOR);
1427 MODULE_ALIAS("devname:kvm");