arch/x86/pci/vmd.c

   1 /*
   2  * Volume Management Device driver
   3  * Copyright (c) 2015, Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  */
  14
  15 #include <linux/device.h>
  16 #include <linux/interrupt.h>
  17 #include <linux/irq.h>
  18 #include <linux/kernel.h>
  19 #include <linux/module.h>
  20 #include <linux/msi.h>
  21 #include <linux/pci.h>
  22 #include <linux/rculist.h>
  23 #include <linux/rcupdate.h>
  24
  25 #include <asm/irqdomain.h>
  26 #include <asm/device.h>
  27 #include <asm/msi.h>
  28 #include <asm/msidef.h>
  29
  30 #define VMD_CFGBAR      0
  31 #define VMD_MEMBAR1     2
  32 #define VMD_MEMBAR2     4
  33
  34 /*
  35  * Lock for manipulating VMD IRQ lists.
  36  */
  37 static DEFINE_RAW_SPINLOCK(list_lock);
  38
  39 /**
  40  * struct vmd_irq - private data to map driver IRQ to the VMD shared vector
  41  * @node:       list item for parent traversal.
  42  * @rcu:        RCU callback item for freeing.
  43  * @irq:        back pointer to parent.
  44  * @virq:       the virtual IRQ value provided to the requesting driver.
  45  *
  46  * Every MSI/MSI-X IRQ requested for a device in a VMD domain will be mapped to
  47  * a VMD IRQ using this structure.
  48  */
  49 struct vmd_irq {
  50         struct list_head        node;
  51         struct rcu_head         rcu;
  52         struct vmd_irq_list     *irq;
  53         unsigned int            virq;
  54 };
  55
  56 /**
  57  * struct vmd_irq_list - list of driver requested IRQs mapping to a VMD vector
  58  * @irq_list:   the list of irq's the VMD one demuxes to.
  59  * @count:      number of child IRQs assigned to this vector; used to track
  60  *              sharing.
  61  */
  62 struct vmd_irq_list {
  63         struct list_head        irq_list;
  64         unsigned int            count;
  65 };
  66
  67 struct vmd_dev {
  68         struct pci_dev          *dev;
  69
  70         spinlock_t              cfg_lock;
  71         char __iomem            *cfgbar;
  72
  73         int msix_count;
  74         struct vmd_irq_list     *irqs;
  75
  76         struct pci_sysdata      sysdata;
  77         struct resource         resources[3];
  78         struct irq_domain       *irq_domain;
  79         struct pci_bus          *bus;
  80
  81 #ifdef CONFIG_X86_DEV_DMA_OPS
  82         struct dma_map_ops      dma_ops;
  83         struct dma_domain       dma_domain;
  84 #endif
  85 };
  86
  87 static inline struct vmd_dev *vmd_from_bus(struct pci_bus *bus)
  88 {
  89         return container_of(bus->sysdata, struct vmd_dev, sysdata);
  90 }
  91
  92 static inline unsigned int index_from_irqs(struct vmd_dev *vmd,
  93                                            struct vmd_irq_list *irqs)
  94 {
  95         return irqs - vmd->irqs;
  96 }
  97
  98 /*
  99  * Drivers managing a device in a VMD domain allocate their own IRQs as before,
 100  * but the MSI entry for the hardware it's driving will be programmed with a
 101  * destination ID for the VMD MSI-X table.  The VMD muxes interrupts in its
 102  * domain into one of its own, and the VMD driver de-muxes these for the
 103  * handlers sharing that VMD IRQ.  The vmd irq_domain provides the operations
 104  * and irq_chip to set this up.
 105  */
 106 static void vmd_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
 107 {
 108         struct vmd_irq *vmdirq = data->chip_data;
 109         struct vmd_irq_list *irq = vmdirq->irq;
 110         struct vmd_dev *vmd = irq_data_get_irq_handler_data(data);
 111
 112         msg->address_hi = MSI_ADDR_BASE_HI;
 113         msg->address_lo = MSI_ADDR_BASE_LO |
 114                           MSI_ADDR_DEST_ID(index_from_irqs(vmd, irq));
 115         msg->data = 0;
 116 }
 117
 118 /*
 119  * We rely on MSI_FLAG_USE_DEF_CHIP_OPS to set the IRQ mask/unmask ops.
 120  */
 121 static void vmd_irq_enable(struct irq_data *data)
 122 {
 123         struct vmd_irq *vmdirq = data->chip_data;
 124         unsigned long flags;
 125
 126         raw_spin_lock_irqsave(&list_lock, flags);
 127         list_add_tail_rcu(&vmdirq->node, &vmdirq->irq->irq_list);
 128         raw_spin_unlock_irqrestore(&list_lock, flags);
 129
 130         data->chip->irq_unmask(data);
 131 }
 132
 133 static void vmd_irq_disable(struct irq_data *data)
 134 {
 135         struct vmd_irq *vmdirq = data->chip_data;
 136         unsigned long flags;
 137
 138         data->chip->irq_mask(data);
 139
 140         raw_spin_lock_irqsave(&list_lock, flags);
 141         list_del_rcu(&vmdirq->node);
 142         INIT_LIST_HEAD_RCU(&vmdirq->node);
 143         raw_spin_unlock_irqrestore(&list_lock, flags);
 144 }
 145
 146 /*
 147  * XXX: Stubbed until we develop acceptable way to not create conflicts with
 148  * other devices sharing the same vector.
 149  */
 150 static int vmd_irq_set_affinity(struct irq_data *data,
 151                                 const struct cpumask *dest, bool force)
 152 {
 153         return -EINVAL;
 154 }
 155
 156 static struct irq_chip vmd_msi_controller = {
 157         .name                   = "VMD-MSI",
 158         .irq_enable             = vmd_irq_enable,
 159         .irq_disable            = vmd_irq_disable,
 160         .irq_compose_msi_msg    = vmd_compose_msi_msg,
 161         .irq_set_affinity       = vmd_irq_set_affinity,
 162 };
 163
 164 static irq_hw_number_t vmd_get_hwirq(struct msi_domain_info *info,
 165                                      msi_alloc_info_t *arg)
 166 {
 167         return 0;
 168 }
 169
 170 /*
 171  * XXX: We can be even smarter selecting the best IRQ once we solve the
 172  * affinity problem.
 173  */
 174 static struct vmd_irq_list *vmd_next_irq(struct vmd_dev *vmd, struct msi_desc *desc)
 175 {
 176         int i, best = 1;
 177         unsigned long flags;
 178
 179         if (!desc->msi_attrib.is_msix || vmd->msix_count == 1)
 180                 return &vmd->irqs[0];
 181
 182         raw_spin_lock_irqsave(&list_lock, flags);
 183         for (i = 1; i < vmd->msix_count; i++)
 184                 if (vmd->irqs[i].count < vmd->irqs[best].count)
 185                         best = i;
 186         vmd->irqs[best].count++;
 187         raw_spin_unlock_irqrestore(&list_lock, flags);
 188
 189         return &vmd->irqs[best];
 190 }
 191
 192 static int vmd_msi_init(struct irq_domain *domain, struct msi_domain_info *info,
 193                         unsigned int virq, irq_hw_number_t hwirq,
 194                         msi_alloc_info_t *arg)
 195 {
 196         struct msi_desc *desc = arg->desc;
 197         struct vmd_dev *vmd = vmd_from_bus(msi_desc_to_pci_dev(desc)->bus);
 198         struct vmd_irq *vmdirq = kzalloc(sizeof(*vmdirq), GFP_KERNEL);
 199         unsigned int index, vector;
 200
 201         if (!vmdirq)
 202                 return -ENOMEM;
 203
 204         INIT_LIST_HEAD(&vmdirq->node);
 205         vmdirq->irq = vmd_next_irq(vmd, desc);
 206         vmdirq->virq = virq;
 207         index = index_from_irqs(vmd, vmdirq->irq);
 208         vector = pci_irq_vector(vmd->dev, index);
 209
 210         irq_domain_set_info(domain, virq, vector, info->chip, vmdirq,
 211                             handle_untracked_irq, vmd, NULL);
 212         return 0;
 213 }
 214
 215 static void vmd_msi_free(struct irq_domain *domain,
 216                         struct msi_domain_info *info, unsigned int virq)
 217 {
 218         struct vmd_irq *vmdirq = irq_get_chip_data(virq);
 219         unsigned long flags;
 220
 221         synchronize_rcu();
 222
 223         /* XXX: Potential optimization to rebalance */
 224         raw_spin_lock_irqsave(&list_lock, flags);
 225         vmdirq->irq->count--;
 226         raw_spin_unlock_irqrestore(&list_lock, flags);
 227
 228         kfree_rcu(vmdirq, rcu);
 229 }
 230
 231 static int vmd_msi_prepare(struct irq_domain *domain, struct device *dev,
 232                            int nvec, msi_alloc_info_t *arg)
 233 {
 234         struct pci_dev *pdev = to_pci_dev(dev);
 235         struct vmd_dev *vmd = vmd_from_bus(pdev->bus);
 236
 237         if (nvec > vmd->msix_count)
 238                 return vmd->msix_count;
 239
 240         memset(arg, 0, sizeof(*arg));
 241         return 0;
 242 }
 243
 244 static void vmd_set_desc(msi_alloc_info_t *arg, struct msi_desc *desc)
 245 {
 246         arg->desc = desc;
 247 }
 248
 249 static struct msi_domain_ops vmd_msi_domain_ops = {
 250         .get_hwirq      = vmd_get_hwirq,
 251         .msi_init       = vmd_msi_init,
 252         .msi_free       = vmd_msi_free,
 253         .msi_prepare    = vmd_msi_prepare,
 254         .set_desc       = vmd_set_desc,
 255 };
 256
 257 static struct msi_domain_info vmd_msi_domain_info = {
 258         .flags          = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
 259                           MSI_FLAG_PCI_MSIX,
 260         .ops            = &vmd_msi_domain_ops,
 261         .chip           = &vmd_msi_controller,
 262 };
 263
 264 #ifdef CONFIG_X86_DEV_DMA_OPS
 265 /*
 266  * VMD replaces the requester ID with its own.  DMA mappings for devices in a
 267  * VMD domain need to be mapped for the VMD, not the device requiring
 268  * the mapping.
 269  */
 270 static struct device *to_vmd_dev(struct device *dev)
 271 {
 272         struct pci_dev *pdev = to_pci_dev(dev);
 273         struct vmd_dev *vmd = vmd_from_bus(pdev->bus);
 274
 275         return &vmd->dev->dev;
 276 }
 277
 278 static struct dma_map_ops *vmd_dma_ops(struct device *dev)
 279 {
 280         return get_dma_ops(to_vmd_dev(dev));
 281 }
 282
 283 static void *vmd_alloc(struct device *dev, size_t size, dma_addr_t *addr,
 284                        gfp_t flag, unsigned long attrs)
 285 {
 286         return vmd_dma_ops(dev)->alloc(to_vmd_dev(dev), size, addr, flag,
 287                                        attrs);
 288 }
 289
 290 static void vmd_free(struct device *dev, size_t size, void *vaddr,
 291                      dma_addr_t addr, unsigned long attrs)
 292 {
 293         return vmd_dma_ops(dev)->free(to_vmd_dev(dev), size, vaddr, addr,
 294                                       attrs);
 295 }
 296
 297 static int vmd_mmap(struct device *dev, struct vm_area_struct *vma,
 298                     void *cpu_addr, dma_addr_t addr, size_t size,
 299                     unsigned long attrs)
 300 {
 301         return vmd_dma_ops(dev)->mmap(to_vmd_dev(dev), vma, cpu_addr, addr,
 302                                       size, attrs);
 303 }
 304
 305 static int vmd_get_sgtable(struct device *dev, struct sg_table *sgt,
 306                            void *cpu_addr, dma_addr_t addr, size_t size,
 307                            unsigned long attrs)
 308 {
 309         return vmd_dma_ops(dev)->get_sgtable(to_vmd_dev(dev), sgt, cpu_addr,
 310                                              addr, size, attrs);
 311 }
 312
 313 static dma_addr_t vmd_map_page(struct device *dev, struct page *page,
 314                                unsigned long offset, size_t size,
 315                                enum dma_data_direction dir,
 316                                unsigned long attrs)
 317 {
 318         return vmd_dma_ops(dev)->map_page(to_vmd_dev(dev), page, offset, size,
 319                                           dir, attrs);
 320 }
 321
 322 static void vmd_unmap_page(struct device *dev, dma_addr_t addr, size_t size,
 323                            enum dma_data_direction dir, unsigned long attrs)
 324 {
 325         vmd_dma_ops(dev)->unmap_page(to_vmd_dev(dev), addr, size, dir, attrs);
 326 }
 327
 328 static int vmd_map_sg(struct device *dev, struct scatterlist *sg, int nents,
 329                       enum dma_data_direction dir, unsigned long attrs)
 330 {
 331         return vmd_dma_ops(dev)->map_sg(to_vmd_dev(dev), sg, nents, dir, attrs);
 332 }
 333
 334 static void vmd_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
 335                          enum dma_data_direction dir, unsigned long attrs)
 336 {
 337         vmd_dma_ops(dev)->unmap_sg(to_vmd_dev(dev), sg, nents, dir, attrs);
 338 }
 339
 340 static void vmd_sync_single_for_cpu(struct device *dev, dma_addr_t addr,
 341                                     size_t size, enum dma_data_direction dir)
 342 {
 343         vmd_dma_ops(dev)->sync_single_for_cpu(to_vmd_dev(dev), addr, size, dir);
 344 }
 345
 346 static void vmd_sync_single_for_device(struct device *dev, dma_addr_t addr,
 347                                        size_t size, enum dma_data_direction dir)
 348 {
 349         vmd_dma_ops(dev)->sync_single_for_device(to_vmd_dev(dev), addr, size,
 350                                                  dir);
 351 }
 352
 353 static void vmd_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
 354                                 int nents, enum dma_data_direction dir)
 355 {
 356         vmd_dma_ops(dev)->sync_sg_for_cpu(to_vmd_dev(dev), sg, nents, dir);
 357 }
 358
 359 static void vmd_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
 360                                    int nents, enum dma_data_direction dir)
 361 {
 362         vmd_dma_ops(dev)->sync_sg_for_device(to_vmd_dev(dev), sg, nents, dir);
 363 }
 364
 365 static int vmd_mapping_error(struct device *dev, dma_addr_t addr)
 366 {
 367         return vmd_dma_ops(dev)->mapping_error(to_vmd_dev(dev), addr);
 368 }
 369
 370 static int vmd_dma_supported(struct device *dev, u64 mask)
 371 {
 372         return vmd_dma_ops(dev)->dma_supported(to_vmd_dev(dev), mask);
 373 }
 374
 375 #ifdef ARCH_HAS_DMA_GET_REQUIRED_MASK
 376 static u64 vmd_get_required_mask(struct device *dev)
 377 {
 378         return vmd_dma_ops(dev)->get_required_mask(to_vmd_dev(dev));
 379 }
 380 #endif
 381
 382 static void vmd_teardown_dma_ops(struct vmd_dev *vmd)
 383 {
 384         struct dma_domain *domain = &vmd->dma_domain;
 385
 386         if (get_dma_ops(&vmd->dev->dev))
 387                 del_dma_domain(domain);
 388 }
 389
 390 #define ASSIGN_VMD_DMA_OPS(source, dest, fn)    \
 391         do {                                    \
 392                 if (source->fn)                 \
 393                         dest->fn = vmd_##fn;    \
 394         } while (0)
 395
 396 static void vmd_setup_dma_ops(struct vmd_dev *vmd)
 397 {
 398         const struct dma_map_ops *source = get_dma_ops(&vmd->dev->dev);
 399         struct dma_map_ops *dest = &vmd->dma_ops;
 400         struct dma_domain *domain = &vmd->dma_domain;
 401
 402         domain->domain_nr = vmd->sysdata.domain;
 403         domain->dma_ops = dest;
 404
 405         if (!source)
 406                 return;
 407         ASSIGN_VMD_DMA_OPS(source, dest, alloc);
 408         ASSIGN_VMD_DMA_OPS(source, dest, free);
 409         ASSIGN_VMD_DMA_OPS(source, dest, mmap);
 410         ASSIGN_VMD_DMA_OPS(source, dest, get_sgtable);
 411         ASSIGN_VMD_DMA_OPS(source, dest, map_page);
 412         ASSIGN_VMD_DMA_OPS(source, dest, unmap_page);
 413         ASSIGN_VMD_DMA_OPS(source, dest, map_sg);
 414         ASSIGN_VMD_DMA_OPS(source, dest, unmap_sg);
 415         ASSIGN_VMD_DMA_OPS(source, dest, sync_single_for_cpu);
 416         ASSIGN_VMD_DMA_OPS(source, dest, sync_single_for_device);
 417         ASSIGN_VMD_DMA_OPS(source, dest, sync_sg_for_cpu);
 418         ASSIGN_VMD_DMA_OPS(source, dest, sync_sg_for_device);
 419         ASSIGN_VMD_DMA_OPS(source, dest, mapping_error);
 420         ASSIGN_VMD_DMA_OPS(source, dest, dma_supported);
 421 #ifdef ARCH_HAS_DMA_GET_REQUIRED_MASK
 422         ASSIGN_VMD_DMA_OPS(source, dest, get_required_mask);
 423 #endif
 424         add_dma_domain(domain);
 425 }
 426 #undef ASSIGN_VMD_DMA_OPS
 427 #else
 428 static void vmd_teardown_dma_ops(struct vmd_dev *vmd) {}
 429 static void vmd_setup_dma_ops(struct vmd_dev *vmd) {}
 430 #endif
 431
 432 static char __iomem *vmd_cfg_addr(struct vmd_dev *vmd, struct pci_bus *bus,
 433                                   unsigned int devfn, int reg, int len)
 434 {
 435         char __iomem *addr = vmd->cfgbar +
 436                              (bus->number << 20) + (devfn << 12) + reg;
 437
 438         if ((addr - vmd->cfgbar) + len >=
 439             resource_size(&vmd->dev->resource[VMD_CFGBAR]))
 440                 return NULL;
 441
 442         return addr;
 443 }
 444
 445 /*
 446  * CPU may deadlock if config space is not serialized on some versions of this
 447  * hardware, so all config space access is done under a spinlock.
 448  */
 449 static int vmd_pci_read(struct pci_bus *bus, unsigned int devfn, int reg,
 450                         int len, u32 *value)
 451 {
 452         struct vmd_dev *vmd = vmd_from_bus(bus);
 453         char __iomem *addr = vmd_cfg_addr(vmd, bus, devfn, reg, len);
 454         unsigned long flags;
 455         int ret = 0;
 456
 457         if (!addr)
 458                 return -EFAULT;
 459
 460         spin_lock_irqsave(&vmd->cfg_lock, flags);
 461         switch (len) {
 462         case 1:
 463                 *value = readb(addr);
 464                 break;
 465         case 2:
 466                 *value = readw(addr);
 467                 break;
 468         case 4:
 469                 *value = readl(addr);
 470                 break;
 471         default:
 472                 ret = -EINVAL;
 473                 break;
 474         }
 475         spin_unlock_irqrestore(&vmd->cfg_lock, flags);
 476         return ret;
 477 }
 478
 479 /*
 480  * VMD h/w converts non-posted config writes to posted memory writes. The
 481  * read-back in this function forces the completion so it returns only after
 482  * the config space was written, as expected.
 483  */
 484 static int vmd_pci_write(struct pci_bus *bus, unsigned int devfn, int reg,
 485                          int len, u32 value)
 486 {
 487         struct vmd_dev *vmd = vmd_from_bus(bus);
 488         char __iomem *addr = vmd_cfg_addr(vmd, bus, devfn, reg, len);
 489         unsigned long flags;
 490         int ret = 0;
 491
 492         if (!addr)
 493                 return -EFAULT;
 494
 495         spin_lock_irqsave(&vmd->cfg_lock, flags);
 496         switch (len) {
 497         case 1:
 498                 writeb(value, addr);
 499                 readb(addr);
 500                 break;
 501         case 2:
 502                 writew(value, addr);
 503                 readw(addr);
 504                 break;
 505         case 4:
 506                 writel(value, addr);
 507                 readl(addr);
 508                 break;
 509         default:
 510                 ret = -EINVAL;
 511                 break;
 512         }
 513         spin_unlock_irqrestore(&vmd->cfg_lock, flags);
 514         return ret;
 515 }
 516
 517 static struct pci_ops vmd_ops = {
 518         .read           = vmd_pci_read,
 519         .write          = vmd_pci_write,
 520 };
 521
 522 static void vmd_attach_resources(struct vmd_dev *vmd)
 523 {
 524         vmd->dev->resource[VMD_MEMBAR1].child = &vmd->resources[1];
 525         vmd->dev->resource[VMD_MEMBAR2].child = &vmd->resources[2];
 526 }
 527
 528 static void vmd_detach_resources(struct vmd_dev *vmd)
 529 {
 530         vmd->dev->resource[VMD_MEMBAR1].child = NULL;
 531         vmd->dev->resource[VMD_MEMBAR2].child = NULL;
 532 }
 533
 534 /*
 535  * VMD domains start at 0x1000 to not clash with ACPI _SEG domains.
 536  */
 537 static int vmd_find_free_domain(void)
 538 {
 539         int domain = 0xffff;
 540         struct pci_bus *bus = NULL;
 541
 542         while ((bus = pci_find_next_bus(bus)) != NULL)
 543                 domain = max_t(int, domain, pci_domain_nr(bus));
 544         return domain + 1;
 545 }
 546
 547 static int vmd_enable_domain(struct vmd_dev *vmd)
 548 {
 549         struct pci_sysdata *sd = &vmd->sysdata;
 550         struct resource *res;
 551         u32 upper_bits;
 552         unsigned long flags;
 553         LIST_HEAD(resources);
 554
 555         res = &vmd->dev->resource[VMD_CFGBAR];
 556         vmd->resources[0] = (struct resource) {
 557                 .name  = "VMD CFGBAR",
 558                 .start = 0,
 559                 .end   = (resource_size(res) >> 20) - 1,
 560                 .flags = IORESOURCE_BUS | IORESOURCE_PCI_FIXED,
 561         };
 562
 563         /*
 564          * If the window is below 4GB, clear IORESOURCE_MEM_64 so we can
 565          * put 32-bit resources in the window.
 566          *
 567          * There's no hardware reason why a 64-bit window *couldn't*
 568          * contain a 32-bit resource, but pbus_size_mem() computes the
 569          * bridge window size assuming a 64-bit window will contain no
 570          * 32-bit resources.  __pci_assign_resource() enforces that
 571          * artificial restriction to make sure everything will fit.
 572          *
 573          * The only way we could use a 64-bit non-prefechable MEMBAR is
 574          * if its address is <4GB so that we can convert it to a 32-bit
 575          * resource.  To be visible to the host OS, all VMD endpoints must
 576          * be initially configured by platform BIOS, which includes setting
 577          * up these resources.  We can assume the device is configured
 578          * according to the platform needs.
 579          */
 580         res = &vmd->dev->resource[VMD_MEMBAR1];
 581         upper_bits = upper_32_bits(res->end);
 582         flags = res->flags & ~IORESOURCE_SIZEALIGN;
 583         if (!upper_bits)
 584                 flags &= ~IORESOURCE_MEM_64;
 585         vmd->resources[1] = (struct resource) {
 586                 .name  = "VMD MEMBAR1",
 587                 .start = res->start,
 588                 .end   = res->end,
 589                 .flags = flags,
 590                 .parent = res,
 591         };
 592
 593         res = &vmd->dev->resource[VMD_MEMBAR2];
 594         upper_bits = upper_32_bits(res->end);
 595         flags = res->flags & ~IORESOURCE_SIZEALIGN;
 596         if (!upper_bits)
 597                 flags &= ~IORESOURCE_MEM_64;
 598         vmd->resources[2] = (struct resource) {
 599                 .name  = "VMD MEMBAR2",
 600                 .start = res->start + 0x2000,
 601                 .end   = res->end,
 602                 .flags = flags,
 603                 .parent = res,
 604         };
 605
 606         sd->domain = vmd_find_free_domain();
 607         if (sd->domain < 0)
 608                 return sd->domain;
 609
 610         sd->node = pcibus_to_node(vmd->dev->bus);
 611
 612         vmd->irq_domain = pci_msi_create_irq_domain(NULL, &vmd_msi_domain_info,
 613                                                     x86_vector_domain);
 614         if (!vmd->irq_domain)
 615                 return -ENODEV;
 616
 617         pci_add_resource(&resources, &vmd->resources[0]);
 618         pci_add_resource(&resources, &vmd->resources[1]);
 619         pci_add_resource(&resources, &vmd->resources[2]);
 620         vmd->bus = pci_create_root_bus(&vmd->dev->dev, 0, &vmd_ops, sd,
 621                                        &resources);
 622         if (!vmd->bus) {
 623                 pci_free_resource_list(&resources);
 624                 irq_domain_remove(vmd->irq_domain);
 625                 return -ENODEV;
 626         }
 627
 628         vmd_attach_resources(vmd);
 629         vmd_setup_dma_ops(vmd);
 630         dev_set_msi_domain(&vmd->bus->dev, vmd->irq_domain);
 631         pci_rescan_bus(vmd->bus);
 632
 633         WARN(sysfs_create_link(&vmd->dev->dev.kobj, &vmd->bus->dev.kobj,
 634                                "domain"), "Can't create symlink to domain\n");
 635         return 0;
 636 }
 637
 638 static irqreturn_t vmd_irq(int irq, void *data)
 639 {
 640         struct vmd_irq_list *irqs = data;
 641         struct vmd_irq *vmdirq;
 642
 643         rcu_read_lock();
 644         list_for_each_entry_rcu(vmdirq, &irqs->irq_list, node)
 645                 generic_handle_irq(vmdirq->virq);
 646         rcu_read_unlock();
 647
 648         return IRQ_HANDLED;
 649 }
 650
 651 static int vmd_probe(struct pci_dev *dev, const struct pci_device_id *id)
 652 {
 653         struct vmd_dev *vmd;
 654         int i, err;
 655
 656         if (resource_size(&dev->resource[VMD_CFGBAR]) < (1 << 20))
 657                 return -ENOMEM;
 658
 659         vmd = devm_kzalloc(&dev->dev, sizeof(*vmd), GFP_KERNEL);
 660         if (!vmd)
 661                 return -ENOMEM;
 662
 663         vmd->dev = dev;
 664         err = pcim_enable_device(dev);
 665         if (err < 0)
 666                 return err;
 667
 668         vmd->cfgbar = pcim_iomap(dev, VMD_CFGBAR, 0);
 669         if (!vmd->cfgbar)
 670                 return -ENOMEM;
 671
 672         pci_set_master(dev);
 673         if (dma_set_mask_and_coherent(&dev->dev, DMA_BIT_MASK(64)) &&
 674             dma_set_mask_and_coherent(&dev->dev, DMA_BIT_MASK(32)))
 675                 return -ENODEV;
 676
 677         vmd->msix_count = pci_msix_vec_count(dev);
 678         if (vmd->msix_count < 0)
 679                 return -ENODEV;
 680
 681         vmd->msix_count = pci_alloc_irq_vectors(dev, 1, vmd->msix_count,
 682                                         PCI_IRQ_MSIX | PCI_IRQ_AFFINITY);
 683         if (vmd->msix_count < 0)
 684                 return vmd->msix_count;
 685
 686         vmd->irqs = devm_kcalloc(&dev->dev, vmd->msix_count, sizeof(*vmd->irqs),
 687                                  GFP_KERNEL);
 688         if (!vmd->irqs)
 689                 return -ENOMEM;
 690
 691         for (i = 0; i < vmd->msix_count; i++) {
 692                 INIT_LIST_HEAD(&vmd->irqs[i].irq_list);
 693                 err = devm_request_irq(&dev->dev, pci_irq_vector(dev, i),
 694                                        vmd_irq, 0, "vmd", &vmd->irqs[i]);
 695                 if (err)
 696                         return err;
 697         }
 698
 699         spin_lock_init(&vmd->cfg_lock);
 700         pci_set_drvdata(dev, vmd);
 701         err = vmd_enable_domain(vmd);
 702         if (err)
 703                 return err;
 704
 705         dev_info(&vmd->dev->dev, "Bound to PCI domain %04x\n",
 706                  vmd->sysdata.domain);
 707         return 0;
 708 }
 709
 710 static void vmd_remove(struct pci_dev *dev)
 711 {
 712         struct vmd_dev *vmd = pci_get_drvdata(dev);
 713
 714         vmd_detach_resources(vmd);
 715         pci_set_drvdata(dev, NULL);
 716         sysfs_remove_link(&vmd->dev->dev.kobj, "domain");
 717         pci_stop_root_bus(vmd->bus);
 718         pci_remove_root_bus(vmd->bus);
 719         vmd_teardown_dma_ops(vmd);
 720         irq_domain_remove(vmd->irq_domain);
 721 }
 722
 723 #ifdef CONFIG_PM
 724 static int vmd_suspend(struct device *dev)
 725 {
 726         struct pci_dev *pdev = to_pci_dev(dev);
 727
 728         pci_save_state(pdev);
 729         return 0;
 730 }
 731
 732 static int vmd_resume(struct device *dev)
 733 {
 734         struct pci_dev *pdev = to_pci_dev(dev);
 735
 736         pci_restore_state(pdev);
 737         return 0;
 738 }
 739 #endif
 740 static SIMPLE_DEV_PM_OPS(vmd_dev_pm_ops, vmd_suspend, vmd_resume);
 741
 742 static const struct pci_device_id vmd_ids[] = {
 743         {PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x201d),},
 744         {0,}
 745 };
 746 MODULE_DEVICE_TABLE(pci, vmd_ids);
 747
 748 static struct pci_driver vmd_drv = {
 749         .name           = "vmd",
 750         .id_table       = vmd_ids,
 751         .probe          = vmd_probe,
 752         .remove         = vmd_remove,
 753         .driver         = {
 754                 .pm     = &vmd_dev_pm_ops,
 755         },
 756 };
 757 module_pci_driver(vmd_drv);
 758
 759 MODULE_AUTHOR("Intel Corporation");
 760 MODULE_LICENSE("GPL v2");
 761 MODULE_VERSION("0.6");