drivers/iommu/dmar.c

   1 /*
   2  * Copyright (c) 2006, Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along with
  14  * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  15  * Place - Suite 330, Boston, MA 02111-1307 USA.
  16  *
  17  * Copyright (C) 2006-2008 Intel Corporation
  18  * Author: Ashok Raj <ashok.raj@intel.com>
  19  * Author: Shaohua Li <shaohua.li@intel.com>
  20  * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
  21  *
  22  * This file implements early detection/parsing of Remapping Devices
  23  * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
  24  * tables.
  25  *
  26  * These routines are used by both DMA-remapping and Interrupt-remapping
  27  */
  28
  29 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt /* has to precede printk.h */
  30
  31 #include <linux/pci.h>
  32 #include <linux/dmar.h>
  33 #include <linux/iova.h>
  34 #include <linux/intel-iommu.h>
  35 #include <linux/timer.h>
  36 #include <linux/irq.h>
  37 #include <linux/interrupt.h>
  38 #include <linux/tboot.h>
  39 #include <linux/dmi.h>
  40 #include <linux/slab.h>
  41 #include <asm/irq_remapping.h>
  42 #include <asm/iommu_table.h>
  43
  44 #include "irq_remapping.h"
  45
  46 /* No locks are needed as DMA remapping hardware unit
  47  * list is constructed at boot time and hotplug of
  48  * these units are not supported by the architecture.
  49  */
  50 LIST_HEAD(dmar_drhd_units);
  51
  52 struct acpi_table_header * __initdata dmar_tbl;
  53 static acpi_size dmar_tbl_size;
  54
  55 static void __init dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
  56 {
  57         /*
  58          * add INCLUDE_ALL at the tail, so scan the list will find it at
  59          * the very end.
  60          */
  61         if (drhd->include_all)
  62                 list_add_tail(&drhd->list, &dmar_drhd_units);
  63         else
  64                 list_add(&drhd->list, &dmar_drhd_units);
  65 }
  66
  67 static int __init dmar_parse_one_dev_scope(struct acpi_dmar_device_scope *scope,
  68                                            struct pci_dev **dev, u16 segment)
  69 {
  70         struct pci_bus *bus;
  71         struct pci_dev *pdev = NULL;
  72         struct acpi_dmar_pci_path *path;
  73         int count;
  74
  75         bus = pci_find_bus(segment, scope->bus);
  76         path = (struct acpi_dmar_pci_path *)(scope + 1);
  77         count = (scope->length - sizeof(struct acpi_dmar_device_scope))
  78                 / sizeof(struct acpi_dmar_pci_path);
  79
  80         while (count) {
  81                 if (pdev)
  82                         pci_dev_put(pdev);
  83                 /*
  84                  * Some BIOSes list non-exist devices in DMAR table, just
  85                  * ignore it
  86                  */
  87                 if (!bus) {
  88                         pr_warn("Device scope bus [%d] not found\n", scope->bus);
  89                         break;
  90                 }
  91                 pdev = pci_get_slot(bus, PCI_DEVFN(path->dev, path->fn));
  92                 if (!pdev) {
  93                         /* warning will be printed below */
  94                         break;
  95                 }
  96                 path ++;
  97                 count --;
  98                 bus = pdev->subordinate;
  99         }
 100         if (!pdev) {
 101                 pr_warn("Device scope device [%04x:%02x:%02x.%02x] not found\n",
 102                         segment, scope->bus, path->dev, path->fn);
 103                 *dev = NULL;
 104                 return 0;
 105         }
 106         if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT && \
 107                         pdev->subordinate) || (scope->entry_type == \
 108                         ACPI_DMAR_SCOPE_TYPE_BRIDGE && !pdev->subordinate)) {
 109                 pci_dev_put(pdev);
 110                 pr_warn("Device scope type does not match for %s\n",
 111                         pci_name(pdev));
 112                 return -EINVAL;
 113         }
 114         *dev = pdev;
 115         return 0;
 116 }
 117
 118 int __init dmar_parse_dev_scope(void *start, void *end, int *cnt,
 119                                 struct pci_dev ***devices, u16 segment)
 120 {
 121         struct acpi_dmar_device_scope *scope;
 122         void * tmp = start;
 123         int index;
 124         int ret;
 125
 126         *cnt = 0;
 127         while (start < end) {
 128                 scope = start;
 129                 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
 130                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
 131                         (*cnt)++;
 132                 else if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_IOAPIC &&
 133                         scope->entry_type != ACPI_DMAR_SCOPE_TYPE_HPET) {
 134                         pr_warn("Unsupported device scope\n");
 135                 }
 136                 start += scope->length;
 137         }
 138         if (*cnt == 0)
 139                 return 0;
 140
 141         *devices = kcalloc(*cnt, sizeof(struct pci_dev *), GFP_KERNEL);
 142         if (!*devices)
 143                 return -ENOMEM;
 144
 145         start = tmp;
 146         index = 0;
 147         while (start < end) {
 148                 scope = start;
 149                 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
 150                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE) {
 151                         ret = dmar_parse_one_dev_scope(scope,
 152                                 &(*devices)[index], segment);
 153                         if (ret) {
 154                                 kfree(*devices);
 155                                 return ret;
 156                         }
 157                         index ++;
 158                 }
 159                 start += scope->length;
 160         }
 161
 162         return 0;
 163 }
 164
 165 /**
 166  * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
 167  * structure which uniquely represent one DMA remapping hardware unit
 168  * present in the platform
 169  */
 170 static int __init
 171 dmar_parse_one_drhd(struct acpi_dmar_header *header)
 172 {
 173         struct acpi_dmar_hardware_unit *drhd;
 174         struct dmar_drhd_unit *dmaru;
 175         int ret = 0;
 176
 177         drhd = (struct acpi_dmar_hardware_unit *)header;
 178         dmaru = kzalloc(sizeof(*dmaru), GFP_KERNEL);
 179         if (!dmaru)
 180                 return -ENOMEM;
 181
 182         dmaru->hdr = header;
 183         dmaru->reg_base_addr = drhd->address;
 184         dmaru->segment = drhd->segment;
 185         dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
 186
 187         ret = alloc_iommu(dmaru);
 188         if (ret) {
 189                 kfree(dmaru);
 190                 return ret;
 191         }
 192         dmar_register_drhd_unit(dmaru);
 193         return 0;
 194 }
 195
 196 static int __init dmar_parse_dev(struct dmar_drhd_unit *dmaru)
 197 {
 198         struct acpi_dmar_hardware_unit *drhd;
 199         int ret = 0;
 200
 201         drhd = (struct acpi_dmar_hardware_unit *) dmaru->hdr;
 202
 203         if (dmaru->include_all)
 204                 return 0;
 205
 206         ret = dmar_parse_dev_scope((void *)(drhd + 1),
 207                                 ((void *)drhd) + drhd->header.length,
 208                                 &dmaru->devices_cnt, &dmaru->devices,
 209                                 drhd->segment);
 210         if (ret) {
 211                 list_del(&dmaru->list);
 212                 kfree(dmaru);
 213         }
 214         return ret;
 215 }
 216
 217 #ifdef CONFIG_ACPI_NUMA
 218 static int __init
 219 dmar_parse_one_rhsa(struct acpi_dmar_header *header)
 220 {
 221         struct acpi_dmar_rhsa *rhsa;
 222         struct dmar_drhd_unit *drhd;
 223
 224         rhsa = (struct acpi_dmar_rhsa *)header;
 225         for_each_drhd_unit(drhd) {
 226                 if (drhd->reg_base_addr == rhsa->base_address) {
 227                         int node = acpi_map_pxm_to_node(rhsa->proximity_domain);
 228
 229                         if (!node_online(node))
 230                                 node = -1;
 231                         drhd->iommu->node = node;
 232                         return 0;
 233                 }
 234         }
 235         WARN_TAINT(
 236                 1, TAINT_FIRMWARE_WORKAROUND,
 237                 "Your BIOS is broken; RHSA refers to non-existent DMAR unit at %llx\n"
 238                 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 239                 drhd->reg_base_addr,
 240                 dmi_get_system_info(DMI_BIOS_VENDOR),
 241                 dmi_get_system_info(DMI_BIOS_VERSION),
 242                 dmi_get_system_info(DMI_PRODUCT_VERSION));
 243
 244         return 0;
 245 }
 246 #endif
 247
 248 static void __init
 249 dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
 250 {
 251         struct acpi_dmar_hardware_unit *drhd;
 252         struct acpi_dmar_reserved_memory *rmrr;
 253         struct acpi_dmar_atsr *atsr;
 254         struct acpi_dmar_rhsa *rhsa;
 255
 256         switch (header->type) {
 257         case ACPI_DMAR_TYPE_HARDWARE_UNIT:
 258                 drhd = container_of(header, struct acpi_dmar_hardware_unit,
 259                                     header);
 260                 pr_info("DRHD base: %#016Lx flags: %#x\n",
 261                         (unsigned long long)drhd->address, drhd->flags);
 262                 break;
 263         case ACPI_DMAR_TYPE_RESERVED_MEMORY:
 264                 rmrr = container_of(header, struct acpi_dmar_reserved_memory,
 265                                     header);
 266                 pr_info("RMRR base: %#016Lx end: %#016Lx\n",
 267                         (unsigned long long)rmrr->base_address,
 268                         (unsigned long long)rmrr->end_address);
 269                 break;
 270         case ACPI_DMAR_TYPE_ATSR:
 271                 atsr = container_of(header, struct acpi_dmar_atsr, header);
 272                 pr_info("ATSR flags: %#x\n", atsr->flags);
 273                 break;
 274         case ACPI_DMAR_HARDWARE_AFFINITY:
 275                 rhsa = container_of(header, struct acpi_dmar_rhsa, header);
 276                 pr_info("RHSA base: %#016Lx proximity domain: %#x\n",
 277                        (unsigned long long)rhsa->base_address,
 278                        rhsa->proximity_domain);
 279                 break;
 280         }
 281 }
 282
 283 /**
 284  * dmar_table_detect - checks to see if the platform supports DMAR devices
 285  */
 286 static int __init dmar_table_detect(void)
 287 {
 288         acpi_status status = AE_OK;
 289
 290         /* if we could find DMAR table, then there are DMAR devices */
 291         status = acpi_get_table_with_size(ACPI_SIG_DMAR, 0,
 292                                 (struct acpi_table_header **)&dmar_tbl,
 293                                 &dmar_tbl_size);
 294
 295         if (ACPI_SUCCESS(status) && !dmar_tbl) {
 296                 pr_warn("Unable to map DMAR\n");
 297                 status = AE_NOT_FOUND;
 298         }
 299
 300         return (ACPI_SUCCESS(status) ? 1 : 0);
 301 }
 302
 303 /**
 304  * parse_dmar_table - parses the DMA reporting table
 305  */
 306 static int __init
 307 parse_dmar_table(void)
 308 {
 309         struct acpi_table_dmar *dmar;
 310         struct acpi_dmar_header *entry_header;
 311         int ret = 0;
 312
 313         /*
 314          * Do it again, earlier dmar_tbl mapping could be mapped with
 315          * fixed map.
 316          */
 317         dmar_table_detect();
 318
 319         /*
 320          * ACPI tables may not be DMA protected by tboot, so use DMAR copy
 321          * SINIT saved in SinitMleData in TXT heap (which is DMA protected)
 322          */
 323         dmar_tbl = tboot_get_dmar_table(dmar_tbl);
 324
 325         dmar = (struct acpi_table_dmar *)dmar_tbl;
 326         if (!dmar)
 327                 return -ENODEV;
 328
 329         if (dmar->width < PAGE_SHIFT - 1) {
 330                 pr_warn("Invalid DMAR haw\n");
 331                 return -EINVAL;
 332         }
 333
 334         pr_info("Host address width %d\n", dmar->width + 1);
 335
 336         entry_header = (struct acpi_dmar_header *)(dmar + 1);
 337         while (((unsigned long)entry_header) <
 338                         (((unsigned long)dmar) + dmar_tbl->length)) {
 339                 /* Avoid looping forever on bad ACPI tables */
 340                 if (entry_header->length == 0) {
 341                         pr_warn("Invalid 0-length structure\n");
 342                         ret = -EINVAL;
 343                         break;
 344                 }
 345
 346                 dmar_table_print_dmar_entry(entry_header);
 347
 348                 switch (entry_header->type) {
 349                 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
 350                         ret = dmar_parse_one_drhd(entry_header);
 351                         break;
 352                 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
 353                         ret = dmar_parse_one_rmrr(entry_header);
 354                         break;
 355                 case ACPI_DMAR_TYPE_ATSR:
 356                         ret = dmar_parse_one_atsr(entry_header);
 357                         break;
 358                 case ACPI_DMAR_HARDWARE_AFFINITY:
 359 #ifdef CONFIG_ACPI_NUMA
 360                         ret = dmar_parse_one_rhsa(entry_header);
 361 #endif
 362                         break;
 363                 default:
 364                         pr_warn("Unknown DMAR structure type %d\n",
 365                                 entry_header->type);
 366                         ret = 0; /* for forward compatibility */
 367                         break;
 368                 }
 369                 if (ret)
 370                         break;
 371
 372                 entry_header = ((void *)entry_header + entry_header->length);
 373         }
 374         return ret;
 375 }
 376
 377 static int dmar_pci_device_match(struct pci_dev *devices[], int cnt,
 378                           struct pci_dev *dev)
 379 {
 380         int index;
 381
 382         while (dev) {
 383                 for (index = 0; index < cnt; index++)
 384                         if (dev == devices[index])
 385                                 return 1;
 386
 387                 /* Check our parent */
 388                 dev = dev->bus->self;
 389         }
 390
 391         return 0;
 392 }
 393
 394 struct dmar_drhd_unit *
 395 dmar_find_matched_drhd_unit(struct pci_dev *dev)
 396 {
 397         struct dmar_drhd_unit *dmaru = NULL;
 398         struct acpi_dmar_hardware_unit *drhd;
 399
 400         dev = pci_physfn(dev);
 401
 402         list_for_each_entry(dmaru, &dmar_drhd_units, list) {
 403                 drhd = container_of(dmaru->hdr,
 404                                     struct acpi_dmar_hardware_unit,
 405                                     header);
 406
 407                 if (dmaru->include_all &&
 408                     drhd->segment == pci_domain_nr(dev->bus))
 409                         return dmaru;
 410
 411                 if (dmar_pci_device_match(dmaru->devices,
 412                                           dmaru->devices_cnt, dev))
 413                         return dmaru;
 414         }
 415
 416         return NULL;
 417 }
 418
 419 int __init dmar_dev_scope_init(void)
 420 {
 421         static int dmar_dev_scope_initialized;
 422         struct dmar_drhd_unit *drhd, *drhd_n;
 423         int ret = -ENODEV;
 424
 425         if (dmar_dev_scope_initialized)
 426                 return dmar_dev_scope_initialized;
 427
 428         if (list_empty(&dmar_drhd_units))
 429                 goto fail;
 430
 431         list_for_each_entry_safe(drhd, drhd_n, &dmar_drhd_units, list) {
 432                 ret = dmar_parse_dev(drhd);
 433                 if (ret)
 434                         goto fail;
 435         }
 436
 437         ret = dmar_parse_rmrr_atsr_dev();
 438         if (ret)
 439                 goto fail;
 440
 441         dmar_dev_scope_initialized = 1;
 442         return 0;
 443
 444 fail:
 445         dmar_dev_scope_initialized = ret;
 446         return ret;
 447 }
 448
 449
 450 int __init dmar_table_init(void)
 451 {
 452         static int dmar_table_initialized;
 453         int ret;
 454
 455         if (dmar_table_initialized)
 456                 return 0;
 457
 458         dmar_table_initialized = 1;
 459
 460         ret = parse_dmar_table();
 461         if (ret) {
 462                 if (ret != -ENODEV)
 463                         pr_info("parse DMAR table failure.\n");
 464                 return ret;
 465         }
 466
 467         if (list_empty(&dmar_drhd_units)) {
 468                 pr_info("No DMAR devices found\n");
 469                 return -ENODEV;
 470         }
 471
 472         return 0;
 473 }
 474
 475 static void warn_invalid_dmar(u64 addr, const char *message)
 476 {
 477         WARN_TAINT_ONCE(
 478                 1, TAINT_FIRMWARE_WORKAROUND,
 479                 "Your BIOS is broken; DMAR reported at address %llx%s!\n"
 480                 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 481                 addr, message,
 482                 dmi_get_system_info(DMI_BIOS_VENDOR),
 483                 dmi_get_system_info(DMI_BIOS_VERSION),
 484                 dmi_get_system_info(DMI_PRODUCT_VERSION));
 485 }
 486
 487 int __init check_zero_address(void)
 488 {
 489         struct acpi_table_dmar *dmar;
 490         struct acpi_dmar_header *entry_header;
 491         struct acpi_dmar_hardware_unit *drhd;
 492
 493         dmar = (struct acpi_table_dmar *)dmar_tbl;
 494         entry_header = (struct acpi_dmar_header *)(dmar + 1);
 495
 496         while (((unsigned long)entry_header) <
 497                         (((unsigned long)dmar) + dmar_tbl->length)) {
 498                 /* Avoid looping forever on bad ACPI tables */
 499                 if (entry_header->length == 0) {
 500                         pr_warn("Invalid 0-length structure\n");
 501                         return 0;
 502                 }
 503
 504                 if (entry_header->type == ACPI_DMAR_TYPE_HARDWARE_UNIT) {
 505                         void __iomem *addr;
 506                         u64 cap, ecap;
 507
 508                         drhd = (void *)entry_header;
 509                         if (!drhd->address) {
 510                                 warn_invalid_dmar(0, "");
 511                                 goto failed;
 512                         }
 513
 514                         addr = early_ioremap(drhd->address, VTD_PAGE_SIZE);
 515                         if (!addr ) {
 516                                 printk("IOMMU: can't validate: %llx\n", drhd->address);
 517                                 goto failed;
 518                         }
 519                         cap = dmar_readq(addr + DMAR_CAP_REG);
 520                         ecap = dmar_readq(addr + DMAR_ECAP_REG);
 521                         early_iounmap(addr, VTD_PAGE_SIZE);
 522                         if (cap == (uint64_t)-1 && ecap == (uint64_t)-1) {
 523                                 warn_invalid_dmar(drhd->address,
 524                                                   " returns all ones");
 525                                 goto failed;
 526                         }
 527                 }
 528
 529                 entry_header = ((void *)entry_header + entry_header->length);
 530         }
 531         return 1;
 532
 533 failed:
 534         return 0;
 535 }
 536
 537 int __init detect_intel_iommu(void)
 538 {
 539         int ret;
 540
 541         ret = dmar_table_detect();
 542         if (ret)
 543                 ret = check_zero_address();
 544         {
 545                 struct acpi_table_dmar *dmar;
 546
 547                 dmar = (struct acpi_table_dmar *) dmar_tbl;
 548
 549                 if (ret && irq_remapping_enabled && cpu_has_x2apic &&
 550                     dmar->flags & 0x1)
 551                         pr_info("Queued invalidation will be enabled to support x2apic and Intr-remapping.\n");
 552
 553                 if (ret && !no_iommu && !iommu_detected && !dmar_disabled) {
 554                         iommu_detected = 1;
 555                         /* Make sure ACS will be enabled */
 556                         pci_request_acs();
 557                 }
 558
 559 #ifdef CONFIG_X86
 560                 if (ret)
 561                         x86_init.iommu.iommu_init = intel_iommu_init;
 562 #endif
 563         }
 564         early_acpi_os_unmap_memory(dmar_tbl, dmar_tbl_size);
 565         dmar_tbl = NULL;
 566
 567         return ret ? 1 : -ENODEV;
 568 }
 569
 570
 571 static void unmap_iommu(struct intel_iommu *iommu)
 572 {
 573         iounmap(iommu->reg);
 574         release_mem_region(iommu->reg_phys, iommu->reg_size);
 575 }
 576
 577 /**
 578  * map_iommu: map the iommu's registers
 579  * @iommu: the iommu to map
 580  * @phys_addr: the physical address of the base resgister
 581  *
 582  * Memory map the iommu's registers.  Start w/ a single page, and
 583  * possibly expand if that turns out to be insufficent.
 584  */
 585 static int map_iommu(struct intel_iommu *iommu, u64 phys_addr)
 586 {
 587         int map_size, err=0;
 588
 589         iommu->reg_phys = phys_addr;
 590         iommu->reg_size = VTD_PAGE_SIZE;
 591
 592         if (!request_mem_region(iommu->reg_phys, iommu->reg_size, iommu->name)) {
 593                 pr_err("IOMMU: can't reserve memory\n");
 594                 err = -EBUSY;
 595                 goto out;
 596         }
 597
 598         iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
 599         if (!iommu->reg) {
 600                 pr_err("IOMMU: can't map the region\n");
 601                 err = -ENOMEM;
 602                 goto release;
 603         }
 604
 605         iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
 606         iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
 607
 608         if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {
 609                 err = -EINVAL;
 610                 warn_invalid_dmar(phys_addr, " returns all ones");
 611                 goto unmap;
 612         }
 613
 614         /* the registers might be more than one page */
 615         map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
 616                          cap_max_fault_reg_offset(iommu->cap));
 617         map_size = VTD_PAGE_ALIGN(map_size);
 618         if (map_size > iommu->reg_size) {
 619                 iounmap(iommu->reg);
 620                 release_mem_region(iommu->reg_phys, iommu->reg_size);
 621                 iommu->reg_size = map_size;
 622                 if (!request_mem_region(iommu->reg_phys, iommu->reg_size,
 623                                         iommu->name)) {
 624                         pr_err("IOMMU: can't reserve memory\n");
 625                         err = -EBUSY;
 626                         goto out;
 627                 }
 628                 iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
 629                 if (!iommu->reg) {
 630                         pr_err("IOMMU: can't map the region\n");
 631                         err = -ENOMEM;
 632                         goto release;
 633                 }
 634         }
 635         err = 0;
 636         goto out;
 637
 638 unmap:
 639         iounmap(iommu->reg);
 640 release:
 641         release_mem_region(iommu->reg_phys, iommu->reg_size);
 642 out:
 643         return err;
 644 }
 645
 646 int alloc_iommu(struct dmar_drhd_unit *drhd)
 647 {
 648         struct intel_iommu *iommu;
 649         u32 ver, sts;
 650         static int iommu_allocated = 0;
 651         int agaw = 0;
 652         int msagaw = 0;
 653         int err;
 654
 655         if (!drhd->reg_base_addr) {
 656                 warn_invalid_dmar(0, "");
 657                 return -EINVAL;
 658         }
 659
 660         iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
 661         if (!iommu)
 662                 return -ENOMEM;
 663
 664         iommu->seq_id = iommu_allocated++;
 665         sprintf (iommu->name, "dmar%d", iommu->seq_id);
 666
 667         err = map_iommu(iommu, drhd->reg_base_addr);
 668         if (err) {
 669                 pr_err("IOMMU: failed to map %s\n", iommu->name);
 670                 goto error;
 671         }
 672
 673         err = -EINVAL;
 674         agaw = iommu_calculate_agaw(iommu);
 675         if (agaw < 0) {
 676                 pr_err("Cannot get a valid agaw for iommu (seq_id = %d)\n",
 677                         iommu->seq_id);
 678                 goto err_unmap;
 679         }
 680         msagaw = iommu_calculate_max_sagaw(iommu);
 681         if (msagaw < 0) {
 682                 pr_err("Cannot get a valid max agaw for iommu (seq_id = %d)\n",
 683                         iommu->seq_id);
 684                 goto err_unmap;
 685         }
 686         iommu->agaw = agaw;
 687         iommu->msagaw = msagaw;
 688
 689         iommu->node = -1;
 690
 691         ver = readl(iommu->reg + DMAR_VER_REG);
 692         pr_info("IOMMU %d: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n",
 693                 iommu->seq_id,
 694                 (unsigned long long)drhd->reg_base_addr,
 695                 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
 696                 (unsigned long long)iommu->cap,
 697                 (unsigned long long)iommu->ecap);
 698
 699         /* Reflect status in gcmd */
 700         sts = readl(iommu->reg + DMAR_GSTS_REG);
 701         if (sts & DMA_GSTS_IRES)
 702                 iommu->gcmd |= DMA_GCMD_IRE;
 703         if (sts & DMA_GSTS_TES)
 704                 iommu->gcmd |= DMA_GCMD_TE;
 705         if (sts & DMA_GSTS_QIES)
 706                 iommu->gcmd |= DMA_GCMD_QIE;
 707
 708         raw_spin_lock_init(&iommu->register_lock);
 709
 710         drhd->iommu = iommu;
 711         return 0;
 712
 713  err_unmap:
 714         unmap_iommu(iommu);
 715  error:
 716         kfree(iommu);
 717         return err;
 718 }
 719
 720 void free_iommu(struct intel_iommu *iommu)
 721 {
 722         if (!iommu)
 723                 return;
 724
 725         free_dmar_iommu(iommu);
 726
 727         if (iommu->reg)
 728                 unmap_iommu(iommu);
 729
 730         kfree(iommu);
 731 }
 732
 733 /*
 734  * Reclaim all the submitted descriptors which have completed its work.
 735  */
 736 static inline void reclaim_free_desc(struct q_inval *qi)
 737 {
 738         while (qi->desc_status[qi->free_tail] == QI_DONE ||
 739                qi->desc_status[qi->free_tail] == QI_ABORT) {
 740                 qi->desc_status[qi->free_tail] = QI_FREE;
 741                 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
 742                 qi->free_cnt++;
 743         }
 744 }
 745
 746 static int qi_check_fault(struct intel_iommu *iommu, int index)
 747 {
 748         u32 fault;
 749         int head, tail;
 750         struct q_inval *qi = iommu->qi;
 751         int wait_index = (index + 1) % QI_LENGTH;
 752
 753         if (qi->desc_status[wait_index] == QI_ABORT)
 754                 return -EAGAIN;
 755
 756         fault = readl(iommu->reg + DMAR_FSTS_REG);
 757
 758         /*
 759          * If IQE happens, the head points to the descriptor associated
 760          * with the error. No new descriptors are fetched until the IQE
 761          * is cleared.
 762          */
 763         if (fault & DMA_FSTS_IQE) {
 764                 head = readl(iommu->reg + DMAR_IQH_REG);
 765                 if ((head >> DMAR_IQ_SHIFT) == index) {
 766                         pr_err("VT-d detected invalid descriptor: "
 767                                 "low=%llx, high=%llx\n",
 768                                 (unsigned long long)qi->desc[index].low,
 769                                 (unsigned long long)qi->desc[index].high);
 770                         memcpy(&qi->desc[index], &qi->desc[wait_index],
 771                                         sizeof(struct qi_desc));
 772                         __iommu_flush_cache(iommu, &qi->desc[index],
 773                                         sizeof(struct qi_desc));
 774                         writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
 775                         return -EINVAL;
 776                 }
 777         }
 778
 779         /*
 780          * If ITE happens, all pending wait_desc commands are aborted.
 781          * No new descriptors are fetched until the ITE is cleared.
 782          */
 783         if (fault & DMA_FSTS_ITE) {
 784                 head = readl(iommu->reg + DMAR_IQH_REG);
 785                 head = ((head >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
 786                 head |= 1;
 787                 tail = readl(iommu->reg + DMAR_IQT_REG);
 788                 tail = ((tail >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
 789
 790                 writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
 791
 792                 do {
 793                         if (qi->desc_status[head] == QI_IN_USE)
 794                                 qi->desc_status[head] = QI_ABORT;
 795                         head = (head - 2 + QI_LENGTH) % QI_LENGTH;
 796                 } while (head != tail);
 797
 798                 if (qi->desc_status[wait_index] == QI_ABORT)
 799                         return -EAGAIN;
 800         }
 801
 802         if (fault & DMA_FSTS_ICE)
 803                 writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
 804
 805         return 0;
 806 }
 807
 808 /*
 809  * Submit the queued invalidation descriptor to the remapping
 810  * hardware unit and wait for its completion.
 811  */
 812 int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
 813 {
 814         int rc;
 815         struct q_inval *qi = iommu->qi;
 816         struct qi_desc *hw, wait_desc;
 817         int wait_index, index;
 818         unsigned long flags;
 819
 820         if (!qi)
 821                 return 0;
 822
 823         hw = qi->desc;
 824
 825 restart:
 826         rc = 0;
 827
 828         raw_spin_lock_irqsave(&qi->q_lock, flags);
 829         while (qi->free_cnt < 3) {
 830                 raw_spin_unlock_irqrestore(&qi->q_lock, flags);
 831                 cpu_relax();
 832                 raw_spin_lock_irqsave(&qi->q_lock, flags);
 833         }
 834
 835         index = qi->free_head;
 836         wait_index = (index + 1) % QI_LENGTH;
 837
 838         qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
 839
 840         hw[index] = *desc;
 841
 842         wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
 843                         QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
 844         wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
 845
 846         hw[wait_index] = wait_desc;
 847
 848         __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
 849         __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
 850
 851         qi->free_head = (qi->free_head + 2) % QI_LENGTH;
 852         qi->free_cnt -= 2;
 853
 854         /*
 855          * update the HW tail register indicating the presence of
 856          * new descriptors.
 857          */
 858         writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg + DMAR_IQT_REG);
 859
 860         while (qi->desc_status[wait_index] != QI_DONE) {
 861                 /*
 862                  * We will leave the interrupts disabled, to prevent interrupt
 863                  * context to queue another cmd while a cmd is already submitted
 864                  * and waiting for completion on this cpu. This is to avoid
 865                  * a deadlock where the interrupt context can wait indefinitely
 866                  * for free slots in the queue.
 867                  */
 868                 rc = qi_check_fault(iommu, index);
 869                 if (rc)
 870                         break;
 871
 872                 raw_spin_unlock(&qi->q_lock);
 873                 cpu_relax();
 874                 raw_spin_lock(&qi->q_lock);
 875         }
 876
 877         qi->desc_status[index] = QI_DONE;
 878
 879         reclaim_free_desc(qi);
 880         raw_spin_unlock_irqrestore(&qi->q_lock, flags);
 881
 882         if (rc == -EAGAIN)
 883                 goto restart;
 884
 885         return rc;
 886 }
 887
 888 /*
 889  * Flush the global interrupt entry cache.
 890  */
 891 void qi_global_iec(struct intel_iommu *iommu)
 892 {
 893         struct qi_desc desc;
 894
 895         desc.low = QI_IEC_TYPE;
 896         desc.high = 0;
 897
 898         /* should never fail */
 899         qi_submit_sync(&desc, iommu);
 900 }
 901
 902 void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
 903                       u64 type)
 904 {
 905         struct qi_desc desc;
 906
 907         desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
 908                         | QI_CC_GRAN(type) | QI_CC_TYPE;
 909         desc.high = 0;
 910
 911         qi_submit_sync(&desc, iommu);
 912 }
 913
 914 void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
 915                     unsigned int size_order, u64 type)
 916 {
 917         u8 dw = 0, dr = 0;
 918
 919         struct qi_desc desc;
 920         int ih = 0;
 921
 922         if (cap_write_drain(iommu->cap))
 923                 dw = 1;
 924
 925         if (cap_read_drain(iommu->cap))
 926                 dr = 1;
 927
 928         desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
 929                 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
 930         desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
 931                 | QI_IOTLB_AM(size_order);
 932
 933         qi_submit_sync(&desc, iommu);
 934 }
 935
 936 void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep,
 937                         u64 addr, unsigned mask)
 938 {
 939         struct qi_desc desc;
 940
 941         if (mask) {
 942                 BUG_ON(addr & ((1 << (VTD_PAGE_SHIFT + mask)) - 1));
 943                 addr |= (1 << (VTD_PAGE_SHIFT + mask - 1)) - 1;
 944                 desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
 945         } else
 946                 desc.high = QI_DEV_IOTLB_ADDR(addr);
 947
 948         if (qdep >= QI_DEV_IOTLB_MAX_INVS)
 949                 qdep = 0;
 950
 951         desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
 952                    QI_DIOTLB_TYPE;
 953
 954         qi_submit_sync(&desc, iommu);
 955 }
 956
 957 /*
 958  * Disable Queued Invalidation interface.
 959  */
 960 void dmar_disable_qi(struct intel_iommu *iommu)
 961 {
 962         unsigned long flags;
 963         u32 sts;
 964         cycles_t start_time = get_cycles();
 965
 966         if (!ecap_qis(iommu->ecap))
 967                 return;
 968
 969         raw_spin_lock_irqsave(&iommu->register_lock, flags);
 970
 971         sts =  dmar_readq(iommu->reg + DMAR_GSTS_REG);
 972         if (!(sts & DMA_GSTS_QIES))
 973                 goto end;
 974
 975         /*
 976          * Give a chance to HW to complete the pending invalidation requests.
 977          */
 978         while ((readl(iommu->reg + DMAR_IQT_REG) !=
 979                 readl(iommu->reg + DMAR_IQH_REG)) &&
 980                 (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
 981                 cpu_relax();
 982
 983         iommu->gcmd &= ~DMA_GCMD_QIE;
 984         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
 985
 986         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
 987                       !(sts & DMA_GSTS_QIES), sts);
 988 end:
 989         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
 990 }
 991
 992 /*
 993  * Enable queued invalidation.
 994  */
 995 static void __dmar_enable_qi(struct intel_iommu *iommu)
 996 {
 997         u32 sts;
 998         unsigned long flags;
 999         struct q_inval *qi = iommu->qi;
1000
1001         qi->free_head = qi->free_tail = 0;
1002         qi->free_cnt = QI_LENGTH;
1003
1004         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1005
1006         /* write zero to the tail reg */
1007         writel(0, iommu->reg + DMAR_IQT_REG);
1008
1009         dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
1010
1011         iommu->gcmd |= DMA_GCMD_QIE;
1012         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1013
1014         /* Make sure hardware complete it */
1015         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
1016
1017         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1018 }
1019
1020 /*
1021  * Enable Queued Invalidation interface. This is a must to support
1022  * interrupt-remapping. Also used by DMA-remapping, which replaces
1023  * register based IOTLB invalidation.
1024  */
1025 int dmar_enable_qi(struct intel_iommu *iommu)
1026 {
1027         struct q_inval *qi;
1028         struct page *desc_page;
1029
1030         if (!ecap_qis(iommu->ecap))
1031                 return -ENOENT;
1032
1033         /*
1034          * queued invalidation is already setup and enabled.
1035          */
1036         if (iommu->qi)
1037                 return 0;
1038
1039         iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
1040         if (!iommu->qi)
1041                 return -ENOMEM;
1042
1043         qi = iommu->qi;
1044
1045
1046         desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, 0);
1047         if (!desc_page) {
1048                 kfree(qi);
1049                 iommu->qi = 0;
1050                 return -ENOMEM;
1051         }
1052
1053         qi->desc = page_address(desc_page);
1054
1055         qi->desc_status = kzalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC);
1056         if (!qi->desc_status) {
1057                 free_page((unsigned long) qi->desc);
1058                 kfree(qi);
1059                 iommu->qi = 0;
1060                 return -ENOMEM;
1061         }
1062
1063         qi->free_head = qi->free_tail = 0;
1064         qi->free_cnt = QI_LENGTH;
1065
1066         raw_spin_lock_init(&qi->q_lock);
1067
1068         __dmar_enable_qi(iommu);
1069
1070         return 0;
1071 }
1072
1073 /* iommu interrupt handling. Most stuff are MSI-like. */
1074
1075 enum faulttype {
1076         DMA_REMAP,
1077         INTR_REMAP,
1078         UNKNOWN,
1079 };
1080
1081 static const char *dma_remap_fault_reasons[] =
1082 {
1083         "Software",
1084         "Present bit in root entry is clear",
1085         "Present bit in context entry is clear",
1086         "Invalid context entry",
1087         "Access beyond MGAW",
1088         "PTE Write access is not set",
1089         "PTE Read access is not set",
1090         "Next page table ptr is invalid",
1091         "Root table address invalid",
1092         "Context table ptr is invalid",
1093         "non-zero reserved fields in RTP",
1094         "non-zero reserved fields in CTP",
1095         "non-zero reserved fields in PTE",
1096         "PCE for translation request specifies blocking",
1097 };
1098
1099 static const char *irq_remap_fault_reasons[] =
1100 {
1101         "Detected reserved fields in the decoded interrupt-remapped request",
1102         "Interrupt index exceeded the interrupt-remapping table size",
1103         "Present field in the IRTE entry is clear",
1104         "Error accessing interrupt-remapping table pointed by IRTA_REG",
1105         "Detected reserved fields in the IRTE entry",
1106         "Blocked a compatibility format interrupt request",
1107         "Blocked an interrupt request due to source-id verification failure",
1108 };
1109
1110 #define MAX_FAULT_REASON_IDX    (ARRAY_SIZE(fault_reason_strings) - 1)
1111
1112 const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
1113 {
1114         if (fault_reason >= 0x20 && (fault_reason - 0x20 <
1115                                         ARRAY_SIZE(irq_remap_fault_reasons))) {
1116                 *fault_type = INTR_REMAP;
1117                 return irq_remap_fault_reasons[fault_reason - 0x20];
1118         } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1119                 *fault_type = DMA_REMAP;
1120                 return dma_remap_fault_reasons[fault_reason];
1121         } else {
1122                 *fault_type = UNKNOWN;
1123                 return "Unknown";
1124         }
1125 }
1126
1127 void dmar_msi_unmask(struct irq_data *data)
1128 {
1129         struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1130         unsigned long flag;
1131
1132         /* unmask it */
1133         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1134         writel(0, iommu->reg + DMAR_FECTL_REG);
1135         /* Read a reg to force flush the post write */
1136         readl(iommu->reg + DMAR_FECTL_REG);
1137         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1138 }
1139
1140 void dmar_msi_mask(struct irq_data *data)
1141 {
1142         unsigned long flag;
1143         struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1144
1145         /* mask it */
1146         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1147         writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG);
1148         /* Read a reg to force flush the post write */
1149         readl(iommu->reg + DMAR_FECTL_REG);
1150         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1151 }
1152
1153 void dmar_msi_write(int irq, struct msi_msg *msg)
1154 {
1155         struct intel_iommu *iommu = irq_get_handler_data(irq);
1156         unsigned long flag;
1157
1158         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1159         writel(msg->data, iommu->reg + DMAR_FEDATA_REG);
1160         writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG);
1161         writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG);
1162         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1163 }
1164
1165 void dmar_msi_read(int irq, struct msi_msg *msg)
1166 {
1167         struct intel_iommu *iommu = irq_get_handler_data(irq);
1168         unsigned long flag;
1169
1170         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1171         msg->data = readl(iommu->reg + DMAR_FEDATA_REG);
1172         msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG);
1173         msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG);
1174         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1175 }
1176
1177 static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1178                 u8 fault_reason, u16 source_id, unsigned long long addr)
1179 {
1180         const char *reason;
1181         int fault_type;
1182
1183         reason = dmar_get_fault_reason(fault_reason, &fault_type);
1184
1185         if (fault_type == INTR_REMAP)
1186                 pr_err("INTR-REMAP: Request device [[%02x:%02x.%d] "
1187                        "fault index %llx\n"
1188                         "INTR-REMAP:[fault reason %02d] %s\n",
1189                         (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1190                         PCI_FUNC(source_id & 0xFF), addr >> 48,
1191                         fault_reason, reason);
1192         else
1193                 pr_err("DMAR:[%s] Request device [%02x:%02x.%d] "
1194                        "fault addr %llx \n"
1195                        "DMAR:[fault reason %02d] %s\n",
1196                        (type ? "DMA Read" : "DMA Write"),
1197                        (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1198                        PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
1199         return 0;
1200 }
1201
1202 #define PRIMARY_FAULT_REG_LEN (16)
1203 irqreturn_t dmar_fault(int irq, void *dev_id)
1204 {
1205         struct intel_iommu *iommu = dev_id;
1206         int reg, fault_index;
1207         u32 fault_status;
1208         unsigned long flag;
1209
1210         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1211         fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1212         if (fault_status)
1213                 pr_err("DRHD: handling fault status reg %x\n", fault_status);
1214
1215         /* TBD: ignore advanced fault log currently */
1216         if (!(fault_status & DMA_FSTS_PPF))
1217                 goto unlock_exit;
1218
1219         fault_index = dma_fsts_fault_record_index(fault_status);
1220         reg = cap_fault_reg_offset(iommu->cap);
1221         while (1) {
1222                 u8 fault_reason;
1223                 u16 source_id;
1224                 u64 guest_addr;
1225                 int type;
1226                 u32 data;
1227
1228                 /* highest 32 bits */
1229                 data = readl(iommu->reg + reg +
1230                                 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1231                 if (!(data & DMA_FRCD_F))
1232                         break;
1233
1234                 fault_reason = dma_frcd_fault_reason(data);
1235                 type = dma_frcd_type(data);
1236
1237                 data = readl(iommu->reg + reg +
1238                                 fault_index * PRIMARY_FAULT_REG_LEN + 8);
1239                 source_id = dma_frcd_source_id(data);
1240
1241                 guest_addr = dmar_readq(iommu->reg + reg +
1242                                 fault_index * PRIMARY_FAULT_REG_LEN);
1243                 guest_addr = dma_frcd_page_addr(guest_addr);
1244                 /* clear the fault */
1245                 writel(DMA_FRCD_F, iommu->reg + reg +
1246                         fault_index * PRIMARY_FAULT_REG_LEN + 12);
1247
1248                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1249
1250                 dmar_fault_do_one(iommu, type, fault_reason,
1251                                 source_id, guest_addr);
1252
1253                 fault_index++;
1254                 if (fault_index >= cap_num_fault_regs(iommu->cap))
1255                         fault_index = 0;
1256                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1257         }
1258
1259         writel(DMA_FSTS_PFO | DMA_FSTS_PPF, iommu->reg + DMAR_FSTS_REG);
1260
1261 unlock_exit:
1262         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1263         return IRQ_HANDLED;
1264 }
1265
1266 int dmar_set_interrupt(struct intel_iommu *iommu)
1267 {
1268         int irq, ret;
1269
1270         /*
1271          * Check if the fault interrupt is already initialized.
1272          */
1273         if (iommu->irq)
1274                 return 0;
1275
1276         irq = create_irq();
1277         if (!irq) {
1278                 pr_err("IOMMU: no free vectors\n");
1279                 return -EINVAL;
1280         }
1281
1282         irq_set_handler_data(irq, iommu);
1283         iommu->irq = irq;
1284
1285         ret = arch_setup_dmar_msi(irq);
1286         if (ret) {
1287                 irq_set_handler_data(irq, NULL);
1288                 iommu->irq = 0;
1289                 destroy_irq(irq);
1290                 return ret;
1291         }
1292
1293         ret = request_irq(irq, dmar_fault, IRQF_NO_THREAD, iommu->name, iommu);
1294         if (ret)
1295                 pr_err("IOMMU: can't request irq\n");
1296         return ret;
1297 }
1298
1299 int __init enable_drhd_fault_handling(void)
1300 {
1301         struct dmar_drhd_unit *drhd;
1302
1303         /*
1304          * Enable fault control interrupt.
1305          */
1306         for_each_drhd_unit(drhd) {
1307                 int ret;
1308                 struct intel_iommu *iommu = drhd->iommu;
1309                 u32 fault_status;
1310                 ret = dmar_set_interrupt(iommu);
1311
1312                 if (ret) {
1313                         pr_err("DRHD %Lx: failed to enable fault, interrupt, ret %d\n",
1314                                (unsigned long long)drhd->reg_base_addr, ret);
1315                         return -1;
1316                 }
1317
1318                 /*
1319                  * Clear any previous faults.
1320                  */
1321                 dmar_fault(iommu->irq, iommu);
1322                 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1323                 writel(fault_status, iommu->reg + DMAR_FSTS_REG);
1324         }
1325
1326         return 0;
1327 }
1328
1329 /*
1330  * Re-enable Queued Invalidation interface.
1331  */
1332 int dmar_reenable_qi(struct intel_iommu *iommu)
1333 {
1334         if (!ecap_qis(iommu->ecap))
1335                 return -ENOENT;
1336
1337         if (!iommu->qi)
1338                 return -ENOENT;
1339
1340         /*
1341          * First disable queued invalidation.
1342          */
1343         dmar_disable_qi(iommu);
1344         /*
1345          * Then enable queued invalidation again. Since there is no pending
1346          * invalidation requests now, it's safe to re-enable queued
1347          * invalidation.
1348          */
1349         __dmar_enable_qi(iommu);
1350
1351         return 0;
1352 }
1353
1354 /*
1355  * Check interrupt remapping support in DMAR table description.
1356  */
1357 int __init dmar_ir_support(void)
1358 {
1359         struct acpi_table_dmar *dmar;
1360         dmar = (struct acpi_table_dmar *)dmar_tbl;
1361         if (!dmar)
1362                 return 0;
1363         return dmar->flags & 0x1;
1364 }
1365 IOMMU_INIT_POST(detect_intel_iommu);