drivers/pci/dmar.c

   1 /*
   2  * Copyright (c) 2006, Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along with
  14  * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  15  * Place - Suite 330, Boston, MA 02111-1307 USA.
  16  *
  17  * Copyright (C) 2006-2008 Intel Corporation
  18  * Author: Ashok Raj <ashok.raj@intel.com>
  19  * Author: Shaohua Li <shaohua.li@intel.com>
  20  * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
  21  *
  22  * This file implements early detection/parsing of Remapping Devices
  23  * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
  24  * tables.
  25  *
  26  * These routines are used by both DMA-remapping and Interrupt-remapping
  27  */
  28
  29 #include <linux/pci.h>
  30 #include <linux/dmar.h>
  31 #include <linux/iova.h>
  32 #include <linux/intel-iommu.h>
  33 #include <linux/timer.h>
  34 #include <linux/irq.h>
  35 #include <linux/interrupt.h>
  36 #include <linux/tboot.h>
  37 #include <linux/dmi.h>
  38
  39 #define PREFIX "DMAR: "
  40
  41 /* No locks are needed as DMA remapping hardware unit
  42  * list is constructed at boot time and hotplug of
  43  * these units are not supported by the architecture.
  44  */
  45 LIST_HEAD(dmar_drhd_units);
  46
  47 static struct acpi_table_header * __initdata dmar_tbl;
  48 static acpi_size dmar_tbl_size;
  49
  50 static void __init dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
  51 {
  52         /*
  53          * add INCLUDE_ALL at the tail, so scan the list will find it at
  54          * the very end.
  55          */
  56         if (drhd->include_all)
  57                 list_add_tail(&drhd->list, &dmar_drhd_units);
  58         else
  59                 list_add(&drhd->list, &dmar_drhd_units);
  60 }
  61
  62 static int __init dmar_parse_one_dev_scope(struct acpi_dmar_device_scope *scope,
  63                                            struct pci_dev **dev, u16 segment)
  64 {
  65         struct pci_bus *bus;
  66         struct pci_dev *pdev = NULL;
  67         struct acpi_dmar_pci_path *path;
  68         int count;
  69
  70         bus = pci_find_bus(segment, scope->bus);
  71         path = (struct acpi_dmar_pci_path *)(scope + 1);
  72         count = (scope->length - sizeof(struct acpi_dmar_device_scope))
  73                 / sizeof(struct acpi_dmar_pci_path);
  74
  75         while (count) {
  76                 if (pdev)
  77                         pci_dev_put(pdev);
  78                 /*
  79                  * Some BIOSes list non-exist devices in DMAR table, just
  80                  * ignore it
  81                  */
  82                 if (!bus) {
  83                         printk(KERN_WARNING
  84                         PREFIX "Device scope bus [%d] not found\n",
  85                         scope->bus);
  86                         break;
  87                 }
  88                 pdev = pci_get_slot(bus, PCI_DEVFN(path->dev, path->fn));
  89                 if (!pdev) {
  90                         printk(KERN_WARNING PREFIX
  91                         "Device scope device [%04x:%02x:%02x.%02x] not found\n",
  92                                 segment, bus->number, path->dev, path->fn);
  93                         break;
  94                 }
  95                 path ++;
  96                 count --;
  97                 bus = pdev->subordinate;
  98         }
  99         if (!pdev) {
 100                 printk(KERN_WARNING PREFIX
 101                 "Device scope device [%04x:%02x:%02x.%02x] not found\n",
 102                 segment, scope->bus, path->dev, path->fn);
 103                 *dev = NULL;
 104                 return 0;
 105         }
 106         if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT && \
 107                         pdev->subordinate) || (scope->entry_type == \
 108                         ACPI_DMAR_SCOPE_TYPE_BRIDGE && !pdev->subordinate)) {
 109                 pci_dev_put(pdev);
 110                 printk(KERN_WARNING PREFIX
 111                         "Device scope type does not match for %s\n",
 112                          pci_name(pdev));
 113                 return -EINVAL;
 114         }
 115         *dev = pdev;
 116         return 0;
 117 }
 118
 119 static int __init dmar_parse_dev_scope(void *start, void *end, int *cnt,
 120                                        struct pci_dev ***devices, u16 segment)
 121 {
 122         struct acpi_dmar_device_scope *scope;
 123         void * tmp = start;
 124         int index;
 125         int ret;
 126
 127         *cnt = 0;
 128         while (start < end) {
 129                 scope = start;
 130                 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
 131                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
 132                         (*cnt)++;
 133                 else if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_IOAPIC) {
 134                         printk(KERN_WARNING PREFIX
 135                                "Unsupported device scope\n");
 136                 }
 137                 start += scope->length;
 138         }
 139         if (*cnt == 0)
 140                 return 0;
 141
 142         *devices = kcalloc(*cnt, sizeof(struct pci_dev *), GFP_KERNEL);
 143         if (!*devices)
 144                 return -ENOMEM;
 145
 146         start = tmp;
 147         index = 0;
 148         while (start < end) {
 149                 scope = start;
 150                 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
 151                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE) {
 152                         ret = dmar_parse_one_dev_scope(scope,
 153                                 &(*devices)[index], segment);
 154                         if (ret) {
 155                                 kfree(*devices);
 156                                 return ret;
 157                         }
 158                         index ++;
 159                 }
 160                 start += scope->length;
 161         }
 162
 163         return 0;
 164 }
 165
 166 /**
 167  * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
 168  * structure which uniquely represent one DMA remapping hardware unit
 169  * present in the platform
 170  */
 171 static int __init
 172 dmar_parse_one_drhd(struct acpi_dmar_header *header)
 173 {
 174         struct acpi_dmar_hardware_unit *drhd;
 175         struct dmar_drhd_unit *dmaru;
 176         int ret = 0;
 177
 178         drhd = (struct acpi_dmar_hardware_unit *)header;
 179         dmaru = kzalloc(sizeof(*dmaru), GFP_KERNEL);
 180         if (!dmaru)
 181                 return -ENOMEM;
 182
 183         dmaru->hdr = header;
 184         dmaru->reg_base_addr = drhd->address;
 185         dmaru->segment = drhd->segment;
 186         dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
 187
 188         ret = alloc_iommu(dmaru);
 189         if (ret) {
 190                 kfree(dmaru);
 191                 return ret;
 192         }
 193         dmar_register_drhd_unit(dmaru);
 194         return 0;
 195 }
 196
 197 static int __init dmar_parse_dev(struct dmar_drhd_unit *dmaru)
 198 {
 199         struct acpi_dmar_hardware_unit *drhd;
 200         int ret = 0;
 201
 202         drhd = (struct acpi_dmar_hardware_unit *) dmaru->hdr;
 203
 204         if (dmaru->include_all)
 205                 return 0;
 206
 207         ret = dmar_parse_dev_scope((void *)(drhd + 1),
 208                                 ((void *)drhd) + drhd->header.length,
 209                                 &dmaru->devices_cnt, &dmaru->devices,
 210                                 drhd->segment);
 211         if (ret) {
 212                 list_del(&dmaru->list);
 213                 kfree(dmaru);
 214         }
 215         return ret;
 216 }
 217
 218 #ifdef CONFIG_DMAR
 219 LIST_HEAD(dmar_rmrr_units);
 220
 221 static void __init dmar_register_rmrr_unit(struct dmar_rmrr_unit *rmrr)
 222 {
 223         list_add(&rmrr->list, &dmar_rmrr_units);
 224 }
 225
 226
 227 static int __init
 228 dmar_parse_one_rmrr(struct acpi_dmar_header *header)
 229 {
 230         struct acpi_dmar_reserved_memory *rmrr;
 231         struct dmar_rmrr_unit *rmrru;
 232
 233         rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
 234         if (!rmrru)
 235                 return -ENOMEM;
 236
 237         rmrru->hdr = header;
 238         rmrr = (struct acpi_dmar_reserved_memory *)header;
 239         rmrru->base_address = rmrr->base_address;
 240         rmrru->end_address = rmrr->end_address;
 241
 242         dmar_register_rmrr_unit(rmrru);
 243         return 0;
 244 }
 245
 246 static int __init
 247 rmrr_parse_dev(struct dmar_rmrr_unit *rmrru)
 248 {
 249         struct acpi_dmar_reserved_memory *rmrr;
 250         int ret;
 251
 252         rmrr = (struct acpi_dmar_reserved_memory *) rmrru->hdr;
 253         ret = dmar_parse_dev_scope((void *)(rmrr + 1),
 254                 ((void *)rmrr) + rmrr->header.length,
 255                 &rmrru->devices_cnt, &rmrru->devices, rmrr->segment);
 256
 257         if (ret || (rmrru->devices_cnt == 0)) {
 258                 list_del(&rmrru->list);
 259                 kfree(rmrru);
 260         }
 261         return ret;
 262 }
 263
 264 static LIST_HEAD(dmar_atsr_units);
 265
 266 static int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr)
 267 {
 268         struct acpi_dmar_atsr *atsr;
 269         struct dmar_atsr_unit *atsru;
 270
 271         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
 272         atsru = kzalloc(sizeof(*atsru), GFP_KERNEL);
 273         if (!atsru)
 274                 return -ENOMEM;
 275
 276         atsru->hdr = hdr;
 277         atsru->include_all = atsr->flags & 0x1;
 278
 279         list_add(&atsru->list, &dmar_atsr_units);
 280
 281         return 0;
 282 }
 283
 284 static int __init atsr_parse_dev(struct dmar_atsr_unit *atsru)
 285 {
 286         int rc;
 287         struct acpi_dmar_atsr *atsr;
 288
 289         if (atsru->include_all)
 290                 return 0;
 291
 292         atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
 293         rc = dmar_parse_dev_scope((void *)(atsr + 1),
 294                                 (void *)atsr + atsr->header.length,
 295                                 &atsru->devices_cnt, &atsru->devices,
 296                                 atsr->segment);
 297         if (rc || !atsru->devices_cnt) {
 298                 list_del(&atsru->list);
 299                 kfree(atsru);
 300         }
 301
 302         return rc;
 303 }
 304
 305 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
 306 {
 307         int i;
 308         struct pci_bus *bus;
 309         struct acpi_dmar_atsr *atsr;
 310         struct dmar_atsr_unit *atsru;
 311
 312         list_for_each_entry(atsru, &dmar_atsr_units, list) {
 313                 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
 314                 if (atsr->segment == pci_domain_nr(dev->bus))
 315                         goto found;
 316         }
 317
 318         return 0;
 319
 320 found:
 321         for (bus = dev->bus; bus; bus = bus->parent) {
 322                 struct pci_dev *bridge = bus->self;
 323
 324                 if (!bridge || !pci_is_pcie(bridge) ||
 325                     bridge->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
 326                         return 0;
 327
 328                 if (bridge->pcie_type == PCI_EXP_TYPE_ROOT_PORT) {
 329                         for (i = 0; i < atsru->devices_cnt; i++)
 330                                 if (atsru->devices[i] == bridge)
 331                                         return 1;
 332                         break;
 333                 }
 334         }
 335
 336         if (atsru->include_all)
 337                 return 1;
 338
 339         return 0;
 340 }
 341 #endif
 342
 343 #ifdef CONFIG_ACPI_NUMA
 344 static int __init
 345 dmar_parse_one_rhsa(struct acpi_dmar_header *header)
 346 {
 347         struct acpi_dmar_rhsa *rhsa;
 348         struct dmar_drhd_unit *drhd;
 349
 350         rhsa = (struct acpi_dmar_rhsa *)header;
 351         for_each_drhd_unit(drhd) {
 352                 if (drhd->reg_base_addr == rhsa->base_address) {
 353                         int node = acpi_map_pxm_to_node(rhsa->proximity_domain);
 354
 355                         if (!node_online(node))
 356                                 node = -1;
 357                         drhd->iommu->node = node;
 358                         return 0;
 359                 }
 360         }
 361         WARN(1, "Your BIOS is broken; RHSA refers to non-existent DMAR unit at %llx\n"
 362              "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 363              drhd->reg_base_addr,
 364              dmi_get_system_info(DMI_BIOS_VENDOR),
 365              dmi_get_system_info(DMI_BIOS_VERSION),
 366              dmi_get_system_info(DMI_PRODUCT_VERSION));
 367
 368         return 0;
 369 }
 370 #endif
 371
 372 static void __init
 373 dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
 374 {
 375         struct acpi_dmar_hardware_unit *drhd;
 376         struct acpi_dmar_reserved_memory *rmrr;
 377         struct acpi_dmar_atsr *atsr;
 378         struct acpi_dmar_rhsa *rhsa;
 379
 380         switch (header->type) {
 381         case ACPI_DMAR_TYPE_HARDWARE_UNIT:
 382                 drhd = container_of(header, struct acpi_dmar_hardware_unit,
 383                                     header);
 384                 printk (KERN_INFO PREFIX
 385                         "DRHD base: %#016Lx flags: %#x\n",
 386                         (unsigned long long)drhd->address, drhd->flags);
 387                 break;
 388         case ACPI_DMAR_TYPE_RESERVED_MEMORY:
 389                 rmrr = container_of(header, struct acpi_dmar_reserved_memory,
 390                                     header);
 391                 printk (KERN_INFO PREFIX
 392                         "RMRR base: %#016Lx end: %#016Lx\n",
 393                         (unsigned long long)rmrr->base_address,
 394                         (unsigned long long)rmrr->end_address);
 395                 break;
 396         case ACPI_DMAR_TYPE_ATSR:
 397                 atsr = container_of(header, struct acpi_dmar_atsr, header);
 398                 printk(KERN_INFO PREFIX "ATSR flags: %#x\n", atsr->flags);
 399                 break;
 400         case ACPI_DMAR_HARDWARE_AFFINITY:
 401                 rhsa = container_of(header, struct acpi_dmar_rhsa, header);
 402                 printk(KERN_INFO PREFIX "RHSA base: %#016Lx proximity domain: %#x\n",
 403                        (unsigned long long)rhsa->base_address,
 404                        rhsa->proximity_domain);
 405                 break;
 406         }
 407 }
 408
 409 /**
 410  * dmar_table_detect - checks to see if the platform supports DMAR devices
 411  */
 412 static int __init dmar_table_detect(void)
 413 {
 414         acpi_status status = AE_OK;
 415
 416         /* if we could find DMAR table, then there are DMAR devices */
 417         status = acpi_get_table_with_size(ACPI_SIG_DMAR, 0,
 418                                 (struct acpi_table_header **)&dmar_tbl,
 419                                 &dmar_tbl_size);
 420
 421         if (ACPI_SUCCESS(status) && !dmar_tbl) {
 422                 printk (KERN_WARNING PREFIX "Unable to map DMAR\n");
 423                 status = AE_NOT_FOUND;
 424         }
 425
 426         return (ACPI_SUCCESS(status) ? 1 : 0);
 427 }
 428
 429 /**
 430  * parse_dmar_table - parses the DMA reporting table
 431  */
 432 static int __init
 433 parse_dmar_table(void)
 434 {
 435         struct acpi_table_dmar *dmar;
 436         struct acpi_dmar_header *entry_header;
 437         int ret = 0;
 438
 439         /*
 440          * Do it again, earlier dmar_tbl mapping could be mapped with
 441          * fixed map.
 442          */
 443         dmar_table_detect();
 444
 445         /*
 446          * ACPI tables may not be DMA protected by tboot, so use DMAR copy
 447          * SINIT saved in SinitMleData in TXT heap (which is DMA protected)
 448          */
 449         dmar_tbl = tboot_get_dmar_table(dmar_tbl);
 450
 451         dmar = (struct acpi_table_dmar *)dmar_tbl;
 452         if (!dmar)
 453                 return -ENODEV;
 454
 455         if (dmar->width < PAGE_SHIFT - 1) {
 456                 printk(KERN_WARNING PREFIX "Invalid DMAR haw\n");
 457                 return -EINVAL;
 458         }
 459
 460         printk (KERN_INFO PREFIX "Host address width %d\n",
 461                 dmar->width + 1);
 462
 463         entry_header = (struct acpi_dmar_header *)(dmar + 1);
 464         while (((unsigned long)entry_header) <
 465                         (((unsigned long)dmar) + dmar_tbl->length)) {
 466                 /* Avoid looping forever on bad ACPI tables */
 467                 if (entry_header->length == 0) {
 468                         printk(KERN_WARNING PREFIX
 469                                 "Invalid 0-length structure\n");
 470                         ret = -EINVAL;
 471                         break;
 472                 }
 473
 474                 dmar_table_print_dmar_entry(entry_header);
 475
 476                 switch (entry_header->type) {
 477                 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
 478                         ret = dmar_parse_one_drhd(entry_header);
 479                         break;
 480                 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
 481 #ifdef CONFIG_DMAR
 482                         ret = dmar_parse_one_rmrr(entry_header);
 483 #endif
 484                         break;
 485                 case ACPI_DMAR_TYPE_ATSR:
 486 #ifdef CONFIG_DMAR
 487                         ret = dmar_parse_one_atsr(entry_header);
 488 #endif
 489                         break;
 490                 case ACPI_DMAR_HARDWARE_AFFINITY:
 491 #ifdef CONFIG_ACPI_NUMA
 492                         ret = dmar_parse_one_rhsa(entry_header);
 493 #endif
 494                         break;
 495                 default:
 496                         printk(KERN_WARNING PREFIX
 497                                 "Unknown DMAR structure type %d\n",
 498                                 entry_header->type);
 499                         ret = 0; /* for forward compatibility */
 500                         break;
 501                 }
 502                 if (ret)
 503                         break;
 504
 505                 entry_header = ((void *)entry_header + entry_header->length);
 506         }
 507         return ret;
 508 }
 509
 510 int dmar_pci_device_match(struct pci_dev *devices[], int cnt,
 511                           struct pci_dev *dev)
 512 {
 513         int index;
 514
 515         while (dev) {
 516                 for (index = 0; index < cnt; index++)
 517                         if (dev == devices[index])
 518                                 return 1;
 519
 520                 /* Check our parent */
 521                 dev = dev->bus->self;
 522         }
 523
 524         return 0;
 525 }
 526
 527 struct dmar_drhd_unit *
 528 dmar_find_matched_drhd_unit(struct pci_dev *dev)
 529 {
 530         struct dmar_drhd_unit *dmaru = NULL;
 531         struct acpi_dmar_hardware_unit *drhd;
 532
 533         list_for_each_entry(dmaru, &dmar_drhd_units, list) {
 534                 drhd = container_of(dmaru->hdr,
 535                                     struct acpi_dmar_hardware_unit,
 536                                     header);
 537
 538                 if (dmaru->include_all &&
 539                     drhd->segment == pci_domain_nr(dev->bus))
 540                         return dmaru;
 541
 542                 if (dmar_pci_device_match(dmaru->devices,
 543                                           dmaru->devices_cnt, dev))
 544                         return dmaru;
 545         }
 546
 547         return NULL;
 548 }
 549
 550 int __init dmar_dev_scope_init(void)
 551 {
 552         struct dmar_drhd_unit *drhd, *drhd_n;
 553         int ret = -ENODEV;
 554
 555         list_for_each_entry_safe(drhd, drhd_n, &dmar_drhd_units, list) {
 556                 ret = dmar_parse_dev(drhd);
 557                 if (ret)
 558                         return ret;
 559         }
 560
 561 #ifdef CONFIG_DMAR
 562         {
 563                 struct dmar_rmrr_unit *rmrr, *rmrr_n;
 564                 struct dmar_atsr_unit *atsr, *atsr_n;
 565
 566                 list_for_each_entry_safe(rmrr, rmrr_n, &dmar_rmrr_units, list) {
 567                         ret = rmrr_parse_dev(rmrr);
 568                         if (ret)
 569                                 return ret;
 570                 }
 571
 572                 list_for_each_entry_safe(atsr, atsr_n, &dmar_atsr_units, list) {
 573                         ret = atsr_parse_dev(atsr);
 574                         if (ret)
 575                                 return ret;
 576                 }
 577         }
 578 #endif
 579
 580         return ret;
 581 }
 582
 583
 584 int __init dmar_table_init(void)
 585 {
 586         static int dmar_table_initialized;
 587         int ret;
 588
 589         if (dmar_table_initialized)
 590                 return 0;
 591
 592         dmar_table_initialized = 1;
 593
 594         ret = parse_dmar_table();
 595         if (ret) {
 596                 if (ret != -ENODEV)
 597                         printk(KERN_INFO PREFIX "parse DMAR table failure.\n");
 598                 return ret;
 599         }
 600
 601         if (list_empty(&dmar_drhd_units)) {
 602                 printk(KERN_INFO PREFIX "No DMAR devices found\n");
 603                 return -ENODEV;
 604         }
 605
 606 #ifdef CONFIG_DMAR
 607         if (list_empty(&dmar_rmrr_units))
 608                 printk(KERN_INFO PREFIX "No RMRR found\n");
 609
 610         if (list_empty(&dmar_atsr_units))
 611                 printk(KERN_INFO PREFIX "No ATSR found\n");
 612 #endif
 613
 614         return 0;
 615 }
 616
 617 static int bios_warned;
 618
 619 int __init check_zero_address(void)
 620 {
 621         struct acpi_table_dmar *dmar;
 622         struct acpi_dmar_header *entry_header;
 623         struct acpi_dmar_hardware_unit *drhd;
 624
 625         dmar = (struct acpi_table_dmar *)dmar_tbl;
 626         entry_header = (struct acpi_dmar_header *)(dmar + 1);
 627
 628         while (((unsigned long)entry_header) <
 629                         (((unsigned long)dmar) + dmar_tbl->length)) {
 630                 /* Avoid looping forever on bad ACPI tables */
 631                 if (entry_header->length == 0) {
 632                         printk(KERN_WARNING PREFIX
 633                                 "Invalid 0-length structure\n");
 634                         return 0;
 635                 }
 636
 637                 if (entry_header->type == ACPI_DMAR_TYPE_HARDWARE_UNIT) {
 638                         void __iomem *addr;
 639                         u64 cap, ecap;
 640
 641                         drhd = (void *)entry_header;
 642                         if (!drhd->address) {
 643                                 /* Promote an attitude of violence to a BIOS engineer today */
 644                                 WARN(1, "Your BIOS is broken; DMAR reported at address zero!\n"
 645                                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 646                                      dmi_get_system_info(DMI_BIOS_VENDOR),
 647                                      dmi_get_system_info(DMI_BIOS_VERSION),
 648                                      dmi_get_system_info(DMI_PRODUCT_VERSION));
 649                                 bios_warned = 1;
 650                                 goto failed;
 651                         }
 652
 653                         addr = early_ioremap(drhd->address, VTD_PAGE_SIZE);
 654                         if (!addr ) {
 655                                 printk("IOMMU: can't validate: %llx\n", drhd->address);
 656                                 goto failed;
 657                         }
 658                         cap = dmar_readq(addr + DMAR_CAP_REG);
 659                         ecap = dmar_readq(addr + DMAR_ECAP_REG);
 660                         early_iounmap(addr, VTD_PAGE_SIZE);
 661                         if (cap == (uint64_t)-1 && ecap == (uint64_t)-1) {
 662                                 /* Promote an attitude of violence to a BIOS engineer today */
 663                                 WARN(1, "Your BIOS is broken; DMAR reported at address %llx returns all ones!\n"
 664                                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 665                                       drhd->address,
 666                                       dmi_get_system_info(DMI_BIOS_VENDOR),
 667                                       dmi_get_system_info(DMI_BIOS_VERSION),
 668                                       dmi_get_system_info(DMI_PRODUCT_VERSION));
 669                                 bios_warned = 1;
 670                                 goto failed;
 671                         }
 672                 }
 673
 674                 entry_header = ((void *)entry_header + entry_header->length);
 675         }
 676         return 1;
 677
 678 failed:
 679 #ifdef CONFIG_DMAR
 680         dmar_disabled = 1;
 681 #endif
 682         return 0;
 683 }
 684
 685 void __init detect_intel_iommu(void)
 686 {
 687         int ret;
 688
 689         ret = dmar_table_detect();
 690         if (ret)
 691                 ret = check_zero_address();
 692         {
 693 #ifdef CONFIG_INTR_REMAP
 694                 struct acpi_table_dmar *dmar;
 695                 /*
 696                  * for now we will disable dma-remapping when interrupt
 697                  * remapping is enabled.
 698                  * When support for queued invalidation for IOTLB invalidation
 699                  * is added, we will not need this any more.
 700                  */
 701                 dmar = (struct acpi_table_dmar *) dmar_tbl;
 702                 if (ret && cpu_has_x2apic && dmar->flags & 0x1)
 703                         printk(KERN_INFO
 704                                "Queued invalidation will be enabled to support "
 705                                "x2apic and Intr-remapping.\n");
 706 #endif
 707 #ifdef CONFIG_DMAR
 708                 if (ret && !no_iommu && !iommu_detected && !dmar_disabled) {
 709                         iommu_detected = 1;
 710                         /* Make sure ACS will be enabled */
 711                         pci_request_acs();
 712                 }
 713 #endif
 714 #ifdef CONFIG_X86
 715                 if (ret)
 716                         x86_init.iommu.iommu_init = intel_iommu_init;
 717 #endif
 718         }
 719         early_acpi_os_unmap_memory(dmar_tbl, dmar_tbl_size);
 720         dmar_tbl = NULL;
 721 }
 722
 723
 724 int alloc_iommu(struct dmar_drhd_unit *drhd)
 725 {
 726         struct intel_iommu *iommu;
 727         int map_size;
 728         u32 ver;
 729         static int iommu_allocated = 0;
 730         int agaw = 0;
 731         int msagaw = 0;
 732
 733         if (!drhd->reg_base_addr) {
 734                 if (!bios_warned) {
 735                         WARN(1, "Your BIOS is broken; DMAR reported at address zero!\n"
 736                              "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 737                              dmi_get_system_info(DMI_BIOS_VENDOR),
 738                              dmi_get_system_info(DMI_BIOS_VERSION),
 739                              dmi_get_system_info(DMI_PRODUCT_VERSION));
 740                         bios_warned = 1;
 741                 }
 742                 return -EINVAL;
 743         }
 744
 745         iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
 746         if (!iommu)
 747                 return -ENOMEM;
 748
 749         iommu->seq_id = iommu_allocated++;
 750         sprintf (iommu->name, "dmar%d", iommu->seq_id);
 751
 752         iommu->reg = ioremap(drhd->reg_base_addr, VTD_PAGE_SIZE);
 753         if (!iommu->reg) {
 754                 printk(KERN_ERR "IOMMU: can't map the region\n");
 755                 goto error;
 756         }
 757         iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
 758         iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
 759
 760         if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {
 761                 if (!bios_warned) {
 762                         /* Promote an attitude of violence to a BIOS engineer today */
 763                         WARN(1, "Your BIOS is broken; DMAR reported at address %llx returns all ones!\n"
 764                              "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 765                              drhd->reg_base_addr,
 766                              dmi_get_system_info(DMI_BIOS_VENDOR),
 767                              dmi_get_system_info(DMI_BIOS_VERSION),
 768                              dmi_get_system_info(DMI_PRODUCT_VERSION));
 769                         bios_warned = 1;
 770                 }
 771                 goto err_unmap;
 772         }
 773
 774 #ifdef CONFIG_DMAR
 775         agaw = iommu_calculate_agaw(iommu);
 776         if (agaw < 0) {
 777                 printk(KERN_ERR
 778                        "Cannot get a valid agaw for iommu (seq_id = %d)\n",
 779                        iommu->seq_id);
 780                 goto err_unmap;
 781         }
 782         msagaw = iommu_calculate_max_sagaw(iommu);
 783         if (msagaw < 0) {
 784                 printk(KERN_ERR
 785                         "Cannot get a valid max agaw for iommu (seq_id = %d)\n",
 786                         iommu->seq_id);
 787                 goto err_unmap;
 788         }
 789 #endif
 790         iommu->agaw = agaw;
 791         iommu->msagaw = msagaw;
 792
 793         iommu->node = -1;
 794
 795         /* the registers might be more than one page */
 796         map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
 797                 cap_max_fault_reg_offset(iommu->cap));
 798         map_size = VTD_PAGE_ALIGN(map_size);
 799         if (map_size > VTD_PAGE_SIZE) {
 800                 iounmap(iommu->reg);
 801                 iommu->reg = ioremap(drhd->reg_base_addr, map_size);
 802                 if (!iommu->reg) {
 803                         printk(KERN_ERR "IOMMU: can't map the region\n");
 804                         goto error;
 805                 }
 806         }
 807
 808         ver = readl(iommu->reg + DMAR_VER_REG);
 809         pr_info("IOMMU %d: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n",
 810                 iommu->seq_id,
 811                 (unsigned long long)drhd->reg_base_addr,
 812                 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
 813                 (unsigned long long)iommu->cap,
 814                 (unsigned long long)iommu->ecap);
 815
 816         spin_lock_init(&iommu->register_lock);
 817
 818         drhd->iommu = iommu;
 819         return 0;
 820
 821  err_unmap:
 822         iounmap(iommu->reg);
 823  error:
 824         kfree(iommu);
 825         return -1;
 826 }
 827
 828 void free_iommu(struct intel_iommu *iommu)
 829 {
 830         if (!iommu)
 831                 return;
 832
 833 #ifdef CONFIG_DMAR
 834         free_dmar_iommu(iommu);
 835 #endif
 836
 837         if (iommu->reg)
 838                 iounmap(iommu->reg);
 839         kfree(iommu);
 840 }
 841
 842 /*
 843  * Reclaim all the submitted descriptors which have completed its work.
 844  */
 845 static inline void reclaim_free_desc(struct q_inval *qi)
 846 {
 847         while (qi->desc_status[qi->free_tail] == QI_DONE ||
 848                qi->desc_status[qi->free_tail] == QI_ABORT) {
 849                 qi->desc_status[qi->free_tail] = QI_FREE;
 850                 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
 851                 qi->free_cnt++;
 852         }
 853 }
 854
 855 static int qi_check_fault(struct intel_iommu *iommu, int index)
 856 {
 857         u32 fault;
 858         int head, tail;
 859         struct q_inval *qi = iommu->qi;
 860         int wait_index = (index + 1) % QI_LENGTH;
 861
 862         if (qi->desc_status[wait_index] == QI_ABORT)
 863                 return -EAGAIN;
 864
 865         fault = readl(iommu->reg + DMAR_FSTS_REG);
 866
 867         /*
 868          * If IQE happens, the head points to the descriptor associated
 869          * with the error. No new descriptors are fetched until the IQE
 870          * is cleared.
 871          */
 872         if (fault & DMA_FSTS_IQE) {
 873                 head = readl(iommu->reg + DMAR_IQH_REG);
 874                 if ((head >> DMAR_IQ_SHIFT) == index) {
 875                         printk(KERN_ERR "VT-d detected invalid descriptor: "
 876                                 "low=%llx, high=%llx\n",
 877                                 (unsigned long long)qi->desc[index].low,
 878                                 (unsigned long long)qi->desc[index].high);
 879                         memcpy(&qi->desc[index], &qi->desc[wait_index],
 880                                         sizeof(struct qi_desc));
 881                         __iommu_flush_cache(iommu, &qi->desc[index],
 882                                         sizeof(struct qi_desc));
 883                         writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
 884                         return -EINVAL;
 885                 }
 886         }
 887
 888         /*
 889          * If ITE happens, all pending wait_desc commands are aborted.
 890          * No new descriptors are fetched until the ITE is cleared.
 891          */
 892         if (fault & DMA_FSTS_ITE) {
 893                 head = readl(iommu->reg + DMAR_IQH_REG);
 894                 head = ((head >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
 895                 head |= 1;
 896                 tail = readl(iommu->reg + DMAR_IQT_REG);
 897                 tail = ((tail >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
 898
 899                 writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
 900
 901                 do {
 902                         if (qi->desc_status[head] == QI_IN_USE)
 903                                 qi->desc_status[head] = QI_ABORT;
 904                         head = (head - 2 + QI_LENGTH) % QI_LENGTH;
 905                 } while (head != tail);
 906
 907                 if (qi->desc_status[wait_index] == QI_ABORT)
 908                         return -EAGAIN;
 909         }
 910
 911         if (fault & DMA_FSTS_ICE)
 912                 writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
 913
 914         return 0;
 915 }
 916
 917 /*
 918  * Submit the queued invalidation descriptor to the remapping
 919  * hardware unit and wait for its completion.
 920  */
 921 int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
 922 {
 923         int rc;
 924         struct q_inval *qi = iommu->qi;
 925         struct qi_desc *hw, wait_desc;
 926         int wait_index, index;
 927         unsigned long flags;
 928
 929         if (!qi)
 930                 return 0;
 931
 932         hw = qi->desc;
 933
 934 restart:
 935         rc = 0;
 936
 937         spin_lock_irqsave(&qi->q_lock, flags);
 938         while (qi->free_cnt < 3) {
 939                 spin_unlock_irqrestore(&qi->q_lock, flags);
 940                 cpu_relax();
 941                 spin_lock_irqsave(&qi->q_lock, flags);
 942         }
 943
 944         index = qi->free_head;
 945         wait_index = (index + 1) % QI_LENGTH;
 946
 947         qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
 948
 949         hw[index] = *desc;
 950
 951         wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
 952                         QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
 953         wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
 954
 955         hw[wait_index] = wait_desc;
 956
 957         __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
 958         __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
 959
 960         qi->free_head = (qi->free_head + 2) % QI_LENGTH;
 961         qi->free_cnt -= 2;
 962
 963         /*
 964          * update the HW tail register indicating the presence of
 965          * new descriptors.
 966          */
 967         writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg + DMAR_IQT_REG);
 968
 969         while (qi->desc_status[wait_index] != QI_DONE) {
 970                 /*
 971                  * We will leave the interrupts disabled, to prevent interrupt
 972                  * context to queue another cmd while a cmd is already submitted
 973                  * and waiting for completion on this cpu. This is to avoid
 974                  * a deadlock where the interrupt context can wait indefinitely
 975                  * for free slots in the queue.
 976                  */
 977                 rc = qi_check_fault(iommu, index);
 978                 if (rc)
 979                         break;
 980
 981                 spin_unlock(&qi->q_lock);
 982                 cpu_relax();
 983                 spin_lock(&qi->q_lock);
 984         }
 985
 986         qi->desc_status[index] = QI_DONE;
 987
 988         reclaim_free_desc(qi);
 989         spin_unlock_irqrestore(&qi->q_lock, flags);
 990
 991         if (rc == -EAGAIN)
 992                 goto restart;
 993
 994         return rc;
 995 }
 996
 997 /*
 998  * Flush the global interrupt entry cache.
 999  */
1000 void qi_global_iec(struct intel_iommu *iommu)
1001 {
1002         struct qi_desc desc;
1003
1004         desc.low = QI_IEC_TYPE;
1005         desc.high = 0;
1006
1007         /* should never fail */
1008         qi_submit_sync(&desc, iommu);
1009 }
1010
1011 void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
1012                       u64 type)
1013 {
1014         struct qi_desc desc;
1015
1016         desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
1017                         | QI_CC_GRAN(type) | QI_CC_TYPE;
1018         desc.high = 0;
1019
1020         qi_submit_sync(&desc, iommu);
1021 }
1022
1023 void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
1024                     unsigned int size_order, u64 type)
1025 {
1026         u8 dw = 0, dr = 0;
1027
1028         struct qi_desc desc;
1029         int ih = 0;
1030
1031         if (cap_write_drain(iommu->cap))
1032                 dw = 1;
1033
1034         if (cap_read_drain(iommu->cap))
1035                 dr = 1;
1036
1037         desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
1038                 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
1039         desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
1040                 | QI_IOTLB_AM(size_order);
1041
1042         qi_submit_sync(&desc, iommu);
1043 }
1044
1045 void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep,
1046                         u64 addr, unsigned mask)
1047 {
1048         struct qi_desc desc;
1049
1050         if (mask) {
1051                 BUG_ON(addr & ((1 << (VTD_PAGE_SHIFT + mask)) - 1));
1052                 addr |= (1 << (VTD_PAGE_SHIFT + mask - 1)) - 1;
1053                 desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
1054         } else
1055                 desc.high = QI_DEV_IOTLB_ADDR(addr);
1056
1057         if (qdep >= QI_DEV_IOTLB_MAX_INVS)
1058                 qdep = 0;
1059
1060         desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
1061                    QI_DIOTLB_TYPE;
1062
1063         qi_submit_sync(&desc, iommu);
1064 }
1065
1066 /*
1067  * Disable Queued Invalidation interface.
1068  */
1069 void dmar_disable_qi(struct intel_iommu *iommu)
1070 {
1071         unsigned long flags;
1072         u32 sts;
1073         cycles_t start_time = get_cycles();
1074
1075         if (!ecap_qis(iommu->ecap))
1076                 return;
1077
1078         spin_lock_irqsave(&iommu->register_lock, flags);
1079
1080         sts =  dmar_readq(iommu->reg + DMAR_GSTS_REG);
1081         if (!(sts & DMA_GSTS_QIES))
1082                 goto end;
1083
1084         /*
1085          * Give a chance to HW to complete the pending invalidation requests.
1086          */
1087         while ((readl(iommu->reg + DMAR_IQT_REG) !=
1088                 readl(iommu->reg + DMAR_IQH_REG)) &&
1089                 (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
1090                 cpu_relax();
1091
1092         iommu->gcmd &= ~DMA_GCMD_QIE;
1093         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1094
1095         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
1096                       !(sts & DMA_GSTS_QIES), sts);
1097 end:
1098         spin_unlock_irqrestore(&iommu->register_lock, flags);
1099 }
1100
1101 /*
1102  * Enable queued invalidation.
1103  */
1104 static void __dmar_enable_qi(struct intel_iommu *iommu)
1105 {
1106         u32 sts;
1107         unsigned long flags;
1108         struct q_inval *qi = iommu->qi;
1109
1110         qi->free_head = qi->free_tail = 0;
1111         qi->free_cnt = QI_LENGTH;
1112
1113         spin_lock_irqsave(&iommu->register_lock, flags);
1114
1115         /* write zero to the tail reg */
1116         writel(0, iommu->reg + DMAR_IQT_REG);
1117
1118         dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
1119
1120         iommu->gcmd |= DMA_GCMD_QIE;
1121         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1122
1123         /* Make sure hardware complete it */
1124         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
1125
1126         spin_unlock_irqrestore(&iommu->register_lock, flags);
1127 }
1128
1129 /*
1130  * Enable Queued Invalidation interface. This is a must to support
1131  * interrupt-remapping. Also used by DMA-remapping, which replaces
1132  * register based IOTLB invalidation.
1133  */
1134 int dmar_enable_qi(struct intel_iommu *iommu)
1135 {
1136         struct q_inval *qi;
1137         struct page *desc_page;
1138
1139         if (!ecap_qis(iommu->ecap))
1140                 return -ENOENT;
1141
1142         /*
1143          * queued invalidation is already setup and enabled.
1144          */
1145         if (iommu->qi)
1146                 return 0;
1147
1148         iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
1149         if (!iommu->qi)
1150                 return -ENOMEM;
1151
1152         qi = iommu->qi;
1153
1154
1155         desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, 0);
1156         if (!desc_page) {
1157                 kfree(qi);
1158                 iommu->qi = 0;
1159                 return -ENOMEM;
1160         }
1161
1162         qi->desc = page_address(desc_page);
1163
1164         qi->desc_status = kmalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC);
1165         if (!qi->desc_status) {
1166                 free_page((unsigned long) qi->desc);
1167                 kfree(qi);
1168                 iommu->qi = 0;
1169                 return -ENOMEM;
1170         }
1171
1172         qi->free_head = qi->free_tail = 0;
1173         qi->free_cnt = QI_LENGTH;
1174
1175         spin_lock_init(&qi->q_lock);
1176
1177         __dmar_enable_qi(iommu);
1178
1179         return 0;
1180 }
1181
1182 /* iommu interrupt handling. Most stuff are MSI-like. */
1183
1184 enum faulttype {
1185         DMA_REMAP,
1186         INTR_REMAP,
1187         UNKNOWN,
1188 };
1189
1190 static const char *dma_remap_fault_reasons[] =
1191 {
1192         "Software",
1193         "Present bit in root entry is clear",
1194         "Present bit in context entry is clear",
1195         "Invalid context entry",
1196         "Access beyond MGAW",
1197         "PTE Write access is not set",
1198         "PTE Read access is not set",
1199         "Next page table ptr is invalid",
1200         "Root table address invalid",
1201         "Context table ptr is invalid",
1202         "non-zero reserved fields in RTP",
1203         "non-zero reserved fields in CTP",
1204         "non-zero reserved fields in PTE",
1205 };
1206
1207 static const char *intr_remap_fault_reasons[] =
1208 {
1209         "Detected reserved fields in the decoded interrupt-remapped request",
1210         "Interrupt index exceeded the interrupt-remapping table size",
1211         "Present field in the IRTE entry is clear",
1212         "Error accessing interrupt-remapping table pointed by IRTA_REG",
1213         "Detected reserved fields in the IRTE entry",
1214         "Blocked a compatibility format interrupt request",
1215         "Blocked an interrupt request due to source-id verification failure",
1216 };
1217
1218 #define MAX_FAULT_REASON_IDX    (ARRAY_SIZE(fault_reason_strings) - 1)
1219
1220 const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
1221 {
1222         if (fault_reason >= 0x20 && (fault_reason <= 0x20 +
1223                                      ARRAY_SIZE(intr_remap_fault_reasons))) {
1224                 *fault_type = INTR_REMAP;
1225                 return intr_remap_fault_reasons[fault_reason - 0x20];
1226         } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1227                 *fault_type = DMA_REMAP;
1228                 return dma_remap_fault_reasons[fault_reason];
1229         } else {
1230                 *fault_type = UNKNOWN;
1231                 return "Unknown";
1232         }
1233 }
1234
1235 void dmar_msi_unmask(unsigned int irq)
1236 {
1237         struct intel_iommu *iommu = get_irq_data(irq);
1238         unsigned long flag;
1239
1240         /* unmask it */
1241         spin_lock_irqsave(&iommu->register_lock, flag);
1242         writel(0, iommu->reg + DMAR_FECTL_REG);
1243         /* Read a reg to force flush the post write */
1244         readl(iommu->reg + DMAR_FECTL_REG);
1245         spin_unlock_irqrestore(&iommu->register_lock, flag);
1246 }
1247
1248 void dmar_msi_mask(unsigned int irq)
1249 {
1250         unsigned long flag;
1251         struct intel_iommu *iommu = get_irq_data(irq);
1252
1253         /* mask it */
1254         spin_lock_irqsave(&iommu->register_lock, flag);
1255         writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG);
1256         /* Read a reg to force flush the post write */
1257         readl(iommu->reg + DMAR_FECTL_REG);
1258         spin_unlock_irqrestore(&iommu->register_lock, flag);
1259 }
1260
1261 void dmar_msi_write(int irq, struct msi_msg *msg)
1262 {
1263         struct intel_iommu *iommu = get_irq_data(irq);
1264         unsigned long flag;
1265
1266         spin_lock_irqsave(&iommu->register_lock, flag);
1267         writel(msg->data, iommu->reg + DMAR_FEDATA_REG);
1268         writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG);
1269         writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG);
1270         spin_unlock_irqrestore(&iommu->register_lock, flag);
1271 }
1272
1273 void dmar_msi_read(int irq, struct msi_msg *msg)
1274 {
1275         struct intel_iommu *iommu = get_irq_data(irq);
1276         unsigned long flag;
1277
1278         spin_lock_irqsave(&iommu->register_lock, flag);
1279         msg->data = readl(iommu->reg + DMAR_FEDATA_REG);
1280         msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG);
1281         msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG);
1282         spin_unlock_irqrestore(&iommu->register_lock, flag);
1283 }
1284
1285 static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1286                 u8 fault_reason, u16 source_id, unsigned long long addr)
1287 {
1288         const char *reason;
1289         int fault_type;
1290
1291         reason = dmar_get_fault_reason(fault_reason, &fault_type);
1292
1293         if (fault_type == INTR_REMAP)
1294                 printk(KERN_ERR "INTR-REMAP: Request device [[%02x:%02x.%d] "
1295                        "fault index %llx\n"
1296                         "INTR-REMAP:[fault reason %02d] %s\n",
1297                         (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1298                         PCI_FUNC(source_id & 0xFF), addr >> 48,
1299                         fault_reason, reason);
1300         else
1301                 printk(KERN_ERR
1302                        "DMAR:[%s] Request device [%02x:%02x.%d] "
1303                        "fault addr %llx \n"
1304                        "DMAR:[fault reason %02d] %s\n",
1305                        (type ? "DMA Read" : "DMA Write"),
1306                        (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1307                        PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
1308         return 0;
1309 }
1310
1311 #define PRIMARY_FAULT_REG_LEN (16)
1312 irqreturn_t dmar_fault(int irq, void *dev_id)
1313 {
1314         struct intel_iommu *iommu = dev_id;
1315         int reg, fault_index;
1316         u32 fault_status;
1317         unsigned long flag;
1318
1319         spin_lock_irqsave(&iommu->register_lock, flag);
1320         fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1321         if (fault_status)
1322                 printk(KERN_ERR "DRHD: handling fault status reg %x\n",
1323                        fault_status);
1324
1325         /* TBD: ignore advanced fault log currently */
1326         if (!(fault_status & DMA_FSTS_PPF))
1327                 goto clear_rest;
1328
1329         fault_index = dma_fsts_fault_record_index(fault_status);
1330         reg = cap_fault_reg_offset(iommu->cap);
1331         while (1) {
1332                 u8 fault_reason;
1333                 u16 source_id;
1334                 u64 guest_addr;
1335                 int type;
1336                 u32 data;
1337
1338                 /* highest 32 bits */
1339                 data = readl(iommu->reg + reg +
1340                                 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1341                 if (!(data & DMA_FRCD_F))
1342                         break;
1343
1344                 fault_reason = dma_frcd_fault_reason(data);
1345                 type = dma_frcd_type(data);
1346
1347                 data = readl(iommu->reg + reg +
1348                                 fault_index * PRIMARY_FAULT_REG_LEN + 8);
1349                 source_id = dma_frcd_source_id(data);
1350
1351                 guest_addr = dmar_readq(iommu->reg + reg +
1352                                 fault_index * PRIMARY_FAULT_REG_LEN);
1353                 guest_addr = dma_frcd_page_addr(guest_addr);
1354                 /* clear the fault */
1355                 writel(DMA_FRCD_F, iommu->reg + reg +
1356                         fault_index * PRIMARY_FAULT_REG_LEN + 12);
1357
1358                 spin_unlock_irqrestore(&iommu->register_lock, flag);
1359
1360                 dmar_fault_do_one(iommu, type, fault_reason,
1361                                 source_id, guest_addr);
1362
1363                 fault_index++;
1364                 if (fault_index >= cap_num_fault_regs(iommu->cap))
1365                         fault_index = 0;
1366                 spin_lock_irqsave(&iommu->register_lock, flag);
1367         }
1368 clear_rest:
1369         /* clear all the other faults */
1370         fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1371         writel(fault_status, iommu->reg + DMAR_FSTS_REG);
1372
1373         spin_unlock_irqrestore(&iommu->register_lock, flag);
1374         return IRQ_HANDLED;
1375 }
1376
1377 int dmar_set_interrupt(struct intel_iommu *iommu)
1378 {
1379         int irq, ret;
1380
1381         /*
1382          * Check if the fault interrupt is already initialized.
1383          */
1384         if (iommu->irq)
1385                 return 0;
1386
1387         irq = create_irq();
1388         if (!irq) {
1389                 printk(KERN_ERR "IOMMU: no free vectors\n");
1390                 return -EINVAL;
1391         }
1392
1393         set_irq_data(irq, iommu);
1394         iommu->irq = irq;
1395
1396         ret = arch_setup_dmar_msi(irq);
1397         if (ret) {
1398                 set_irq_data(irq, NULL);
1399                 iommu->irq = 0;
1400                 destroy_irq(irq);
1401                 return ret;
1402         }
1403
1404         ret = request_irq(irq, dmar_fault, 0, iommu->name, iommu);
1405         if (ret)
1406                 printk(KERN_ERR "IOMMU: can't request irq\n");
1407         return ret;
1408 }
1409
1410 int __init enable_drhd_fault_handling(void)
1411 {
1412         struct dmar_drhd_unit *drhd;
1413
1414         /*
1415          * Enable fault control interrupt.
1416          */
1417         for_each_drhd_unit(drhd) {
1418                 int ret;
1419                 struct intel_iommu *iommu = drhd->iommu;
1420                 ret = dmar_set_interrupt(iommu);
1421
1422                 if (ret) {
1423                         printk(KERN_ERR "DRHD %Lx: failed to enable fault, "
1424                                " interrupt, ret %d\n",
1425                                (unsigned long long)drhd->reg_base_addr, ret);
1426                         return -1;
1427                 }
1428         }
1429
1430         return 0;
1431 }
1432
1433 /*
1434  * Re-enable Queued Invalidation interface.
1435  */
1436 int dmar_reenable_qi(struct intel_iommu *iommu)
1437 {
1438         if (!ecap_qis(iommu->ecap))
1439                 return -ENOENT;
1440
1441         if (!iommu->qi)
1442                 return -ENOENT;
1443
1444         /*
1445          * First disable queued invalidation.
1446          */
1447         dmar_disable_qi(iommu);
1448         /*
1449          * Then enable queued invalidation again. Since there is no pending
1450          * invalidation requests now, it's safe to re-enable queued
1451          * invalidation.
1452          */
1453         __dmar_enable_qi(iommu);
1454
1455         return 0;
1456 }
1457
1458 /*
1459  * Check interrupt remapping support in DMAR table description.
1460  */
1461 int __init dmar_ir_support(void)
1462 {
1463         struct acpi_table_dmar *dmar;
1464         dmar = (struct acpi_table_dmar *)dmar_tbl;
1465         return dmar->flags & 0x1;
1466 }