drivers/iommu/intel-iommu.c

   1 /*
   2  * Copyright © 2006-2014 Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * Authors: David Woodhouse <dwmw2@infradead.org>,
  14  *          Ashok Raj <ashok.raj@intel.com>,
  15  *          Shaohua Li <shaohua.li@intel.com>,
  16  *          Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
  17  *          Fenghua Yu <fenghua.yu@intel.com>
  18  *          Joerg Roedel <jroedel@suse.de>
  19  */
  20
  21 #define pr_fmt(fmt)     "DMAR: " fmt
  22
  23 #include <linux/init.h>
  24 #include <linux/bitmap.h>
  25 #include <linux/debugfs.h>
  26 #include <linux/export.h>
  27 #include <linux/slab.h>
  28 #include <linux/irq.h>
  29 #include <linux/interrupt.h>
  30 #include <linux/spinlock.h>
  31 #include <linux/pci.h>
  32 #include <linux/dmar.h>
  33 #include <linux/dma-mapping.h>
  34 #include <linux/mempool.h>
  35 #include <linux/memory.h>
  36 #include <linux/timer.h>
  37 #include <linux/iova.h>
  38 #include <linux/iommu.h>
  39 #include <linux/intel-iommu.h>
  40 #include <linux/syscore_ops.h>
  41 #include <linux/tboot.h>
  42 #include <linux/dmi.h>
  43 #include <linux/pci-ats.h>
  44 #include <linux/memblock.h>
  45 #include <linux/dma-contiguous.h>
  46 #include <linux/crash_dump.h>
  47 #include <asm/irq_remapping.h>
  48 #include <asm/cacheflush.h>
  49 #include <asm/iommu.h>
  50
  51 #include "irq_remapping.h"
  52
  53 #define ROOT_SIZE               VTD_PAGE_SIZE
  54 #define CONTEXT_SIZE            VTD_PAGE_SIZE
  55
  56 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
  57 #define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
  58 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
  59 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
  60
  61 #define IOAPIC_RANGE_START      (0xfee00000)
  62 #define IOAPIC_RANGE_END        (0xfeefffff)
  63 #define IOVA_START_ADDR         (0x1000)
  64
  65 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
  66
  67 #define MAX_AGAW_WIDTH 64
  68 #define MAX_AGAW_PFN_WIDTH      (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
  69
  70 #define __DOMAIN_MAX_PFN(gaw)  ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
  71 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
  72
  73 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
  74    to match. That way, we can use 'unsigned long' for PFNs with impunity. */
  75 #define DOMAIN_MAX_PFN(gaw)     ((unsigned long) min_t(uint64_t, \
  76                                 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
  77 #define DOMAIN_MAX_ADDR(gaw)    (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
  78
  79 /* IO virtual address start page frame number */
  80 #define IOVA_START_PFN          (1)
  81
  82 #define IOVA_PFN(addr)          ((addr) >> PAGE_SHIFT)
  83 #define DMA_32BIT_PFN           IOVA_PFN(DMA_BIT_MASK(32))
  84 #define DMA_64BIT_PFN           IOVA_PFN(DMA_BIT_MASK(64))
  85
  86 /* page table handling */
  87 #define LEVEL_STRIDE            (9)
  88 #define LEVEL_MASK              (((u64)1 << LEVEL_STRIDE) - 1)
  89
  90 /*
  91  * This bitmap is used to advertise the page sizes our hardware support
  92  * to the IOMMU core, which will then use this information to split
  93  * physically contiguous memory regions it is mapping into page sizes
  94  * that we support.
  95  *
  96  * Traditionally the IOMMU core just handed us the mappings directly,
  97  * after making sure the size is an order of a 4KiB page and that the
  98  * mapping has natural alignment.
  99  *
 100  * To retain this behavior, we currently advertise that we support
 101  * all page sizes that are an order of 4KiB.
 102  *
 103  * If at some point we'd like to utilize the IOMMU core's new behavior,
 104  * we could change this to advertise the real page sizes we support.
 105  */
 106 #define INTEL_IOMMU_PGSIZES     (~0xFFFUL)
 107
 108 static inline int agaw_to_level(int agaw)
 109 {
 110         return agaw + 2;
 111 }
 112
 113 static inline int agaw_to_width(int agaw)
 114 {
 115         return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
 116 }
 117
 118 static inline int width_to_agaw(int width)
 119 {
 120         return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
 121 }
 122
 123 static inline unsigned int level_to_offset_bits(int level)
 124 {
 125         return (level - 1) * LEVEL_STRIDE;
 126 }
 127
 128 static inline int pfn_level_offset(unsigned long pfn, int level)
 129 {
 130         return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
 131 }
 132
 133 static inline unsigned long level_mask(int level)
 134 {
 135         return -1UL << level_to_offset_bits(level);
 136 }
 137
 138 static inline unsigned long level_size(int level)
 139 {
 140         return 1UL << level_to_offset_bits(level);
 141 }
 142
 143 static inline unsigned long align_to_level(unsigned long pfn, int level)
 144 {
 145         return (pfn + level_size(level) - 1) & level_mask(level);
 146 }
 147
 148 static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
 149 {
 150         return  1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
 151 }
 152
 153 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
 154    are never going to work. */
 155 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
 156 {
 157         return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
 158 }
 159
 160 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
 161 {
 162         return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
 163 }
 164 static inline unsigned long page_to_dma_pfn(struct page *pg)
 165 {
 166         return mm_to_dma_pfn(page_to_pfn(pg));
 167 }
 168 static inline unsigned long virt_to_dma_pfn(void *p)
 169 {
 170         return page_to_dma_pfn(virt_to_page(p));
 171 }
 172
 173 /* global iommu list, set NULL for ignored DMAR units */
 174 static struct intel_iommu **g_iommus;
 175
 176 static void __init check_tylersburg_isoch(void);
 177 static int rwbf_quirk;
 178
 179 /*
 180  * set to 1 to panic kernel if can't successfully enable VT-d
 181  * (used when kernel is launched w/ TXT)
 182  */
 183 static int force_on = 0;
 184
 185 /*
 186  * 0: Present
 187  * 1-11: Reserved
 188  * 12-63: Context Ptr (12 - (haw-1))
 189  * 64-127: Reserved
 190  */
 191 struct root_entry {
 192         u64     lo;
 193         u64     hi;
 194 };
 195 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
 196
 197 /*
 198  * Take a root_entry and return the Lower Context Table Pointer (LCTP)
 199  * if marked present.
 200  */
 201 static phys_addr_t root_entry_lctp(struct root_entry *re)
 202 {
 203         if (!(re->lo & 1))
 204                 return 0;
 205
 206         return re->lo & VTD_PAGE_MASK;
 207 }
 208
 209 /*
 210  * Take a root_entry and return the Upper Context Table Pointer (UCTP)
 211  * if marked present.
 212  */
 213 static phys_addr_t root_entry_uctp(struct root_entry *re)
 214 {
 215         if (!(re->hi & 1))
 216                 return 0;
 217
 218         return re->hi & VTD_PAGE_MASK;
 219 }
 220 /*
 221  * low 64 bits:
 222  * 0: present
 223  * 1: fault processing disable
 224  * 2-3: translation type
 225  * 12-63: address space root
 226  * high 64 bits:
 227  * 0-2: address width
 228  * 3-6: aval
 229  * 8-23: domain id
 230  */
 231 struct context_entry {
 232         u64 lo;
 233         u64 hi;
 234 };
 235
 236 static inline void context_clear_pasid_enable(struct context_entry *context)
 237 {
 238         context->lo &= ~(1ULL << 11);
 239 }
 240
 241 static inline bool context_pasid_enabled(struct context_entry *context)
 242 {
 243         return !!(context->lo & (1ULL << 11));
 244 }
 245
 246 static inline void context_set_copied(struct context_entry *context)
 247 {
 248         context->hi |= (1ull << 3);
 249 }
 250
 251 static inline bool context_copied(struct context_entry *context)
 252 {
 253         return !!(context->hi & (1ULL << 3));
 254 }
 255
 256 static inline bool __context_present(struct context_entry *context)
 257 {
 258         return (context->lo & 1);
 259 }
 260
 261 static inline bool context_present(struct context_entry *context)
 262 {
 263         return context_pasid_enabled(context) ?
 264              __context_present(context) :
 265              __context_present(context) && !context_copied(context);
 266 }
 267
 268 static inline void context_set_present(struct context_entry *context)
 269 {
 270         context->lo |= 1;
 271 }
 272
 273 static inline void context_set_fault_enable(struct context_entry *context)
 274 {
 275         context->lo &= (((u64)-1) << 2) | 1;
 276 }
 277
 278 static inline void context_set_translation_type(struct context_entry *context,
 279                                                 unsigned long value)
 280 {
 281         context->lo &= (((u64)-1) << 4) | 3;
 282         context->lo |= (value & 3) << 2;
 283 }
 284
 285 static inline void context_set_address_root(struct context_entry *context,
 286                                             unsigned long value)
 287 {
 288         context->lo &= ~VTD_PAGE_MASK;
 289         context->lo |= value & VTD_PAGE_MASK;
 290 }
 291
 292 static inline void context_set_address_width(struct context_entry *context,
 293                                              unsigned long value)
 294 {
 295         context->hi |= value & 7;
 296 }
 297
 298 static inline void context_set_domain_id(struct context_entry *context,
 299                                          unsigned long value)
 300 {
 301         context->hi |= (value & ((1 << 16) - 1)) << 8;
 302 }
 303
 304 static inline int context_domain_id(struct context_entry *c)
 305 {
 306         return((c->hi >> 8) & 0xffff);
 307 }
 308
 309 static inline void context_clear_entry(struct context_entry *context)
 310 {
 311         context->lo = 0;
 312         context->hi = 0;
 313 }
 314
 315 /*
 316  * 0: readable
 317  * 1: writable
 318  * 2-6: reserved
 319  * 7: super page
 320  * 8-10: available
 321  * 11: snoop behavior
 322  * 12-63: Host physcial address
 323  */
 324 struct dma_pte {
 325         u64 val;
 326 };
 327
 328 static inline void dma_clear_pte(struct dma_pte *pte)
 329 {
 330         pte->val = 0;
 331 }
 332
 333 static inline u64 dma_pte_addr(struct dma_pte *pte)
 334 {
 335 #ifdef CONFIG_64BIT
 336         return pte->val & VTD_PAGE_MASK;
 337 #else
 338         /* Must have a full atomic 64-bit read */
 339         return  __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
 340 #endif
 341 }
 342
 343 static inline bool dma_pte_present(struct dma_pte *pte)
 344 {
 345         return (pte->val & 3) != 0;
 346 }
 347
 348 static inline bool dma_pte_superpage(struct dma_pte *pte)
 349 {
 350         return (pte->val & DMA_PTE_LARGE_PAGE);
 351 }
 352
 353 static inline int first_pte_in_page(struct dma_pte *pte)
 354 {
 355         return !((unsigned long)pte & ~VTD_PAGE_MASK);
 356 }
 357
 358 /*
 359  * This domain is a statically identity mapping domain.
 360  *      1. This domain creats a static 1:1 mapping to all usable memory.
 361  *      2. It maps to each iommu if successful.
 362  *      3. Each iommu mapps to this domain if successful.
 363  */
 364 static struct dmar_domain *si_domain;
 365 static int hw_pass_through = 1;
 366
 367 /* domain represents a virtual machine, more than one devices
 368  * across iommus may be owned in one domain, e.g. kvm guest.
 369  */
 370 #define DOMAIN_FLAG_VIRTUAL_MACHINE     (1 << 0)
 371
 372 /* si_domain contains mulitple devices */
 373 #define DOMAIN_FLAG_STATIC_IDENTITY     (1 << 1)
 374
 375 struct dmar_domain {
 376         int     id;                     /* domain id */
 377         int     nid;                    /* node id */
 378         DECLARE_BITMAP(iommu_bmp, DMAR_UNITS_SUPPORTED);
 379                                         /* bitmap of iommus this domain uses*/
 380
 381         struct list_head devices;       /* all devices' list */
 382         struct iova_domain iovad;       /* iova's that belong to this domain */
 383
 384         struct dma_pte  *pgd;           /* virtual address */
 385         int             gaw;            /* max guest address width */
 386
 387         /* adjusted guest address width, 0 is level 2 30-bit */
 388         int             agaw;
 389
 390         int             flags;          /* flags to find out type of domain */
 391
 392         int             iommu_coherency;/* indicate coherency of iommu access */
 393         int             iommu_snooping; /* indicate snooping control feature*/
 394         int             iommu_count;    /* reference count of iommu */
 395         int             iommu_superpage;/* Level of superpages supported:
 396                                            0 == 4KiB (no superpages), 1 == 2MiB,
 397                                            2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
 398         spinlock_t      iommu_lock;     /* protect iommu set in domain */
 399         u64             max_addr;       /* maximum mapped address */
 400
 401         struct iommu_domain domain;     /* generic domain data structure for
 402                                            iommu core */
 403 };
 404
 405 /* PCI domain-device relationship */
 406 struct device_domain_info {
 407         struct list_head link;  /* link to domain siblings */
 408         struct list_head global; /* link to global list */
 409         u8 bus;                 /* PCI bus number */
 410         u8 devfn;               /* PCI devfn number */
 411         struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
 412         struct intel_iommu *iommu; /* IOMMU used by this device */
 413         struct dmar_domain *domain; /* pointer to domain */
 414 };
 415
 416 struct dmar_rmrr_unit {
 417         struct list_head list;          /* list of rmrr units   */
 418         struct acpi_dmar_header *hdr;   /* ACPI header          */
 419         u64     base_address;           /* reserved base address*/
 420         u64     end_address;            /* reserved end address */
 421         struct dmar_dev_scope *devices; /* target devices */
 422         int     devices_cnt;            /* target device count */
 423 };
 424
 425 struct dmar_atsr_unit {
 426         struct list_head list;          /* list of ATSR units */
 427         struct acpi_dmar_header *hdr;   /* ACPI header */
 428         struct dmar_dev_scope *devices; /* target devices */
 429         int devices_cnt;                /* target device count */
 430         u8 include_all:1;               /* include all ports */
 431 };
 432
 433 static LIST_HEAD(dmar_atsr_units);
 434 static LIST_HEAD(dmar_rmrr_units);
 435
 436 #define for_each_rmrr_units(rmrr) \
 437         list_for_each_entry(rmrr, &dmar_rmrr_units, list)
 438
 439 static void flush_unmaps_timeout(unsigned long data);
 440
 441 static DEFINE_TIMER(unmap_timer,  flush_unmaps_timeout, 0, 0);
 442
 443 #define HIGH_WATER_MARK 250
 444 struct deferred_flush_tables {
 445         int next;
 446         struct iova *iova[HIGH_WATER_MARK];
 447         struct dmar_domain *domain[HIGH_WATER_MARK];
 448         struct page *freelist[HIGH_WATER_MARK];
 449 };
 450
 451 static struct deferred_flush_tables *deferred_flush;
 452
 453 /* bitmap for indexing intel_iommus */
 454 static int g_num_of_iommus;
 455
 456 static DEFINE_SPINLOCK(async_umap_flush_lock);
 457 static LIST_HEAD(unmaps_to_do);
 458
 459 static int timer_on;
 460 static long list_size;
 461
 462 static void domain_exit(struct dmar_domain *domain);
 463 static void domain_remove_dev_info(struct dmar_domain *domain);
 464 static void domain_remove_one_dev_info(struct dmar_domain *domain,
 465                                        struct device *dev);
 466 static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
 467                                            struct device *dev);
 468 static int domain_detach_iommu(struct dmar_domain *domain,
 469                                struct intel_iommu *iommu);
 470
 471 #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
 472 int dmar_disabled = 0;
 473 #else
 474 int dmar_disabled = 1;
 475 #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
 476
 477 int intel_iommu_enabled = 0;
 478 EXPORT_SYMBOL_GPL(intel_iommu_enabled);
 479
 480 static int dmar_map_gfx = 1;
 481 static int dmar_forcedac;
 482 static int intel_iommu_strict;
 483 static int intel_iommu_superpage = 1;
 484 static int intel_iommu_ecs = 1;
 485
 486 /* We only actually use ECS when PASID support (on the new bit 40)
 487  * is also advertised. Some early implementations — the ones with
 488  * PASID support on bit 28 — have issues even when we *only* use
 489  * extended root/context tables. */
 490 #define ecs_enabled(iommu) (intel_iommu_ecs && ecap_ecs(iommu->ecap) && \
 491                             ecap_pasid(iommu->ecap))
 492
 493 int intel_iommu_gfx_mapped;
 494 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
 495
 496 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
 497 static DEFINE_SPINLOCK(device_domain_lock);
 498 static LIST_HEAD(device_domain_list);
 499
 500 static const struct iommu_ops intel_iommu_ops;
 501
 502 static bool translation_pre_enabled(struct intel_iommu *iommu)
 503 {
 504         return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
 505 }
 506
 507 static void clear_translation_pre_enabled(struct intel_iommu *iommu)
 508 {
 509         iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
 510 }
 511
 512 static void init_translation_status(struct intel_iommu *iommu)
 513 {
 514         u32 gsts;
 515
 516         gsts = readl(iommu->reg + DMAR_GSTS_REG);
 517         if (gsts & DMA_GSTS_TES)
 518                 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
 519 }
 520
 521 /* Convert generic 'struct iommu_domain to private struct dmar_domain */
 522 static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
 523 {
 524         return container_of(dom, struct dmar_domain, domain);
 525 }
 526
 527 static int __init intel_iommu_setup(char *str)
 528 {
 529         if (!str)
 530                 return -EINVAL;
 531         while (*str) {
 532                 if (!strncmp(str, "on", 2)) {
 533                         dmar_disabled = 0;
 534                         pr_info("IOMMU enabled\n");
 535                 } else if (!strncmp(str, "off", 3)) {
 536                         dmar_disabled = 1;
 537                         pr_info("IOMMU disabled\n");
 538                 } else if (!strncmp(str, "igfx_off", 8)) {
 539                         dmar_map_gfx = 0;
 540                         pr_info("Disable GFX device mapping\n");
 541                 } else if (!strncmp(str, "forcedac", 8)) {
 542                         pr_info("Forcing DAC for PCI devices\n");
 543                         dmar_forcedac = 1;
 544                 } else if (!strncmp(str, "strict", 6)) {
 545                         pr_info("Disable batched IOTLB flush\n");
 546                         intel_iommu_strict = 1;
 547                 } else if (!strncmp(str, "sp_off", 6)) {
 548                         pr_info("Disable supported super page\n");
 549                         intel_iommu_superpage = 0;
 550                 } else if (!strncmp(str, "ecs_off", 7)) {
 551                         printk(KERN_INFO
 552                                 "Intel-IOMMU: disable extended context table support\n");
 553                         intel_iommu_ecs = 0;
 554                 }
 555
 556                 str += strcspn(str, ",");
 557                 while (*str == ',')
 558                         str++;
 559         }
 560         return 0;
 561 }
 562 __setup("intel_iommu=", intel_iommu_setup);
 563
 564 static struct kmem_cache *iommu_domain_cache;
 565 static struct kmem_cache *iommu_devinfo_cache;
 566
 567 static inline void *alloc_pgtable_page(int node)
 568 {
 569         struct page *page;
 570         void *vaddr = NULL;
 571
 572         page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
 573         if (page)
 574                 vaddr = page_address(page);
 575         return vaddr;
 576 }
 577
 578 static inline void free_pgtable_page(void *vaddr)
 579 {
 580         free_page((unsigned long)vaddr);
 581 }
 582
 583 static inline void *alloc_domain_mem(void)
 584 {
 585         return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
 586 }
 587
 588 static void free_domain_mem(void *vaddr)
 589 {
 590         kmem_cache_free(iommu_domain_cache, vaddr);
 591 }
 592
 593 static inline void * alloc_devinfo_mem(void)
 594 {
 595         return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
 596 }
 597
 598 static inline void free_devinfo_mem(void *vaddr)
 599 {
 600         kmem_cache_free(iommu_devinfo_cache, vaddr);
 601 }
 602
 603 static inline int domain_type_is_vm(struct dmar_domain *domain)
 604 {
 605         return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
 606 }
 607
 608 static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
 609 {
 610         return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
 611                                 DOMAIN_FLAG_STATIC_IDENTITY);
 612 }
 613
 614 static inline int domain_pfn_supported(struct dmar_domain *domain,
 615                                        unsigned long pfn)
 616 {
 617         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 618
 619         return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
 620 }
 621
 622 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
 623 {
 624         unsigned long sagaw;
 625         int agaw = -1;
 626
 627         sagaw = cap_sagaw(iommu->cap);
 628         for (agaw = width_to_agaw(max_gaw);
 629              agaw >= 0; agaw--) {
 630                 if (test_bit(agaw, &sagaw))
 631                         break;
 632         }
 633
 634         return agaw;
 635 }
 636
 637 /*
 638  * Calculate max SAGAW for each iommu.
 639  */
 640 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
 641 {
 642         return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
 643 }
 644
 645 /*
 646  * calculate agaw for each iommu.
 647  * "SAGAW" may be different across iommus, use a default agaw, and
 648  * get a supported less agaw for iommus that don't support the default agaw.
 649  */
 650 int iommu_calculate_agaw(struct intel_iommu *iommu)
 651 {
 652         return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
 653 }
 654
 655 /* This functionin only returns single iommu in a domain */
 656 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
 657 {
 658         int iommu_id;
 659
 660         /* si_domain and vm domain should not get here. */
 661         BUG_ON(domain_type_is_vm_or_si(domain));
 662         iommu_id = find_first_bit(domain->iommu_bmp, g_num_of_iommus);
 663         if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
 664                 return NULL;
 665
 666         return g_iommus[iommu_id];
 667 }
 668
 669 static void domain_update_iommu_coherency(struct dmar_domain *domain)
 670 {
 671         struct dmar_drhd_unit *drhd;
 672         struct intel_iommu *iommu;
 673         bool found = false;
 674         int i;
 675
 676         domain->iommu_coherency = 1;
 677
 678         for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
 679                 found = true;
 680                 if (!ecap_coherent(g_iommus[i]->ecap)) {
 681                         domain->iommu_coherency = 0;
 682                         break;
 683                 }
 684         }
 685         if (found)
 686                 return;
 687
 688         /* No hardware attached; use lowest common denominator */
 689         rcu_read_lock();
 690         for_each_active_iommu(iommu, drhd) {
 691                 if (!ecap_coherent(iommu->ecap)) {
 692                         domain->iommu_coherency = 0;
 693                         break;
 694                 }
 695         }
 696         rcu_read_unlock();
 697 }
 698
 699 static int domain_update_iommu_snooping(struct intel_iommu *skip)
 700 {
 701         struct dmar_drhd_unit *drhd;
 702         struct intel_iommu *iommu;
 703         int ret = 1;
 704
 705         rcu_read_lock();
 706         for_each_active_iommu(iommu, drhd) {
 707                 if (iommu != skip) {
 708                         if (!ecap_sc_support(iommu->ecap)) {
 709                                 ret = 0;
 710                                 break;
 711                         }
 712                 }
 713         }
 714         rcu_read_unlock();
 715
 716         return ret;
 717 }
 718
 719 static int domain_update_iommu_superpage(struct intel_iommu *skip)
 720 {
 721         struct dmar_drhd_unit *drhd;
 722         struct intel_iommu *iommu;
 723         int mask = 0xf;
 724
 725         if (!intel_iommu_superpage) {
 726                 return 0;
 727         }
 728
 729         /* set iommu_superpage to the smallest common denominator */
 730         rcu_read_lock();
 731         for_each_active_iommu(iommu, drhd) {
 732                 if (iommu != skip) {
 733                         mask &= cap_super_page_val(iommu->cap);
 734                         if (!mask)
 735                                 break;
 736                 }
 737         }
 738         rcu_read_unlock();
 739
 740         return fls(mask);
 741 }
 742
 743 /* Some capabilities may be different across iommus */
 744 static void domain_update_iommu_cap(struct dmar_domain *domain)
 745 {
 746         domain_update_iommu_coherency(domain);
 747         domain->iommu_snooping = domain_update_iommu_snooping(NULL);
 748         domain->iommu_superpage = domain_update_iommu_superpage(NULL);
 749 }
 750
 751 static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu,
 752                                                        u8 bus, u8 devfn, int alloc)
 753 {
 754         struct root_entry *root = &iommu->root_entry[bus];
 755         struct context_entry *context;
 756         u64 *entry;
 757
 758         if (ecs_enabled(iommu)) {
 759                 if (devfn >= 0x80) {
 760                         devfn -= 0x80;
 761                         entry = &root->hi;
 762                 }
 763                 devfn *= 2;
 764         }
 765         entry = &root->lo;
 766         if (*entry & 1)
 767                 context = phys_to_virt(*entry & VTD_PAGE_MASK);
 768         else {
 769                 unsigned long phy_addr;
 770                 if (!alloc)
 771                         return NULL;
 772
 773                 context = alloc_pgtable_page(iommu->node);
 774                 if (!context)
 775                         return NULL;
 776
 777                 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
 778                 phy_addr = virt_to_phys((void *)context);
 779                 *entry = phy_addr | 1;
 780                 __iommu_flush_cache(iommu, entry, sizeof(*entry));
 781         }
 782         return &context[devfn];
 783 }
 784
 785 static int iommu_dummy(struct device *dev)
 786 {
 787         return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
 788 }
 789
 790 static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
 791 {
 792         struct dmar_drhd_unit *drhd = NULL;
 793         struct intel_iommu *iommu;
 794         struct device *tmp;
 795         struct pci_dev *ptmp, *pdev = NULL;
 796         u16 segment = 0;
 797         int i;
 798
 799         if (iommu_dummy(dev))
 800                 return NULL;
 801
 802         if (dev_is_pci(dev)) {
 803                 pdev = to_pci_dev(dev);
 804                 segment = pci_domain_nr(pdev->bus);
 805         } else if (has_acpi_companion(dev))
 806                 dev = &ACPI_COMPANION(dev)->dev;
 807
 808         rcu_read_lock();
 809         for_each_active_iommu(iommu, drhd) {
 810                 if (pdev && segment != drhd->segment)
 811                         continue;
 812
 813                 for_each_active_dev_scope(drhd->devices,
 814                                           drhd->devices_cnt, i, tmp) {
 815                         if (tmp == dev) {
 816                                 *bus = drhd->devices[i].bus;
 817                                 *devfn = drhd->devices[i].devfn;
 818                                 goto out;
 819                         }
 820
 821                         if (!pdev || !dev_is_pci(tmp))
 822                                 continue;
 823
 824                         ptmp = to_pci_dev(tmp);
 825                         if (ptmp->subordinate &&
 826                             ptmp->subordinate->number <= pdev->bus->number &&
 827                             ptmp->subordinate->busn_res.end >= pdev->bus->number)
 828                                 goto got_pdev;
 829                 }
 830
 831                 if (pdev && drhd->include_all) {
 832                 got_pdev:
 833                         *bus = pdev->bus->number;
 834                         *devfn = pdev->devfn;
 835                         goto out;
 836                 }
 837         }
 838         iommu = NULL;
 839  out:
 840         rcu_read_unlock();
 841
 842         return iommu;
 843 }
 844
 845 static void domain_flush_cache(struct dmar_domain *domain,
 846                                void *addr, int size)
 847 {
 848         if (!domain->iommu_coherency)
 849                 clflush_cache_range(addr, size);
 850 }
 851
 852 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
 853 {
 854         struct context_entry *context;
 855         int ret = 0;
 856         unsigned long flags;
 857
 858         spin_lock_irqsave(&iommu->lock, flags);
 859         context = iommu_context_addr(iommu, bus, devfn, 0);
 860         if (context)
 861                 ret = context_present(context);
 862         spin_unlock_irqrestore(&iommu->lock, flags);
 863         return ret;
 864 }
 865
 866 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
 867 {
 868         struct context_entry *context;
 869         unsigned long flags;
 870
 871         spin_lock_irqsave(&iommu->lock, flags);
 872         context = iommu_context_addr(iommu, bus, devfn, 0);
 873         if (context) {
 874                 context_clear_entry(context);
 875                 __iommu_flush_cache(iommu, context, sizeof(*context));
 876         }
 877         spin_unlock_irqrestore(&iommu->lock, flags);
 878 }
 879
 880 static void free_context_table(struct intel_iommu *iommu)
 881 {
 882         int i;
 883         unsigned long flags;
 884         struct context_entry *context;
 885
 886         spin_lock_irqsave(&iommu->lock, flags);
 887         if (!iommu->root_entry) {
 888                 goto out;
 889         }
 890         for (i = 0; i < ROOT_ENTRY_NR; i++) {
 891                 context = iommu_context_addr(iommu, i, 0, 0);
 892                 if (context)
 893                         free_pgtable_page(context);
 894
 895                 if (!ecs_enabled(iommu))
 896                         continue;
 897
 898                 context = iommu_context_addr(iommu, i, 0x80, 0);
 899                 if (context)
 900                         free_pgtable_page(context);
 901
 902         }
 903         free_pgtable_page(iommu->root_entry);
 904         iommu->root_entry = NULL;
 905 out:
 906         spin_unlock_irqrestore(&iommu->lock, flags);
 907 }
 908
 909 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
 910                                       unsigned long pfn, int *target_level)
 911 {
 912         struct dma_pte *parent, *pte = NULL;
 913         int level = agaw_to_level(domain->agaw);
 914         int offset;
 915
 916         BUG_ON(!domain->pgd);
 917
 918         if (!domain_pfn_supported(domain, pfn))
 919                 /* Address beyond IOMMU's addressing capabilities. */
 920                 return NULL;
 921
 922         parent = domain->pgd;
 923
 924         while (1) {
 925                 void *tmp_page;
 926
 927                 offset = pfn_level_offset(pfn, level);
 928                 pte = &parent[offset];
 929                 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
 930                         break;
 931                 if (level == *target_level)
 932                         break;
 933
 934                 if (!dma_pte_present(pte)) {
 935                         uint64_t pteval;
 936
 937                         tmp_page = alloc_pgtable_page(domain->nid);
 938
 939                         if (!tmp_page)
 940                                 return NULL;
 941
 942                         domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
 943                         pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
 944                         if (cmpxchg64(&pte->val, 0ULL, pteval))
 945                                 /* Someone else set it while we were thinking; use theirs. */
 946                                 free_pgtable_page(tmp_page);
 947                         else
 948                                 domain_flush_cache(domain, pte, sizeof(*pte));
 949                 }
 950                 if (level == 1)
 951                         break;
 952
 953                 parent = phys_to_virt(dma_pte_addr(pte));
 954                 level--;
 955         }
 956
 957         if (!*target_level)
 958                 *target_level = level;
 959
 960         return pte;
 961 }
 962
 963
 964 /* return address's pte at specific level */
 965 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
 966                                          unsigned long pfn,
 967                                          int level, int *large_page)
 968 {
 969         struct dma_pte *parent, *pte = NULL;
 970         int total = agaw_to_level(domain->agaw);
 971         int offset;
 972
 973         parent = domain->pgd;
 974         while (level <= total) {
 975                 offset = pfn_level_offset(pfn, total);
 976                 pte = &parent[offset];
 977                 if (level == total)
 978                         return pte;
 979
 980                 if (!dma_pte_present(pte)) {
 981                         *large_page = total;
 982                         break;
 983                 }
 984
 985                 if (dma_pte_superpage(pte)) {
 986                         *large_page = total;
 987                         return pte;
 988                 }
 989
 990                 parent = phys_to_virt(dma_pte_addr(pte));
 991                 total--;
 992         }
 993         return NULL;
 994 }
 995
 996 /* clear last level pte, a tlb flush should be followed */
 997 static void dma_pte_clear_range(struct dmar_domain *domain,
 998                                 unsigned long start_pfn,
 999                                 unsigned long last_pfn)
1000 {
1001         unsigned int large_page = 1;
1002         struct dma_pte *first_pte, *pte;
1003
1004         BUG_ON(!domain_pfn_supported(domain, start_pfn));
1005         BUG_ON(!domain_pfn_supported(domain, last_pfn));
1006         BUG_ON(start_pfn > last_pfn);
1007
1008         /* we don't need lock here; nobody else touches the iova range */
1009         do {
1010                 large_page = 1;
1011                 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
1012                 if (!pte) {
1013                         start_pfn = align_to_level(start_pfn + 1, large_page + 1);
1014                         continue;
1015                 }
1016                 do {
1017                         dma_clear_pte(pte);
1018                         start_pfn += lvl_to_nr_pages(large_page);
1019                         pte++;
1020                 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
1021
1022                 domain_flush_cache(domain, first_pte,
1023                                    (void *)pte - (void *)first_pte);
1024
1025         } while (start_pfn && start_pfn <= last_pfn);
1026 }
1027
1028 static void dma_pte_free_level(struct dmar_domain *domain, int level,
1029                                struct dma_pte *pte, unsigned long pfn,
1030                                unsigned long start_pfn, unsigned long last_pfn)
1031 {
1032         pfn = max(start_pfn, pfn);
1033         pte = &pte[pfn_level_offset(pfn, level)];
1034
1035         do {
1036                 unsigned long level_pfn;
1037                 struct dma_pte *level_pte;
1038
1039                 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
1040                         goto next;
1041
1042                 level_pfn = pfn & level_mask(level - 1);
1043                 level_pte = phys_to_virt(dma_pte_addr(pte));
1044
1045                 if (level > 2)
1046                         dma_pte_free_level(domain, level - 1, level_pte,
1047                                            level_pfn, start_pfn, last_pfn);
1048
1049                 /* If range covers entire pagetable, free it */
1050                 if (!(start_pfn > level_pfn ||
1051                       last_pfn < level_pfn + level_size(level) - 1)) {
1052                         dma_clear_pte(pte);
1053                         domain_flush_cache(domain, pte, sizeof(*pte));
1054                         free_pgtable_page(level_pte);
1055                 }
1056 next:
1057                 pfn += level_size(level);
1058         } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1059 }
1060
1061 /* free page table pages. last level pte should already be cleared */
1062 static void dma_pte_free_pagetable(struct dmar_domain *domain,
1063                                    unsigned long start_pfn,
1064                                    unsigned long last_pfn)
1065 {
1066         BUG_ON(!domain_pfn_supported(domain, start_pfn));
1067         BUG_ON(!domain_pfn_supported(domain, last_pfn));
1068         BUG_ON(start_pfn > last_pfn);
1069
1070         dma_pte_clear_range(domain, start_pfn, last_pfn);
1071
1072         /* We don't need lock here; nobody else touches the iova range */
1073         dma_pte_free_level(domain, agaw_to_level(domain->agaw),
1074                            domain->pgd, 0, start_pfn, last_pfn);
1075
1076         /* free pgd */
1077         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1078                 free_pgtable_page(domain->pgd);
1079                 domain->pgd = NULL;
1080         }
1081 }
1082
1083 /* When a page at a given level is being unlinked from its parent, we don't
1084    need to *modify* it at all. All we need to do is make a list of all the
1085    pages which can be freed just as soon as we've flushed the IOTLB and we
1086    know the hardware page-walk will no longer touch them.
1087    The 'pte' argument is the *parent* PTE, pointing to the page that is to
1088    be freed. */
1089 static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1090                                             int level, struct dma_pte *pte,
1091                                             struct page *freelist)
1092 {
1093         struct page *pg;
1094
1095         pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1096         pg->freelist = freelist;
1097         freelist = pg;
1098
1099         if (level == 1)
1100                 return freelist;
1101
1102         pte = page_address(pg);
1103         do {
1104                 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1105                         freelist = dma_pte_list_pagetables(domain, level - 1,
1106                                                            pte, freelist);
1107                 pte++;
1108         } while (!first_pte_in_page(pte));
1109
1110         return freelist;
1111 }
1112
1113 static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1114                                         struct dma_pte *pte, unsigned long pfn,
1115                                         unsigned long start_pfn,
1116                                         unsigned long last_pfn,
1117                                         struct page *freelist)
1118 {
1119         struct dma_pte *first_pte = NULL, *last_pte = NULL;
1120
1121         pfn = max(start_pfn, pfn);
1122         pte = &pte[pfn_level_offset(pfn, level)];
1123
1124         do {
1125                 unsigned long level_pfn;
1126
1127                 if (!dma_pte_present(pte))
1128                         goto next;
1129
1130                 level_pfn = pfn & level_mask(level);
1131
1132                 /* If range covers entire pagetable, free it */
1133                 if (start_pfn <= level_pfn &&
1134                     last_pfn >= level_pfn + level_size(level) - 1) {
1135                         /* These suborbinate page tables are going away entirely. Don't
1136                            bother to clear them; we're just going to *free* them. */
1137                         if (level > 1 && !dma_pte_superpage(pte))
1138                                 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1139
1140                         dma_clear_pte(pte);
1141                         if (!first_pte)
1142                                 first_pte = pte;
1143                         last_pte = pte;
1144                 } else if (level > 1) {
1145                         /* Recurse down into a level that isn't *entirely* obsolete */
1146                         freelist = dma_pte_clear_level(domain, level - 1,
1147                                                        phys_to_virt(dma_pte_addr(pte)),
1148                                                        level_pfn, start_pfn, last_pfn,
1149                                                        freelist);
1150                 }
1151 next:
1152                 pfn += level_size(level);
1153         } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1154
1155         if (first_pte)
1156                 domain_flush_cache(domain, first_pte,
1157                                    (void *)++last_pte - (void *)first_pte);
1158
1159         return freelist;
1160 }
1161
1162 /* We can't just free the pages because the IOMMU may still be walking
1163    the page tables, and may have cached the intermediate levels. The
1164    pages can only be freed after the IOTLB flush has been done. */
1165 struct page *domain_unmap(struct dmar_domain *domain,
1166                           unsigned long start_pfn,
1167                           unsigned long last_pfn)
1168 {
1169         struct page *freelist = NULL;
1170
1171         BUG_ON(!domain_pfn_supported(domain, start_pfn));
1172         BUG_ON(!domain_pfn_supported(domain, last_pfn));
1173         BUG_ON(start_pfn > last_pfn);
1174
1175         /* we don't need lock here; nobody else touches the iova range */
1176         freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1177                                        domain->pgd, 0, start_pfn, last_pfn, NULL);
1178
1179         /* free pgd */
1180         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1181                 struct page *pgd_page = virt_to_page(domain->pgd);
1182                 pgd_page->freelist = freelist;
1183                 freelist = pgd_page;
1184
1185                 domain->pgd = NULL;
1186         }
1187
1188         return freelist;
1189 }
1190
1191 void dma_free_pagelist(struct page *freelist)
1192 {
1193         struct page *pg;
1194
1195         while ((pg = freelist)) {
1196                 freelist = pg->freelist;
1197                 free_pgtable_page(page_address(pg));
1198         }
1199 }
1200
1201 /* iommu handling */
1202 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1203 {
1204         struct root_entry *root;
1205         unsigned long flags;
1206
1207         root = (struct root_entry *)alloc_pgtable_page(iommu->node);
1208         if (!root) {
1209                 pr_err("Allocating root entry for %s failed\n",
1210                         iommu->name);
1211                 return -ENOMEM;
1212         }
1213
1214         __iommu_flush_cache(iommu, root, ROOT_SIZE);
1215
1216         spin_lock_irqsave(&iommu->lock, flags);
1217         iommu->root_entry = root;
1218         spin_unlock_irqrestore(&iommu->lock, flags);
1219
1220         return 0;
1221 }
1222
1223 static void iommu_set_root_entry(struct intel_iommu *iommu)
1224 {
1225         u64 addr;
1226         u32 sts;
1227         unsigned long flag;
1228
1229         addr = virt_to_phys(iommu->root_entry);
1230         if (ecs_enabled(iommu))
1231                 addr |= DMA_RTADDR_RTT;
1232
1233         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1234         dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
1235
1236         writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
1237
1238         /* Make sure hardware complete it */
1239         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1240                       readl, (sts & DMA_GSTS_RTPS), sts);
1241
1242         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1243 }
1244
1245 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1246 {
1247         u32 val;
1248         unsigned long flag;
1249
1250         if (!rwbf_quirk && !cap_rwbf(iommu->cap))
1251                 return;
1252
1253         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1254         writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
1255
1256         /* Make sure hardware complete it */
1257         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1258                       readl, (!(val & DMA_GSTS_WBFS)), val);
1259
1260         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1261 }
1262
1263 /* return value determine if we need a write buffer flush */
1264 static void __iommu_flush_context(struct intel_iommu *iommu,
1265                                   u16 did, u16 source_id, u8 function_mask,
1266                                   u64 type)
1267 {
1268         u64 val = 0;
1269         unsigned long flag;
1270
1271         switch (type) {
1272         case DMA_CCMD_GLOBAL_INVL:
1273                 val = DMA_CCMD_GLOBAL_INVL;
1274                 break;
1275         case DMA_CCMD_DOMAIN_INVL:
1276                 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1277                 break;
1278         case DMA_CCMD_DEVICE_INVL:
1279                 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1280                         | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1281                 break;
1282         default:
1283                 BUG();
1284         }
1285         val |= DMA_CCMD_ICC;
1286
1287         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1288         dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1289
1290         /* Make sure hardware complete it */
1291         IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1292                 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1293
1294         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1295 }
1296
1297 /* return value determine if we need a write buffer flush */
1298 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1299                                 u64 addr, unsigned int size_order, u64 type)
1300 {
1301         int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1302         u64 val = 0, val_iva = 0;
1303         unsigned long flag;
1304
1305         switch (type) {
1306         case DMA_TLB_GLOBAL_FLUSH:
1307                 /* global flush doesn't need set IVA_REG */
1308                 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1309                 break;
1310         case DMA_TLB_DSI_FLUSH:
1311                 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1312                 break;
1313         case DMA_TLB_PSI_FLUSH:
1314                 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1315                 /* IH bit is passed in as part of address */
1316                 val_iva = size_order | addr;
1317                 break;
1318         default:
1319                 BUG();
1320         }
1321         /* Note: set drain read/write */
1322 #if 0
1323         /*
1324          * This is probably to be super secure.. Looks like we can
1325          * ignore it without any impact.
1326          */
1327         if (cap_read_drain(iommu->cap))
1328                 val |= DMA_TLB_READ_DRAIN;
1329 #endif
1330         if (cap_write_drain(iommu->cap))
1331                 val |= DMA_TLB_WRITE_DRAIN;
1332
1333         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1334         /* Note: Only uses first TLB reg currently */
1335         if (val_iva)
1336                 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1337         dmar_writeq(iommu->reg + tlb_offset + 8, val);
1338
1339         /* Make sure hardware complete it */
1340         IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1341                 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1342
1343         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1344
1345         /* check IOTLB invalidation granularity */
1346         if (DMA_TLB_IAIG(val) == 0)
1347                 pr_err("Flush IOTLB failed\n");
1348         if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1349                 pr_debug("TLB flush request %Lx, actual %Lx\n",
1350                         (unsigned long long)DMA_TLB_IIRG(type),
1351                         (unsigned long long)DMA_TLB_IAIG(val));
1352 }
1353
1354 static struct device_domain_info *
1355 iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1356                          u8 bus, u8 devfn)
1357 {
1358         bool found = false;
1359         unsigned long flags;
1360         struct device_domain_info *info;
1361         struct pci_dev *pdev;
1362
1363         if (!ecap_dev_iotlb_support(iommu->ecap))
1364                 return NULL;
1365
1366         if (!iommu->qi)
1367                 return NULL;
1368
1369         spin_lock_irqsave(&device_domain_lock, flags);
1370         list_for_each_entry(info, &domain->devices, link)
1371                 if (info->iommu == iommu && info->bus == bus &&
1372                     info->devfn == devfn) {
1373                         found = true;
1374                         break;
1375                 }
1376         spin_unlock_irqrestore(&device_domain_lock, flags);
1377
1378         if (!found || !info->dev || !dev_is_pci(info->dev))
1379                 return NULL;
1380
1381         pdev = to_pci_dev(info->dev);
1382
1383         if (!pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS))
1384                 return NULL;
1385
1386         if (!dmar_find_matched_atsr_unit(pdev))
1387                 return NULL;
1388
1389         return info;
1390 }
1391
1392 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1393 {
1394         if (!info || !dev_is_pci(info->dev))
1395                 return;
1396
1397         pci_enable_ats(to_pci_dev(info->dev), VTD_PAGE_SHIFT);
1398 }
1399
1400 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1401 {
1402         if (!info->dev || !dev_is_pci(info->dev) ||
1403             !pci_ats_enabled(to_pci_dev(info->dev)))
1404                 return;
1405
1406         pci_disable_ats(to_pci_dev(info->dev));
1407 }
1408
1409 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1410                                   u64 addr, unsigned mask)
1411 {
1412         u16 sid, qdep;
1413         unsigned long flags;
1414         struct device_domain_info *info;
1415
1416         spin_lock_irqsave(&device_domain_lock, flags);
1417         list_for_each_entry(info, &domain->devices, link) {
1418                 struct pci_dev *pdev;
1419                 if (!info->dev || !dev_is_pci(info->dev))
1420                         continue;
1421
1422                 pdev = to_pci_dev(info->dev);
1423                 if (!pci_ats_enabled(pdev))
1424                         continue;
1425
1426                 sid = info->bus << 8 | info->devfn;
1427                 qdep = pci_ats_queue_depth(pdev);
1428                 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1429         }
1430         spin_unlock_irqrestore(&device_domain_lock, flags);
1431 }
1432
1433 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
1434                                   unsigned long pfn, unsigned int pages, int ih, int map)
1435 {
1436         unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1437         uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1438
1439         BUG_ON(pages == 0);
1440
1441         if (ih)
1442                 ih = 1 << 6;
1443         /*
1444          * Fallback to domain selective flush if no PSI support or the size is
1445          * too big.
1446          * PSI requires page size to be 2 ^ x, and the base address is naturally
1447          * aligned to the size
1448          */
1449         if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1450                 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1451                                                 DMA_TLB_DSI_FLUSH);
1452         else
1453                 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
1454                                                 DMA_TLB_PSI_FLUSH);
1455
1456         /*
1457          * In caching mode, changes of pages from non-present to present require
1458          * flush. However, device IOTLB doesn't need to be flushed in this case.
1459          */
1460         if (!cap_caching_mode(iommu->cap) || !map)
1461                 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
1462 }
1463
1464 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1465 {
1466         u32 pmen;
1467         unsigned long flags;
1468
1469         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1470         pmen = readl(iommu->reg + DMAR_PMEN_REG);
1471         pmen &= ~DMA_PMEN_EPM;
1472         writel(pmen, iommu->reg + DMAR_PMEN_REG);
1473
1474         /* wait for the protected region status bit to clear */
1475         IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1476                 readl, !(pmen & DMA_PMEN_PRS), pmen);
1477
1478         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1479 }
1480
1481 static void iommu_enable_translation(struct intel_iommu *iommu)
1482 {
1483         u32 sts;
1484         unsigned long flags;
1485
1486         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1487         iommu->gcmd |= DMA_GCMD_TE;
1488         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1489
1490         /* Make sure hardware complete it */
1491         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1492                       readl, (sts & DMA_GSTS_TES), sts);
1493
1494         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1495 }
1496
1497 static void iommu_disable_translation(struct intel_iommu *iommu)
1498 {
1499         u32 sts;
1500         unsigned long flag;
1501
1502         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1503         iommu->gcmd &= ~DMA_GCMD_TE;
1504         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1505
1506         /* Make sure hardware complete it */
1507         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1508                       readl, (!(sts & DMA_GSTS_TES)), sts);
1509
1510         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1511 }
1512
1513
1514 static int iommu_init_domains(struct intel_iommu *iommu)
1515 {
1516         unsigned long ndomains;
1517         unsigned long nlongs;
1518
1519         ndomains = cap_ndoms(iommu->cap);
1520         pr_debug("%s: Number of Domains supported <%ld>\n",
1521                  iommu->name, ndomains);
1522         nlongs = BITS_TO_LONGS(ndomains);
1523
1524         spin_lock_init(&iommu->lock);
1525
1526         /* TBD: there might be 64K domains,
1527          * consider other allocation for future chip
1528          */
1529         iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1530         if (!iommu->domain_ids) {
1531                 pr_err("%s: Allocating domain id array failed\n",
1532                        iommu->name);
1533                 return -ENOMEM;
1534         }
1535         iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1536                         GFP_KERNEL);
1537         if (!iommu->domains) {
1538                 pr_err("%s: Allocating domain array failed\n",
1539                        iommu->name);
1540                 kfree(iommu->domain_ids);
1541                 iommu->domain_ids = NULL;
1542                 return -ENOMEM;
1543         }
1544
1545         /*
1546          * if Caching mode is set, then invalid translations are tagged
1547          * with domainid 0. Hence we need to pre-allocate it.
1548          */
1549         if (cap_caching_mode(iommu->cap))
1550                 set_bit(0, iommu->domain_ids);
1551         return 0;
1552 }
1553
1554 static void disable_dmar_iommu(struct intel_iommu *iommu)
1555 {
1556         struct dmar_domain *domain;
1557         int i;
1558
1559         if ((iommu->domains) && (iommu->domain_ids)) {
1560                 for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
1561                         /*
1562                          * Domain id 0 is reserved for invalid translation
1563                          * if hardware supports caching mode.
1564                          */
1565                         if (cap_caching_mode(iommu->cap) && i == 0)
1566                                 continue;
1567
1568                         domain = iommu->domains[i];
1569                         clear_bit(i, iommu->domain_ids);
1570                         if (domain_detach_iommu(domain, iommu) == 0 &&
1571                             !domain_type_is_vm(domain))
1572                                 domain_exit(domain);
1573                 }
1574         }
1575
1576         if (iommu->gcmd & DMA_GCMD_TE)
1577                 iommu_disable_translation(iommu);
1578 }
1579
1580 static void free_dmar_iommu(struct intel_iommu *iommu)
1581 {
1582         if ((iommu->domains) && (iommu->domain_ids)) {
1583                 kfree(iommu->domains);
1584                 kfree(iommu->domain_ids);
1585                 iommu->domains = NULL;
1586                 iommu->domain_ids = NULL;
1587         }
1588
1589         g_iommus[iommu->seq_id] = NULL;
1590
1591         /* free context mapping */
1592         free_context_table(iommu);
1593 }
1594
1595 static struct dmar_domain *alloc_domain(int flags)
1596 {
1597         /* domain id for virtual machine, it won't be set in context */
1598         static atomic_t vm_domid = ATOMIC_INIT(0);
1599         struct dmar_domain *domain;
1600
1601         domain = alloc_domain_mem();
1602         if (!domain)
1603                 return NULL;
1604
1605         memset(domain, 0, sizeof(*domain));
1606         domain->nid = -1;
1607         domain->flags = flags;
1608         spin_lock_init(&domain->iommu_lock);
1609         INIT_LIST_HEAD(&domain->devices);
1610         if (flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
1611                 domain->id = atomic_inc_return(&vm_domid);
1612
1613         return domain;
1614 }
1615
1616 static int __iommu_attach_domain(struct dmar_domain *domain,
1617                                  struct intel_iommu *iommu)
1618 {
1619         int num;
1620         unsigned long ndomains;
1621
1622         ndomains = cap_ndoms(iommu->cap);
1623         num = find_first_zero_bit(iommu->domain_ids, ndomains);
1624         if (num < ndomains) {
1625                 set_bit(num, iommu->domain_ids);
1626                 iommu->domains[num] = domain;
1627         } else {
1628                 num = -ENOSPC;
1629         }
1630
1631         return num;
1632 }
1633
1634 static int iommu_attach_domain(struct dmar_domain *domain,
1635                                struct intel_iommu *iommu)
1636 {
1637         int num;
1638         unsigned long flags;
1639
1640         spin_lock_irqsave(&iommu->lock, flags);
1641         num = __iommu_attach_domain(domain, iommu);
1642         spin_unlock_irqrestore(&iommu->lock, flags);
1643         if (num < 0)
1644                 pr_err("%s: No free domain ids\n", iommu->name);
1645
1646         return num;
1647 }
1648
1649 static int iommu_attach_vm_domain(struct dmar_domain *domain,
1650                                   struct intel_iommu *iommu)
1651 {
1652         int num;
1653         unsigned long ndomains;
1654
1655         ndomains = cap_ndoms(iommu->cap);
1656         for_each_set_bit(num, iommu->domain_ids, ndomains)
1657                 if (iommu->domains[num] == domain)
1658                         return num;
1659
1660         return __iommu_attach_domain(domain, iommu);
1661 }
1662
1663 static void iommu_detach_domain(struct dmar_domain *domain,
1664                                 struct intel_iommu *iommu)
1665 {
1666         unsigned long flags;
1667         int num, ndomains;
1668
1669         spin_lock_irqsave(&iommu->lock, flags);
1670         if (domain_type_is_vm_or_si(domain)) {
1671                 ndomains = cap_ndoms(iommu->cap);
1672                 for_each_set_bit(num, iommu->domain_ids, ndomains) {
1673                         if (iommu->domains[num] == domain) {
1674                                 clear_bit(num, iommu->domain_ids);
1675                                 iommu->domains[num] = NULL;
1676                                 break;
1677                         }
1678                 }
1679         } else {
1680                 clear_bit(domain->id, iommu->domain_ids);
1681                 iommu->domains[domain->id] = NULL;
1682         }
1683         spin_unlock_irqrestore(&iommu->lock, flags);
1684 }
1685
1686 static void domain_attach_iommu(struct dmar_domain *domain,
1687                                struct intel_iommu *iommu)
1688 {
1689         unsigned long flags;
1690
1691         spin_lock_irqsave(&domain->iommu_lock, flags);
1692         if (!test_and_set_bit(iommu->seq_id, domain->iommu_bmp)) {
1693                 domain->iommu_count++;
1694                 if (domain->iommu_count == 1)
1695                         domain->nid = iommu->node;
1696                 domain_update_iommu_cap(domain);
1697         }
1698         spin_unlock_irqrestore(&domain->iommu_lock, flags);
1699 }
1700
1701 static int domain_detach_iommu(struct dmar_domain *domain,
1702                                struct intel_iommu *iommu)
1703 {
1704         unsigned long flags;
1705         int count = INT_MAX;
1706
1707         spin_lock_irqsave(&domain->iommu_lock, flags);
1708         if (test_and_clear_bit(iommu->seq_id, domain->iommu_bmp)) {
1709                 count = --domain->iommu_count;
1710                 domain_update_iommu_cap(domain);
1711         }
1712         spin_unlock_irqrestore(&domain->iommu_lock, flags);
1713
1714         return count;
1715 }
1716
1717 static struct iova_domain reserved_iova_list;
1718 static struct lock_class_key reserved_rbtree_key;
1719
1720 static int dmar_init_reserved_ranges(void)
1721 {
1722         struct pci_dev *pdev = NULL;
1723         struct iova *iova;
1724         int i;
1725
1726         init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN,
1727                         DMA_32BIT_PFN);
1728
1729         lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1730                 &reserved_rbtree_key);
1731
1732         /* IOAPIC ranges shouldn't be accessed by DMA */
1733         iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1734                 IOVA_PFN(IOAPIC_RANGE_END));
1735         if (!iova) {
1736                 pr_err("Reserve IOAPIC range failed\n");
1737                 return -ENODEV;
1738         }
1739
1740         /* Reserve all PCI MMIO to avoid peer-to-peer access */
1741         for_each_pci_dev(pdev) {
1742                 struct resource *r;
1743
1744                 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1745                         r = &pdev->resource[i];
1746                         if (!r->flags || !(r->flags & IORESOURCE_MEM))
1747                                 continue;
1748                         iova = reserve_iova(&reserved_iova_list,
1749                                             IOVA_PFN(r->start),
1750                                             IOVA_PFN(r->end));
1751                         if (!iova) {
1752                                 pr_err("Reserve iova failed\n");
1753                                 return -ENODEV;
1754                         }
1755                 }
1756         }
1757         return 0;
1758 }
1759
1760 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1761 {
1762         copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1763 }
1764
1765 static inline int guestwidth_to_adjustwidth(int gaw)
1766 {
1767         int agaw;
1768         int r = (gaw - 12) % 9;
1769
1770         if (r == 0)
1771                 agaw = gaw;
1772         else
1773                 agaw = gaw + 9 - r;
1774         if (agaw > 64)
1775                 agaw = 64;
1776         return agaw;
1777 }
1778
1779 static int domain_init(struct dmar_domain *domain, int guest_width)
1780 {
1781         struct intel_iommu *iommu;
1782         int adjust_width, agaw;
1783         unsigned long sagaw;
1784
1785         init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
1786                         DMA_32BIT_PFN);
1787         domain_reserve_special_ranges(domain);
1788
1789         /* calculate AGAW */
1790         iommu = domain_get_iommu(domain);
1791         if (guest_width > cap_mgaw(iommu->cap))
1792                 guest_width = cap_mgaw(iommu->cap);
1793         domain->gaw = guest_width;
1794         adjust_width = guestwidth_to_adjustwidth(guest_width);
1795         agaw = width_to_agaw(adjust_width);
1796         sagaw = cap_sagaw(iommu->cap);
1797         if (!test_bit(agaw, &sagaw)) {
1798                 /* hardware doesn't support it, choose a bigger one */
1799                 pr_debug("Hardware doesn't support agaw %d\n", agaw);
1800                 agaw = find_next_bit(&sagaw, 5, agaw);
1801                 if (agaw >= 5)
1802                         return -ENODEV;
1803         }
1804         domain->agaw = agaw;
1805
1806         if (ecap_coherent(iommu->ecap))
1807                 domain->iommu_coherency = 1;
1808         else
1809                 domain->iommu_coherency = 0;
1810
1811         if (ecap_sc_support(iommu->ecap))
1812                 domain->iommu_snooping = 1;
1813         else
1814                 domain->iommu_snooping = 0;
1815
1816         if (intel_iommu_superpage)
1817                 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1818         else
1819                 domain->iommu_superpage = 0;
1820
1821         domain->nid = iommu->node;
1822
1823         /* always allocate the top pgd */
1824         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1825         if (!domain->pgd)
1826                 return -ENOMEM;
1827         __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1828         return 0;
1829 }
1830
1831 static void domain_exit(struct dmar_domain *domain)
1832 {
1833         struct page *freelist = NULL;
1834         int i;
1835
1836         /* Domain 0 is reserved, so dont process it */
1837         if (!domain)
1838                 return;
1839
1840         /* Flush any lazy unmaps that may reference this domain */
1841         if (!intel_iommu_strict)
1842                 flush_unmaps_timeout(0);
1843
1844         /* remove associated devices */
1845         domain_remove_dev_info(domain);
1846
1847         /* destroy iovas */
1848         put_iova_domain(&domain->iovad);
1849
1850         freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1851
1852         /* clear attached or cached domains */
1853         rcu_read_lock();
1854         for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus)
1855                 iommu_detach_domain(domain, g_iommus[i]);
1856         rcu_read_unlock();
1857
1858         dma_free_pagelist(freelist);
1859
1860         free_domain_mem(domain);
1861 }
1862
1863 static int domain_context_mapping_one(struct dmar_domain *domain,
1864                                       struct intel_iommu *iommu,
1865                                       u8 bus, u8 devfn, int translation)
1866 {
1867         struct context_entry *context;
1868         unsigned long flags;
1869         struct dma_pte *pgd;
1870         int id;
1871         int agaw;
1872         struct device_domain_info *info = NULL;
1873
1874         pr_debug("Set context mapping for %02x:%02x.%d\n",
1875                 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1876
1877         BUG_ON(!domain->pgd);
1878         BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1879                translation != CONTEXT_TT_MULTI_LEVEL);
1880
1881         spin_lock_irqsave(&iommu->lock, flags);
1882         context = iommu_context_addr(iommu, bus, devfn, 1);
1883         spin_unlock_irqrestore(&iommu->lock, flags);
1884         if (!context)
1885                 return -ENOMEM;
1886         spin_lock_irqsave(&iommu->lock, flags);
1887         if (context_present(context)) {
1888                 spin_unlock_irqrestore(&iommu->lock, flags);
1889                 return 0;
1890         }
1891
1892         context_clear_entry(context);
1893
1894         id = domain->id;
1895         pgd = domain->pgd;
1896
1897         if (domain_type_is_vm_or_si(domain)) {
1898                 if (domain_type_is_vm(domain)) {
1899                         id = iommu_attach_vm_domain(domain, iommu);
1900                         if (id < 0) {
1901                                 spin_unlock_irqrestore(&iommu->lock, flags);
1902                                 pr_err("%s: No free domain ids\n", iommu->name);
1903                                 return -EFAULT;
1904                         }
1905                 }
1906
1907                 /* Skip top levels of page tables for
1908                  * iommu which has less agaw than default.
1909                  * Unnecessary for PT mode.
1910                  */
1911                 if (translation != CONTEXT_TT_PASS_THROUGH) {
1912                         for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1913                                 pgd = phys_to_virt(dma_pte_addr(pgd));
1914                                 if (!dma_pte_present(pgd)) {
1915                                         spin_unlock_irqrestore(&iommu->lock, flags);
1916                                         return -ENOMEM;
1917                                 }
1918                         }
1919                 }
1920         }
1921
1922         context_set_domain_id(context, id);
1923
1924         if (translation != CONTEXT_TT_PASS_THROUGH) {
1925                 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
1926                 translation = info ? CONTEXT_TT_DEV_IOTLB :
1927                                      CONTEXT_TT_MULTI_LEVEL;
1928         }
1929         /*
1930          * In pass through mode, AW must be programmed to indicate the largest
1931          * AGAW value supported by hardware. And ASR is ignored by hardware.
1932          */
1933         if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
1934                 context_set_address_width(context, iommu->msagaw);
1935         else {
1936                 context_set_address_root(context, virt_to_phys(pgd));
1937                 context_set_address_width(context, iommu->agaw);
1938         }
1939
1940         context_set_translation_type(context, translation);
1941         context_set_fault_enable(context);
1942         context_set_present(context);
1943         domain_flush_cache(domain, context, sizeof(*context));
1944
1945         /*
1946          * It's a non-present to present mapping. If hardware doesn't cache
1947          * non-present entry we only need to flush the write-buffer. If the
1948          * _does_ cache non-present entries, then it does so in the special
1949          * domain #0, which we have to flush:
1950          */
1951         if (cap_caching_mode(iommu->cap)) {
1952                 iommu->flush.flush_context(iommu, 0,
1953                                            (((u16)bus) << 8) | devfn,
1954                                            DMA_CCMD_MASK_NOBIT,
1955                                            DMA_CCMD_DEVICE_INVL);
1956                 iommu->flush.flush_iotlb(iommu, id, 0, 0, DMA_TLB_DSI_FLUSH);
1957         } else {
1958                 iommu_flush_write_buffer(iommu);
1959         }
1960         iommu_enable_dev_iotlb(info);
1961         spin_unlock_irqrestore(&iommu->lock, flags);
1962
1963         domain_attach_iommu(domain, iommu);
1964
1965         return 0;
1966 }
1967
1968 struct domain_context_mapping_data {
1969         struct dmar_domain *domain;
1970         struct intel_iommu *iommu;
1971         int translation;
1972 };
1973
1974 static int domain_context_mapping_cb(struct pci_dev *pdev,
1975                                      u16 alias, void *opaque)
1976 {
1977         struct domain_context_mapping_data *data = opaque;
1978
1979         return domain_context_mapping_one(data->domain, data->iommu,
1980                                           PCI_BUS_NUM(alias), alias & 0xff,
1981                                           data->translation);
1982 }
1983
1984 static int
1985 domain_context_mapping(struct dmar_domain *domain, struct device *dev,
1986                        int translation)
1987 {
1988         struct intel_iommu *iommu;
1989         u8 bus, devfn;
1990         struct domain_context_mapping_data data;
1991
1992         iommu = device_to_iommu(dev, &bus, &devfn);
1993         if (!iommu)
1994                 return -ENODEV;
1995
1996         if (!dev_is_pci(dev))
1997                 return domain_context_mapping_one(domain, iommu, bus, devfn,
1998                                                   translation);
1999
2000         data.domain = domain;
2001         data.iommu = iommu;
2002         data.translation = translation;
2003
2004         return pci_for_each_dma_alias(to_pci_dev(dev),
2005                                       &domain_context_mapping_cb, &data);
2006 }
2007
2008 static int domain_context_mapped_cb(struct pci_dev *pdev,
2009                                     u16 alias, void *opaque)
2010 {
2011         struct intel_iommu *iommu = opaque;
2012
2013         return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
2014 }
2015
2016 static int domain_context_mapped(struct device *dev)
2017 {
2018         struct intel_iommu *iommu;
2019         u8 bus, devfn;
2020
2021         iommu = device_to_iommu(dev, &bus, &devfn);
2022         if (!iommu)
2023                 return -ENODEV;
2024
2025         if (!dev_is_pci(dev))
2026                 return device_context_mapped(iommu, bus, devfn);
2027
2028         return !pci_for_each_dma_alias(to_pci_dev(dev),
2029                                        domain_context_mapped_cb, iommu);
2030 }
2031
2032 /* Returns a number of VTD pages, but aligned to MM page size */
2033 static inline unsigned long aligned_nrpages(unsigned long host_addr,
2034                                             size_t size)
2035 {
2036         host_addr &= ~PAGE_MASK;
2037         return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2038 }
2039
2040 /* Return largest possible superpage level for a given mapping */
2041 static inline int hardware_largepage_caps(struct dmar_domain *domain,
2042                                           unsigned long iov_pfn,
2043                                           unsigned long phy_pfn,
2044                                           unsigned long pages)
2045 {
2046         int support, level = 1;
2047         unsigned long pfnmerge;
2048
2049         support = domain->iommu_superpage;
2050
2051         /* To use a large page, the virtual *and* physical addresses
2052            must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2053            of them will mean we have to use smaller pages. So just
2054            merge them and check both at once. */
2055         pfnmerge = iov_pfn | phy_pfn;
2056
2057         while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2058                 pages >>= VTD_STRIDE_SHIFT;
2059                 if (!pages)
2060                         break;
2061                 pfnmerge >>= VTD_STRIDE_SHIFT;
2062                 level++;
2063                 support--;
2064         }
2065         return level;
2066 }
2067
2068 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2069                             struct scatterlist *sg, unsigned long phys_pfn,
2070                             unsigned long nr_pages, int prot)
2071 {
2072         struct dma_pte *first_pte = NULL, *pte = NULL;
2073         phys_addr_t uninitialized_var(pteval);
2074         unsigned long sg_res = 0;
2075         unsigned int largepage_lvl = 0;
2076         unsigned long lvl_pages = 0;
2077
2078         BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
2079
2080         if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2081                 return -EINVAL;
2082
2083         prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2084
2085         if (!sg) {
2086                 sg_res = nr_pages;
2087                 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2088         }
2089
2090         while (nr_pages > 0) {
2091                 uint64_t tmp;
2092
2093                 if (!sg_res) {
2094                         sg_res = aligned_nrpages(sg->offset, sg->length);
2095                         sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
2096                         sg->dma_length = sg->length;
2097                         pteval = page_to_phys(sg_page(sg)) | prot;
2098                         phys_pfn = pteval >> VTD_PAGE_SHIFT;
2099                 }
2100
2101                 if (!pte) {
2102                         largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2103
2104                         first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
2105                         if (!pte)
2106                                 return -ENOMEM;
2107                         /* It is large page*/
2108                         if (largepage_lvl > 1) {
2109                                 pteval |= DMA_PTE_LARGE_PAGE;
2110                                 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2111                                 /*
2112                                  * Ensure that old small page tables are
2113                                  * removed to make room for superpage,
2114                                  * if they exist.
2115                                  */
2116                                 dma_pte_free_pagetable(domain, iov_pfn,
2117                                                        iov_pfn + lvl_pages - 1);
2118                         } else {
2119                                 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
2120                         }
2121
2122                 }
2123                 /* We don't need lock here, nobody else
2124                  * touches the iova range
2125                  */
2126                 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
2127                 if (tmp) {
2128                         static int dumps = 5;
2129                         pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2130                                 iov_pfn, tmp, (unsigned long long)pteval);
2131                         if (dumps) {
2132                                 dumps--;
2133                                 debug_dma_dump_mappings(NULL);
2134                         }
2135                         WARN_ON(1);
2136                 }
2137
2138                 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2139
2140                 BUG_ON(nr_pages < lvl_pages);
2141                 BUG_ON(sg_res < lvl_pages);
2142
2143                 nr_pages -= lvl_pages;
2144                 iov_pfn += lvl_pages;
2145                 phys_pfn += lvl_pages;
2146                 pteval += lvl_pages * VTD_PAGE_SIZE;
2147                 sg_res -= lvl_pages;
2148
2149                 /* If the next PTE would be the first in a new page, then we
2150                    need to flush the cache on the entries we've just written.
2151                    And then we'll need to recalculate 'pte', so clear it and
2152                    let it get set again in the if (!pte) block above.
2153
2154                    If we're done (!nr_pages) we need to flush the cache too.
2155
2156                    Also if we've been setting superpages, we may need to
2157                    recalculate 'pte' and switch back to smaller pages for the
2158                    end of the mapping, if the trailing size is not enough to
2159                    use another superpage (i.e. sg_res < lvl_pages). */
2160                 pte++;
2161                 if (!nr_pages || first_pte_in_page(pte) ||
2162                     (largepage_lvl > 1 && sg_res < lvl_pages)) {
2163                         domain_flush_cache(domain, first_pte,
2164                                            (void *)pte - (void *)first_pte);
2165                         pte = NULL;
2166                 }
2167
2168                 if (!sg_res && nr_pages)
2169                         sg = sg_next(sg);
2170         }
2171         return 0;
2172 }
2173
2174 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2175                                     struct scatterlist *sg, unsigned long nr_pages,
2176                                     int prot)
2177 {
2178         return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2179 }
2180
2181 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2182                                      unsigned long phys_pfn, unsigned long nr_pages,
2183                                      int prot)
2184 {
2185         return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
2186 }
2187
2188 static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
2189 {
2190         if (!iommu)
2191                 return;
2192
2193         clear_context_table(iommu, bus, devfn);
2194         iommu->flush.flush_context(iommu, 0, 0, 0,
2195                                            DMA_CCMD_GLOBAL_INVL);
2196         iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2197 }
2198
2199 static inline void unlink_domain_info(struct device_domain_info *info)
2200 {
2201         assert_spin_locked(&device_domain_lock);
2202         list_del(&info->link);
2203         list_del(&info->global);
2204         if (info->dev)
2205                 info->dev->archdata.iommu = NULL;
2206 }
2207
2208 static void domain_remove_dev_info(struct dmar_domain *domain)
2209 {
2210         struct device_domain_info *info, *tmp;
2211         unsigned long flags;
2212
2213         spin_lock_irqsave(&device_domain_lock, flags);
2214         list_for_each_entry_safe(info, tmp, &domain->devices, link) {
2215                 unlink_domain_info(info);
2216                 spin_unlock_irqrestore(&device_domain_lock, flags);
2217
2218                 iommu_disable_dev_iotlb(info);
2219                 iommu_detach_dev(info->iommu, info->bus, info->devfn);
2220
2221                 if (domain_type_is_vm(domain)) {
2222                         iommu_detach_dependent_devices(info->iommu, info->dev);
2223                         domain_detach_iommu(domain, info->iommu);
2224                 }
2225
2226                 free_devinfo_mem(info);
2227                 spin_lock_irqsave(&device_domain_lock, flags);
2228         }
2229         spin_unlock_irqrestore(&device_domain_lock, flags);
2230 }
2231
2232 /*
2233  * find_domain
2234  * Note: we use struct device->archdata.iommu stores the info
2235  */
2236 static struct dmar_domain *find_domain(struct device *dev)
2237 {
2238         struct device_domain_info *info;
2239
2240         /* No lock here, assumes no domain exit in normal case */
2241         info = dev->archdata.iommu;
2242         if (info)
2243                 return info->domain;
2244         return NULL;
2245 }
2246
2247 static inline struct device_domain_info *
2248 dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2249 {
2250         struct device_domain_info *info;
2251
2252         list_for_each_entry(info, &device_domain_list, global)
2253                 if (info->iommu->segment == segment && info->bus == bus &&
2254                     info->devfn == devfn)
2255                         return info;
2256
2257         return NULL;
2258 }
2259
2260 static struct dmar_domain *dmar_insert_dev_info(struct intel_iommu *iommu,
2261                                                 int bus, int devfn,
2262                                                 struct device *dev,
2263                                                 struct dmar_domain *domain)
2264 {
2265         struct dmar_domain *found = NULL;
2266         struct device_domain_info *info;
2267         unsigned long flags;
2268
2269         info = alloc_devinfo_mem();
2270         if (!info)
2271                 return NULL;
2272
2273         info->bus = bus;
2274         info->devfn = devfn;
2275         info->dev = dev;
2276         info->domain = domain;
2277         info->iommu = iommu;
2278
2279         spin_lock_irqsave(&device_domain_lock, flags);
2280         if (dev)
2281                 found = find_domain(dev);
2282         else {
2283                 struct device_domain_info *info2;
2284                 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
2285                 if (info2)
2286                         found = info2->domain;
2287         }
2288         if (found) {
2289                 spin_unlock_irqrestore(&device_domain_lock, flags);
2290                 free_devinfo_mem(info);
2291                 /* Caller must free the original domain */
2292                 return found;
2293         }
2294
2295         list_add(&info->link, &domain->devices);
2296         list_add(&info->global, &device_domain_list);
2297         if (dev)
2298                 dev->archdata.iommu = info;
2299         spin_unlock_irqrestore(&device_domain_lock, flags);
2300
2301         return domain;
2302 }
2303
2304 static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2305 {
2306         *(u16 *)opaque = alias;
2307         return 0;
2308 }
2309
2310 /* domain is initialized */
2311 static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
2312 {
2313         struct dmar_domain *domain, *tmp;
2314         struct intel_iommu *iommu;
2315         struct device_domain_info *info;
2316         u16 dma_alias;
2317         unsigned long flags;
2318         u8 bus, devfn;
2319
2320         domain = find_domain(dev);
2321         if (domain)
2322                 return domain;
2323
2324         iommu = device_to_iommu(dev, &bus, &devfn);
2325         if (!iommu)
2326                 return NULL;
2327
2328         if (dev_is_pci(dev)) {
2329                 struct pci_dev *pdev = to_pci_dev(dev);
2330
2331                 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2332
2333                 spin_lock_irqsave(&device_domain_lock, flags);
2334                 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2335                                                       PCI_BUS_NUM(dma_alias),
2336                                                       dma_alias & 0xff);
2337                 if (info) {
2338                         iommu = info->iommu;
2339                         domain = info->domain;
2340                 }
2341                 spin_unlock_irqrestore(&device_domain_lock, flags);
2342
2343                 /* DMA alias already has a domain, uses it */
2344                 if (info)
2345                         goto found_domain;
2346         }
2347
2348         /* Allocate and initialize new domain for the device */
2349         domain = alloc_domain(0);
2350         if (!domain)
2351                 return NULL;
2352         domain->id = iommu_attach_domain(domain, iommu);
2353         if (domain->id < 0) {
2354                 free_domain_mem(domain);
2355                 return NULL;
2356         }
2357         domain_attach_iommu(domain, iommu);
2358         if (domain_init(domain, gaw)) {
2359                 domain_exit(domain);
2360                 return NULL;
2361         }
2362
2363         /* register PCI DMA alias device */
2364         if (dev_is_pci(dev)) {
2365                 tmp = dmar_insert_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2366                                            dma_alias & 0xff, NULL, domain);
2367
2368                 if (!tmp || tmp != domain) {
2369                         domain_exit(domain);
2370                         domain = tmp;
2371                 }
2372
2373                 if (!domain)
2374                         return NULL;
2375         }
2376
2377 found_domain:
2378         tmp = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
2379
2380         if (!tmp || tmp != domain) {
2381                 domain_exit(domain);
2382                 domain = tmp;
2383         }
2384
2385         return domain;
2386 }
2387
2388 static int iommu_identity_mapping;
2389 #define IDENTMAP_ALL            1
2390 #define IDENTMAP_GFX            2
2391 #define IDENTMAP_AZALIA         4
2392
2393 static int iommu_domain_identity_map(struct dmar_domain *domain,
2394                                      unsigned long long start,
2395                                      unsigned long long end)
2396 {
2397         unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2398         unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2399
2400         if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2401                           dma_to_mm_pfn(last_vpfn))) {
2402                 pr_err("Reserving iova failed\n");
2403                 return -ENOMEM;
2404         }
2405
2406         pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
2407                  start, end, domain->id);
2408         /*
2409          * RMRR range might have overlap with physical memory range,
2410          * clear it first
2411          */
2412         dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2413
2414         return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2415                                   last_vpfn - first_vpfn + 1,
2416                                   DMA_PTE_READ|DMA_PTE_WRITE);
2417 }
2418
2419 static int iommu_prepare_identity_map(struct device *dev,
2420                                       unsigned long long start,
2421                                       unsigned long long end)
2422 {
2423         struct dmar_domain *domain;
2424         int ret;
2425
2426         domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2427         if (!domain)
2428                 return -ENOMEM;
2429
2430         /* For _hardware_ passthrough, don't bother. But for software
2431            passthrough, we do it anyway -- it may indicate a memory
2432            range which is reserved in E820, so which didn't get set
2433            up to start with in si_domain */
2434         if (domain == si_domain && hw_pass_through) {
2435                 pr_warn("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2436                         dev_name(dev), start, end);
2437                 return 0;
2438         }
2439
2440         pr_info("Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2441                 dev_name(dev), start, end);
2442
2443         if (end < start) {
2444                 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2445                         "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2446                         dmi_get_system_info(DMI_BIOS_VENDOR),
2447                         dmi_get_system_info(DMI_BIOS_VERSION),
2448                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2449                 ret = -EIO;
2450                 goto error;
2451         }
2452
2453         if (end >> agaw_to_width(domain->agaw)) {
2454                 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2455                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2456                      agaw_to_width(domain->agaw),
2457                      dmi_get_system_info(DMI_BIOS_VENDOR),
2458                      dmi_get_system_info(DMI_BIOS_VERSION),
2459                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2460                 ret = -EIO;
2461                 goto error;
2462         }
2463
2464         ret = iommu_domain_identity_map(domain, start, end);
2465         if (ret)
2466                 goto error;
2467
2468         /* context entry init */
2469         ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL);
2470         if (ret)
2471                 goto error;
2472
2473         return 0;
2474
2475  error:
2476         domain_exit(domain);
2477         return ret;
2478 }
2479
2480 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2481                                          struct device *dev)
2482 {
2483         if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2484                 return 0;
2485         return iommu_prepare_identity_map(dev, rmrr->base_address,
2486                                           rmrr->end_address);
2487 }
2488
2489 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
2490 static inline void iommu_prepare_isa(void)
2491 {
2492         struct pci_dev *pdev;
2493         int ret;
2494
2495         pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2496         if (!pdev)
2497                 return;
2498
2499         pr_info("Prepare 0-16MiB unity mapping for LPC\n");
2500         ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
2501
2502         if (ret)
2503                 pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
2504
2505         pci_dev_put(pdev);
2506 }
2507 #else
2508 static inline void iommu_prepare_isa(void)
2509 {
2510         return;
2511 }
2512 #endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
2513
2514 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2515
2516 static int __init si_domain_init(int hw)
2517 {
2518         struct dmar_drhd_unit *drhd;
2519         struct intel_iommu *iommu;
2520         int nid, ret = 0;
2521         bool first = true;
2522
2523         si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2524         if (!si_domain)
2525                 return -EFAULT;
2526
2527         for_each_active_iommu(iommu, drhd) {
2528                 ret = iommu_attach_domain(si_domain, iommu);
2529                 if (ret < 0) {
2530                         domain_exit(si_domain);
2531                         return -EFAULT;
2532                 } else if (first) {
2533                         si_domain->id = ret;
2534                         first = false;
2535                 } else if (si_domain->id != ret) {
2536                         domain_exit(si_domain);
2537                         return -EFAULT;
2538                 }
2539                 domain_attach_iommu(si_domain, iommu);
2540         }
2541
2542         if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2543                 domain_exit(si_domain);
2544                 return -EFAULT;
2545         }
2546
2547         pr_debug("Identity mapping domain is domain %d\n",
2548                  si_domain->id);
2549
2550         if (hw)
2551                 return 0;
2552
2553         for_each_online_node(nid) {
2554                 unsigned long start_pfn, end_pfn;
2555                 int i;
2556
2557                 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2558                         ret = iommu_domain_identity_map(si_domain,
2559                                         PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2560                         if (ret)
2561                                 return ret;
2562                 }
2563         }
2564
2565         return 0;
2566 }
2567
2568 static int identity_mapping(struct device *dev)
2569 {
2570         struct device_domain_info *info;
2571
2572         if (likely(!iommu_identity_mapping))
2573                 return 0;
2574
2575         info = dev->archdata.iommu;
2576         if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2577                 return (info->domain == si_domain);
2578
2579         return 0;
2580 }
2581
2582 static int domain_add_dev_info(struct dmar_domain *domain,
2583                                struct device *dev, int translation)
2584 {
2585         struct dmar_domain *ndomain;
2586         struct intel_iommu *iommu;
2587         u8 bus, devfn;
2588         int ret;
2589
2590         iommu = device_to_iommu(dev, &bus, &devfn);
2591         if (!iommu)
2592                 return -ENODEV;
2593
2594         ndomain = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
2595         if (ndomain != domain)
2596                 return -EBUSY;
2597
2598         ret = domain_context_mapping(domain, dev, translation);
2599         if (ret) {
2600                 domain_remove_one_dev_info(domain, dev);
2601                 return ret;
2602         }
2603
2604         return 0;
2605 }
2606
2607 static bool device_has_rmrr(struct device *dev)
2608 {
2609         struct dmar_rmrr_unit *rmrr;
2610         struct device *tmp;
2611         int i;
2612
2613         rcu_read_lock();
2614         for_each_rmrr_units(rmrr) {
2615                 /*
2616                  * Return TRUE if this RMRR contains the device that
2617                  * is passed in.
2618                  */
2619                 for_each_active_dev_scope(rmrr->devices,
2620                                           rmrr->devices_cnt, i, tmp)
2621                         if (tmp == dev) {
2622                                 rcu_read_unlock();
2623                                 return true;
2624                         }
2625         }
2626         rcu_read_unlock();
2627         return false;
2628 }
2629
2630 /*
2631  * There are a couple cases where we need to restrict the functionality of
2632  * devices associated with RMRRs.  The first is when evaluating a device for
2633  * identity mapping because problems exist when devices are moved in and out
2634  * of domains and their respective RMRR information is lost.  This means that
2635  * a device with associated RMRRs will never be in a "passthrough" domain.
2636  * The second is use of the device through the IOMMU API.  This interface
2637  * expects to have full control of the IOVA space for the device.  We cannot
2638  * satisfy both the requirement that RMRR access is maintained and have an
2639  * unencumbered IOVA space.  We also have no ability to quiesce the device's
2640  * use of the RMRR space or even inform the IOMMU API user of the restriction.
2641  * We therefore prevent devices associated with an RMRR from participating in
2642  * the IOMMU API, which eliminates them from device assignment.
2643  *
2644  * In both cases we assume that PCI USB devices with RMRRs have them largely
2645  * for historical reasons and that the RMRR space is not actively used post
2646  * boot.  This exclusion may change if vendors begin to abuse it.
2647  *
2648  * The same exception is made for graphics devices, with the requirement that
2649  * any use of the RMRR regions will be torn down before assigning the device
2650  * to a guest.
2651  */
2652 static bool device_is_rmrr_locked(struct device *dev)
2653 {
2654         if (!device_has_rmrr(dev))
2655                 return false;
2656
2657         if (dev_is_pci(dev)) {
2658                 struct pci_dev *pdev = to_pci_dev(dev);
2659
2660                 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
2661                         return false;
2662         }
2663
2664         return true;
2665 }
2666
2667 static int iommu_should_identity_map(struct device *dev, int startup)
2668 {
2669
2670         if (dev_is_pci(dev)) {
2671                 struct pci_dev *pdev = to_pci_dev(dev);
2672
2673                 if (device_is_rmrr_locked(dev))
2674                         return 0;
2675
2676                 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2677                         return 1;
2678
2679                 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2680                         return 1;
2681
2682                 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2683                         return 0;
2684
2685                 /*
2686                  * We want to start off with all devices in the 1:1 domain, and
2687                  * take them out later if we find they can't access all of memory.
2688                  *
2689                  * However, we can't do this for PCI devices behind bridges,
2690                  * because all PCI devices behind the same bridge will end up
2691                  * with the same source-id on their transactions.
2692                  *
2693                  * Practically speaking, we can't change things around for these
2694                  * devices at run-time, because we can't be sure there'll be no
2695                  * DMA transactions in flight for any of their siblings.
2696                  *
2697                  * So PCI devices (unless they're on the root bus) as well as
2698                  * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2699                  * the 1:1 domain, just in _case_ one of their siblings turns out
2700                  * not to be able to map all of memory.
2701                  */
2702                 if (!pci_is_pcie(pdev)) {
2703                         if (!pci_is_root_bus(pdev->bus))
2704                                 return 0;
2705                         if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2706                                 return 0;
2707                 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
2708                         return 0;
2709         } else {
2710                 if (device_has_rmrr(dev))
2711                         return 0;
2712         }
2713
2714         /*
2715          * At boot time, we don't yet know if devices will be 64-bit capable.
2716          * Assume that they will — if they turn out not to be, then we can
2717          * take them out of the 1:1 domain later.
2718          */
2719         if (!startup) {
2720                 /*
2721                  * If the device's dma_mask is less than the system's memory
2722                  * size then this is not a candidate for identity mapping.
2723                  */
2724                 u64 dma_mask = *dev->dma_mask;
2725
2726                 if (dev->coherent_dma_mask &&
2727                     dev->coherent_dma_mask < dma_mask)
2728                         dma_mask = dev->coherent_dma_mask;
2729
2730                 return dma_mask >= dma_get_required_mask(dev);
2731         }
2732
2733         return 1;
2734 }
2735
2736 static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2737 {
2738         int ret;
2739
2740         if (!iommu_should_identity_map(dev, 1))
2741                 return 0;
2742
2743         ret = domain_add_dev_info(si_domain, dev,
2744                                   hw ? CONTEXT_TT_PASS_THROUGH :
2745                                        CONTEXT_TT_MULTI_LEVEL);
2746         if (!ret)
2747                 pr_info("%s identity mapping for device %s\n",
2748                         hw ? "Hardware" : "Software", dev_name(dev));
2749         else if (ret == -ENODEV)
2750                 /* device not associated with an iommu */
2751                 ret = 0;
2752
2753         return ret;
2754 }
2755
2756
2757 static int __init iommu_prepare_static_identity_mapping(int hw)
2758 {
2759         struct pci_dev *pdev = NULL;
2760         struct dmar_drhd_unit *drhd;
2761         struct intel_iommu *iommu;
2762         struct device *dev;
2763         int i;
2764         int ret = 0;
2765
2766         for_each_pci_dev(pdev) {
2767                 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2768                 if (ret)
2769                         return ret;
2770         }
2771
2772         for_each_active_iommu(iommu, drhd)
2773                 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
2774                         struct acpi_device_physical_node *pn;
2775                         struct acpi_device *adev;
2776
2777                         if (dev->bus != &acpi_bus_type)
2778                                 continue;
2779
2780                         adev= to_acpi_device(dev);
2781                         mutex_lock(&adev->physical_node_lock);
2782                         list_for_each_entry(pn, &adev->physical_node_list, node) {
2783                                 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
2784                                 if (ret)
2785                                         break;
2786                         }
2787                         mutex_unlock(&adev->physical_node_lock);
2788                         if (ret)
2789                                 return ret;
2790                 }
2791
2792         return 0;
2793 }
2794
2795 static void intel_iommu_init_qi(struct intel_iommu *iommu)
2796 {
2797         /*
2798          * Start from the sane iommu hardware state.
2799          * If the queued invalidation is already initialized by us
2800          * (for example, while enabling interrupt-remapping) then
2801          * we got the things already rolling from a sane state.
2802          */
2803         if (!iommu->qi) {
2804                 /*
2805                  * Clear any previous faults.
2806                  */
2807                 dmar_fault(-1, iommu);
2808                 /*
2809                  * Disable queued invalidation if supported and already enabled
2810                  * before OS handover.
2811                  */
2812                 dmar_disable_qi(iommu);
2813         }
2814
2815         if (dmar_enable_qi(iommu)) {
2816                 /*
2817                  * Queued Invalidate not enabled, use Register Based Invalidate
2818                  */
2819                 iommu->flush.flush_context = __iommu_flush_context;
2820                 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
2821                 pr_info("%s: Using Register based invalidation\n",
2822                         iommu->name);
2823         } else {
2824                 iommu->flush.flush_context = qi_flush_context;
2825                 iommu->flush.flush_iotlb = qi_flush_iotlb;
2826                 pr_info("%s: Using Queued invalidation\n", iommu->name);
2827         }
2828 }
2829
2830 static int copy_context_table(struct intel_iommu *iommu,
2831                               struct root_entry *old_re,
2832                               struct context_entry **tbl,
2833                               int bus, bool ext)
2834 {
2835         struct context_entry *old_ce = NULL, *new_ce = NULL, ce;
2836         int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
2837         phys_addr_t old_ce_phys;
2838
2839         tbl_idx = ext ? bus * 2 : bus;
2840
2841         for (devfn = 0; devfn < 256; devfn++) {
2842                 /* First calculate the correct index */
2843                 idx = (ext ? devfn * 2 : devfn) % 256;
2844
2845                 if (idx == 0) {
2846                         /* First save what we may have and clean up */
2847                         if (new_ce) {
2848                                 tbl[tbl_idx] = new_ce;
2849                                 __iommu_flush_cache(iommu, new_ce,
2850                                                     VTD_PAGE_SIZE);
2851                                 pos = 1;
2852                         }
2853
2854                         if (old_ce)
2855                                 iounmap(old_ce);
2856
2857                         ret = 0;
2858                         if (devfn < 0x80)
2859                                 old_ce_phys = root_entry_lctp(old_re);
2860                         else
2861                                 old_ce_phys = root_entry_uctp(old_re);
2862
2863                         if (!old_ce_phys) {
2864                                 if (ext && devfn == 0) {
2865                                         /* No LCTP, try UCTP */
2866                                         devfn = 0x7f;
2867                                         continue;
2868                                 } else {
2869                                         goto out;
2870                                 }
2871                         }
2872
2873                         ret = -ENOMEM;
2874                         old_ce = ioremap_cache(old_ce_phys, PAGE_SIZE);
2875                         if (!old_ce)
2876                                 goto out;
2877
2878                         new_ce = alloc_pgtable_page(iommu->node);
2879                         if (!new_ce)
2880                                 goto out_unmap;
2881
2882                         ret = 0;
2883                 }
2884
2885                 /* Now copy the context entry */
2886                 ce = old_ce[idx];
2887
2888                 if (!__context_present(&ce))
2889                         continue;
2890
2891                 did = context_domain_id(&ce);
2892                 if (did >= 0 && did < cap_ndoms(iommu->cap))
2893                         set_bit(did, iommu->domain_ids);
2894
2895                 /*
2896                  * We need a marker for copied context entries. This
2897                  * marker needs to work for the old format as well as
2898                  * for extended context entries.
2899                  *
2900                  * Bit 67 of the context entry is used. In the old
2901                  * format this bit is available to software, in the
2902                  * extended format it is the PGE bit, but PGE is ignored
2903                  * by HW if PASIDs are disabled (and thus still
2904                  * available).
2905                  *
2906                  * So disable PASIDs first and then mark the entry
2907                  * copied. This means that we don't copy PASID
2908                  * translations from the old kernel, but this is fine as
2909                  * faults there are not fatal.
2910                  */
2911                 context_clear_pasid_enable(&ce);
2912                 context_set_copied(&ce);
2913
2914                 new_ce[idx] = ce;
2915         }
2916
2917         tbl[tbl_idx + pos] = new_ce;
2918
2919         __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
2920
2921 out_unmap:
2922         iounmap(old_ce);
2923
2924 out:
2925         return ret;
2926 }
2927
2928 static int copy_translation_tables(struct intel_iommu *iommu)
2929 {
2930         struct context_entry **ctxt_tbls;
2931         struct root_entry *old_rt;
2932         phys_addr_t old_rt_phys;
2933         int ctxt_table_entries;
2934         unsigned long flags;
2935         u64 rtaddr_reg;
2936         int bus, ret;
2937         bool new_ext, ext;
2938
2939         rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
2940         ext        = !!(rtaddr_reg & DMA_RTADDR_RTT);
2941         new_ext    = !!ecap_ecs(iommu->ecap);
2942
2943         /*
2944          * The RTT bit can only be changed when translation is disabled,
2945          * but disabling translation means to open a window for data
2946          * corruption. So bail out and don't copy anything if we would
2947          * have to change the bit.
2948          */
2949         if (new_ext != ext)
2950                 return -EINVAL;
2951
2952         old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
2953         if (!old_rt_phys)
2954                 return -EINVAL;
2955
2956         old_rt = ioremap_cache(old_rt_phys, PAGE_SIZE);
2957         if (!old_rt)
2958                 return -ENOMEM;
2959
2960         /* This is too big for the stack - allocate it from slab */
2961         ctxt_table_entries = ext ? 512 : 256;
2962         ret = -ENOMEM;
2963         ctxt_tbls = kzalloc(ctxt_table_entries * sizeof(void *), GFP_KERNEL);
2964         if (!ctxt_tbls)
2965                 goto out_unmap;
2966
2967         for (bus = 0; bus < 256; bus++) {
2968                 ret = copy_context_table(iommu, &old_rt[bus],
2969                                          ctxt_tbls, bus, ext);
2970                 if (ret) {
2971                         pr_err("%s: Failed to copy context table for bus %d\n",
2972                                 iommu->name, bus);
2973                         continue;
2974                 }
2975         }
2976
2977         spin_lock_irqsave(&iommu->lock, flags);
2978
2979         /* Context tables are copied, now write them to the root_entry table */
2980         for (bus = 0; bus < 256; bus++) {
2981                 int idx = ext ? bus * 2 : bus;
2982                 u64 val;
2983
2984                 if (ctxt_tbls[idx]) {
2985                         val = virt_to_phys(ctxt_tbls[idx]) | 1;
2986                         iommu->root_entry[bus].lo = val;
2987                 }
2988
2989                 if (!ext || !ctxt_tbls[idx + 1])
2990                         continue;
2991
2992                 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
2993                 iommu->root_entry[bus].hi = val;
2994         }
2995
2996         spin_unlock_irqrestore(&iommu->lock, flags);
2997
2998         kfree(ctxt_tbls);
2999
3000         __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
3001
3002         ret = 0;
3003
3004 out_unmap:
3005         iounmap(old_rt);
3006
3007         return ret;
3008 }
3009
3010 static int __init init_dmars(void)
3011 {
3012         struct dmar_drhd_unit *drhd;
3013         struct dmar_rmrr_unit *rmrr;
3014         bool copied_tables = false;
3015         struct device *dev;
3016         struct intel_iommu *iommu;
3017         int i, ret;
3018
3019         /*
3020          * for each drhd
3021          *    allocate root
3022          *    initialize and program root entry to not present
3023          * endfor
3024          */
3025         for_each_drhd_unit(drhd) {
3026                 /*
3027                  * lock not needed as this is only incremented in the single
3028                  * threaded kernel __init code path all other access are read
3029                  * only
3030                  */
3031                 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
3032                         g_num_of_iommus++;
3033                         continue;
3034                 }
3035                 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
3036         }
3037
3038         /* Preallocate enough resources for IOMMU hot-addition */
3039         if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
3040                 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
3041
3042         g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
3043                         GFP_KERNEL);
3044         if (!g_iommus) {
3045                 pr_err("Allocating global iommu array failed\n");
3046                 ret = -ENOMEM;
3047                 goto error;
3048         }
3049
3050         deferred_flush = kzalloc(g_num_of_iommus *
3051                 sizeof(struct deferred_flush_tables), GFP_KERNEL);
3052         if (!deferred_flush) {
3053                 ret = -ENOMEM;
3054                 goto free_g_iommus;
3055         }
3056
3057         for_each_active_iommu(iommu, drhd) {
3058                 g_iommus[iommu->seq_id] = iommu;
3059
3060                 intel_iommu_init_qi(iommu);
3061
3062                 ret = iommu_init_domains(iommu);
3063                 if (ret)
3064                         goto free_iommu;
3065
3066                 init_translation_status(iommu);
3067
3068                 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
3069                         iommu_disable_translation(iommu);
3070                         clear_translation_pre_enabled(iommu);
3071                         pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
3072                                 iommu->name);
3073                 }
3074
3075                 /*
3076                  * TBD:
3077                  * we could share the same root & context tables
3078                  * among all IOMMU's. Need to Split it later.
3079                  */
3080                 ret = iommu_alloc_root_entry(iommu);
3081                 if (ret)
3082                         goto free_iommu;
3083
3084                 if (translation_pre_enabled(iommu)) {
3085                         pr_info("Translation already enabled - trying to copy translation structures\n");
3086
3087                         ret = copy_translation_tables(iommu);
3088                         if (ret) {
3089                                 /*
3090                                  * We found the IOMMU with translation
3091                                  * enabled - but failed to copy over the
3092                                  * old root-entry table. Try to proceed
3093                                  * by disabling translation now and
3094                                  * allocating a clean root-entry table.
3095                                  * This might cause DMAR faults, but
3096                                  * probably the dump will still succeed.
3097                                  */
3098                                 pr_err("Failed to copy translation tables from previous kernel for %s\n",
3099                                        iommu->name);
3100                                 iommu_disable_translation(iommu);
3101                                 clear_translation_pre_enabled(iommu);
3102                         } else {
3103                                 pr_info("Copied translation tables from previous kernel for %s\n",
3104                                         iommu->name);
3105                                 copied_tables = true;
3106                         }
3107                 }
3108
3109                 iommu_flush_write_buffer(iommu);
3110                 iommu_set_root_entry(iommu);
3111                 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3112                 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3113
3114                 if (!ecap_pass_through(iommu->ecap))
3115                         hw_pass_through = 0;
3116         }
3117
3118         if (iommu_pass_through)
3119                 iommu_identity_mapping |= IDENTMAP_ALL;
3120
3121 #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
3122         iommu_identity_mapping |= IDENTMAP_GFX;
3123 #endif
3124
3125         if (iommu_identity_mapping) {
3126                 ret = si_domain_init(hw_pass_through);
3127                 if (ret)
3128                         goto free_iommu;
3129         }
3130
3131         check_tylersburg_isoch();
3132
3133         /*
3134          * If we copied translations from a previous kernel in the kdump
3135          * case, we can not assign the devices to domains now, as that
3136          * would eliminate the old mappings. So skip this part and defer
3137          * the assignment to device driver initialization time.
3138          */
3139         if (copied_tables)
3140                 goto domains_done;
3141
3142         /*
3143          * If pass through is not set or not enabled, setup context entries for
3144          * identity mappings for rmrr, gfx, and isa and may fall back to static
3145          * identity mapping if iommu_identity_mapping is set.
3146          */
3147         if (iommu_identity_mapping) {
3148                 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
3149                 if (ret) {
3150                         pr_crit("Failed to setup IOMMU pass-through\n");
3151                         goto free_iommu;
3152                 }
3153         }
3154         /*
3155          * For each rmrr
3156          *   for each dev attached to rmrr
3157          *   do
3158          *     locate drhd for dev, alloc domain for dev
3159          *     allocate free domain
3160          *     allocate page table entries for rmrr
3161          *     if context not allocated for bus
3162          *           allocate and init context
3163          *           set present in root table for this bus
3164          *     init context with domain, translation etc
3165          *    endfor
3166          * endfor
3167          */
3168         pr_info("Setting RMRR:\n");
3169         for_each_rmrr_units(rmrr) {
3170                 /* some BIOS lists non-exist devices in DMAR table. */
3171                 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3172                                           i, dev) {
3173                         ret = iommu_prepare_rmrr_dev(rmrr, dev);
3174                         if (ret)
3175                                 pr_err("Mapping reserved region failed\n");
3176                 }
3177         }
3178
3179         iommu_prepare_isa();
3180
3181 domains_done:
3182
3183         /*
3184          * for each drhd
3185          *   enable fault log
3186          *   global invalidate context cache
3187          *   global invalidate iotlb
3188          *   enable translation
3189          */
3190         for_each_iommu(iommu, drhd) {
3191                 if (drhd->ignored) {
3192                         /*
3193                          * we always have to disable PMRs or DMA may fail on
3194                          * this device
3195                          */
3196                         if (force_on)
3197                                 iommu_disable_protect_mem_regions(iommu);
3198                         continue;
3199                 }
3200
3201                 iommu_flush_write_buffer(iommu);
3202
3203                 ret = dmar_set_interrupt(iommu);
3204                 if (ret)
3205                         goto free_iommu;
3206
3207                 if (!translation_pre_enabled(iommu))
3208                         iommu_enable_translation(iommu);
3209
3210                 iommu_disable_protect_mem_regions(iommu);
3211         }
3212
3213         return 0;
3214
3215 free_iommu:
3216         for_each_active_iommu(iommu, drhd) {
3217                 disable_dmar_iommu(iommu);
3218                 free_dmar_iommu(iommu);
3219         }
3220         kfree(deferred_flush);
3221 free_g_iommus:
3222         kfree(g_iommus);
3223 error:
3224         return ret;
3225 }
3226
3227 /* This takes a number of _MM_ pages, not VTD pages */
3228 static struct iova *intel_alloc_iova(struct device *dev,
3229                                      struct dmar_domain *domain,
3230                                      unsigned long nrpages, uint64_t dma_mask)
3231 {
3232         struct iova *iova = NULL;
3233
3234         /* Restrict dma_mask to the width that the iommu can handle */
3235         dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
3236
3237         if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
3238                 /*
3239                  * First try to allocate an io virtual address in
3240                  * DMA_BIT_MASK(32) and if that fails then try allocating
3241                  * from higher range
3242                  */
3243                 iova = alloc_iova(&domain->iovad, nrpages,
3244                                   IOVA_PFN(DMA_BIT_MASK(32)), 1);
3245                 if (iova)
3246                         return iova;
3247         }
3248         iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
3249         if (unlikely(!iova)) {
3250                 pr_err("Allocating %ld-page iova for %s failed",
3251                        nrpages, dev_name(dev));
3252                 return NULL;
3253         }
3254
3255         return iova;
3256 }
3257
3258 static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
3259 {
3260         struct dmar_domain *domain;
3261         int ret;
3262
3263         domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
3264         if (!domain) {
3265                 pr_err("Allocating domain for %s failed\n",
3266                        dev_name(dev));
3267                 return NULL;
3268         }
3269
3270         /* make sure context mapping is ok */
3271         if (unlikely(!domain_context_mapped(dev))) {
3272                 ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL);
3273                 if (ret) {
3274                         pr_err("Domain context map for %s failed\n",
3275                                dev_name(dev));
3276                         return NULL;
3277                 }
3278         }
3279
3280         return domain;
3281 }
3282
3283 static inline struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
3284 {
3285         struct device_domain_info *info;
3286
3287         /* No lock here, assumes no domain exit in normal case */
3288         info = dev->archdata.iommu;
3289         if (likely(info))
3290                 return info->domain;
3291
3292         return __get_valid_domain_for_dev(dev);
3293 }
3294
3295 /* Check if the dev needs to go through non-identity map and unmap process.*/
3296 static int iommu_no_mapping(struct device *dev)
3297 {
3298         int found;
3299
3300         if (iommu_dummy(dev))
3301                 return 1;
3302
3303         if (!iommu_identity_mapping)
3304                 return 0;
3305
3306         found = identity_mapping(dev);
3307         if (found) {
3308                 if (iommu_should_identity_map(dev, 0))
3309                         return 1;
3310                 else {
3311                         /*
3312                          * 32 bit DMA is removed from si_domain and fall back
3313                          * to non-identity mapping.
3314                          */
3315                         domain_remove_one_dev_info(si_domain, dev);
3316                         pr_info("32bit %s uses non-identity mapping\n",
3317                                 dev_name(dev));
3318                         return 0;
3319                 }
3320         } else {
3321                 /*
3322                  * In case of a detached 64 bit DMA device from vm, the device
3323                  * is put into si_domain for identity mapping.
3324                  */
3325                 if (iommu_should_identity_map(dev, 0)) {
3326                         int ret;
3327                         ret = domain_add_dev_info(si_domain, dev,
3328                                                   hw_pass_through ?
3329                                                   CONTEXT_TT_PASS_THROUGH :
3330                                                   CONTEXT_TT_MULTI_LEVEL);
3331                         if (!ret) {
3332                                 pr_info("64bit %s uses identity mapping\n",
3333                                         dev_name(dev));
3334                                 return 1;
3335                         }
3336                 }
3337         }
3338
3339         return 0;
3340 }
3341
3342 static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
3343                                      size_t size, int dir, u64 dma_mask)
3344 {
3345         struct dmar_domain *domain;
3346         phys_addr_t start_paddr;
3347         struct iova *iova;
3348         int prot = 0;
3349         int ret;
3350         struct intel_iommu *iommu;
3351         unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
3352
3353         BUG_ON(dir == DMA_NONE);
3354
3355         if (iommu_no_mapping(dev))
3356                 return paddr;
3357
3358         domain = get_valid_domain_for_dev(dev);
3359         if (!domain)
3360                 return 0;
3361
3362         iommu = domain_get_iommu(domain);
3363         size = aligned_nrpages(paddr, size);
3364
3365         iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
3366         if (!iova)
3367                 goto error;
3368
3369         /*
3370          * Check if DMAR supports zero-length reads on write only
3371          * mappings..
3372          */
3373         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3374                         !cap_zlr(iommu->cap))
3375                 prot |= DMA_PTE_READ;
3376         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3377                 prot |= DMA_PTE_WRITE;
3378         /*
3379          * paddr - (paddr + size) might be partial page, we should map the whole
3380          * page.  Note: if two part of one page are separately mapped, we
3381          * might have two guest_addr mapping to the same host paddr, but this
3382          * is not a big problem
3383          */
3384         ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
3385                                  mm_to_dma_pfn(paddr_pfn), size, prot);
3386         if (ret)
3387                 goto error;
3388
3389         /* it's a non-present to present mapping. Only flush if caching mode */
3390         if (cap_caching_mode(iommu->cap))
3391                 iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 0, 1);
3392         else
3393                 iommu_flush_write_buffer(iommu);
3394
3395         start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
3396         start_paddr += paddr & ~PAGE_MASK;
3397         return start_paddr;
3398
3399 error:
3400         if (iova)
3401                 __free_iova(&domain->iovad, iova);
3402         pr_err("Device %s request: %zx@%llx dir %d --- failed\n",
3403                 dev_name(dev), size, (unsigned long long)paddr, dir);
3404         return 0;
3405 }
3406
3407 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3408                                  unsigned long offset, size_t size,
3409                                  enum dma_data_direction dir,
3410                                  struct dma_attrs *attrs)
3411 {
3412         return __intel_map_single(dev, page_to_phys(page) + offset, size,
3413                                   dir, *dev->dma_mask);
3414 }
3415
3416 static void flush_unmaps(void)
3417 {
3418         int i, j;
3419
3420         timer_on = 0;
3421
3422         /* just flush them all */
3423         for (i = 0; i < g_num_of_iommus; i++) {
3424                 struct intel_iommu *iommu = g_iommus[i];
3425                 if (!iommu)
3426                         continue;
3427
3428                 if (!deferred_flush[i].next)
3429                         continue;
3430
3431                 /* In caching mode, global flushes turn emulation expensive */
3432                 if (!cap_caching_mode(iommu->cap))
3433                         iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3434                                          DMA_TLB_GLOBAL_FLUSH);
3435                 for (j = 0; j < deferred_flush[i].next; j++) {
3436                         unsigned long mask;
3437                         struct iova *iova = deferred_flush[i].iova[j];
3438                         struct dmar_domain *domain = deferred_flush[i].domain[j];
3439
3440                         /* On real hardware multiple invalidations are expensive */
3441                         if (cap_caching_mode(iommu->cap))
3442                                 iommu_flush_iotlb_psi(iommu, domain->id,
3443                                         iova->pfn_lo, iova_size(iova),
3444                                         !deferred_flush[i].freelist[j], 0);
3445                         else {
3446                                 mask = ilog2(mm_to_dma_pfn(iova_size(iova)));
3447                                 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
3448                                                 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
3449                         }
3450                         __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
3451                         if (deferred_flush[i].freelist[j])
3452                                 dma_free_pagelist(deferred_flush[i].freelist[j]);
3453                 }
3454                 deferred_flush[i].next = 0;
3455         }
3456
3457         list_size = 0;
3458 }
3459
3460 static void flush_unmaps_timeout(unsigned long data)
3461 {
3462         unsigned long flags;
3463
3464         spin_lock_irqsave(&async_umap_flush_lock, flags);
3465         flush_unmaps();
3466         spin_unlock_irqrestore(&async_umap_flush_lock, flags);
3467 }
3468
3469 static void add_unmap(struct dmar_domain *dom, struct iova *iova, struct page *freelist)
3470 {
3471         unsigned long flags;
3472         int next, iommu_id;
3473         struct intel_iommu *iommu;
3474
3475         spin_lock_irqsave(&async_umap_flush_lock, flags);
3476         if (list_size == HIGH_WATER_MARK)
3477                 flush_unmaps();
3478
3479         iommu = domain_get_iommu(dom);
3480         iommu_id = iommu->seq_id;
3481
3482         next = deferred_flush[iommu_id].next;
3483         deferred_flush[iommu_id].domain[next] = dom;
3484         deferred_flush[iommu_id].iova[next] = iova;
3485         deferred_flush[iommu_id].freelist[next] = freelist;
3486         deferred_flush[iommu_id].next++;
3487
3488         if (!timer_on) {
3489                 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
3490                 timer_on = 1;
3491         }
3492         list_size++;
3493         spin_unlock_irqrestore(&async_umap_flush_lock, flags);
3494 }
3495
3496 static void intel_unmap(struct device *dev, dma_addr_t dev_addr)
3497 {
3498         struct dmar_domain *domain;
3499         unsigned long start_pfn, last_pfn;
3500         struct iova *iova;
3501         struct intel_iommu *iommu;
3502         struct page *freelist;
3503
3504         if (iommu_no_mapping(dev))
3505                 return;
3506
3507         domain = find_domain(dev);
3508         BUG_ON(!domain);
3509
3510         iommu = domain_get_iommu(domain);
3511
3512         iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
3513         if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
3514                       (unsigned long long)dev_addr))
3515                 return;
3516
3517         start_pfn = mm_to_dma_pfn(iova->pfn_lo);
3518         last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
3519
3520         pr_debug("Device %s unmapping: pfn %lx-%lx\n",
3521                  dev_name(dev), start_pfn, last_pfn);
3522
3523         freelist = domain_unmap(domain, start_pfn, last_pfn);
3524
3525         if (intel_iommu_strict) {
3526                 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
3527                                       last_pfn - start_pfn + 1, !freelist, 0);
3528                 /* free iova */
3529                 __free_iova(&domain->iovad, iova);
3530                 dma_free_pagelist(freelist);
3531         } else {
3532                 add_unmap(domain, iova, freelist);
3533                 /*
3534                  * queue up the release of the unmap to save the 1/6th of the
3535                  * cpu used up by the iotlb flush operation...
3536                  */
3537         }
3538 }
3539
3540 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3541                              size_t size, enum dma_data_direction dir,
3542                              struct dma_attrs *attrs)
3543 {
3544         intel_unmap(dev, dev_addr);
3545 }
3546
3547 static void *intel_alloc_coherent(struct device *dev, size_t size,
3548                                   dma_addr_t *dma_handle, gfp_t flags,
3549                                   struct dma_attrs *attrs)
3550 {
3551         struct page *page = NULL;
3552         int order;
3553
3554         size = PAGE_ALIGN(size);
3555         order = get_order(size);
3556
3557         if (!iommu_no_mapping(dev))
3558                 flags &= ~(GFP_DMA | GFP_DMA32);
3559         else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3560                 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
3561                         flags |= GFP_DMA;
3562                 else
3563                         flags |= GFP_DMA32;
3564         }
3565
3566         if (flags & __GFP_WAIT) {
3567                 unsigned int count = size >> PAGE_SHIFT;
3568
3569                 page = dma_alloc_from_contiguous(dev, count, order);
3570                 if (page && iommu_no_mapping(dev) &&
3571                     page_to_phys(page) + size > dev->coherent_dma_mask) {
3572                         dma_release_from_contiguous(dev, page, count);
3573                         page = NULL;
3574                 }
3575         }
3576
3577         if (!page)
3578                 page = alloc_pages(flags, order);
3579         if (!page)
3580                 return NULL;
3581         memset(page_address(page), 0, size);
3582
3583         *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
3584                                          DMA_BIDIRECTIONAL,
3585                                          dev->coherent_dma_mask);
3586         if (*dma_handle)
3587                 return page_address(page);
3588         if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3589                 __free_pages(page, order);
3590
3591         return NULL;
3592 }
3593
3594 static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
3595                                 dma_addr_t dma_handle, struct dma_attrs *attrs)
3596 {
3597         int order;
3598         struct page *page = virt_to_page(vaddr);
3599
3600         size = PAGE_ALIGN(size);
3601         order = get_order(size);
3602
3603         intel_unmap(dev, dma_handle);
3604         if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3605                 __free_pages(page, order);
3606 }
3607
3608 static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
3609                            int nelems, enum dma_data_direction dir,
3610                            struct dma_attrs *attrs)
3611 {
3612         intel_unmap(dev, sglist[0].dma_address);
3613 }
3614
3615 static int intel_nontranslate_map_sg(struct device *hddev,
3616         struct scatterlist *sglist, int nelems, int dir)
3617 {
3618         int i;
3619         struct scatterlist *sg;
3620
3621         for_each_sg(sglist, sg, nelems, i) {
3622                 BUG_ON(!sg_page(sg));
3623                 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
3624                 sg->dma_length = sg->length;
3625         }
3626         return nelems;
3627 }
3628
3629 static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
3630                         enum dma_data_direction dir, struct dma_attrs *attrs)
3631 {
3632         int i;
3633         struct dmar_domain *domain;
3634         size_t size = 0;
3635         int prot = 0;
3636         struct iova *iova = NULL;
3637         int ret;
3638         struct scatterlist *sg;
3639         unsigned long start_vpfn;
3640         struct intel_iommu *iommu;
3641
3642         BUG_ON(dir == DMA_NONE);
3643         if (iommu_no_mapping(dev))
3644                 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
3645
3646         domain = get_valid_domain_for_dev(dev);
3647         if (!domain)
3648                 return 0;
3649
3650         iommu = domain_get_iommu(domain);
3651
3652         for_each_sg(sglist, sg, nelems, i)
3653                 size += aligned_nrpages(sg->offset, sg->length);
3654
3655         iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3656                                 *dev->dma_mask);
3657         if (!iova) {
3658                 sglist->dma_length = 0;
3659                 return 0;
3660         }
3661
3662         /*
3663          * Check if DMAR supports zero-length reads on write only
3664          * mappings..
3665          */
3666         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3667                         !cap_zlr(iommu->cap))
3668                 prot |= DMA_PTE_READ;
3669         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3670                 prot |= DMA_PTE_WRITE;
3671
3672         start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
3673
3674         ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
3675         if (unlikely(ret)) {
3676                 dma_pte_free_pagetable(domain, start_vpfn,
3677                                        start_vpfn + size - 1);
3678                 __free_iova(&domain->iovad, iova);
3679                 return 0;
3680         }
3681
3682         /* it's a non-present to present mapping. Only flush if caching mode */
3683         if (cap_caching_mode(iommu->cap))
3684                 iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 0, 1);
3685         else
3686                 iommu_flush_write_buffer(iommu);
3687
3688         return nelems;
3689 }
3690
3691 static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3692 {
3693         return !dma_addr;
3694 }
3695
3696 struct dma_map_ops intel_dma_ops = {
3697         .alloc = intel_alloc_coherent,
3698         .free = intel_free_coherent,
3699         .map_sg = intel_map_sg,
3700         .unmap_sg = intel_unmap_sg,
3701         .map_page = intel_map_page,
3702         .unmap_page = intel_unmap_page,
3703         .mapping_error = intel_mapping_error,
3704 };
3705
3706 static inline int iommu_domain_cache_init(void)
3707 {
3708         int ret = 0;
3709
3710         iommu_domain_cache = kmem_cache_create("iommu_domain",
3711                                          sizeof(struct dmar_domain),
3712                                          0,
3713                                          SLAB_HWCACHE_ALIGN,
3714
3715                                          NULL);
3716         if (!iommu_domain_cache) {
3717                 pr_err("Couldn't create iommu_domain cache\n");
3718                 ret = -ENOMEM;
3719         }
3720
3721         return ret;
3722 }
3723
3724 static inline int iommu_devinfo_cache_init(void)
3725 {
3726         int ret = 0;
3727
3728         iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3729                                          sizeof(struct device_domain_info),
3730                                          0,
3731                                          SLAB_HWCACHE_ALIGN,
3732                                          NULL);
3733         if (!iommu_devinfo_cache) {
3734                 pr_err("Couldn't create devinfo cache\n");
3735                 ret = -ENOMEM;
3736         }
3737
3738         return ret;
3739 }
3740
3741 static int __init iommu_init_mempool(void)
3742 {
3743         int ret;
3744         ret = iommu_iova_cache_init();
3745         if (ret)
3746                 return ret;
3747
3748         ret = iommu_domain_cache_init();
3749         if (ret)
3750                 goto domain_error;
3751
3752         ret = iommu_devinfo_cache_init();
3753         if (!ret)
3754                 return ret;
3755
3756         kmem_cache_destroy(iommu_domain_cache);
3757 domain_error:
3758         iommu_iova_cache_destroy();
3759
3760         return -ENOMEM;
3761 }
3762
3763 static void __init iommu_exit_mempool(void)
3764 {
3765         kmem_cache_destroy(iommu_devinfo_cache);
3766         kmem_cache_destroy(iommu_domain_cache);
3767         iommu_iova_cache_destroy();
3768 }
3769
3770 static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3771 {
3772         struct dmar_drhd_unit *drhd;
3773         u32 vtbar;
3774         int rc;
3775
3776         /* We know that this device on this chipset has its own IOMMU.
3777          * If we find it under a different IOMMU, then the BIOS is lying
3778          * to us. Hope that the IOMMU for this device is actually
3779          * disabled, and it needs no translation...
3780          */
3781         rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3782         if (rc) {
3783                 /* "can't" happen */
3784                 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3785                 return;
3786         }
3787         vtbar &= 0xffff0000;
3788
3789         /* we know that the this iommu should be at offset 0xa000 from vtbar */
3790         drhd = dmar_find_matched_drhd_unit(pdev);
3791         if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3792                             TAINT_FIRMWARE_WORKAROUND,
3793                             "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3794                 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3795 }
3796 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3797
3798 static void __init init_no_remapping_devices(void)
3799 {
3800         struct dmar_drhd_unit *drhd;
3801         struct device *dev;
3802         int i;
3803
3804         for_each_drhd_unit(drhd) {
3805                 if (!drhd->include_all) {
3806                         for_each_active_dev_scope(drhd->devices,
3807                                                   drhd->devices_cnt, i, dev)
3808                                 break;
3809                         /* ignore DMAR unit if no devices exist */
3810                         if (i == drhd->devices_cnt)
3811                                 drhd->ignored = 1;
3812                 }
3813         }
3814
3815         for_each_active_drhd_unit(drhd) {
3816                 if (drhd->include_all)
3817                         continue;
3818
3819                 for_each_active_dev_scope(drhd->devices,
3820                                           drhd->devices_cnt, i, dev)
3821                         if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
3822                                 break;
3823                 if (i < drhd->devices_cnt)
3824                         continue;
3825
3826                 /* This IOMMU has *only* gfx devices. Either bypass it or
3827                    set the gfx_mapped flag, as appropriate */
3828                 if (dmar_map_gfx) {
3829                         intel_iommu_gfx_mapped = 1;
3830                 } else {
3831                         drhd->ignored = 1;
3832                         for_each_active_dev_scope(drhd->devices,
3833                                                   drhd->devices_cnt, i, dev)
3834                                 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3835                 }
3836         }
3837 }
3838
3839 #ifdef CONFIG_SUSPEND
3840 static int init_iommu_hw(void)
3841 {
3842         struct dmar_drhd_unit *drhd;
3843         struct intel_iommu *iommu = NULL;
3844
3845         for_each_active_iommu(iommu, drhd)
3846                 if (iommu->qi)
3847                         dmar_reenable_qi(iommu);
3848
3849         for_each_iommu(iommu, drhd) {
3850                 if (drhd->ignored) {
3851                         /*
3852                          * we always have to disable PMRs or DMA may fail on
3853                          * this device
3854                          */
3855                         if (force_on)
3856                                 iommu_disable_protect_mem_regions(iommu);
3857                         continue;
3858                 }
3859
3860                 iommu_flush_write_buffer(iommu);
3861
3862                 iommu_set_root_entry(iommu);
3863
3864                 iommu->flush.flush_context(iommu, 0, 0, 0,
3865                                            DMA_CCMD_GLOBAL_INVL);
3866                 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3867                 iommu_enable_translation(iommu);
3868                 iommu_disable_protect_mem_regions(iommu);
3869         }
3870
3871         return 0;
3872 }
3873
3874 static void iommu_flush_all(void)
3875 {
3876         struct dmar_drhd_unit *drhd;
3877         struct intel_iommu *iommu;
3878
3879         for_each_active_iommu(iommu, drhd) {
3880                 iommu->flush.flush_context(iommu, 0, 0, 0,
3881                                            DMA_CCMD_GLOBAL_INVL);
3882                 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3883                                          DMA_TLB_GLOBAL_FLUSH);
3884         }
3885 }
3886
3887 static int iommu_suspend(void)
3888 {
3889         struct dmar_drhd_unit *drhd;
3890         struct intel_iommu *iommu = NULL;
3891         unsigned long flag;
3892
3893         for_each_active_iommu(iommu, drhd) {
3894                 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3895                                                  GFP_ATOMIC);
3896                 if (!iommu->iommu_state)
3897                         goto nomem;
3898         }
3899
3900         iommu_flush_all();
3901
3902         for_each_active_iommu(iommu, drhd) {
3903                 iommu_disable_translation(iommu);
3904
3905                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
3906
3907                 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3908                         readl(iommu->reg + DMAR_FECTL_REG);
3909                 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3910                         readl(iommu->reg + DMAR_FEDATA_REG);
3911                 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3912                         readl(iommu->reg + DMAR_FEADDR_REG);
3913                 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3914                         readl(iommu->reg + DMAR_FEUADDR_REG);
3915
3916                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
3917         }
3918         return 0;
3919
3920 nomem:
3921         for_each_active_iommu(iommu, drhd)
3922                 kfree(iommu->iommu_state);
3923
3924         return -ENOMEM;
3925 }
3926
3927 static void iommu_resume(void)
3928 {
3929         struct dmar_drhd_unit *drhd;
3930         struct intel_iommu *iommu = NULL;
3931         unsigned long flag;
3932
3933         if (init_iommu_hw()) {
3934                 if (force_on)
3935                         panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
3936                 else
3937                         WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
3938                 return;
3939         }
3940
3941         for_each_active_iommu(iommu, drhd) {
3942
3943                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
3944
3945                 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3946                         iommu->reg + DMAR_FECTL_REG);
3947                 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3948                         iommu->reg + DMAR_FEDATA_REG);
3949                 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3950                         iommu->reg + DMAR_FEADDR_REG);
3951                 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3952                         iommu->reg + DMAR_FEUADDR_REG);
3953
3954                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
3955         }
3956
3957         for_each_active_iommu(iommu, drhd)
3958                 kfree(iommu->iommu_state);
3959 }
3960
3961 static struct syscore_ops iommu_syscore_ops = {
3962         .resume         = iommu_resume,
3963         .suspend        = iommu_suspend,
3964 };
3965
3966 static void __init init_iommu_pm_ops(void)
3967 {
3968         register_syscore_ops(&iommu_syscore_ops);
3969 }
3970
3971 #else
3972 static inline void init_iommu_pm_ops(void) {}
3973 #endif  /* CONFIG_PM */
3974
3975
3976 int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
3977 {
3978         struct acpi_dmar_reserved_memory *rmrr;
3979         struct dmar_rmrr_unit *rmrru;
3980
3981         rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
3982         if (!rmrru)
3983                 return -ENOMEM;
3984
3985         rmrru->hdr = header;
3986         rmrr = (struct acpi_dmar_reserved_memory *)header;
3987         rmrru->base_address = rmrr->base_address;
3988         rmrru->end_address = rmrr->end_address;
3989         rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
3990                                 ((void *)rmrr) + rmrr->header.length,
3991                                 &rmrru->devices_cnt);
3992         if (rmrru->devices_cnt && rmrru->devices == NULL) {
3993                 kfree(rmrru);
3994                 return -ENOMEM;
3995         }
3996
3997         list_add(&rmrru->list, &dmar_rmrr_units);
3998
3999         return 0;
4000 }
4001
4002 static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
4003 {
4004         struct dmar_atsr_unit *atsru;
4005         struct acpi_dmar_atsr *tmp;
4006
4007         list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4008                 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
4009                 if (atsr->segment != tmp->segment)
4010                         continue;
4011                 if (atsr->header.length != tmp->header.length)
4012                         continue;
4013                 if (memcmp(atsr, tmp, atsr->header.length) == 0)
4014                         return atsru;
4015         }
4016
4017         return NULL;
4018 }
4019
4020 int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4021 {
4022         struct acpi_dmar_atsr *atsr;
4023         struct dmar_atsr_unit *atsru;
4024
4025         if (system_state != SYSTEM_BOOTING && !intel_iommu_enabled)
4026                 return 0;
4027
4028         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4029         atsru = dmar_find_atsr(atsr);
4030         if (atsru)
4031                 return 0;
4032
4033         atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
4034         if (!atsru)
4035                 return -ENOMEM;
4036
4037         /*
4038          * If memory is allocated from slab by ACPI _DSM method, we need to
4039          * copy the memory content because the memory buffer will be freed
4040          * on return.
4041          */
4042         atsru->hdr = (void *)(atsru + 1);
4043         memcpy(atsru->hdr, hdr, hdr->length);
4044         atsru->include_all = atsr->flags & 0x1;
4045         if (!atsru->include_all) {
4046                 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
4047                                 (void *)atsr + atsr->header.length,
4048                                 &atsru->devices_cnt);
4049                 if (atsru->devices_cnt && atsru->devices == NULL) {
4050                         kfree(atsru);
4051                         return -ENOMEM;
4052                 }
4053         }
4054
4055         list_add_rcu(&atsru->list, &dmar_atsr_units);
4056
4057         return 0;
4058 }
4059
4060 static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
4061 {
4062         dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
4063         kfree(atsru);
4064 }
4065
4066 int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4067 {
4068         struct acpi_dmar_atsr *atsr;
4069         struct dmar_atsr_unit *atsru;
4070
4071         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4072         atsru = dmar_find_atsr(atsr);
4073         if (atsru) {
4074                 list_del_rcu(&atsru->list);
4075                 synchronize_rcu();
4076                 intel_iommu_free_atsr(atsru);
4077         }
4078
4079         return 0;
4080 }
4081
4082 int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4083 {
4084         int i;
4085         struct device *dev;
4086         struct acpi_dmar_atsr *atsr;
4087         struct dmar_atsr_unit *atsru;
4088
4089         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4090         atsru = dmar_find_atsr(atsr);
4091         if (!atsru)
4092                 return 0;
4093
4094         if (!atsru->include_all && atsru->devices && atsru->devices_cnt)
4095                 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
4096                                           i, dev)
4097                         return -EBUSY;
4098
4099         return 0;
4100 }
4101
4102 static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
4103 {
4104         int sp, ret = 0;
4105         struct intel_iommu *iommu = dmaru->iommu;
4106
4107         if (g_iommus[iommu->seq_id])
4108                 return 0;
4109
4110         if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
4111                 pr_warn("%s: Doesn't support hardware pass through.\n",
4112                         iommu->name);
4113                 return -ENXIO;
4114         }
4115         if (!ecap_sc_support(iommu->ecap) &&
4116             domain_update_iommu_snooping(iommu)) {
4117                 pr_warn("%s: Doesn't support snooping.\n",
4118                         iommu->name);
4119                 return -ENXIO;
4120         }
4121         sp = domain_update_iommu_superpage(iommu) - 1;
4122         if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
4123                 pr_warn("%s: Doesn't support large page.\n",
4124                         iommu->name);
4125                 return -ENXIO;
4126         }
4127
4128         /*
4129          * Disable translation if already enabled prior to OS handover.
4130          */
4131         if (iommu->gcmd & DMA_GCMD_TE)
4132                 iommu_disable_translation(iommu);
4133
4134         g_iommus[iommu->seq_id] = iommu;
4135         ret = iommu_init_domains(iommu);
4136         if (ret == 0)
4137                 ret = iommu_alloc_root_entry(iommu);
4138         if (ret)
4139                 goto out;
4140
4141         if (dmaru->ignored) {
4142                 /*
4143                  * we always have to disable PMRs or DMA may fail on this device
4144                  */
4145                 if (force_on)
4146                         iommu_disable_protect_mem_regions(iommu);
4147                 return 0;
4148         }
4149
4150         intel_iommu_init_qi(iommu);
4151         iommu_flush_write_buffer(iommu);
4152         ret = dmar_set_interrupt(iommu);
4153         if (ret)
4154                 goto disable_iommu;
4155
4156         iommu_set_root_entry(iommu);
4157         iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
4158         iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4159         iommu_enable_translation(iommu);
4160
4161         if (si_domain) {
4162                 ret = iommu_attach_domain(si_domain, iommu);
4163                 if (ret < 0 || si_domain->id != ret)
4164                         goto disable_iommu;
4165                 domain_attach_iommu(si_domain, iommu);
4166         }
4167
4168         iommu_disable_protect_mem_regions(iommu);
4169         return 0;
4170
4171 disable_iommu:
4172         disable_dmar_iommu(iommu);
4173 out:
4174         free_dmar_iommu(iommu);
4175         return ret;
4176 }
4177
4178 int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
4179 {
4180         int ret = 0;
4181         struct intel_iommu *iommu = dmaru->iommu;
4182
4183         if (!intel_iommu_enabled)
4184                 return 0;
4185         if (iommu == NULL)
4186                 return -EINVAL;
4187
4188         if (insert) {
4189                 ret = intel_iommu_add(dmaru);
4190         } else {
4191                 disable_dmar_iommu(iommu);
4192                 free_dmar_iommu(iommu);
4193         }
4194
4195         return ret;
4196 }
4197
4198 static void intel_iommu_free_dmars(void)
4199 {
4200         struct dmar_rmrr_unit *rmrru, *rmrr_n;
4201         struct dmar_atsr_unit *atsru, *atsr_n;
4202
4203         list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
4204                 list_del(&rmrru->list);
4205                 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
4206                 kfree(rmrru);
4207         }
4208
4209         list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
4210                 list_del(&atsru->list);
4211                 intel_iommu_free_atsr(atsru);
4212         }
4213 }
4214
4215 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
4216 {
4217         int i, ret = 1;
4218         struct pci_bus *bus;
4219         struct pci_dev *bridge = NULL;
4220         struct device *tmp;
4221         struct acpi_dmar_atsr *atsr;
4222         struct dmar_atsr_unit *atsru;
4223
4224         dev = pci_physfn(dev);
4225         for (bus = dev->bus; bus; bus = bus->parent) {
4226                 bridge = bus->self;
4227                 if (!bridge || !pci_is_pcie(bridge) ||
4228                     pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
4229                         return 0;
4230                 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
4231                         break;
4232         }
4233         if (!bridge)
4234                 return 0;
4235
4236         rcu_read_lock();
4237         list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4238                 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4239                 if (atsr->segment != pci_domain_nr(dev->bus))
4240                         continue;
4241
4242                 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
4243                         if (tmp == &bridge->dev)
4244                                 goto out;
4245
4246                 if (atsru->include_all)
4247                         goto out;
4248         }
4249         ret = 0;
4250 out:
4251         rcu_read_unlock();
4252
4253         return ret;
4254 }
4255
4256 int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
4257 {
4258         int ret = 0;
4259         struct dmar_rmrr_unit *rmrru;
4260         struct dmar_atsr_unit *atsru;
4261         struct acpi_dmar_atsr *atsr;
4262         struct acpi_dmar_reserved_memory *rmrr;
4263
4264         if (!intel_iommu_enabled && system_state != SYSTEM_BOOTING)
4265                 return 0;
4266
4267         list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
4268                 rmrr = container_of(rmrru->hdr,
4269                                     struct acpi_dmar_reserved_memory, header);
4270                 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4271                         ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
4272                                 ((void *)rmrr) + rmrr->header.length,
4273                                 rmrr->segment, rmrru->devices,
4274                                 rmrru->devices_cnt);
4275                         if(ret < 0)
4276                                 return ret;
4277                 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
4278                         dmar_remove_dev_scope(info, rmrr->segment,
4279                                 rmrru->devices, rmrru->devices_cnt);
4280                 }
4281         }
4282
4283         list_for_each_entry(atsru, &dmar_atsr_units, list) {
4284                 if (atsru->include_all)
4285                         continue;
4286
4287                 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4288                 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4289                         ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
4290                                         (void *)atsr + atsr->header.length,
4291                                         atsr->segment, atsru->devices,
4292                                         atsru->devices_cnt);
4293                         if (ret > 0)
4294                                 break;
4295                         else if(ret < 0)
4296                                 return ret;
4297                 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
4298                         if (dmar_remove_dev_scope(info, atsr->segment,
4299                                         atsru->devices, atsru->devices_cnt))
4300                                 break;
4301                 }
4302         }
4303
4304         return 0;
4305 }
4306
4307 /*
4308  * Here we only respond to action of unbound device from driver.
4309  *
4310  * Added device is not attached to its DMAR domain here yet. That will happen
4311  * when mapping the device to iova.
4312  */
4313 static int device_notifier(struct notifier_block *nb,
4314                                   unsigned long action, void *data)
4315 {
4316         struct device *dev = data;
4317         struct dmar_domain *domain;
4318
4319         if (iommu_dummy(dev))
4320                 return 0;
4321
4322         if (action != BUS_NOTIFY_REMOVED_DEVICE)
4323                 return 0;
4324
4325         domain = find_domain(dev);
4326         if (!domain)
4327                 return 0;
4328
4329         down_read(&dmar_global_lock);
4330         domain_remove_one_dev_info(domain, dev);
4331         if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
4332                 domain_exit(domain);
4333         up_read(&dmar_global_lock);
4334
4335         return 0;
4336 }
4337
4338 static struct notifier_block device_nb = {
4339         .notifier_call = device_notifier,
4340 };
4341
4342 static int intel_iommu_memory_notifier(struct notifier_block *nb,
4343                                        unsigned long val, void *v)
4344 {
4345         struct memory_notify *mhp = v;
4346         unsigned long long start, end;
4347         unsigned long start_vpfn, last_vpfn;
4348
4349         switch (val) {
4350         case MEM_GOING_ONLINE:
4351                 start = mhp->start_pfn << PAGE_SHIFT;
4352                 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4353                 if (iommu_domain_identity_map(si_domain, start, end)) {
4354                         pr_warn("Failed to build identity map for [%llx-%llx]\n",
4355                                 start, end);
4356                         return NOTIFY_BAD;
4357                 }
4358                 break;
4359
4360         case MEM_OFFLINE:
4361         case MEM_CANCEL_ONLINE:
4362                 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4363                 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4364                 while (start_vpfn <= last_vpfn) {
4365                         struct iova *iova;
4366                         struct dmar_drhd_unit *drhd;
4367                         struct intel_iommu *iommu;
4368                         struct page *freelist;
4369
4370                         iova = find_iova(&si_domain->iovad, start_vpfn);
4371                         if (iova == NULL) {
4372                                 pr_debug("Failed get IOVA for PFN %lx\n",
4373                                          start_vpfn);
4374                                 break;
4375                         }
4376
4377                         iova = split_and_remove_iova(&si_domain->iovad, iova,
4378                                                      start_vpfn, last_vpfn);
4379                         if (iova == NULL) {
4380                                 pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
4381                                         start_vpfn, last_vpfn);
4382                                 return NOTIFY_BAD;
4383                         }
4384
4385                         freelist = domain_unmap(si_domain, iova->pfn_lo,
4386                                                iova->pfn_hi);
4387
4388                         rcu_read_lock();
4389                         for_each_active_iommu(iommu, drhd)
4390                                 iommu_flush_iotlb_psi(iommu, si_domain->id,
4391                                         iova->pfn_lo, iova_size(iova),
4392                                         !freelist, 0);
4393                         rcu_read_unlock();
4394                         dma_free_pagelist(freelist);
4395
4396                         start_vpfn = iova->pfn_hi + 1;
4397                         free_iova_mem(iova);
4398                 }
4399                 break;
4400         }
4401
4402         return NOTIFY_OK;
4403 }
4404
4405 static struct notifier_block intel_iommu_memory_nb = {
4406         .notifier_call = intel_iommu_memory_notifier,
4407         .priority = 0
4408 };
4409
4410
4411 static ssize_t intel_iommu_show_version(struct device *dev,
4412                                         struct device_attribute *attr,
4413                                         char *buf)
4414 {
4415         struct intel_iommu *iommu = dev_get_drvdata(dev);
4416         u32 ver = readl(iommu->reg + DMAR_VER_REG);
4417         return sprintf(buf, "%d:%d\n",
4418                        DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4419 }
4420 static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4421
4422 static ssize_t intel_iommu_show_address(struct device *dev,
4423                                         struct device_attribute *attr,
4424                                         char *buf)
4425 {
4426         struct intel_iommu *iommu = dev_get_drvdata(dev);
4427         return sprintf(buf, "%llx\n", iommu->reg_phys);
4428 }
4429 static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4430
4431 static ssize_t intel_iommu_show_cap(struct device *dev,
4432                                     struct device_attribute *attr,
4433                                     char *buf)
4434 {
4435         struct intel_iommu *iommu = dev_get_drvdata(dev);
4436         return sprintf(buf, "%llx\n", iommu->cap);
4437 }
4438 static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4439
4440 static ssize_t intel_iommu_show_ecap(struct device *dev,
4441                                     struct device_attribute *attr,
4442                                     char *buf)
4443 {
4444         struct intel_iommu *iommu = dev_get_drvdata(dev);
4445         return sprintf(buf, "%llx\n", iommu->ecap);
4446 }
4447 static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4448
4449 static struct attribute *intel_iommu_attrs[] = {
4450         &dev_attr_version.attr,
4451         &dev_attr_address.attr,
4452         &dev_attr_cap.attr,
4453         &dev_attr_ecap.attr,
4454         NULL,
4455 };
4456
4457 static struct attribute_group intel_iommu_group = {
4458         .name = "intel-iommu",
4459         .attrs = intel_iommu_attrs,
4460 };
4461
4462 const struct attribute_group *intel_iommu_groups[] = {
4463         &intel_iommu_group,
4464         NULL,
4465 };
4466
4467 int __init intel_iommu_init(void)
4468 {
4469         int ret = -ENODEV;
4470         struct dmar_drhd_unit *drhd;
4471         struct intel_iommu *iommu;
4472
4473         /* VT-d is required for a TXT/tboot launch, so enforce that */
4474         force_on = tboot_force_iommu();
4475
4476         if (iommu_init_mempool()) {
4477                 if (force_on)
4478                         panic("tboot: Failed to initialize iommu memory\n");
4479                 return -ENOMEM;
4480         }
4481
4482         down_write(&dmar_global_lock);
4483         if (dmar_table_init()) {
4484                 if (force_on)
4485                         panic("tboot: Failed to initialize DMAR table\n");
4486                 goto out_free_dmar;
4487         }
4488
4489         if (dmar_dev_scope_init() < 0) {
4490                 if (force_on)
4491                         panic("tboot: Failed to initialize DMAR device scope\n");
4492                 goto out_free_dmar;
4493         }
4494
4495         if (no_iommu || dmar_disabled)
4496                 goto out_free_dmar;
4497
4498         if (list_empty(&dmar_rmrr_units))
4499                 pr_info("No RMRR found\n");
4500
4501         if (list_empty(&dmar_atsr_units))
4502                 pr_info("No ATSR found\n");
4503
4504         if (dmar_init_reserved_ranges()) {
4505                 if (force_on)
4506                         panic("tboot: Failed to reserve iommu ranges\n");
4507                 goto out_free_reserved_range;
4508         }
4509
4510         init_no_remapping_devices();
4511
4512         ret = init_dmars();
4513         if (ret) {
4514                 if (force_on)
4515                         panic("tboot: Failed to initialize DMARs\n");
4516                 pr_err("Initialization failed\n");
4517                 goto out_free_reserved_range;
4518         }
4519         up_write(&dmar_global_lock);
4520         pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
4521
4522         init_timer(&unmap_timer);
4523 #ifdef CONFIG_SWIOTLB
4524         swiotlb = 0;
4525 #endif
4526         dma_ops = &intel_dma_ops;
4527
4528         init_iommu_pm_ops();
4529
4530         for_each_active_iommu(iommu, drhd)
4531                 iommu->iommu_dev = iommu_device_create(NULL, iommu,
4532                                                        intel_iommu_groups,
4533                                                        iommu->name);
4534
4535         bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
4536         bus_register_notifier(&pci_bus_type, &device_nb);
4537         if (si_domain && !hw_pass_through)
4538                 register_memory_notifier(&intel_iommu_memory_nb);
4539
4540         intel_iommu_enabled = 1;
4541
4542         return 0;
4543
4544 out_free_reserved_range:
4545         put_iova_domain(&reserved_iova_list);
4546 out_free_dmar:
4547         intel_iommu_free_dmars();
4548         up_write(&dmar_global_lock);
4549         iommu_exit_mempool();
4550         return ret;
4551 }
4552
4553 static int iommu_detach_dev_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4554 {
4555         struct intel_iommu *iommu = opaque;
4556
4557         iommu_detach_dev(iommu, PCI_BUS_NUM(alias), alias & 0xff);
4558         return 0;
4559 }
4560
4561 /*
4562  * NB - intel-iommu lacks any sort of reference counting for the users of
4563  * dependent devices.  If multiple endpoints have intersecting dependent
4564  * devices, unbinding the driver from any one of them will possibly leave
4565  * the others unable to operate.
4566  */
4567 static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
4568                                            struct device *dev)
4569 {
4570         if (!iommu || !dev || !dev_is_pci(dev))
4571                 return;
4572
4573         pci_for_each_dma_alias(to_pci_dev(dev), &iommu_detach_dev_cb, iommu);
4574 }
4575
4576 static void domain_remove_one_dev_info(struct dmar_domain *domain,
4577                                        struct device *dev)
4578 {
4579         struct device_domain_info *info, *tmp;
4580         struct intel_iommu *iommu;
4581         unsigned long flags;
4582         bool found = false;
4583         u8 bus, devfn;
4584
4585         iommu = device_to_iommu(dev, &bus, &devfn);
4586         if (!iommu)
4587                 return;
4588
4589         spin_lock_irqsave(&device_domain_lock, flags);
4590         list_for_each_entry_safe(info, tmp, &domain->devices, link) {
4591                 if (info->iommu == iommu && info->bus == bus &&
4592                     info->devfn == devfn) {
4593                         unlink_domain_info(info);
4594                         spin_unlock_irqrestore(&device_domain_lock, flags);
4595
4596                         iommu_disable_dev_iotlb(info);
4597                         iommu_detach_dev(iommu, info->bus, info->devfn);
4598                         iommu_detach_dependent_devices(iommu, dev);
4599                         free_devinfo_mem(info);
4600
4601                         spin_lock_irqsave(&device_domain_lock, flags);
4602
4603                         if (found)
4604                                 break;
4605                         else
4606                                 continue;
4607                 }
4608
4609                 /* if there is no other devices under the same iommu
4610                  * owned by this domain, clear this iommu in iommu_bmp
4611                  * update iommu count and coherency
4612                  */
4613                 if (info->iommu == iommu)
4614                         found = true;
4615         }
4616
4617         spin_unlock_irqrestore(&device_domain_lock, flags);
4618
4619         if (found == 0) {
4620                 domain_detach_iommu(domain, iommu);
4621                 if (!domain_type_is_vm_or_si(domain))
4622                         iommu_detach_domain(domain, iommu);
4623         }
4624 }
4625
4626 static int md_domain_init(struct dmar_domain *domain, int guest_width)
4627 {
4628         int adjust_width;
4629
4630         init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
4631                         DMA_32BIT_PFN);
4632         domain_reserve_special_ranges(domain);
4633
4634         /* calculate AGAW */
4635         domain->gaw = guest_width;
4636         adjust_width = guestwidth_to_adjustwidth(guest_width);
4637         domain->agaw = width_to_agaw(adjust_width);
4638
4639         domain->iommu_coherency = 0;
4640         domain->iommu_snooping = 0;
4641         domain->iommu_superpage = 0;
4642         domain->max_addr = 0;
4643
4644         /* always allocate the top pgd */
4645         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
4646         if (!domain->pgd)
4647                 return -ENOMEM;
4648         domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4649         return 0;
4650 }
4651
4652 static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
4653 {
4654         struct dmar_domain *dmar_domain;
4655         struct iommu_domain *domain;
4656
4657         if (type != IOMMU_DOMAIN_UNMANAGED)
4658                 return NULL;
4659
4660         dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
4661         if (!dmar_domain) {
4662                 pr_err("Can't allocate dmar_domain\n");
4663                 return NULL;
4664         }
4665         if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
4666                 pr_err("Domain initialization failed\n");
4667                 domain_exit(dmar_domain);
4668                 return NULL;
4669         }
4670         domain_update_iommu_cap(dmar_domain);
4671
4672         domain = &dmar_domain->domain;
4673         domain->geometry.aperture_start = 0;
4674         domain->geometry.aperture_end   = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4675         domain->geometry.force_aperture = true;
4676
4677         return domain;
4678 }
4679
4680 static void intel_iommu_domain_free(struct iommu_domain *domain)
4681 {
4682         domain_exit(to_dmar_domain(domain));
4683 }
4684
4685 static int intel_iommu_attach_device(struct iommu_domain *domain,
4686                                      struct device *dev)
4687 {
4688         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4689         struct intel_iommu *iommu;
4690         int addr_width;
4691         u8 bus, devfn;
4692
4693         if (device_is_rmrr_locked(dev)) {
4694                 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement.  Contact your platform vendor.\n");
4695                 return -EPERM;
4696         }
4697
4698         /* normally dev is not mapped */
4699         if (unlikely(domain_context_mapped(dev))) {
4700                 struct dmar_domain *old_domain;
4701
4702                 old_domain = find_domain(dev);
4703                 if (old_domain) {
4704                         if (domain_type_is_vm_or_si(dmar_domain))
4705                                 domain_remove_one_dev_info(old_domain, dev);
4706                         else
4707                                 domain_remove_dev_info(old_domain);
4708
4709                         if (!domain_type_is_vm_or_si(old_domain) &&
4710                              list_empty(&old_domain->devices))
4711                                 domain_exit(old_domain);
4712                 }
4713         }
4714
4715         iommu = device_to_iommu(dev, &bus, &devfn);
4716         if (!iommu)
4717                 return -ENODEV;
4718
4719         /* check if this iommu agaw is sufficient for max mapped address */
4720         addr_width = agaw_to_width(iommu->agaw);
4721         if (addr_width > cap_mgaw(iommu->cap))
4722                 addr_width = cap_mgaw(iommu->cap);
4723
4724         if (dmar_domain->max_addr > (1LL << addr_width)) {
4725                 pr_err("%s: iommu width (%d) is not "
4726                        "sufficient for the mapped address (%llx)\n",
4727                        __func__, addr_width, dmar_domain->max_addr);
4728                 return -EFAULT;
4729         }
4730         dmar_domain->gaw = addr_width;
4731
4732         /*
4733          * Knock out extra levels of page tables if necessary
4734          */
4735         while (iommu->agaw < dmar_domain->agaw) {
4736                 struct dma_pte *pte;
4737
4738                 pte = dmar_domain->pgd;
4739                 if (dma_pte_present(pte)) {
4740                         dmar_domain->pgd = (struct dma_pte *)
4741                                 phys_to_virt(dma_pte_addr(pte));
4742                         free_pgtable_page(pte);
4743                 }
4744                 dmar_domain->agaw--;
4745         }
4746
4747         return domain_add_dev_info(dmar_domain, dev, CONTEXT_TT_MULTI_LEVEL);
4748 }
4749
4750 static void intel_iommu_detach_device(struct iommu_domain *domain,
4751                                       struct device *dev)
4752 {
4753         domain_remove_one_dev_info(to_dmar_domain(domain), dev);
4754 }
4755
4756 static int intel_iommu_map(struct iommu_domain *domain,
4757                            unsigned long iova, phys_addr_t hpa,
4758                            size_t size, int iommu_prot)
4759 {
4760         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4761         u64 max_addr;
4762         int prot = 0;
4763         int ret;
4764
4765         if (iommu_prot & IOMMU_READ)
4766                 prot |= DMA_PTE_READ;
4767         if (iommu_prot & IOMMU_WRITE)
4768                 prot |= DMA_PTE_WRITE;
4769         if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
4770                 prot |= DMA_PTE_SNP;
4771
4772         max_addr = iova + size;
4773         if (dmar_domain->max_addr < max_addr) {
4774                 u64 end;
4775
4776                 /* check if minimum agaw is sufficient for mapped address */
4777                 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
4778                 if (end < max_addr) {
4779                         pr_err("%s: iommu width (%d) is not "
4780                                "sufficient for the mapped address (%llx)\n",
4781                                __func__, dmar_domain->gaw, max_addr);
4782                         return -EFAULT;
4783                 }
4784                 dmar_domain->max_addr = max_addr;
4785         }
4786         /* Round up size to next multiple of PAGE_SIZE, if it and
4787            the low bits of hpa would take us onto the next page */
4788         size = aligned_nrpages(hpa, size);
4789         ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
4790                                  hpa >> VTD_PAGE_SHIFT, size, prot);
4791         return ret;
4792 }
4793
4794 static size_t intel_iommu_unmap(struct iommu_domain *domain,
4795                                 unsigned long iova, size_t size)
4796 {
4797         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4798         struct page *freelist = NULL;
4799         struct intel_iommu *iommu;
4800         unsigned long start_pfn, last_pfn;
4801         unsigned int npages;
4802         int iommu_id, num, ndomains, level = 0;
4803
4804         /* Cope with horrid API which requires us to unmap more than the
4805            size argument if it happens to be a large-page mapping. */
4806         if (!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level))
4807                 BUG();
4808
4809         if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
4810                 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
4811
4812         start_pfn = iova >> VTD_PAGE_SHIFT;
4813         last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
4814
4815         freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
4816
4817         npages = last_pfn - start_pfn + 1;
4818
4819         for_each_set_bit(iommu_id, dmar_domain->iommu_bmp, g_num_of_iommus) {
4820                iommu = g_iommus[iommu_id];
4821
4822                /*
4823                 * find bit position of dmar_domain
4824                 */
4825                ndomains = cap_ndoms(iommu->cap);
4826                for_each_set_bit(num, iommu->domain_ids, ndomains) {
4827                        if (iommu->domains[num] == dmar_domain)
4828                                iommu_flush_iotlb_psi(iommu, num, start_pfn,
4829                                                      npages, !freelist, 0);
4830                }
4831
4832         }
4833
4834         dma_free_pagelist(freelist);
4835
4836         if (dmar_domain->max_addr == iova + size)
4837                 dmar_domain->max_addr = iova;
4838
4839         return size;
4840 }
4841
4842 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
4843                                             dma_addr_t iova)
4844 {
4845         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4846         struct dma_pte *pte;
4847         int level = 0;
4848         u64 phys = 0;
4849
4850         pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
4851         if (pte)
4852                 phys = dma_pte_addr(pte);
4853
4854         return phys;
4855 }
4856
4857 static bool intel_iommu_capable(enum iommu_cap cap)
4858 {
4859         if (cap == IOMMU_CAP_CACHE_COHERENCY)
4860                 return domain_update_iommu_snooping(NULL) == 1;
4861         if (cap == IOMMU_CAP_INTR_REMAP)
4862                 return irq_remapping_enabled == 1;
4863
4864         return false;
4865 }
4866
4867 static int intel_iommu_add_device(struct device *dev)
4868 {
4869         struct intel_iommu *iommu;
4870         struct iommu_group *group;
4871         u8 bus, devfn;
4872
4873         iommu = device_to_iommu(dev, &bus, &devfn);
4874         if (!iommu)
4875                 return -ENODEV;
4876
4877         iommu_device_link(iommu->iommu_dev, dev);
4878
4879         group = iommu_group_get_for_dev(dev);
4880
4881         if (IS_ERR(group))
4882                 return PTR_ERR(group);
4883
4884         iommu_group_put(group);
4885         return 0;
4886 }
4887
4888 static void intel_iommu_remove_device(struct device *dev)
4889 {
4890         struct intel_iommu *iommu;
4891         u8 bus, devfn;
4892
4893         iommu = device_to_iommu(dev, &bus, &devfn);
4894         if (!iommu)
4895                 return;
4896
4897         iommu_group_remove_device(dev);
4898
4899         iommu_device_unlink(iommu->iommu_dev, dev);
4900 }
4901
4902 static const struct iommu_ops intel_iommu_ops = {
4903         .capable        = intel_iommu_capable,
4904         .domain_alloc   = intel_iommu_domain_alloc,
4905         .domain_free    = intel_iommu_domain_free,
4906         .attach_dev     = intel_iommu_attach_device,
4907         .detach_dev     = intel_iommu_detach_device,
4908         .map            = intel_iommu_map,
4909         .unmap          = intel_iommu_unmap,
4910         .map_sg         = default_iommu_map_sg,
4911         .iova_to_phys   = intel_iommu_iova_to_phys,
4912         .add_device     = intel_iommu_add_device,
4913         .remove_device  = intel_iommu_remove_device,
4914         .pgsize_bitmap  = INTEL_IOMMU_PGSIZES,
4915 };
4916
4917 static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
4918 {
4919         /* G4x/GM45 integrated gfx dmar support is totally busted. */
4920         pr_info("Disabling IOMMU for graphics on this chipset\n");
4921         dmar_map_gfx = 0;
4922 }
4923
4924 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
4925 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
4926 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
4927 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
4928 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
4929 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
4930 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
4931
4932 static void quirk_iommu_rwbf(struct pci_dev *dev)
4933 {
4934         /*
4935          * Mobile 4 Series Chipset neglects to set RWBF capability,
4936          * but needs it. Same seems to hold for the desktop versions.
4937          */
4938         pr_info("Forcing write-buffer flush capability\n");
4939         rwbf_quirk = 1;
4940 }
4941
4942 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
4943 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
4944 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
4945 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
4946 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
4947 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
4948 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
4949
4950 #define GGC 0x52
4951 #define GGC_MEMORY_SIZE_MASK    (0xf << 8)
4952 #define GGC_MEMORY_SIZE_NONE    (0x0 << 8)
4953 #define GGC_MEMORY_SIZE_1M      (0x1 << 8)
4954 #define GGC_MEMORY_SIZE_2M      (0x3 << 8)
4955 #define GGC_MEMORY_VT_ENABLED   (0x8 << 8)
4956 #define GGC_MEMORY_SIZE_2M_VT   (0x9 << 8)
4957 #define GGC_MEMORY_SIZE_3M_VT   (0xa << 8)
4958 #define GGC_MEMORY_SIZE_4M_VT   (0xb << 8)
4959
4960 static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
4961 {
4962         unsigned short ggc;
4963
4964         if (pci_read_config_word(dev, GGC, &ggc))
4965                 return;
4966
4967         if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
4968                 pr_info("BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
4969                 dmar_map_gfx = 0;
4970         } else if (dmar_map_gfx) {
4971                 /* we have to ensure the gfx device is idle before we flush */
4972                 pr_info("Disabling batched IOTLB flush on Ironlake\n");
4973                 intel_iommu_strict = 1;
4974        }
4975 }
4976 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
4977 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
4978 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
4979 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
4980
4981 /* On Tylersburg chipsets, some BIOSes have been known to enable the
4982    ISOCH DMAR unit for the Azalia sound device, but not give it any
4983    TLB entries, which causes it to deadlock. Check for that.  We do
4984    this in a function called from init_dmars(), instead of in a PCI
4985    quirk, because we don't want to print the obnoxious "BIOS broken"
4986    message if VT-d is actually disabled.
4987 */
4988 static void __init check_tylersburg_isoch(void)
4989 {
4990         struct pci_dev *pdev;
4991         uint32_t vtisochctrl;
4992
4993         /* If there's no Azalia in the system anyway, forget it. */
4994         pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
4995         if (!pdev)
4996                 return;
4997         pci_dev_put(pdev);
4998
4999         /* System Management Registers. Might be hidden, in which case
5000            we can't do the sanity check. But that's OK, because the
5001            known-broken BIOSes _don't_ actually hide it, so far. */
5002         pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
5003         if (!pdev)
5004                 return;
5005
5006         if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
5007                 pci_dev_put(pdev);
5008                 return;
5009         }
5010
5011         pci_dev_put(pdev);
5012
5013         /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
5014         if (vtisochctrl & 1)
5015                 return;
5016
5017         /* Drop all bits other than the number of TLB entries */
5018         vtisochctrl &= 0x1c;
5019
5020         /* If we have the recommended number of TLB entries (16), fine. */
5021         if (vtisochctrl == 0x10)
5022                 return;
5023
5024         /* Zero TLB entries? You get to ride the short bus to school. */
5025         if (!vtisochctrl) {
5026                 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
5027                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
5028                      dmi_get_system_info(DMI_BIOS_VENDOR),
5029                      dmi_get_system_info(DMI_BIOS_VERSION),
5030                      dmi_get_system_info(DMI_PRODUCT_VERSION));
5031                 iommu_identity_mapping |= IDENTMAP_AZALIA;
5032                 return;
5033         }
5034
5035         pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
5036                vtisochctrl);
5037 }