#include <linux/pci.h>
#include <linux/acpi.h>
#include <linux/amba/bus.h>
+#include <linux/platform_device.h>
#include <linux/pci-ats.h>
#include <linux/bitmap.h>
#include <linux/slab.h>
#include <linux/dma-contiguous.h>
#include <linux/irqdomain.h>
#include <linux/percpu.h>
+#include <linux/iova.h>
#include <asm/irq_remapping.h>
#include <asm/io_apic.h>
#include <asm/apic.h>
#define LOOP_TIMEOUT 100000
+/* IO virtual address start page frame number */
+#define IOVA_START_PFN (1)
+#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
+#define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
+
+/* Reserved IOVA ranges */
+#define MSI_RANGE_START (0xfee00000)
+#define MSI_RANGE_END (0xfeefffff)
+#define HT_RANGE_START (0xfd00000000ULL)
+#define HT_RANGE_END (0xffffffffffULL)
+
/*
* This bitmap is used to advertise the page sizes our hardware support
* to the IOMMU core, which will then use this information to split
LIST_HEAD(hpet_map);
LIST_HEAD(acpihid_map);
+#define FLUSH_QUEUE_SIZE 256
+
+struct flush_queue_entry {
+ unsigned long iova_pfn;
+ unsigned long pages;
+ struct dma_ops_domain *dma_dom;
+};
+
+struct flush_queue {
+ spinlock_t lock;
+ unsigned next;
+ struct flush_queue_entry *entries;
+};
+
+DEFINE_PER_CPU(struct flush_queue, flush_queue);
+
+static atomic_t queue_timer_on;
+static struct timer_list queue_timer;
+
/*
* Domain for untranslated devices - only allocated
* if iommu=pt passed on kernel cmd line.
static int protection_domain_init(struct protection_domain *domain);
static void detach_device(struct device *dev);
-/*
- * For dynamic growth the aperture size is split into ranges of 128MB of
- * DMA address space each. This struct represents one such range.
- */
-struct aperture_range {
-
- spinlock_t bitmap_lock;
-
- /* address allocation bitmap */
- unsigned long *bitmap;
- unsigned long offset;
- unsigned long next_bit;
-
- /*
- * Array of PTE pages for the aperture. In this array we save all the
- * leaf pages of the domain page table used for the aperture. This way
- * we don't need to walk the page table to find a specific PTE. We can
- * just calculate its address in constant time.
- */
- u64 *pte_pages[64];
-};
-
/*
* Data container for a dma_ops specific protection domain
*/
/* generic protection domain information */
struct protection_domain domain;
- /* size of the aperture for the mappings */
- unsigned long aperture_size;
-
- /* aperture index we start searching for free addresses */
- u32 __percpu *next_index;
-
- /* address space relevant data */
- struct aperture_range *aperture[APERTURE_MAX_RANGES];
+ /* IOVA RB-Tree */
+ struct iova_domain iovad;
};
+static struct iova_domain reserved_iova_ranges;
+static struct lock_class_key reserved_rbtree_key;
+
/****************************************************************************
*
* Helper functions
return container_of(dom, struct protection_domain, domain);
}
+static struct dma_ops_domain* to_dma_ops_domain(struct protection_domain *domain)
+{
+ BUG_ON(domain->flags != PD_DMA_OPS_MASK);
+ return container_of(domain, struct dma_ops_domain, domain);
+}
+
static struct iommu_dev_data *alloc_dev_data(u16 devid)
{
struct iommu_dev_data *dev_data;
return dev_data->errata & (1 << erratum) ? true : false;
}
-/*
- * This function actually applies the mapping to the page table of the
- * dma_ops domain.
- */
-static void alloc_unity_mapping(struct dma_ops_domain *dma_dom,
- struct unity_map_entry *e)
-{
- u64 addr;
-
- for (addr = e->address_start; addr < e->address_end;
- addr += PAGE_SIZE) {
- if (addr < dma_dom->aperture_size)
- __set_bit(addr >> PAGE_SHIFT,
- dma_dom->aperture[0]->bitmap);
- }
-}
-
-/*
- * Inits the unity mappings required for a specific device
- */
-static void init_unity_mappings_for_device(struct device *dev,
- struct dma_ops_domain *dma_dom)
-{
- struct unity_map_entry *e;
- int devid;
-
- devid = get_device_id(dev);
- if (devid < 0)
- return;
-
- list_for_each_entry(e, &amd_iommu_unity_map, list) {
- if (!(devid >= e->devid_start && devid <= e->devid_end))
- continue;
- alloc_unity_mapping(dma_dom, e);
- }
-}
-
/*
* This function checks if the driver got a valid device from the caller to
* avoid dereferencing invalid pointers.
static void init_iommu_group(struct device *dev)
{
- struct dma_ops_domain *dma_domain;
- struct iommu_domain *domain;
struct iommu_group *group;
group = iommu_group_get_for_dev(dev);
if (IS_ERR(group))
return;
- domain = iommu_group_default_domain(group);
- if (!domain)
- goto out;
-
- dma_domain = to_pdomain(domain)->priv;
-
- init_unity_mappings_for_device(dev, dma_domain);
-out:
iommu_group_put(group);
}
* Writes the command to the IOMMUs command buffer and informs the
* hardware about the new command.
*/
-static int iommu_queue_command_sync(struct amd_iommu *iommu,
- struct iommu_cmd *cmd,
- bool sync)
+static int __iommu_queue_command_sync(struct amd_iommu *iommu,
+ struct iommu_cmd *cmd,
+ bool sync)
{
u32 left, tail, head, next_tail;
- unsigned long flags;
again:
- spin_lock_irqsave(&iommu->lock, flags);
head = readl(iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
tail = readl(iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
if (left <= 2) {
struct iommu_cmd sync_cmd;
- volatile u64 sem = 0;
int ret;
- build_completion_wait(&sync_cmd, (u64)&sem);
- copy_cmd_to_buffer(iommu, &sync_cmd, tail);
+ iommu->cmd_sem = 0;
- spin_unlock_irqrestore(&iommu->lock, flags);
+ build_completion_wait(&sync_cmd, (u64)&iommu->cmd_sem);
+ copy_cmd_to_buffer(iommu, &sync_cmd, tail);
- if ((ret = wait_on_sem(&sem)) != 0)
+ if ((ret = wait_on_sem(&iommu->cmd_sem)) != 0)
return ret;
goto again;
/* We need to sync now to make sure all commands are processed */
iommu->need_sync = sync;
+ return 0;
+}
+
+static int iommu_queue_command_sync(struct amd_iommu *iommu,
+ struct iommu_cmd *cmd,
+ bool sync)
+{
+ unsigned long flags;
+ int ret;
+
+ spin_lock_irqsave(&iommu->lock, flags);
+ ret = __iommu_queue_command_sync(iommu, cmd, sync);
spin_unlock_irqrestore(&iommu->lock, flags);
- return 0;
+ return ret;
}
static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
static int iommu_completion_wait(struct amd_iommu *iommu)
{
struct iommu_cmd cmd;
- volatile u64 sem = 0;
+ unsigned long flags;
int ret;
if (!iommu->need_sync)
return 0;
- build_completion_wait(&cmd, (u64)&sem);
- ret = iommu_queue_command_sync(iommu, &cmd, false);
+ build_completion_wait(&cmd, (u64)&iommu->cmd_sem);
+
+ spin_lock_irqsave(&iommu->lock, flags);
+
+ iommu->cmd_sem = 0;
+
+ ret = __iommu_queue_command_sync(iommu, &cmd, false);
if (ret)
- return ret;
+ goto out_unlock;
+
+ ret = wait_on_sem(&iommu->cmd_sem);
- return wait_on_sem(&sem);
+out_unlock:
+ spin_unlock_irqrestore(&iommu->lock, flags);
+
+ return ret;
}
static int iommu_flush_dte(struct amd_iommu *iommu, u16 devid)
int i;
for (i = 0; i < amd_iommus_present; ++i) {
- if (!domain->dev_iommu[i])
+ if (domain && !domain->dev_iommu[i])
continue;
/*
static int iommu_map_page(struct protection_domain *dom,
unsigned long bus_addr,
unsigned long phys_addr,
+ unsigned long page_size,
int prot,
- unsigned long page_size)
+ gfp_t gfp)
{
u64 __pte, *pte;
int i, count;
return -EINVAL;
count = PAGE_SIZE_PTE_COUNT(page_size);
- pte = alloc_pte(dom, bus_addr, page_size, NULL, GFP_KERNEL);
+ pte = alloc_pte(dom, bus_addr, page_size, NULL, gfp);
if (!pte)
return -ENOMEM;
/****************************************************************************
*
* The next functions belong to the address allocator for the dma_ops
- * interface functions. They work like the allocators in the other IOMMU
- * drivers. Its basically a bitmap which marks the allocated pages in
- * the aperture. Maybe it could be enhanced in the future to a more
- * efficient allocator.
+ * interface functions.
*
****************************************************************************/
-/*
- * The address allocator core functions.
- *
- * called with domain->lock held
- */
-
-/*
- * Used to reserve address ranges in the aperture (e.g. for exclusion
- * ranges.
- */
-static void dma_ops_reserve_addresses(struct dma_ops_domain *dom,
- unsigned long start_page,
- unsigned int pages)
-{
- unsigned int i, last_page = dom->aperture_size >> PAGE_SHIFT;
-
- if (start_page + pages > last_page)
- pages = last_page - start_page;
-
- for (i = start_page; i < start_page + pages; ++i) {
- int index = i / APERTURE_RANGE_PAGES;
- int page = i % APERTURE_RANGE_PAGES;
- __set_bit(page, dom->aperture[index]->bitmap);
- }
-}
-/*
- * This function is used to add a new aperture range to an existing
- * aperture in case of dma_ops domain allocation or address allocation
- * failure.
- */
-static int alloc_new_range(struct dma_ops_domain *dma_dom,
- bool populate, gfp_t gfp)
+static unsigned long dma_ops_alloc_iova(struct device *dev,
+ struct dma_ops_domain *dma_dom,
+ unsigned int pages, u64 dma_mask)
{
- int index = dma_dom->aperture_size >> APERTURE_RANGE_SHIFT;
- unsigned long i, old_size, pte_pgsize;
- struct aperture_range *range;
- struct amd_iommu *iommu;
- unsigned long flags;
-
-#ifdef CONFIG_IOMMU_STRESS
- populate = false;
-#endif
-
- if (index >= APERTURE_MAX_RANGES)
- return -ENOMEM;
-
- range = kzalloc(sizeof(struct aperture_range), gfp);
- if (!range)
- return -ENOMEM;
-
- range->bitmap = (void *)get_zeroed_page(gfp);
- if (!range->bitmap)
- goto out_free;
+ unsigned long pfn = 0;
- range->offset = dma_dom->aperture_size;
-
- spin_lock_init(&range->bitmap_lock);
-
- if (populate) {
- unsigned long address = dma_dom->aperture_size;
- int i, num_ptes = APERTURE_RANGE_PAGES / 512;
- u64 *pte, *pte_page;
-
- for (i = 0; i < num_ptes; ++i) {
- pte = alloc_pte(&dma_dom->domain, address, PAGE_SIZE,
- &pte_page, gfp);
- if (!pte)
- goto out_free;
-
- range->pte_pages[i] = pte_page;
-
- address += APERTURE_RANGE_SIZE / 64;
- }
- }
+ pages = __roundup_pow_of_two(pages);
- spin_lock_irqsave(&dma_dom->domain.lock, flags);
+ if (dma_mask > DMA_BIT_MASK(32))
+ pfn = alloc_iova_fast(&dma_dom->iovad, pages,
+ IOVA_PFN(DMA_BIT_MASK(32)));
- /* First take the bitmap_lock and then publish the range */
- spin_lock(&range->bitmap_lock);
+ if (!pfn)
+ pfn = alloc_iova_fast(&dma_dom->iovad, pages, IOVA_PFN(dma_mask));
- old_size = dma_dom->aperture_size;
- dma_dom->aperture[index] = range;
- dma_dom->aperture_size += APERTURE_RANGE_SIZE;
-
- /* Reserve address range used for MSI messages */
- if (old_size < MSI_ADDR_BASE_LO &&
- dma_dom->aperture_size > MSI_ADDR_BASE_LO) {
- unsigned long spage;
- int pages;
-
- pages = iommu_num_pages(MSI_ADDR_BASE_LO, 0x10000, PAGE_SIZE);
- spage = MSI_ADDR_BASE_LO >> PAGE_SHIFT;
-
- dma_ops_reserve_addresses(dma_dom, spage, pages);
- }
-
- /* Initialize the exclusion range if necessary */
- for_each_iommu(iommu) {
- if (iommu->exclusion_start &&
- iommu->exclusion_start >= dma_dom->aperture[index]->offset
- && iommu->exclusion_start < dma_dom->aperture_size) {
- unsigned long startpage;
- int pages = iommu_num_pages(iommu->exclusion_start,
- iommu->exclusion_length,
- PAGE_SIZE);
- startpage = iommu->exclusion_start >> PAGE_SHIFT;
- dma_ops_reserve_addresses(dma_dom, startpage, pages);
- }
- }
-
- /*
- * Check for areas already mapped as present in the new aperture
- * range and mark those pages as reserved in the allocator. Such
- * mappings may already exist as a result of requested unity
- * mappings for devices.
- */
- for (i = dma_dom->aperture[index]->offset;
- i < dma_dom->aperture_size;
- i += pte_pgsize) {
- u64 *pte = fetch_pte(&dma_dom->domain, i, &pte_pgsize);
- if (!pte || !IOMMU_PTE_PRESENT(*pte))
- continue;
-
- dma_ops_reserve_addresses(dma_dom, i >> PAGE_SHIFT,
- pte_pgsize >> 12);
- }
-
- update_domain(&dma_dom->domain);
-
- spin_unlock(&range->bitmap_lock);
-
- spin_unlock_irqrestore(&dma_dom->domain.lock, flags);
-
- return 0;
-
-out_free:
- update_domain(&dma_dom->domain);
-
- free_page((unsigned long)range->bitmap);
-
- kfree(range);
-
- return -ENOMEM;
-}
-
-static dma_addr_t dma_ops_aperture_alloc(struct dma_ops_domain *dom,
- struct aperture_range *range,
- unsigned long pages,
- unsigned long dma_mask,
- unsigned long boundary_size,
- unsigned long align_mask,
- bool trylock)
-{
- unsigned long offset, limit, flags;
- dma_addr_t address;
- bool flush = false;
-
- offset = range->offset >> PAGE_SHIFT;
- limit = iommu_device_max_index(APERTURE_RANGE_PAGES, offset,
- dma_mask >> PAGE_SHIFT);
-
- if (trylock) {
- if (!spin_trylock_irqsave(&range->bitmap_lock, flags))
- return -1;
- } else {
- spin_lock_irqsave(&range->bitmap_lock, flags);
- }
-
- address = iommu_area_alloc(range->bitmap, limit, range->next_bit,
- pages, offset, boundary_size, align_mask);
- if (address == -1) {
- /* Nothing found, retry one time */
- address = iommu_area_alloc(range->bitmap, limit,
- 0, pages, offset, boundary_size,
- align_mask);
- flush = true;
- }
-
- if (address != -1)
- range->next_bit = address + pages;
-
- spin_unlock_irqrestore(&range->bitmap_lock, flags);
-
- if (flush) {
- domain_flush_tlb(&dom->domain);
- domain_flush_complete(&dom->domain);
- }
-
- return address;
-}
-
-static unsigned long dma_ops_area_alloc(struct device *dev,
- struct dma_ops_domain *dom,
- unsigned int pages,
- unsigned long align_mask,
- u64 dma_mask)
-{
- unsigned long boundary_size, mask;
- unsigned long address = -1;
- bool first = true;
- u32 start, i;
-
- preempt_disable();
-
- mask = dma_get_seg_boundary(dev);
-
-again:
- start = this_cpu_read(*dom->next_index);
-
- /* Sanity check - is it really necessary? */
- if (unlikely(start > APERTURE_MAX_RANGES)) {
- start = 0;
- this_cpu_write(*dom->next_index, 0);
- }
-
- boundary_size = mask + 1 ? ALIGN(mask + 1, PAGE_SIZE) >> PAGE_SHIFT :
- 1UL << (BITS_PER_LONG - PAGE_SHIFT);
-
- for (i = 0; i < APERTURE_MAX_RANGES; ++i) {
- struct aperture_range *range;
- int index;
-
- index = (start + i) % APERTURE_MAX_RANGES;
-
- range = dom->aperture[index];
-
- if (!range || range->offset >= dma_mask)
- continue;
-
- address = dma_ops_aperture_alloc(dom, range, pages,
- dma_mask, boundary_size,
- align_mask, first);
- if (address != -1) {
- address = range->offset + (address << PAGE_SHIFT);
- this_cpu_write(*dom->next_index, index);
- break;
- }
- }
-
- if (address == -1 && first) {
- first = false;
- goto again;
- }
-
- preempt_enable();
-
- return address;
+ return (pfn << PAGE_SHIFT);
}
-static unsigned long dma_ops_alloc_addresses(struct device *dev,
- struct dma_ops_domain *dom,
- unsigned int pages,
- unsigned long align_mask,
- u64 dma_mask)
+static void dma_ops_free_iova(struct dma_ops_domain *dma_dom,
+ unsigned long address,
+ unsigned int pages)
{
- unsigned long address = -1;
-
- while (address == -1) {
- address = dma_ops_area_alloc(dev, dom, pages,
- align_mask, dma_mask);
-
- if (address == -1 && alloc_new_range(dom, false, GFP_ATOMIC))
- break;
- }
-
- if (unlikely(address == -1))
- address = DMA_ERROR_CODE;
-
- WARN_ON((address + (PAGE_SIZE*pages)) > dom->aperture_size);
-
- return address;
-}
-
-/*
- * The address free function.
- *
- * called with domain->lock held
- */
-static void dma_ops_free_addresses(struct dma_ops_domain *dom,
- unsigned long address,
- unsigned int pages)
-{
- unsigned i = address >> APERTURE_RANGE_SHIFT;
- struct aperture_range *range = dom->aperture[i];
- unsigned long flags;
-
- BUG_ON(i >= APERTURE_MAX_RANGES || range == NULL);
-
-#ifdef CONFIG_IOMMU_STRESS
- if (i < 4)
- return;
-#endif
-
- if (amd_iommu_unmap_flush) {
- domain_flush_tlb(&dom->domain);
- domain_flush_complete(&dom->domain);
- }
-
- address = (address % APERTURE_RANGE_SIZE) >> PAGE_SHIFT;
-
- spin_lock_irqsave(&range->bitmap_lock, flags);
- if (address + pages > range->next_bit)
- range->next_bit = address + pages;
- bitmap_clear(range->bitmap, address, pages);
- spin_unlock_irqrestore(&range->bitmap_lock, flags);
+ pages = __roundup_pow_of_two(pages);
+ address >>= PAGE_SHIFT;
+ free_iova_fast(&dma_dom->iovad, address, pages);
}
/****************************************************************************
*/
static void dma_ops_domain_free(struct dma_ops_domain *dom)
{
- int i;
-
if (!dom)
return;
- free_percpu(dom->next_index);
-
del_domain_from_list(&dom->domain);
- free_pagetable(&dom->domain);
+ put_iova_domain(&dom->iovad);
- for (i = 0; i < APERTURE_MAX_RANGES; ++i) {
- if (!dom->aperture[i])
- continue;
- free_page((unsigned long)dom->aperture[i]->bitmap);
- kfree(dom->aperture[i]);
- }
+ free_pagetable(&dom->domain);
kfree(dom);
}
-static int dma_ops_domain_alloc_apertures(struct dma_ops_domain *dma_dom,
- int max_apertures)
-{
- int ret, i, apertures;
-
- apertures = dma_dom->aperture_size >> APERTURE_RANGE_SHIFT;
- ret = 0;
-
- for (i = apertures; i < max_apertures; ++i) {
- ret = alloc_new_range(dma_dom, false, GFP_KERNEL);
- if (ret)
- break;
- }
-
- return ret;
-}
-
/*
* Allocates a new protection domain usable for the dma_ops functions.
* It also initializes the page table and the address allocator data
static struct dma_ops_domain *dma_ops_domain_alloc(void)
{
struct dma_ops_domain *dma_dom;
- int cpu;
dma_dom = kzalloc(sizeof(struct dma_ops_domain), GFP_KERNEL);
if (!dma_dom)
if (protection_domain_init(&dma_dom->domain))
goto free_dma_dom;
- dma_dom->next_index = alloc_percpu(u32);
- if (!dma_dom->next_index)
- goto free_dma_dom;
-
- dma_dom->domain.mode = PAGE_MODE_2_LEVEL;
+ dma_dom->domain.mode = PAGE_MODE_3_LEVEL;
dma_dom->domain.pt_root = (void *)get_zeroed_page(GFP_KERNEL);
dma_dom->domain.flags = PD_DMA_OPS_MASK;
- dma_dom->domain.priv = dma_dom;
if (!dma_dom->domain.pt_root)
goto free_dma_dom;
- add_domain_to_list(&dma_dom->domain);
+ init_iova_domain(&dma_dom->iovad, PAGE_SIZE,
+ IOVA_START_PFN, DMA_32BIT_PFN);
- if (alloc_new_range(dma_dom, true, GFP_KERNEL))
- goto free_dma_dom;
+ /* Initialize reserved ranges */
+ copy_reserved_iova(&reserved_iova_ranges, &dma_dom->iovad);
- /*
- * mark the first page as allocated so we never return 0 as
- * a valid dma-address. So we can use 0 as error value
- */
- dma_dom->aperture[0]->bitmap[0] = 1;
-
- for_each_possible_cpu(cpu)
- *per_cpu_ptr(dma_dom->next_index, cpu) = 0;
+ add_domain_to_list(&dma_dom->domain);
return dma_dom;
*
*****************************************************************************/
+static void __queue_flush(struct flush_queue *queue)
+{
+ struct protection_domain *domain;
+ unsigned long flags;
+ int idx;
+
+ /* First flush TLB of all known domains */
+ spin_lock_irqsave(&amd_iommu_pd_lock, flags);
+ list_for_each_entry(domain, &amd_iommu_pd_list, list)
+ domain_flush_tlb(domain);
+ spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);
+
+ /* Wait until flushes have completed */
+ domain_flush_complete(NULL);
+
+ for (idx = 0; idx < queue->next; ++idx) {
+ struct flush_queue_entry *entry;
+
+ entry = queue->entries + idx;
+
+ free_iova_fast(&entry->dma_dom->iovad,
+ entry->iova_pfn,
+ entry->pages);
+
+ /* Not really necessary, just to make sure we catch any bugs */
+ entry->dma_dom = NULL;
+ }
+
+ queue->next = 0;
+}
+
+static void queue_flush_all(void)
+{
+ int cpu;
+
+ for_each_possible_cpu(cpu) {
+ struct flush_queue *queue;
+ unsigned long flags;
+
+ queue = per_cpu_ptr(&flush_queue, cpu);
+ spin_lock_irqsave(&queue->lock, flags);
+ if (queue->next > 0)
+ __queue_flush(queue);
+ spin_unlock_irqrestore(&queue->lock, flags);
+ }
+}
+
+static void queue_flush_timeout(unsigned long unsused)
+{
+ atomic_set(&queue_timer_on, 0);
+ queue_flush_all();
+}
+
+static void queue_add(struct dma_ops_domain *dma_dom,
+ unsigned long address, unsigned long pages)
+{
+ struct flush_queue_entry *entry;
+ struct flush_queue *queue;
+ unsigned long flags;
+ int idx;
+
+ pages = __roundup_pow_of_two(pages);
+ address >>= PAGE_SHIFT;
+
+ queue = get_cpu_ptr(&flush_queue);
+ spin_lock_irqsave(&queue->lock, flags);
+
+ if (queue->next == FLUSH_QUEUE_SIZE)
+ __queue_flush(queue);
+
+ idx = queue->next++;
+ entry = queue->entries + idx;
+
+ entry->iova_pfn = address;
+ entry->pages = pages;
+ entry->dma_dom = dma_dom;
+
+ spin_unlock_irqrestore(&queue->lock, flags);
+
+ if (atomic_cmpxchg(&queue_timer_on, 0, 1) == 0)
+ mod_timer(&queue_timer, jiffies + msecs_to_jiffies(10));
+
+ put_cpu_ptr(&flush_queue);
+}
+
+
/*
* In the dma_ops path we only have the struct device. This function
* finds the corresponding IOMMU, the protection domain and the
static struct protection_domain *get_domain(struct device *dev)
{
struct protection_domain *domain;
- struct iommu_domain *io_domain;
if (!check_device(dev))
return ERR_PTR(-EINVAL);
- io_domain = iommu_get_domain_for_dev(dev);
- if (!io_domain)
- return NULL;
-
- domain = to_pdomain(io_domain);
+ domain = get_dev_data(dev)->domain;
if (!dma_ops_domain(domain))
return ERR_PTR(-EBUSY);
{
struct iommu_dev_data *dev_data;
- list_for_each_entry(dev_data, &domain->dev_list, list)
+ list_for_each_entry(dev_data, &domain->dev_list, list) {
set_dte_entry(dev_data->devid, domain, dev_data->ats.enabled);
+
+ if (dev_data->devid == dev_data->alias)
+ continue;
+
+ /* There is an alias, update device table entry for it */
+ set_dte_entry(dev_data->alias, domain, dev_data->ats.enabled);
+ }
}
static void update_domain(struct protection_domain *domain)
domain->updated = false;
}
-/*
- * This function fetches the PTE for a given address in the aperture
- */
-static u64* dma_ops_get_pte(struct dma_ops_domain *dom,
- unsigned long address)
-{
- struct aperture_range *aperture;
- u64 *pte, *pte_page;
-
- aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];
- if (!aperture)
- return NULL;
-
- pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];
- if (!pte) {
- pte = alloc_pte(&dom->domain, address, PAGE_SIZE, &pte_page,
- GFP_ATOMIC);
- aperture->pte_pages[APERTURE_PAGE_INDEX(address)] = pte_page;
- } else
- pte += PM_LEVEL_INDEX(0, address);
-
- update_domain(&dom->domain);
-
- return pte;
-}
-
-/*
- * This is the generic map function. It maps one 4kb page at paddr to
- * the given address in the DMA address space for the domain.
- */
-static dma_addr_t dma_ops_domain_map(struct dma_ops_domain *dom,
- unsigned long address,
- phys_addr_t paddr,
- int direction)
+static int dir2prot(enum dma_data_direction direction)
{
- u64 *pte, __pte;
-
- WARN_ON(address > dom->aperture_size);
-
- paddr &= PAGE_MASK;
-
- pte = dma_ops_get_pte(dom, address);
- if (!pte)
- return DMA_ERROR_CODE;
-
- __pte = paddr | IOMMU_PTE_P | IOMMU_PTE_FC;
-
if (direction == DMA_TO_DEVICE)
- __pte |= IOMMU_PTE_IR;
+ return IOMMU_PROT_IR;
else if (direction == DMA_FROM_DEVICE)
- __pte |= IOMMU_PTE_IW;
+ return IOMMU_PROT_IW;
else if (direction == DMA_BIDIRECTIONAL)
- __pte |= IOMMU_PTE_IR | IOMMU_PTE_IW;
-
- WARN_ON_ONCE(*pte);
-
- *pte = __pte;
-
- return (dma_addr_t)address;
-}
-
-/*
- * The generic unmapping function for on page in the DMA address space.
- */
-static void dma_ops_domain_unmap(struct dma_ops_domain *dom,
- unsigned long address)
-{
- struct aperture_range *aperture;
- u64 *pte;
-
- if (address >= dom->aperture_size)
- return;
-
- aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];
- if (!aperture)
- return;
-
- pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];
- if (!pte)
- return;
-
- pte += PM_LEVEL_INDEX(0, address);
-
- WARN_ON_ONCE(!*pte);
-
- *pte = 0ULL;
+ return IOMMU_PROT_IW | IOMMU_PROT_IR;
+ else
+ return 0;
}
-
/*
* This function contains common code for mapping of a physically
* contiguous memory region into DMA address space. It is used by all
struct dma_ops_domain *dma_dom,
phys_addr_t paddr,
size_t size,
- int dir,
- bool align,
+ enum dma_data_direction direction,
u64 dma_mask)
{
dma_addr_t offset = paddr & ~PAGE_MASK;
dma_addr_t address, start, ret;
unsigned int pages;
- unsigned long align_mask = 0;
+ int prot = 0;
int i;
pages = iommu_num_pages(paddr, size, PAGE_SIZE);
paddr &= PAGE_MASK;
- if (align)
- align_mask = (1UL << get_order(size)) - 1;
-
- address = dma_ops_alloc_addresses(dev, dma_dom, pages, align_mask,
- dma_mask);
-
+ address = dma_ops_alloc_iova(dev, dma_dom, pages, dma_mask);
if (address == DMA_ERROR_CODE)
goto out;
+ prot = dir2prot(direction);
+
start = address;
for (i = 0; i < pages; ++i) {
- ret = dma_ops_domain_map(dma_dom, start, paddr, dir);
- if (ret == DMA_ERROR_CODE)
+ ret = iommu_map_page(&dma_dom->domain, start, paddr,
+ PAGE_SIZE, prot, GFP_ATOMIC);
+ if (ret)
goto out_unmap;
paddr += PAGE_SIZE;
for (--i; i >= 0; --i) {
start -= PAGE_SIZE;
- dma_ops_domain_unmap(dma_dom, start);
+ iommu_unmap_page(&dma_dom->domain, start, PAGE_SIZE);
}
- dma_ops_free_addresses(dma_dom, address, pages);
+ domain_flush_tlb(&dma_dom->domain);
+ domain_flush_complete(&dma_dom->domain);
+
+ dma_ops_free_iova(dma_dom, address, pages);
return DMA_ERROR_CODE;
}
dma_addr_t i, start;
unsigned int pages;
- if ((dma_addr == DMA_ERROR_CODE) ||
- (dma_addr + size > dma_dom->aperture_size))
- return;
-
flush_addr = dma_addr;
pages = iommu_num_pages(dma_addr, size, PAGE_SIZE);
dma_addr &= PAGE_MASK;
start = dma_addr;
for (i = 0; i < pages; ++i) {
- dma_ops_domain_unmap(dma_dom, start);
+ iommu_unmap_page(&dma_dom->domain, start, PAGE_SIZE);
start += PAGE_SIZE;
}
- dma_ops_free_addresses(dma_dom, dma_addr, pages);
+ if (amd_iommu_unmap_flush) {
+ dma_ops_free_iova(dma_dom, dma_addr, pages);
+ domain_flush_tlb(&dma_dom->domain);
+ domain_flush_complete(&dma_dom->domain);
+ } else {
+ queue_add(dma_dom, dma_addr, pages);
+ }
}
/*
static dma_addr_t map_page(struct device *dev, struct page *page,
unsigned long offset, size_t size,
enum dma_data_direction dir,
- struct dma_attrs *attrs)
+ unsigned long attrs)
{
phys_addr_t paddr = page_to_phys(page) + offset;
struct protection_domain *domain;
+ struct dma_ops_domain *dma_dom;
u64 dma_mask;
domain = get_domain(dev);
return DMA_ERROR_CODE;
dma_mask = *dev->dma_mask;
+ dma_dom = to_dma_ops_domain(domain);
- return __map_single(dev, domain->priv, paddr, size, dir, false,
- dma_mask);
+ return __map_single(dev, dma_dom, paddr, size, dir, dma_mask);
}
/*
* The exported unmap_single function for dma_ops.
*/
static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size,
- enum dma_data_direction dir, struct dma_attrs *attrs)
+ enum dma_data_direction dir, unsigned long attrs)
{
struct protection_domain *domain;
+ struct dma_ops_domain *dma_dom;
domain = get_domain(dev);
if (IS_ERR(domain))
return;
- __unmap_single(domain->priv, dma_addr, size, dir);
+ dma_dom = to_dma_ops_domain(domain);
+
+ __unmap_single(dma_dom, dma_addr, size, dir);
+}
+
+static int sg_num_pages(struct device *dev,
+ struct scatterlist *sglist,
+ int nelems)
+{
+ unsigned long mask, boundary_size;
+ struct scatterlist *s;
+ int i, npages = 0;
+
+ mask = dma_get_seg_boundary(dev);
+ boundary_size = mask + 1 ? ALIGN(mask + 1, PAGE_SIZE) >> PAGE_SHIFT :
+ 1UL << (BITS_PER_LONG - PAGE_SHIFT);
+
+ for_each_sg(sglist, s, nelems, i) {
+ int p, n;
+
+ s->dma_address = npages << PAGE_SHIFT;
+ p = npages % boundary_size;
+ n = iommu_num_pages(sg_phys(s), s->length, PAGE_SIZE);
+ if (p + n > boundary_size)
+ npages += boundary_size - p;
+ npages += n;
+ }
+
+ return npages;
}
/*
* lists).
*/
static int map_sg(struct device *dev, struct scatterlist *sglist,
- int nelems, enum dma_data_direction dir,
- struct dma_attrs *attrs)
+ int nelems, enum dma_data_direction direction,
+ unsigned long attrs)
{
+ int mapped_pages = 0, npages = 0, prot = 0, i;
struct protection_domain *domain;
- int i;
+ struct dma_ops_domain *dma_dom;
struct scatterlist *s;
- phys_addr_t paddr;
- int mapped_elems = 0;
+ unsigned long address;
u64 dma_mask;
domain = get_domain(dev);
if (IS_ERR(domain))
return 0;
+ dma_dom = to_dma_ops_domain(domain);
dma_mask = *dev->dma_mask;
+ npages = sg_num_pages(dev, sglist, nelems);
+
+ address = dma_ops_alloc_iova(dev, dma_dom, npages, dma_mask);
+ if (address == DMA_ERROR_CODE)
+ goto out_err;
+
+ prot = dir2prot(direction);
+
+ /* Map all sg entries */
for_each_sg(sglist, s, nelems, i) {
- paddr = sg_phys(s);
+ int j, pages = iommu_num_pages(sg_phys(s), s->length, PAGE_SIZE);
- s->dma_address = __map_single(dev, domain->priv,
- paddr, s->length, dir, false,
- dma_mask);
+ for (j = 0; j < pages; ++j) {
+ unsigned long bus_addr, phys_addr;
+ int ret;
- if (s->dma_address) {
- s->dma_length = s->length;
- mapped_elems++;
- } else
- goto unmap;
+ bus_addr = address + s->dma_address + (j << PAGE_SHIFT);
+ phys_addr = (sg_phys(s) & PAGE_MASK) + (j << PAGE_SHIFT);
+ ret = iommu_map_page(domain, bus_addr, phys_addr, PAGE_SIZE, prot, GFP_ATOMIC);
+ if (ret)
+ goto out_unmap;
+
+ mapped_pages += 1;
+ }
}
- return mapped_elems;
+ /* Everything is mapped - write the right values into s->dma_address */
+ for_each_sg(sglist, s, nelems, i) {
+ s->dma_address += address + s->offset;
+ s->dma_length = s->length;
+ }
+
+ return nelems;
+
+out_unmap:
+ pr_err("%s: IOMMU mapping error in map_sg (io-pages: %d)\n",
+ dev_name(dev), npages);
+
+ for_each_sg(sglist, s, nelems, i) {
+ int j, pages = iommu_num_pages(sg_phys(s), s->length, PAGE_SIZE);
+
+ for (j = 0; j < pages; ++j) {
+ unsigned long bus_addr;
+
+ bus_addr = address + s->dma_address + (j << PAGE_SHIFT);
+ iommu_unmap_page(domain, bus_addr, PAGE_SIZE);
-unmap:
- for_each_sg(sglist, s, mapped_elems, i) {
- if (s->dma_address)
- __unmap_single(domain->priv, s->dma_address,
- s->dma_length, dir);
- s->dma_address = s->dma_length = 0;
+ if (--mapped_pages)
+ goto out_free_iova;
+ }
}
+out_free_iova:
+ free_iova_fast(&dma_dom->iovad, address, npages);
+
+out_err:
return 0;
}
*/
static void unmap_sg(struct device *dev, struct scatterlist *sglist,
int nelems, enum dma_data_direction dir,
- struct dma_attrs *attrs)
+ unsigned long attrs)
{
struct protection_domain *domain;
- struct scatterlist *s;
- int i;
+ struct dma_ops_domain *dma_dom;
+ unsigned long startaddr;
+ int npages = 2;
domain = get_domain(dev);
if (IS_ERR(domain))
return;
- for_each_sg(sglist, s, nelems, i) {
- __unmap_single(domain->priv, s->dma_address,
- s->dma_length, dir);
- s->dma_address = s->dma_length = 0;
- }
+ startaddr = sg_dma_address(sglist) & PAGE_MASK;
+ dma_dom = to_dma_ops_domain(domain);
+ npages = sg_num_pages(dev, sglist, nelems);
+
+ __unmap_single(dma_dom, startaddr, npages << PAGE_SHIFT, dir);
}
/*
*/
static void *alloc_coherent(struct device *dev, size_t size,
dma_addr_t *dma_addr, gfp_t flag,
- struct dma_attrs *attrs)
+ unsigned long attrs)
{
u64 dma_mask = dev->coherent_dma_mask;
struct protection_domain *domain;
+ struct dma_ops_domain *dma_dom;
struct page *page;
domain = get_domain(dev);
} else if (IS_ERR(domain))
return NULL;
+ dma_dom = to_dma_ops_domain(domain);
size = PAGE_ALIGN(size);
dma_mask = dev->coherent_dma_mask;
flag &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);
if (!dma_mask)
dma_mask = *dev->dma_mask;
- *dma_addr = __map_single(dev, domain->priv, page_to_phys(page),
- size, DMA_BIDIRECTIONAL, true, dma_mask);
+ *dma_addr = __map_single(dev, dma_dom, page_to_phys(page),
+ size, DMA_BIDIRECTIONAL, dma_mask);
if (*dma_addr == DMA_ERROR_CODE)
goto out_free;
*/
static void free_coherent(struct device *dev, size_t size,
void *virt_addr, dma_addr_t dma_addr,
- struct dma_attrs *attrs)
+ unsigned long attrs)
{
struct protection_domain *domain;
+ struct dma_ops_domain *dma_dom;
struct page *page;
page = virt_to_page(virt_addr);
if (IS_ERR(domain))
goto free_mem;
- __unmap_single(domain->priv, dma_addr, size, DMA_BIDIRECTIONAL);
+ dma_dom = to_dma_ops_domain(domain);
+
+ __unmap_single(dma_dom, dma_addr, size, DMA_BIDIRECTIONAL);
free_mem:
if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
return check_device(dev);
}
-static int set_dma_mask(struct device *dev, u64 mask)
+static struct dma_map_ops amd_iommu_dma_ops = {
+ .alloc = alloc_coherent,
+ .free = free_coherent,
+ .map_page = map_page,
+ .unmap_page = unmap_page,
+ .map_sg = map_sg,
+ .unmap_sg = unmap_sg,
+ .dma_supported = amd_iommu_dma_supported,
+};
+
+static int init_reserved_iova_ranges(void)
{
- struct protection_domain *domain;
- int max_apertures = 1;
+ struct pci_dev *pdev = NULL;
+ struct iova *val;
- domain = get_domain(dev);
- if (IS_ERR(domain))
- return PTR_ERR(domain);
+ init_iova_domain(&reserved_iova_ranges, PAGE_SIZE,
+ IOVA_START_PFN, DMA_32BIT_PFN);
+
+ lockdep_set_class(&reserved_iova_ranges.iova_rbtree_lock,
+ &reserved_rbtree_key);
- if (mask == DMA_BIT_MASK(64))
- max_apertures = 8;
- else if (mask > DMA_BIT_MASK(32))
- max_apertures = 4;
+ /* MSI memory range */
+ val = reserve_iova(&reserved_iova_ranges,
+ IOVA_PFN(MSI_RANGE_START), IOVA_PFN(MSI_RANGE_END));
+ if (!val) {
+ pr_err("Reserving MSI range failed\n");
+ return -ENOMEM;
+ }
+
+ /* HT memory range */
+ val = reserve_iova(&reserved_iova_ranges,
+ IOVA_PFN(HT_RANGE_START), IOVA_PFN(HT_RANGE_END));
+ if (!val) {
+ pr_err("Reserving HT range failed\n");
+ return -ENOMEM;
+ }
/*
- * To prevent lock contention it doesn't make sense to allocate more
- * apertures than online cpus
+ * Memory used for PCI resources
+ * FIXME: Check whether we can reserve the PCI-hole completly
*/
- if (max_apertures > num_online_cpus())
- max_apertures = num_online_cpus();
+ for_each_pci_dev(pdev) {
+ int i;
+
+ for (i = 0; i < PCI_NUM_RESOURCES; ++i) {
+ struct resource *r = &pdev->resource[i];
+
+ if (!(r->flags & IORESOURCE_MEM))
+ continue;
- if (dma_ops_domain_alloc_apertures(domain->priv, max_apertures))
- dev_err(dev, "Can't allocate %d iommu apertures\n",
- max_apertures);
+ val = reserve_iova(&reserved_iova_ranges,
+ IOVA_PFN(r->start),
+ IOVA_PFN(r->end));
+ if (!val) {
+ pr_err("Reserve pci-resource range failed\n");
+ return -ENOMEM;
+ }
+ }
+ }
return 0;
}
-static struct dma_map_ops amd_iommu_dma_ops = {
- .alloc = alloc_coherent,
- .free = free_coherent,
- .map_page = map_page,
- .unmap_page = unmap_page,
- .map_sg = map_sg,
- .unmap_sg = unmap_sg,
- .dma_supported = amd_iommu_dma_supported,
- .set_dma_mask = set_dma_mask,
-};
-
int __init amd_iommu_init_api(void)
{
- int err = 0;
+ int ret, cpu, err = 0;
+
+ ret = iova_cache_get();
+ if (ret)
+ return ret;
+
+ ret = init_reserved_iova_ranges();
+ if (ret)
+ return ret;
+
+ for_each_possible_cpu(cpu) {
+ struct flush_queue *queue = per_cpu_ptr(&flush_queue, cpu);
+
+ queue->entries = kzalloc(FLUSH_QUEUE_SIZE *
+ sizeof(*queue->entries),
+ GFP_KERNEL);
+ if (!queue->entries)
+ goto out_put_iova;
+
+ spin_lock_init(&queue->lock);
+ }
err = bus_set_iommu(&pci_bus_type, &amd_iommu_ops);
if (err)
if (err)
return err;
#endif
+ err = bus_set_iommu(&platform_bus_type, &amd_iommu_ops);
+ if (err)
+ return err;
return 0;
+
+out_put_iova:
+ for_each_possible_cpu(cpu) {
+ struct flush_queue *queue = per_cpu_ptr(&flush_queue, cpu);
+
+ kfree(queue->entries);
+ }
+
+ return -ENOMEM;
}
int __init amd_iommu_init_dma_ops(void)
{
+ setup_timer(&queue_timer, queue_flush_timeout, 0);
+ atomic_set(&queue_timer_on, 0);
+
swiotlb = iommu_pass_through ? 1 : 0;
iommu_detected = 1;
pr_info("AMD-Vi: Lazy IO/TLB flushing enabled\n");
return 0;
+
}
/*****************************************************************************
static void amd_iommu_domain_free(struct iommu_domain *dom)
{
struct protection_domain *domain;
-
- if (!dom)
- return;
+ struct dma_ops_domain *dma_dom;
domain = to_pdomain(dom);
BUG_ON(domain->dev_cnt != 0);
- if (domain->mode != PAGE_MODE_NONE)
- free_pagetable(domain);
+ if (!dom)
+ return;
- if (domain->flags & PD_IOMMUV2_MASK)
- free_gcr3_table(domain);
+ switch (dom->type) {
+ case IOMMU_DOMAIN_DMA:
+ /*
+ * First make sure the domain is no longer referenced from the
+ * flush queue
+ */
+ queue_flush_all();
+
+ /* Now release the domain */
+ dma_dom = to_dma_ops_domain(domain);
+ dma_ops_domain_free(dma_dom);
+ break;
+ default:
+ if (domain->mode != PAGE_MODE_NONE)
+ free_pagetable(domain);
+
+ if (domain->flags & PD_IOMMUV2_MASK)
+ free_gcr3_table(domain);
- protection_domain_free(domain);
+ protection_domain_free(domain);
+ break;
+ }
}
static void amd_iommu_detach_device(struct iommu_domain *dom,
prot |= IOMMU_PROT_IW;
mutex_lock(&domain->api_lock);
- ret = iommu_map_page(domain, iova, paddr, prot, page_size);
+ ret = iommu_map_page(domain, iova, paddr, page_size, prot, GFP_KERNEL);
mutex_unlock(&domain->api_lock);
return ret;
kfree(entry);
}
+static void amd_iommu_apply_dm_region(struct device *dev,
+ struct iommu_domain *domain,
+ struct iommu_dm_region *region)
+{
+ struct dma_ops_domain *dma_dom = to_dma_ops_domain(to_pdomain(domain));
+ unsigned long start, end;
+
+ start = IOVA_PFN(region->start);
+ end = IOVA_PFN(region->start + region->length);
+
+ WARN_ON_ONCE(reserve_iova(&dma_dom->iovad, start, end) == NULL);
+}
+
static const struct iommu_ops amd_iommu_ops = {
.capable = amd_iommu_capable,
.domain_alloc = amd_iommu_domain_alloc,
.device_group = amd_iommu_device_group,
.get_dm_regions = amd_iommu_get_dm_regions,
.put_dm_regions = amd_iommu_put_dm_regions,
+ .apply_dm_region = amd_iommu_apply_dm_region,
.pgsize_bitmap = AMD_IOMMU_PGSIZES,
};
projects / cascardo / linux.git / blobdiff