#include <asm/proto.h>
#include <asm/iommu.h>
#include <asm/gart.h>
+#include <asm/dma.h>
#include <asm/amd_iommu_proto.h>
#include <asm/amd_iommu_types.h>
#include <asm/amd_iommu.h>
static LIST_HEAD(iommu_pd_list);
static DEFINE_SPINLOCK(iommu_pd_list_lock);
+/* List of all available dev_data structures */
+static LIST_HEAD(dev_data_list);
+static DEFINE_SPINLOCK(dev_data_list_lock);
+
/*
* Domain for untranslated devices - only allocated
* if iommu=pt passed on kernel cmd line.
*
****************************************************************************/
+static struct iommu_dev_data *alloc_dev_data(u16 devid)
+{
+ struct iommu_dev_data *dev_data;
+ unsigned long flags;
+
+ dev_data = kzalloc(sizeof(*dev_data), GFP_KERNEL);
+ if (!dev_data)
+ return NULL;
+
+ dev_data->devid = devid;
+ atomic_set(&dev_data->bind, 0);
+
+ spin_lock_irqsave(&dev_data_list_lock, flags);
+ list_add_tail(&dev_data->dev_data_list, &dev_data_list);
+ spin_unlock_irqrestore(&dev_data_list_lock, flags);
+
+ return dev_data;
+}
+
+static void free_dev_data(struct iommu_dev_data *dev_data)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&dev_data_list_lock, flags);
+ list_del(&dev_data->dev_data_list);
+ spin_unlock_irqrestore(&dev_data_list_lock, flags);
+
+ kfree(dev_data);
+}
+
static inline u16 get_device_id(struct device *dev)
{
struct pci_dev *pdev = to_pci_dev(dev);
{
struct iommu_dev_data *dev_data;
struct pci_dev *pdev;
- u16 devid, alias;
+ u16 alias;
if (dev->archdata.iommu)
return 0;
- dev_data = kzalloc(sizeof(*dev_data), GFP_KERNEL);
+ dev_data = alloc_dev_data(get_device_id(dev));
if (!dev_data)
return -ENOMEM;
dev_data->dev = dev;
- devid = get_device_id(dev);
- alias = amd_iommu_alias_table[devid];
+ alias = amd_iommu_alias_table[dev_data->devid];
pdev = pci_get_bus_and_slot(PCI_BUS(alias), alias & 0xff);
if (pdev)
dev_data->alias = &pdev->dev;
-
- atomic_set(&dev_data->bind, 0);
+ else {
+ free_dev_data(dev_data);
+ return -ENOTSUPP;
+ }
dev->archdata.iommu = dev_data;
-
return 0;
}
+static void iommu_ignore_device(struct device *dev)
+{
+ u16 devid, alias;
+
+ devid = get_device_id(dev);
+ alias = amd_iommu_alias_table[devid];
+
+ memset(&amd_iommu_dev_table[devid], 0, sizeof(struct dev_table_entry));
+ memset(&amd_iommu_dev_table[alias], 0, sizeof(struct dev_table_entry));
+
+ amd_iommu_rlookup_table[devid] = NULL;
+ amd_iommu_rlookup_table[alias] = NULL;
+}
+
static void iommu_uninit_device(struct device *dev)
{
- kfree(dev->archdata.iommu);
+ /*
+ * Nothing to do here - we keep dev_data around for unplugged devices
+ * and reuse it when the device is re-plugged - not doing so would
+ * introduce a ton of races.
+ */
}
void __init amd_iommu_uninit_devices(void)
{
+ struct iommu_dev_data *dev_data, *n;
struct pci_dev *pdev = NULL;
for_each_pci_dev(pdev) {
iommu_uninit_device(&pdev->dev);
}
+
+ /* Free all of our dev_data structures */
+ list_for_each_entry_safe(dev_data, n, &dev_data_list, dev_data_list)
+ free_dev_data(dev_data);
}
int __init amd_iommu_init_devices(void)
continue;
ret = iommu_init_device(&pdev->dev);
- if (ret)
+ if (ret == -ENOTSUPP)
+ iommu_ignore_device(&pdev->dev);
+ else if (ret)
goto out_free;
}
/*
* Command send function for flushing on-device TLB
*/
-static int device_flush_iotlb(struct device *dev, u64 address, size_t size)
+static int device_flush_iotlb(struct iommu_dev_data *dev_data,
+ u64 address, size_t size)
{
- struct pci_dev *pdev = to_pci_dev(dev);
struct amd_iommu *iommu;
struct iommu_cmd cmd;
- u16 devid;
int qdep;
- qdep = pci_ats_queue_depth(pdev);
- devid = get_device_id(dev);
- iommu = amd_iommu_rlookup_table[devid];
+ qdep = dev_data->ats.qdep;
+ iommu = amd_iommu_rlookup_table[dev_data->devid];
- build_inv_iotlb_pages(&cmd, devid, qdep, address, size);
+ build_inv_iotlb_pages(&cmd, dev_data->devid, qdep, address, size);
return iommu_queue_command(iommu, &cmd);
}
/*
* Command send function for invalidating a device table entry
*/
-static int device_flush_dte(struct device *dev)
+static int device_flush_dte(struct iommu_dev_data *dev_data)
{
struct amd_iommu *iommu;
- struct pci_dev *pdev;
- u16 devid;
int ret;
- pdev = to_pci_dev(dev);
- devid = get_device_id(dev);
- iommu = amd_iommu_rlookup_table[devid];
+ iommu = amd_iommu_rlookup_table[dev_data->devid];
- ret = iommu_flush_dte(iommu, devid);
+ ret = iommu_flush_dte(iommu, dev_data->devid);
if (ret)
return ret;
- if (pci_ats_enabled(pdev))
- ret = device_flush_iotlb(dev, 0, ~0UL);
+ if (dev_data->ats.enabled)
+ ret = device_flush_iotlb(dev_data, 0, ~0UL);
return ret;
}
}
list_for_each_entry(dev_data, &domain->dev_list, list) {
- struct pci_dev *pdev = to_pci_dev(dev_data->dev);
- if (!pci_ats_enabled(pdev))
+ if (!dev_data->ats.enabled)
continue;
- ret |= device_flush_iotlb(dev_data->dev, address, size);
+ ret |= device_flush_iotlb(dev_data, address, size);
}
WARN_ON(ret);
spin_lock_irqsave(&domain->lock, flags);
list_for_each_entry(dev_data, &domain->dev_list, list)
- device_flush_dte(dev_data->dev);
+ device_flush_dte(dev_data);
spin_unlock_irqrestore(&domain->lock, flags);
}
struct amd_iommu *iommu;
struct pci_dev *pdev;
bool ats = false;
- u16 devid;
- devid = get_device_id(dev);
- iommu = amd_iommu_rlookup_table[devid];
dev_data = get_dev_data(dev);
+ iommu = amd_iommu_rlookup_table[dev_data->devid];
pdev = to_pci_dev(dev);
- if (amd_iommu_iotlb_sup)
- ats = pci_ats_enabled(pdev);
+ ats = dev_data->ats.enabled;
/* Update data structures */
dev_data->domain = domain;
list_add(&dev_data->list, &domain->dev_list);
- set_dte_entry(devid, domain, ats);
+ set_dte_entry(dev_data->devid, domain, ats);
/* Do reference counting */
domain->dev_iommu[iommu->index] += 1;
domain->dev_cnt += 1;
/* Flush the DTE entry */
- device_flush_dte(dev);
+ device_flush_dte(dev_data);
}
static void do_detach(struct device *dev)
{
struct iommu_dev_data *dev_data;
struct amd_iommu *iommu;
- struct pci_dev *pdev;
- u16 devid;
- devid = get_device_id(dev);
- iommu = amd_iommu_rlookup_table[devid];
dev_data = get_dev_data(dev);
- pdev = to_pci_dev(dev);
+ iommu = amd_iommu_rlookup_table[dev_data->devid];
/* decrease reference counters */
dev_data->domain->dev_iommu[iommu->index] -= 1;
/* Update data structures */
dev_data->domain = NULL;
list_del(&dev_data->list);
- clear_dte_entry(devid);
+ clear_dte_entry(dev_data->devid);
/* Flush the DTE entry */
- device_flush_dte(dev);
+ device_flush_dte(dev_data);
}
/*
struct protection_domain *domain)
{
struct pci_dev *pdev = to_pci_dev(dev);
+ struct iommu_dev_data *dev_data;
unsigned long flags;
int ret;
- if (amd_iommu_iotlb_sup)
- pci_enable_ats(pdev, PAGE_SHIFT);
+ dev_data = get_dev_data(dev);
+
+ if (amd_iommu_iotlb_sup && pci_enable_ats(pdev, PAGE_SHIFT) == 0) {
+ dev_data->ats.enabled = true;
+ dev_data->ats.qdep = pci_ats_queue_depth(pdev);
+ }
write_lock_irqsave(&amd_iommu_devtable_lock, flags);
ret = __attach_device(dev, domain);
*/
static void detach_device(struct device *dev)
{
- struct pci_dev *pdev = to_pci_dev(dev);
+ struct iommu_dev_data *dev_data;
unsigned long flags;
/* lock device table */
__detach_device(dev);
write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
- if (amd_iommu_iotlb_sup && pci_ats_enabled(pdev))
- pci_disable_ats(pdev);
+ dev_data = get_dev_data(dev);
+
+ if (dev_data->ats.enabled) {
+ pci_disable_ats(to_pci_dev(dev));
+ dev_data->ats.enabled = false;
+ }
}
/*
struct protection_domain *dom;
struct iommu_dev_data *dev_data, *alias_data;
unsigned long flags;
- u16 devid, alias;
- devid = get_device_id(dev);
- alias = amd_iommu_alias_table[devid];
dev_data = get_dev_data(dev);
alias_data = get_dev_data(dev_data->alias);
if (!alias_data)
goto out;
}
- device_flush_dte(dev);
iommu_completion_wait(iommu);
out:
{
struct iommu_dev_data *dev_data;
- list_for_each_entry(dev_data, &domain->dev_list, list) {
- struct pci_dev *pdev = to_pci_dev(dev_data->dev);
- u16 devid = get_device_id(dev_data->dev);
- set_dte_entry(devid, domain, pci_ats_enabled(pdev));
- }
+ list_for_each_entry(dev_data, &domain->dev_list, list)
+ set_dte_entry(dev_data->devid, domain, dev_data->ats.enabled);
}
static void update_domain(struct protection_domain *domain)
.dma_supported = amd_iommu_dma_supported,
};
+static unsigned device_dma_ops_init(void)
+{
+ struct pci_dev *pdev = NULL;
+ unsigned unhandled = 0;
+
+ for_each_pci_dev(pdev) {
+ if (!check_device(&pdev->dev)) {
+ unhandled += 1;
+ continue;
+ }
+
+ pdev->dev.archdata.dma_ops = &amd_iommu_dma_ops;
+ }
+
+ return unhandled;
+}
+
/*
* The function which clues the AMD IOMMU driver into dma_ops.
*/
int __init amd_iommu_init_dma_ops(void)
{
struct amd_iommu *iommu;
- int ret;
+ int ret, unhandled;
/*
* first allocate a default protection domain for every IOMMU we
swiotlb = 0;
/* Make the driver finally visible to the drivers */
- dma_ops = &amd_iommu_dma_ops;
+ unhandled = device_dma_ops_init();
+ if (unhandled && max_pfn > MAX_DMA32_PFN) {
+ /* There are unhandled devices - initialize swiotlb for them */
+ swiotlb = 1;
+ }
amd_iommu_stats_init();
if (!iommu)
return;
- device_flush_dte(dev);
iommu_completion_wait(iommu);
}
struct iommu_dev_data *dev_data;
struct amd_iommu *iommu;
int ret;
- u16 devid;
if (!check_device(dev))
return -EINVAL;
dev_data = dev->archdata.iommu;
- devid = get_device_id(dev);
-
- iommu = amd_iommu_rlookup_table[devid];
+ iommu = amd_iommu_rlookup_table[dev_data->devid];
if (!iommu)
return -EINVAL;
projects / cascardo / linux.git / blobdiff