x86/amd-iommu: Use only dev_data for dte and iotlb flushing routines

[cascardo/linux.git] / arch / x86 / kernel / amd_iommu.c

diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c
index 7c3a95e..0a5b465 100644 (file)
--- a/arch/x86/kernel/amd_iommu.c
+++ b/arch/x86/kernel/amd_iommu.c
@@ -45,6 +45,10 @@ static DEFINE_RWLOCK(amd_iommu_devtable_lock);
 static LIST_HEAD(iommu_pd_list);
 static DEFINE_SPINLOCK(iommu_pd_list_lock);
 
+/* List of all available dev_data structures */
+static LIST_HEAD(dev_data_list);
+static DEFINE_SPINLOCK(dev_data_list_lock);
+
 /*
  * Domain for untranslated devices - only allocated
  * if iommu=pt passed on kernel cmd line.
@@ -68,6 +72,36 @@ static void update_domain(struct protection_domain *domain);
  *
  ****************************************************************************/
 
+static struct iommu_dev_data *alloc_dev_data(u16 devid)
+{
+       struct iommu_dev_data *dev_data;
+       unsigned long flags;
+
+       dev_data = kzalloc(sizeof(*dev_data), GFP_KERNEL);
+       if (!dev_data)
+               return NULL;
+
+       dev_data->devid = devid;
+       atomic_set(&dev_data->bind, 0);
+
+       spin_lock_irqsave(&dev_data_list_lock, flags);
+       list_add_tail(&dev_data->dev_data_list, &dev_data_list);
+       spin_unlock_irqrestore(&dev_data_list_lock, flags);
+
+       return dev_data;
+}
+
+static void free_dev_data(struct iommu_dev_data *dev_data)
+{
+       unsigned long flags;
+
+       spin_lock_irqsave(&dev_data_list_lock, flags);
+       list_del(&dev_data->dev_data_list);
+       spin_unlock_irqrestore(&dev_data_list_lock, flags);
+
+       kfree(dev_data);
+}
+
 static inline u16 get_device_id(struct device *dev)
 {
        struct pci_dev *pdev = to_pci_dev(dev);
@@ -139,32 +173,28 @@ static int iommu_init_device(struct device *dev)
 {
        struct iommu_dev_data *dev_data;
        struct pci_dev *pdev;
-       u16 devid, alias;
+       u16 alias;
 
        if (dev->archdata.iommu)
                return 0;
 
-       dev_data = kzalloc(sizeof(*dev_data), GFP_KERNEL);
+       dev_data = alloc_dev_data(get_device_id(dev));
        if (!dev_data)
                return -ENOMEM;
 
        dev_data->dev = dev;
 
-       devid = get_device_id(dev);
-       alias = amd_iommu_alias_table[devid];
+       alias = amd_iommu_alias_table[dev_data->devid];
        pdev = pci_get_bus_and_slot(PCI_BUS(alias), alias & 0xff);
        if (pdev)
                dev_data->alias = &pdev->dev;
        else {
-               kfree(dev_data);
+               free_dev_data(dev_data);
                return -ENOTSUPP;
        }
 
-       atomic_set(&dev_data->bind, 0);
-
        dev->archdata.iommu = dev_data;
 
-
        return 0;
 }
 
@@ -184,11 +214,16 @@ static void iommu_ignore_device(struct device *dev)
 
 static void iommu_uninit_device(struct device *dev)
 {
-       kfree(dev->archdata.iommu);
+       /*
+        * Nothing to do here - we keep dev_data around for unplugged devices
+        * and reuse it when the device is re-plugged - not doing so would
+        * introduce a ton of races.
+        */
 }
 
 void __init amd_iommu_uninit_devices(void)
 {
+       struct iommu_dev_data *dev_data, *n;
        struct pci_dev *pdev = NULL;
 
        for_each_pci_dev(pdev) {
@@ -198,6 +233,10 @@ void __init amd_iommu_uninit_devices(void)
 
                iommu_uninit_device(&pdev->dev);
        }
+
+       /* Free all of our dev_data structures */
+       list_for_each_entry_safe(dev_data, n, &dev_data_list, dev_data_list)
+               free_dev_data(dev_data);
 }
 
 int __init amd_iommu_init_devices(void)
@@ -654,19 +693,17 @@ void iommu_flush_all_caches(struct amd_iommu *iommu)
 /*
  * Command send function for flushing on-device TLB
  */
-static int device_flush_iotlb(struct device *dev, u64 address, size_t size)
+static int device_flush_iotlb(struct iommu_dev_data *dev_data,
+                             u64 address, size_t size)
 {
-       struct pci_dev *pdev = to_pci_dev(dev);
        struct amd_iommu *iommu;
        struct iommu_cmd cmd;
-       u16 devid;
        int qdep;
 
-       qdep  = pci_ats_queue_depth(pdev);
-       devid = get_device_id(dev);
-       iommu = amd_iommu_rlookup_table[devid];
+       qdep     = dev_data->ats.qdep;
+       iommu    = amd_iommu_rlookup_table[dev_data->devid];
 
-       build_inv_iotlb_pages(&cmd, devid, qdep, address, size);
+       build_inv_iotlb_pages(&cmd, dev_data->devid, qdep, address, size);
 
        return iommu_queue_command(iommu, &cmd);
 }
@@ -674,23 +711,19 @@ static int device_flush_iotlb(struct device *dev, u64 address, size_t size)
 /*
  * Command send function for invalidating a device table entry
  */
-static int device_flush_dte(struct device *dev)
+static int device_flush_dte(struct iommu_dev_data *dev_data)
 {
        struct amd_iommu *iommu;
-       struct pci_dev *pdev;
-       u16 devid;
        int ret;
 
-       pdev  = to_pci_dev(dev);
-       devid = get_device_id(dev);
-       iommu = amd_iommu_rlookup_table[devid];
+       iommu = amd_iommu_rlookup_table[dev_data->devid];
 
-       ret = iommu_flush_dte(iommu, devid);
+       ret = iommu_flush_dte(iommu, dev_data->devid);
        if (ret)
                return ret;
 
-       if (pci_ats_enabled(pdev))
-               ret = device_flush_iotlb(dev, 0, ~0UL);
+       if (dev_data->ats.enabled)
+               ret = device_flush_iotlb(dev_data, 0, ~0UL);
 
        return ret;
 }
@@ -721,12 +754,11 @@ static void __domain_flush_pages(struct protection_domain *domain,
        }
 
        list_for_each_entry(dev_data, &domain->dev_list, list) {
-               struct pci_dev *pdev = to_pci_dev(dev_data->dev);
 
-               if (!pci_ats_enabled(pdev))
+               if (!dev_data->ats.enabled)
                        continue;
 
-               ret |= device_flush_iotlb(dev_data->dev, address, size);
+               ret |= device_flush_iotlb(dev_data, address, size);
        }
 
        WARN_ON(ret);
@@ -778,7 +810,7 @@ static void domain_flush_devices(struct protection_domain *domain)
        spin_lock_irqsave(&domain->lock, flags);
 
        list_for_each_entry(dev_data, &domain->dev_list, list)
-               device_flush_dte(dev_data->dev);
+               device_flush_dte(dev_data);
 
        spin_unlock_irqrestore(&domain->lock, flags);
 }
@@ -1532,38 +1564,33 @@ static void do_attach(struct device *dev, struct protection_domain *domain)
        struct amd_iommu *iommu;
        struct pci_dev *pdev;
        bool ats = false;
-       u16 devid;
 
-       devid    = get_device_id(dev);
-       iommu    = amd_iommu_rlookup_table[devid];
        dev_data = get_dev_data(dev);
+       iommu    = amd_iommu_rlookup_table[dev_data->devid];
        pdev     = to_pci_dev(dev);
 
-       if (amd_iommu_iotlb_sup)
-               ats = pci_ats_enabled(pdev);
+       ats = dev_data->ats.enabled;
 
        /* Update data structures */
        dev_data->domain = domain;
        list_add(&dev_data->list, &domain->dev_list);
-       set_dte_entry(devid, domain, ats);
+       set_dte_entry(dev_data->devid, domain, ats);
 
        /* Do reference counting */
        domain->dev_iommu[iommu->index] += 1;
        domain->dev_cnt                 += 1;
 
        /* Flush the DTE entry */
-       device_flush_dte(dev);
+       device_flush_dte(dev_data);
 }
 
 static void do_detach(struct device *dev)
 {
        struct iommu_dev_data *dev_data;
        struct amd_iommu *iommu;
-       u16 devid;
 
-       devid    = get_device_id(dev);
-       iommu    = amd_iommu_rlookup_table[devid];
        dev_data = get_dev_data(dev);
+       iommu    = amd_iommu_rlookup_table[dev_data->devid];
 
        /* decrease reference counters */
        dev_data->domain->dev_iommu[iommu->index] -= 1;
@@ -1572,10 +1599,10 @@ static void do_detach(struct device *dev)
        /* Update data structures */
        dev_data->domain = NULL;
        list_del(&dev_data->list);
-       clear_dte_entry(devid);
+       clear_dte_entry(dev_data->devid);
 
        /* Flush the DTE entry */
-       device_flush_dte(dev);
+       device_flush_dte(dev_data);
 }
 
 /*
@@ -1639,11 +1666,16 @@ static int attach_device(struct device *dev,
                         struct protection_domain *domain)
 {
        struct pci_dev *pdev = to_pci_dev(dev);
+       struct iommu_dev_data *dev_data;
        unsigned long flags;
        int ret;
 
-       if (amd_iommu_iotlb_sup)
-               pci_enable_ats(pdev, PAGE_SHIFT);
+       dev_data = get_dev_data(dev);
+
+       if (amd_iommu_iotlb_sup && pci_enable_ats(pdev, PAGE_SHIFT) == 0) {
+               dev_data->ats.enabled = true;
+               dev_data->ats.qdep    = pci_ats_queue_depth(pdev);
+       }
 
        write_lock_irqsave(&amd_iommu_devtable_lock, flags);
        ret = __attach_device(dev, domain);
@@ -1701,7 +1733,7 @@ static void __detach_device(struct device *dev)
  */
 static void detach_device(struct device *dev)
 {
-       struct pci_dev *pdev = to_pci_dev(dev);
+       struct iommu_dev_data *dev_data;
        unsigned long flags;
 
        /* lock device table */
@@ -1709,8 +1741,12 @@ static void detach_device(struct device *dev)
        __detach_device(dev);
        write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
 
-       if (amd_iommu_iotlb_sup && pci_ats_enabled(pdev))
-               pci_disable_ats(pdev);
+       dev_data = get_dev_data(dev);
+
+       if (dev_data->ats.enabled) {
+               pci_disable_ats(to_pci_dev(dev));
+               dev_data->ats.enabled = false;
+       }
 }
 
 /*
@@ -1722,9 +1758,7 @@ static struct protection_domain *domain_for_device(struct device *dev)
        struct protection_domain *dom;
        struct iommu_dev_data *dev_data, *alias_data;
        unsigned long flags;
-       u16 devid;
 
-       devid      = get_device_id(dev);
        dev_data   = get_dev_data(dev);
        alias_data = get_dev_data(dev_data->alias);
        if (!alias_data)
@@ -1798,7 +1832,6 @@ static int device_change_notifier(struct notifier_block *nb,
                goto out;
        }
 
-       device_flush_dte(dev);
        iommu_completion_wait(iommu);
 
 out:
@@ -1858,11 +1891,8 @@ static void update_device_table(struct protection_domain *domain)
 {
        struct iommu_dev_data *dev_data;
 
-       list_for_each_entry(dev_data, &domain->dev_list, list) {
-               struct pci_dev *pdev = to_pci_dev(dev_data->dev);
-               u16 devid = get_device_id(dev_data->dev);
-               set_dte_entry(devid, domain, pci_ats_enabled(pdev));
-       }
+       list_for_each_entry(dev_data, &domain->dev_list, list)
+               set_dte_entry(dev_data->devid, domain, dev_data->ats.enabled);
 }
 
 static void update_domain(struct protection_domain *domain)
@@ -2605,7 +2635,6 @@ static void amd_iommu_detach_device(struct iommu_domain *dom,
        if (!iommu)
                return;
 
-       device_flush_dte(dev);
        iommu_completion_wait(iommu);
 }
 
@@ -2616,16 +2645,13 @@ static int amd_iommu_attach_device(struct iommu_domain *dom,
        struct iommu_dev_data *dev_data;
        struct amd_iommu *iommu;
        int ret;
-       u16 devid;
 
        if (!check_device(dev))
                return -EINVAL;
 
        dev_data = dev->archdata.iommu;
 
-       devid = get_device_id(dev);
-
-       iommu = amd_iommu_rlookup_table[devid];
+       iommu = amd_iommu_rlookup_table[dev_data->devid];
        if (!iommu)
                return -EINVAL;