}
#define dma_max_pfn(dev) dma_max_pfn(dev)
- static inline int set_arch_dma_coherent_ops(struct device *dev)
- {
- dev->archdata.dma_coherent = true;
- set_dma_ops(dev, &arm_coherent_dma_ops);
- return 0;
- }
- #define set_arch_dma_coherent_ops(dev) set_arch_dma_coherent_ops(dev)
+ #define arch_setup_dma_ops arch_setup_dma_ops
+ extern void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
+ struct iommu_ops *iommu, bool coherent);
+
+ #define arch_teardown_dma_ops arch_teardown_dma_ops
+ extern void arch_teardown_dma_ops(struct device *dev);
+/* do not use this function in a driver */
+static inline bool is_device_dma_coherent(struct device *dev)
+{
+ return dev->archdata.dma_coherent;
+}
+
static inline dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)
{
unsigned int offset = paddr & ~PAGE_MASK;
#include <linux/bootmem.h>
#include <linux/seq_file.h>
#include <linux/screen_info.h>
+ #include <linux/of_iommu.h>
#include <linux/of_platform.h>
#include <linux/init.h>
#include <linux/kexec.h>
* machine from the device tree, if no callback is provided,
* otherwise we would always need an init_machine callback.
*/
+ of_iommu_init();
if (machine_desc->init_machine)
machine_desc->init_machine();
#ifdef CONFIG_OF
mdesc = setup_machine_tags(__atags_pointer, __machine_arch_type);
machine_desc = mdesc;
machine_name = mdesc->name;
+ dump_stack_set_arch_desc("%s", mdesc->name);
if (mdesc->reboot_mode != REBOOT_HARD)
reboot_mode = mdesc->reboot_mode;
* arm_iommu_create_mapping)
*
* Attaches specified io address space mapping to the provided device,
- * this replaces the dma operations (dma_map_ops pointer) with the
- * IOMMU aware version. More than one client might be attached to
- * the same io address space mapping.
+ * More than one client might be attached to the same io address space
+ * mapping.
*/
int arm_iommu_attach_device(struct device *dev,
struct dma_iommu_mapping *mapping)
kref_get(&mapping->kref);
dev->archdata.mapping = mapping;
- set_dma_ops(dev, &iommu_ops);
pr_debug("Attached IOMMU controller to %s device.\n", dev_name(dev));
return 0;
* @dev: valid struct device pointer
*
* Detaches the provided device from a previously attached map.
- * This voids the dma operations (dma_map_ops pointer)
*/
void arm_iommu_detach_device(struct device *dev)
{
iommu_detach_device(mapping->domain, dev);
kref_put(&mapping->kref, release_iommu_mapping);
dev->archdata.mapping = NULL;
- set_dma_ops(dev, NULL);
pr_debug("Detached IOMMU controller from %s device.\n", dev_name(dev));
}
EXPORT_SYMBOL_GPL(arm_iommu_detach_device);
- #endif
+ static struct dma_map_ops *arm_get_iommu_dma_map_ops(bool coherent)
+ {
+ return coherent ? &iommu_coherent_ops : &iommu_ops;
+ }
+
+ static bool arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size,
+ struct iommu_ops *iommu)
+ {
+ struct dma_iommu_mapping *mapping;
+
+ if (!iommu)
+ return false;
+
+ mapping = arm_iommu_create_mapping(dev->bus, dma_base, size);
+ if (IS_ERR(mapping)) {
+ pr_warn("Failed to create %llu-byte IOMMU mapping for device %s\n",
+ size, dev_name(dev));
+ return false;
+ }
+
+ if (arm_iommu_attach_device(dev, mapping)) {
+ pr_warn("Failed to attached device %s to IOMMU_mapping\n",
+ dev_name(dev));
+ arm_iommu_release_mapping(mapping);
+ return false;
+ }
+
+ return true;
+ }
+
+ static void arm_teardown_iommu_dma_ops(struct device *dev)
+ {
+ struct dma_iommu_mapping *mapping = dev->archdata.mapping;
+
+ arm_iommu_detach_device(dev);
+ arm_iommu_release_mapping(mapping);
+ }
+
+ #else
+
+ static bool arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size,
+ struct iommu_ops *iommu)
+ {
+ return false;
+ }
+
+ static void arm_teardown_iommu_dma_ops(struct device *dev) { }
+
+ #define arm_get_iommu_dma_map_ops arm_get_dma_map_ops
+
+ #endif /* CONFIG_ARM_DMA_USE_IOMMU */
+
+ static struct dma_map_ops *arm_get_dma_map_ops(bool coherent)
+ {
+ return coherent ? &arm_coherent_dma_ops : &arm_dma_ops;
+ }
+
+ void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
+ struct iommu_ops *iommu, bool coherent)
+ {
+ struct dma_map_ops *dma_ops;
+
++ dev->archdata.dma_coherent = coherent;
+ if (arm_setup_iommu_dma_ops(dev, dma_base, size, iommu))
+ dma_ops = arm_get_iommu_dma_map_ops(coherent);
+ else
+ dma_ops = arm_get_dma_map_ops(coherent);
+
+ set_dma_ops(dev, dma_ops);
+ }
+
+ void arch_teardown_dma_ops(struct device *dev)
+ {
+ arm_teardown_iommu_dma_ops(dev);
+ }
config OF_IOMMU
def_bool y
- depends on OF
+ depends on OF && IOMMU_API
config FSL_PAMU
bool "Freescale IOMMU support"
select IOMMU_API
config OMAP_IOMMU_DEBUG
- tristate "Export OMAP IOMMU internals in DebugFS"
- depends on OMAP_IOMMU && DEBUG_FS
- help
- Select this to see extensive information about
- the internal state of OMAP IOMMU in debugfs.
+ bool "Export OMAP IOMMU internals in DebugFS"
+ depends on OMAP_IOMMU && DEBUG_FS
+ ---help---
+ Select this to see extensive information about
+ the internal state of OMAP IOMMU in debugfs.
+
+ Say N unless you know you need this.
- Say N unless you know you need this.
+config ROCKCHIP_IOMMU
+ bool "Rockchip IOMMU Support"
+ depends on ARM
+ depends on ARCH_ROCKCHIP || COMPILE_TEST
+ select IOMMU_API
+ select ARM_DMA_USE_IOMMU
+ help
+ Support for IOMMUs found on Rockchip rk32xx SOCs.
+ These IOMMUs allow virtualization of the address space used by most
+ cores within the multimedia subsystem.
+ Say Y here if you are using a Rockchip SoC that includes an IOMMU
+ device.
config TEGRA_IOMMU_GART
bool "Tegra GART IOMMU Support"
hardware included on Tegra SoCs.
config TEGRA_IOMMU_SMMU
- bool "Tegra SMMU IOMMU Support"
- depends on ARCH_TEGRA && TEGRA_AHB
+ bool "NVIDIA Tegra SMMU Support"
+ depends on ARCH_TEGRA
+ depends on TEGRA_AHB
+ depends on TEGRA_MC
select IOMMU_API
help
- Enables support for remapping discontiguous physical memory
- shared with the operating system into contiguous I/O virtual
- space through the SMMU (System Memory Management Unit)
- hardware included on Tegra SoCs.
+ This driver supports the IOMMU hardware (SMMU) found on NVIDIA Tegra
+ SoCs (Tegra30 up to Tegra124).
config EXYNOS_IOMMU
bool "Exynos IOMMU Support"
- depends on ARCH_EXYNOS
+ depends on ARCH_EXYNOS && ARM
select IOMMU_API
select ARM_DMA_USE_IOMMU
help
const struct iommu_ops *ops = cb->ops;
if (!ops->add_device)
- return -ENODEV;
+ return 0;
WARN_ON(dev->iommu_group);
kfree(nb);
return err;
}
- return bus_for_each_dev(bus, NULL, &cb, add_iommu_group);
+
+ err = bus_for_each_dev(bus, NULL, &cb, add_iommu_group);
+ if (err) {
+ bus_unregister_notifier(bus, nb);
+ kfree(nb);
+ return err;
+ }
+
+ return 0;
}
/**
*/
int bus_set_iommu(struct bus_type *bus, const struct iommu_ops *ops)
{
+ int err;
+
if (bus->iommu_ops != NULL)
return -EBUSY;
bus->iommu_ops = ops;
/* Do IOMMU specific setup for this bus-type */
- return iommu_bus_init(bus, ops);
+ err = iommu_bus_init(bus, ops);
+ if (err)
+ bus->iommu_ops = NULL;
+
+ return err;
}
EXPORT_SYMBOL_GPL(bus_set_iommu);
}
EXPORT_SYMBOL_GPL(iommu_unmap);
+size_t default_iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
+ struct scatterlist *sg, unsigned int nents, int prot)
+{
+ struct scatterlist *s;
+ size_t mapped = 0;
+ unsigned int i, min_pagesz;
+ int ret;
+
+ if (unlikely(domain->ops->pgsize_bitmap == 0UL))
+ return 0;
+
+ min_pagesz = 1 << __ffs(domain->ops->pgsize_bitmap);
+
+ for_each_sg(sg, s, nents, i) {
+ phys_addr_t phys = page_to_phys(sg_page(s)) + s->offset;
+
+ /*
+ * We are mapping on IOMMU page boundaries, so offset within
+ * the page must be 0. However, the IOMMU may support pages
+ * smaller than PAGE_SIZE, so s->offset may still represent
+ * an offset of that boundary within the CPU page.
+ */
+ if (!IS_ALIGNED(s->offset, min_pagesz))
+ goto out_err;
+
+ ret = iommu_map(domain, iova + mapped, phys, s->length, prot);
+ if (ret)
+ goto out_err;
+
+ mapped += s->length;
+ }
+
+ return mapped;
+
+out_err:
+ /* undo mappings already done */
+ iommu_unmap(domain, iova, mapped);
+
+ return 0;
+
+}
+EXPORT_SYMBOL_GPL(default_iommu_map_sg);
int iommu_domain_window_enable(struct iommu_domain *domain, u32 wnd_nr,
phys_addr_t paddr, u64 size, int prot)
#include <linux/slab.h>
#include <linux/of_address.h>
#include <linux/of_device.h>
+ #include <linux/of_iommu.h>
#include <linux/of_irq.h>
#include <linux/of_platform.h>
#include <linux/platform_device.h>
}
dev->dev.of_node = of_node_get(np);
- dev->dev.parent = parent;
+ dev->dev.parent = parent ? : &platform_bus;
if (bus_id)
dev_set_name(&dev->dev, "%s", bus_id);
{
u64 dma_addr, paddr, size;
int ret;
+ bool coherent;
+ unsigned long offset;
+ struct iommu_ops *iommu;
/*
* Set default dma-mask to 32 bit. Drivers are expected to setup
if (!dev->dma_mask)
dev->dma_mask = &dev->coherent_dma_mask;
- /*
- * if dma-coherent property exist, call arch hook to setup
- * dma coherent operations.
- */
- if (of_dma_is_coherent(dev->of_node)) {
- set_arch_dma_coherent_ops(dev);
- dev_dbg(dev, "device is dma coherent\n");
- }
-
- /*
- * if dma-ranges property doesn't exist - just return else
- * setup the dma offset
- */
ret = of_dma_get_range(dev->of_node, &dma_addr, &paddr, &size);
if (ret < 0) {
- dev_dbg(dev, "no dma range information to setup\n");
- return;
+ dma_addr = offset = 0;
+ size = dev->coherent_dma_mask;
+ } else {
+ offset = PFN_DOWN(paddr - dma_addr);
+ dev_dbg(dev, "dma_pfn_offset(%#08lx)\n", dev->dma_pfn_offset);
}
+ dev->dma_pfn_offset = offset;
+
+ coherent = of_dma_is_coherent(dev->of_node);
+ dev_dbg(dev, "device is%sdma coherent\n",
+ coherent ? " " : " not ");
+
+ iommu = of_iommu_configure(dev);
+ dev_dbg(dev, "device is%sbehind an iommu\n",
+ iommu ? " " : " not ");
- /* DMA ranges found. Calculate and set dma_pfn_offset */
- dev->dma_pfn_offset = PFN_DOWN(paddr - dma_addr);
- dev_dbg(dev, "dma_pfn_offset(%#08lx)\n", dev->dma_pfn_offset);
+ arch_setup_dma_ops(dev, dma_addr, size, iommu, coherent);
+ }
+
+ static void of_dma_deconfigure(struct device *dev)
+ {
+ arch_teardown_dma_ops(dev);
}
/**
if (!dev)
goto err_clear_flag;
- of_dma_configure(&dev->dev);
dev->dev.bus = &platform_bus_type;
dev->dev.platform_data = platform_data;
-
- /* We do not fill the DMA ops for platform devices by default.
- * This is currently the responsibility of the platform code
- * to do such, possibly using a device notifier
- */
+ of_dma_configure(&dev->dev);
if (of_device_add(dev) != 0) {
+ of_dma_deconfigure(&dev->dev);
platform_device_put(dev);
goto err_clear_flag;
}
/* setup generic device info */
dev->dev.of_node = of_node_get(node);
- dev->dev.parent = parent;
+ dev->dev.parent = parent ? : &platform_bus;
dev->dev.platform_data = platform_data;
if (bus_id)
dev_set_name(&dev->dev, "%s", bus_id);
if (rc)
break;
}
+ of_node_set_flag(root, OF_POPULATED_BUS);
of_node_put(root);
return rc;
*/
void of_platform_depopulate(struct device *parent)
{
- device_for_each_child(parent, NULL, of_platform_device_destroy);
+ if (parent->of_node && of_node_check_flag(parent->of_node, OF_POPULATED_BUS)) {
+ device_for_each_child(parent, NULL, of_platform_device_destroy);
+ of_node_clear_flag(parent->of_node, OF_POPULATED_BUS);
+ }
}
EXPORT_SYMBOL_GPL(of_platform_depopulate);
+#ifdef CONFIG_OF_DYNAMIC
+static int of_platform_notify(struct notifier_block *nb,
+ unsigned long action, void *arg)
+{
+ struct of_reconfig_data *rd = arg;
+ struct platform_device *pdev_parent, *pdev;
+ bool children_left;
+
+ switch (of_reconfig_get_state_change(action, rd)) {
+ case OF_RECONFIG_CHANGE_ADD:
+ /* verify that the parent is a bus */
+ if (!of_node_check_flag(rd->dn->parent, OF_POPULATED_BUS))
+ return NOTIFY_OK; /* not for us */
+
+ /* pdev_parent may be NULL when no bus platform device */
+ pdev_parent = of_find_device_by_node(rd->dn->parent);
+ pdev = of_platform_device_create(rd->dn, NULL,
+ pdev_parent ? &pdev_parent->dev : NULL);
+ of_dev_put(pdev_parent);
+
+ if (pdev == NULL) {
+ pr_err("%s: failed to create for '%s'\n",
+ __func__, rd->dn->full_name);
+ /* of_platform_device_create tosses the error code */
+ return notifier_from_errno(-EINVAL);
+ }
+ break;
+
+ case OF_RECONFIG_CHANGE_REMOVE:
+ /* find our device by node */
+ pdev = of_find_device_by_node(rd->dn);
+ if (pdev == NULL)
+ return NOTIFY_OK; /* no? not meant for us */
+
+ /* unregister takes one ref away */
+ of_platform_device_destroy(&pdev->dev, &children_left);
+
+ /* and put the reference of the find */
+ of_dev_put(pdev);
+ break;
+ }
+
+ return NOTIFY_OK;
+}
+
+static struct notifier_block platform_of_notifier = {
+ .notifier_call = of_platform_notify,
+};
+
+void of_platform_register_reconfig_notifier(void)
+{
+ WARN_ON(of_reconfig_notifier_register(&platform_of_notifier));
+}
+#endif /* CONFIG_OF_DYNAMIC */
+
#endif /* CONFIG_OF_ADDRESS */
#include <linux/errno.h>
#include <linux/err.h>
+ #include <linux/of.h>
#include <linux/types.h>
+#include <linux/scatterlist.h>
#include <trace/events/iommu.h>
#define IOMMU_READ (1 << 0)
#define IOMMU_WRITE (1 << 1)
#define IOMMU_CACHE (1 << 2) /* DMA cache coherency */
-#define IOMMU_EXEC (1 << 3)
+#define IOMMU_NOEXEC (1 << 3)
struct iommu_ops;
struct iommu_group;
IOMMU_CAP_CACHE_COHERENCY, /* IOMMU can enforce cache coherent DMA
transactions */
IOMMU_CAP_INTR_REMAP, /* IOMMU supports interrupt isolation */
+ IOMMU_CAP_NOEXEC, /* IOMMU_NOEXEC flag */
};
/*
* @detach_dev: detach device from an iommu domain
* @map: map a physically contiguous memory region to an iommu domain
* @unmap: unmap a physically contiguous memory region from an iommu domain
+ * @map_sg: map a scatter-gather list of physically contiguous memory chunks
+ * to an iommu domain
* @iova_to_phys: translate iova to physical address
* @add_device: add device to iommu grouping
* @remove_device: remove device from iommu grouping
* @domain_get_attr: Query domain attributes
* @domain_set_attr: Change domain attributes
+ * @of_xlate: add OF master IDs to iommu grouping
* @pgsize_bitmap: bitmap of supported page sizes
+ * @priv: per-instance data private to the iommu driver
*/
struct iommu_ops {
bool (*capable)(enum iommu_cap);
phys_addr_t paddr, size_t size, int prot);
size_t (*unmap)(struct iommu_domain *domain, unsigned long iova,
size_t size);
+ size_t (*map_sg)(struct iommu_domain *domain, unsigned long iova,
+ struct scatterlist *sg, unsigned int nents, int prot);
phys_addr_t (*iova_to_phys)(struct iommu_domain *domain, dma_addr_t iova);
int (*add_device)(struct device *dev);
void (*remove_device)(struct device *dev);
/* Get the numer of window per domain */
u32 (*domain_get_windows)(struct iommu_domain *domain);
+ #ifdef CONFIG_OF_IOMMU
+ int (*of_xlate)(struct device *dev, struct of_phandle_args *args);
+ #endif
+
unsigned long pgsize_bitmap;
+ void *priv;
};
#define IOMMU_GROUP_NOTIFY_ADD_DEVICE 1 /* Device added */
phys_addr_t paddr, size_t size, int prot);
extern size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova,
size_t size);
+extern size_t default_iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
+ struct scatterlist *sg,unsigned int nents,
+ int prot);
extern phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova);
extern void iommu_set_fault_handler(struct iommu_domain *domain,
iommu_fault_handler_t handler, void *token);
return ret;
}
+static inline size_t iommu_map_sg(struct iommu_domain *domain,
+ unsigned long iova, struct scatterlist *sg,
+ unsigned int nents, int prot)
+{
+ return domain->ops->map_sg(domain, iova, sg, nents, prot);
+}
+
#else /* CONFIG_IOMMU_API */
struct iommu_ops {};
return -ENODEV;
}
+static inline size_t iommu_map_sg(struct iommu_domain *domain,
+ unsigned long iova, struct scatterlist *sg,
+ unsigned int nents, int prot)
+{
+ return -ENODEV;
+}
+
static inline int iommu_domain_window_enable(struct iommu_domain *domain,
u32 wnd_nr, phys_addr_t paddr,
u64 size, int prot)
projects / cascardo / linux.git / commitdiff