drivers/gpu/drm/etnaviv/etnaviv_mmu.c

   1 /*
   2  * Copyright (C) 2015 Etnaviv Project
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms of the GNU General Public License version 2 as published by
   6  * the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along with
  14  * this program.  If not, see <http://www.gnu.org/licenses/>.
  15  */
  16
  17 #include "etnaviv_drv.h"
  18 #include "etnaviv_gem.h"
  19 #include "etnaviv_gpu.h"
  20 #include "etnaviv_mmu.h"
  21
  22 static int etnaviv_fault_handler(struct iommu_domain *iommu, struct device *dev,
  23                 unsigned long iova, int flags, void *arg)
  24 {
  25         DBG("*** fault: iova=%08lx, flags=%d", iova, flags);
  26         return 0;
  27 }
  28
  29 int etnaviv_iommu_map(struct etnaviv_iommu *iommu, u32 iova,
  30                 struct sg_table *sgt, unsigned len, int prot)
  31 {
  32         struct iommu_domain *domain = iommu->domain;
  33         struct scatterlist *sg;
  34         unsigned int da = iova;
  35         unsigned int i, j;
  36         int ret;
  37
  38         if (!domain || !sgt)
  39                 return -EINVAL;
  40
  41         for_each_sg(sgt->sgl, sg, sgt->nents, i) {
  42                 u32 pa = sg_dma_address(sg) - sg->offset;
  43                 size_t bytes = sg_dma_len(sg) + sg->offset;
  44
  45                 VERB("map[%d]: %08x %08x(%zx)", i, iova, pa, bytes);
  46
  47                 ret = iommu_map(domain, da, pa, bytes, prot);
  48                 if (ret)
  49                         goto fail;
  50
  51                 da += bytes;
  52         }
  53
  54         return 0;
  55
  56 fail:
  57         da = iova;
  58
  59         for_each_sg(sgt->sgl, sg, i, j) {
  60                 size_t bytes = sg_dma_len(sg) + sg->offset;
  61
  62                 iommu_unmap(domain, da, bytes);
  63                 da += bytes;
  64         }
  65         return ret;
  66 }
  67
  68 int etnaviv_iommu_unmap(struct etnaviv_iommu *iommu, u32 iova,
  69                 struct sg_table *sgt, unsigned len)
  70 {
  71         struct iommu_domain *domain = iommu->domain;
  72         struct scatterlist *sg;
  73         unsigned int da = iova;
  74         int i;
  75
  76         for_each_sg(sgt->sgl, sg, sgt->nents, i) {
  77                 size_t bytes = sg_dma_len(sg) + sg->offset;
  78                 size_t unmapped;
  79
  80                 unmapped = iommu_unmap(domain, da, bytes);
  81                 if (unmapped < bytes)
  82                         return unmapped;
  83
  84                 VERB("unmap[%d]: %08x(%zx)", i, iova, bytes);
  85
  86                 BUG_ON(!PAGE_ALIGNED(bytes));
  87
  88                 da += bytes;
  89         }
  90
  91         return 0;
  92 }
  93
  94 static void etnaviv_iommu_remove_mapping(struct etnaviv_iommu *mmu,
  95         struct etnaviv_vram_mapping *mapping)
  96 {
  97         struct etnaviv_gem_object *etnaviv_obj = mapping->object;
  98
  99         etnaviv_iommu_unmap(mmu, mapping->vram_node.start,
 100                             etnaviv_obj->sgt, etnaviv_obj->base.size);
 101         drm_mm_remove_node(&mapping->vram_node);
 102 }
 103
 104 int etnaviv_iommu_map_gem(struct etnaviv_iommu *mmu,
 105         struct etnaviv_gem_object *etnaviv_obj, u32 memory_base,
 106         struct etnaviv_vram_mapping *mapping)
 107 {
 108         struct etnaviv_vram_mapping *free = NULL;
 109         struct sg_table *sgt = etnaviv_obj->sgt;
 110         struct drm_mm_node *node;
 111         int ret;
 112
 113         lockdep_assert_held(&etnaviv_obj->lock);
 114
 115         mutex_lock(&mmu->lock);
 116
 117         /* v1 MMU can optimize single entry (contiguous) scatterlists */
 118         if (sgt->nents == 1 && !(etnaviv_obj->flags & ETNA_BO_FORCE_MMU)) {
 119                 u32 iova;
 120
 121                 iova = sg_dma_address(sgt->sgl) - memory_base;
 122                 if (iova < 0x80000000 - sg_dma_len(sgt->sgl)) {
 123                         mapping->iova = iova;
 124                         list_add_tail(&mapping->mmu_node, &mmu->mappings);
 125                         mutex_unlock(&mmu->lock);
 126                         return 0;
 127                 }
 128         }
 129
 130         node = &mapping->vram_node;
 131         while (1) {
 132                 struct etnaviv_vram_mapping *m, *n;
 133                 struct list_head list;
 134                 bool found;
 135
 136                 ret = drm_mm_insert_node_in_range(&mmu->mm, node,
 137                         etnaviv_obj->base.size, 0, mmu->last_iova, ~0UL,
 138                         DRM_MM_SEARCH_DEFAULT);
 139
 140                 if (ret != -ENOSPC)
 141                         break;
 142
 143                 /*
 144                  * If we did not search from the start of the MMU region,
 145                  * try again in case there are free slots.
 146                  */
 147                 if (mmu->last_iova) {
 148                         mmu->last_iova = 0;
 149                         mmu->need_flush = true;
 150                         continue;
 151                 }
 152
 153                 /* Try to retire some entries */
 154                 drm_mm_init_scan(&mmu->mm, etnaviv_obj->base.size, 0, 0);
 155
 156                 found = 0;
 157                 INIT_LIST_HEAD(&list);
 158                 list_for_each_entry(free, &mmu->mappings, mmu_node) {
 159                         /* If this vram node has not been used, skip this. */
 160                         if (!free->vram_node.mm)
 161                                 continue;
 162
 163                         /*
 164                          * If the iova is pinned, then it's in-use,
 165                          * so we must keep its mapping.
 166                          */
 167                         if (free->use)
 168                                 continue;
 169
 170                         list_add(&free->scan_node, &list);
 171                         if (drm_mm_scan_add_block(&free->vram_node)) {
 172                                 found = true;
 173                                 break;
 174                         }
 175                 }
 176
 177                 if (!found) {
 178                         /* Nothing found, clean up and fail */
 179                         list_for_each_entry_safe(m, n, &list, scan_node)
 180                                 BUG_ON(drm_mm_scan_remove_block(&m->vram_node));
 181                         break;
 182                 }
 183
 184                 /*
 185                  * drm_mm does not allow any other operations while
 186                  * scanning, so we have to remove all blocks first.
 187                  * If drm_mm_scan_remove_block() returns false, we
 188                  * can leave the block pinned.
 189                  */
 190                 list_for_each_entry_safe(m, n, &list, scan_node)
 191                         if (!drm_mm_scan_remove_block(&m->vram_node))
 192                                 list_del_init(&m->scan_node);
 193
 194                 /*
 195                  * Unmap the blocks which need to be reaped from the MMU.
 196                  * Clear the mmu pointer to prevent the mapping_get finding
 197                  * this mapping.
 198                  */
 199                 list_for_each_entry_safe(m, n, &list, scan_node) {
 200                         etnaviv_iommu_remove_mapping(mmu, m);
 201                         m->mmu = NULL;
 202                         list_del_init(&m->mmu_node);
 203                         list_del_init(&m->scan_node);
 204                 }
 205
 206                 /*
 207                  * We removed enough mappings so that the new allocation will
 208                  * succeed.  Ensure that the MMU will be flushed before the
 209                  * associated commit requesting this mapping, and retry the
 210                  * allocation one more time.
 211                  */
 212                 mmu->need_flush = true;
 213         }
 214
 215         if (ret < 0) {
 216                 mutex_unlock(&mmu->lock);
 217                 return ret;
 218         }
 219
 220         mmu->last_iova = node->start + etnaviv_obj->base.size;
 221         mapping->iova = node->start;
 222         ret = etnaviv_iommu_map(mmu, node->start, sgt, etnaviv_obj->base.size,
 223                                 IOMMU_READ | IOMMU_WRITE);
 224
 225         if (ret < 0) {
 226                 drm_mm_remove_node(node);
 227                 mutex_unlock(&mmu->lock);
 228                 return ret;
 229         }
 230
 231         list_add_tail(&mapping->mmu_node, &mmu->mappings);
 232         mutex_unlock(&mmu->lock);
 233
 234         return ret;
 235 }
 236
 237 void etnaviv_iommu_unmap_gem(struct etnaviv_iommu *mmu,
 238         struct etnaviv_vram_mapping *mapping)
 239 {
 240         WARN_ON(mapping->use);
 241
 242         mutex_lock(&mmu->lock);
 243
 244         /* If the vram node is on the mm, unmap and remove the node */
 245         if (mapping->vram_node.mm == &mmu->mm)
 246                 etnaviv_iommu_remove_mapping(mmu, mapping);
 247
 248         list_del(&mapping->mmu_node);
 249         mutex_unlock(&mmu->lock);
 250 }
 251
 252 void etnaviv_iommu_destroy(struct etnaviv_iommu *mmu)
 253 {
 254         drm_mm_takedown(&mmu->mm);
 255         iommu_domain_free(mmu->domain);
 256         kfree(mmu);
 257 }
 258
 259 struct etnaviv_iommu *etnaviv_iommu_new(struct etnaviv_gpu *gpu,
 260         struct iommu_domain *domain, enum etnaviv_iommu_version version)
 261 {
 262         struct etnaviv_iommu *mmu;
 263
 264         mmu = kzalloc(sizeof(*mmu), GFP_KERNEL);
 265         if (!mmu)
 266                 return ERR_PTR(-ENOMEM);
 267
 268         mmu->domain = domain;
 269         mmu->gpu = gpu;
 270         mmu->version = version;
 271         mutex_init(&mmu->lock);
 272         INIT_LIST_HEAD(&mmu->mappings);
 273
 274         drm_mm_init(&mmu->mm, domain->geometry.aperture_start,
 275                     domain->geometry.aperture_end -
 276                       domain->geometry.aperture_start + 1);
 277
 278         iommu_set_fault_handler(domain, etnaviv_fault_handler, gpu->dev);
 279
 280         return mmu;
 281 }
 282
 283 size_t etnaviv_iommu_dump_size(struct etnaviv_iommu *iommu)
 284 {
 285         struct etnaviv_iommu_ops *ops;
 286
 287         ops = container_of(iommu->domain->ops, struct etnaviv_iommu_ops, ops);
 288
 289         return ops->dump_size(iommu->domain);
 290 }
 291
 292 void etnaviv_iommu_dump(struct etnaviv_iommu *iommu, void *buf)
 293 {
 294         struct etnaviv_iommu_ops *ops;
 295
 296         ops = container_of(iommu->domain->ops, struct etnaviv_iommu_ops, ops);
 297
 298         ops->dump(iommu->domain, buf);
 299 }