drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors: Dave Airlie
  25  *          Alex Deucher
  26  *          Jerome Glisse
  27  */
  28 #include <drm/drmP.h>
  29 #include <drm/amdgpu_drm.h>
  30 #include "amdgpu.h"
  31 #include "amdgpu_trace.h"
  32
  33 /*
  34  * GPUVM
  35  * GPUVM is similar to the legacy gart on older asics, however
  36  * rather than there being a single global gart table
  37  * for the entire GPU, there are multiple VM page tables active
  38  * at any given time.  The VM page tables can contain a mix
  39  * vram pages and system memory pages and system memory pages
  40  * can be mapped as snooped (cached system pages) or unsnooped
  41  * (uncached system pages).
  42  * Each VM has an ID associated with it and there is a page table
  43  * associated with each VMID.  When execting a command buffer,
  44  * the kernel tells the the ring what VMID to use for that command
  45  * buffer.  VMIDs are allocated dynamically as commands are submitted.
  46  * The userspace drivers maintain their own address space and the kernel
  47  * sets up their pages tables accordingly when they submit their
  48  * command buffers and a VMID is assigned.
  49  * Cayman/Trinity support up to 8 active VMs at any given time;
  50  * SI supports 16.
  51  */
  52
  53 /* Special value that no flush is necessary */
  54 #define AMDGPU_VM_NO_FLUSH (~0ll)
  55
  56 /* Local structure. Encapsulate some VM table update parameters to reduce
  57  * the number of function parameters
  58  */
  59 struct amdgpu_vm_update_params {
  60         /* address where to copy page table entries from */
  61         uint64_t src;
  62         /* DMA addresses to use for mapping */
  63         dma_addr_t *pages_addr;
  64         /* indirect buffer to fill with commands */
  65         struct amdgpu_ib *ib;
  66 };
  67
  68 /**
  69  * amdgpu_vm_num_pde - return the number of page directory entries
  70  *
  71  * @adev: amdgpu_device pointer
  72  *
  73  * Calculate the number of page directory entries.
  74  */
  75 static unsigned amdgpu_vm_num_pdes(struct amdgpu_device *adev)
  76 {
  77         return adev->vm_manager.max_pfn >> amdgpu_vm_block_size;
  78 }
  79
  80 /**
  81  * amdgpu_vm_directory_size - returns the size of the page directory in bytes
  82  *
  83  * @adev: amdgpu_device pointer
  84  *
  85  * Calculate the size of the page directory in bytes.
  86  */
  87 static unsigned amdgpu_vm_directory_size(struct amdgpu_device *adev)
  88 {
  89         return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_num_pdes(adev) * 8);
  90 }
  91
  92 /**
  93  * amdgpu_vm_get_pd_bo - add the VM PD to a validation list
  94  *
  95  * @vm: vm providing the BOs
  96  * @validated: head of validation list
  97  * @entry: entry to add
  98  *
  99  * Add the page directory to the list of BOs to
 100  * validate for command submission.
 101  */
 102 void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
 103                          struct list_head *validated,
 104                          struct amdgpu_bo_list_entry *entry)
 105 {
 106         entry->robj = vm->page_directory;
 107         entry->priority = 0;
 108         entry->tv.bo = &vm->page_directory->tbo;
 109         entry->tv.shared = true;
 110         entry->user_pages = NULL;
 111         list_add(&entry->tv.head, validated);
 112 }
 113
 114 /**
 115  * amdgpu_vm_get_bos - add the vm BOs to a duplicates list
 116  *
 117  * @vm: vm providing the BOs
 118  * @duplicates: head of duplicates list
 119  *
 120  * Add the page directory to the BO duplicates list
 121  * for command submission.
 122  */
 123 void amdgpu_vm_get_pt_bos(struct amdgpu_vm *vm, struct list_head *duplicates)
 124 {
 125         unsigned i;
 126
 127         /* add the vm page table to the list */
 128         for (i = 0; i <= vm->max_pde_used; ++i) {
 129                 struct amdgpu_bo_list_entry *entry = &vm->page_tables[i].entry;
 130
 131                 if (!entry->robj)
 132                         continue;
 133
 134                 list_add(&entry->tv.head, duplicates);
 135         }
 136
 137 }
 138
 139 /**
 140  * amdgpu_vm_move_pt_bos_in_lru - move the PT BOs to the LRU tail
 141  *
 142  * @adev: amdgpu device instance
 143  * @vm: vm providing the BOs
 144  *
 145  * Move the PT BOs to the tail of the LRU.
 146  */
 147 void amdgpu_vm_move_pt_bos_in_lru(struct amdgpu_device *adev,
 148                                   struct amdgpu_vm *vm)
 149 {
 150         struct ttm_bo_global *glob = adev->mman.bdev.glob;
 151         unsigned i;
 152
 153         spin_lock(&glob->lru_lock);
 154         for (i = 0; i <= vm->max_pde_used; ++i) {
 155                 struct amdgpu_bo_list_entry *entry = &vm->page_tables[i].entry;
 156
 157                 if (!entry->robj)
 158                         continue;
 159
 160                 ttm_bo_move_to_lru_tail(&entry->robj->tbo);
 161         }
 162         spin_unlock(&glob->lru_lock);
 163 }
 164
 165 /**
 166  * amdgpu_vm_grab_id - allocate the next free VMID
 167  *
 168  * @vm: vm to allocate id for
 169  * @ring: ring we want to submit job to
 170  * @sync: sync object where we add dependencies
 171  * @fence: fence protecting ID from reuse
 172  *
 173  * Allocate an id for the vm, adding fences to the sync obj as necessary.
 174  */
 175 int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
 176                       struct amdgpu_sync *sync, struct fence *fence,
 177                       unsigned *vm_id, uint64_t *vm_pd_addr)
 178 {
 179         uint64_t pd_addr = amdgpu_bo_gpu_offset(vm->page_directory);
 180         struct amdgpu_device *adev = ring->adev;
 181         struct fence *updates = sync->last_vm_update;
 182         struct amdgpu_vm_id *id;
 183         unsigned i = ring->idx;
 184         int r;
 185
 186         mutex_lock(&adev->vm_manager.lock);
 187
 188         /* Check if we can use a VMID already assigned to this VM */
 189         do {
 190                 struct fence *flushed;
 191
 192                 id = vm->ids[i++];
 193                 if (i == AMDGPU_MAX_RINGS)
 194                         i = 0;
 195
 196                 /* Check all the prerequisites to using this VMID */
 197                 if (!id)
 198                         continue;
 199
 200                 if (atomic64_read(&id->owner) != vm->client_id)
 201                         continue;
 202
 203                 if (pd_addr != id->pd_gpu_addr)
 204                         continue;
 205
 206                 if (id->last_user != ring &&
 207                     (!id->last_flush || !fence_is_signaled(id->last_flush)))
 208                         continue;
 209
 210                 flushed  = id->flushed_updates;
 211                 if (updates && (!flushed || fence_is_later(updates, flushed)))
 212                         continue;
 213
 214                 /* Good we can use this VMID */
 215                 if (id->last_user == ring) {
 216                         r = amdgpu_sync_fence(ring->adev, sync,
 217                                               id->first);
 218                         if (r)
 219                                 goto error;
 220                 }
 221
 222                 /* And remember this submission as user of the VMID */
 223                 r = amdgpu_sync_fence(ring->adev, &id->active, fence);
 224                 if (r)
 225                         goto error;
 226
 227                 list_move_tail(&id->list, &adev->vm_manager.ids_lru);
 228                 vm->ids[ring->idx] = id;
 229
 230                 *vm_id = id - adev->vm_manager.ids;
 231                 *vm_pd_addr = AMDGPU_VM_NO_FLUSH;
 232                 trace_amdgpu_vm_grab_id(vm, ring->idx, *vm_id, *vm_pd_addr);
 233
 234                 mutex_unlock(&adev->vm_manager.lock);
 235                 return 0;
 236
 237         } while (i != ring->idx);
 238
 239         id = list_first_entry(&adev->vm_manager.ids_lru,
 240                               struct amdgpu_vm_id,
 241                               list);
 242
 243         if (!amdgpu_sync_is_idle(&id->active)) {
 244                 struct list_head *head = &adev->vm_manager.ids_lru;
 245                 struct amdgpu_vm_id *tmp;
 246
 247                 list_for_each_entry_safe(id, tmp, &adev->vm_manager.ids_lru,
 248                                          list) {
 249                         if (amdgpu_sync_is_idle(&id->active)) {
 250                                 list_move(&id->list, head);
 251                                 head = &id->list;
 252                         }
 253                 }
 254                 id = list_first_entry(&adev->vm_manager.ids_lru,
 255                                       struct amdgpu_vm_id,
 256                                       list);
 257         }
 258
 259         r = amdgpu_sync_cycle_fences(sync, &id->active, fence);
 260         if (r)
 261                 goto error;
 262
 263         fence_put(id->first);
 264         id->first = fence_get(fence);
 265
 266         fence_put(id->last_flush);
 267         id->last_flush = NULL;
 268
 269         fence_put(id->flushed_updates);
 270         id->flushed_updates = fence_get(updates);
 271
 272         id->pd_gpu_addr = pd_addr;
 273
 274         list_move_tail(&id->list, &adev->vm_manager.ids_lru);
 275         id->last_user = ring;
 276         atomic64_set(&id->owner, vm->client_id);
 277         vm->ids[ring->idx] = id;
 278
 279         *vm_id = id - adev->vm_manager.ids;
 280         *vm_pd_addr = pd_addr;
 281         trace_amdgpu_vm_grab_id(vm, ring->idx, *vm_id, *vm_pd_addr);
 282
 283 error:
 284         mutex_unlock(&adev->vm_manager.lock);
 285         return r;
 286 }
 287
 288 /**
 289  * amdgpu_vm_flush - hardware flush the vm
 290  *
 291  * @ring: ring to use for flush
 292  * @vm_id: vmid number to use
 293  * @pd_addr: address of the page directory
 294  *
 295  * Emit a VM flush when it is necessary.
 296  */
 297 int amdgpu_vm_flush(struct amdgpu_ring *ring,
 298                     unsigned vm_id, uint64_t pd_addr,
 299                     uint32_t gds_base, uint32_t gds_size,
 300                     uint32_t gws_base, uint32_t gws_size,
 301                     uint32_t oa_base, uint32_t oa_size)
 302 {
 303         struct amdgpu_device *adev = ring->adev;
 304         struct amdgpu_vm_id *id = &adev->vm_manager.ids[vm_id];
 305         bool gds_switch_needed = ring->funcs->emit_gds_switch && (
 306                 id->gds_base != gds_base ||
 307                 id->gds_size != gds_size ||
 308                 id->gws_base != gws_base ||
 309                 id->gws_size != gws_size ||
 310                 id->oa_base != oa_base ||
 311                 id->oa_size != oa_size);
 312         int r;
 313
 314         if (ring->funcs->emit_pipeline_sync && (
 315             pd_addr != AMDGPU_VM_NO_FLUSH || gds_switch_needed ||
 316                     ring->type == AMDGPU_RING_TYPE_COMPUTE))
 317                 amdgpu_ring_emit_pipeline_sync(ring);
 318
 319         if (ring->funcs->emit_vm_flush &&
 320             pd_addr != AMDGPU_VM_NO_FLUSH) {
 321                 struct fence *fence;
 322
 323                 trace_amdgpu_vm_flush(pd_addr, ring->idx, vm_id);
 324                 amdgpu_ring_emit_vm_flush(ring, vm_id, pd_addr);
 325
 326                 mutex_lock(&adev->vm_manager.lock);
 327                 if ((id->pd_gpu_addr == pd_addr) && (id->last_user == ring)) {
 328                         r = amdgpu_fence_emit(ring, &fence);
 329                         if (r) {
 330                                 mutex_unlock(&adev->vm_manager.lock);
 331                                 return r;
 332                         }
 333                         fence_put(id->last_flush);
 334                         id->last_flush = fence;
 335                 }
 336                 mutex_unlock(&adev->vm_manager.lock);
 337         }
 338
 339         if (gds_switch_needed) {
 340                 id->gds_base = gds_base;
 341                 id->gds_size = gds_size;
 342                 id->gws_base = gws_base;
 343                 id->gws_size = gws_size;
 344                 id->oa_base = oa_base;
 345                 id->oa_size = oa_size;
 346                 amdgpu_ring_emit_gds_switch(ring, vm_id,
 347                                             gds_base, gds_size,
 348                                             gws_base, gws_size,
 349                                             oa_base, oa_size);
 350         }
 351
 352         return 0;
 353 }
 354
 355 /**
 356  * amdgpu_vm_reset_id - reset VMID to zero
 357  *
 358  * @adev: amdgpu device structure
 359  * @vm_id: vmid number to use
 360  *
 361  * Reset saved GDW, GWS and OA to force switch on next flush.
 362  */
 363 void amdgpu_vm_reset_id(struct amdgpu_device *adev, unsigned vm_id)
 364 {
 365         struct amdgpu_vm_id *id = &adev->vm_manager.ids[vm_id];
 366
 367         id->gds_base = 0;
 368         id->gds_size = 0;
 369         id->gws_base = 0;
 370         id->gws_size = 0;
 371         id->oa_base = 0;
 372         id->oa_size = 0;
 373 }
 374
 375 /**
 376  * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo
 377  *
 378  * @vm: requested vm
 379  * @bo: requested buffer object
 380  *
 381  * Find @bo inside the requested vm.
 382  * Search inside the @bos vm list for the requested vm
 383  * Returns the found bo_va or NULL if none is found
 384  *
 385  * Object has to be reserved!
 386  */
 387 struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,
 388                                        struct amdgpu_bo *bo)
 389 {
 390         struct amdgpu_bo_va *bo_va;
 391
 392         list_for_each_entry(bo_va, &bo->va, bo_list) {
 393                 if (bo_va->vm == vm) {
 394                         return bo_va;
 395                 }
 396         }
 397         return NULL;
 398 }
 399
 400 /**
 401  * amdgpu_vm_update_pages - helper to call the right asic function
 402  *
 403  * @adev: amdgpu_device pointer
 404  * @vm_update_params: see amdgpu_vm_update_params definition
 405  * @pe: addr of the page entry
 406  * @addr: dst addr to write into pe
 407  * @count: number of page entries to update
 408  * @incr: increase next addr by incr bytes
 409  * @flags: hw access flags
 410  *
 411  * Traces the parameters and calls the right asic functions
 412  * to setup the page table using the DMA.
 413  */
 414 static void amdgpu_vm_update_pages(struct amdgpu_device *adev,
 415                                    struct amdgpu_vm_update_params
 416                                         *vm_update_params,
 417                                    uint64_t pe, uint64_t addr,
 418                                    unsigned count, uint32_t incr,
 419                                    uint32_t flags)
 420 {
 421         trace_amdgpu_vm_set_page(pe, addr, count, incr, flags);
 422
 423         if (vm_update_params->src) {
 424                 amdgpu_vm_copy_pte(adev, vm_update_params->ib,
 425                         pe, (vm_update_params->src + (addr >> 12) * 8), count);
 426
 427         } else if (vm_update_params->pages_addr) {
 428                 amdgpu_vm_write_pte(adev, vm_update_params->ib,
 429                         vm_update_params->pages_addr,
 430                         pe, addr, count, incr, flags);
 431
 432         } else if (count < 3) {
 433                 amdgpu_vm_write_pte(adev, vm_update_params->ib, NULL, pe, addr,
 434                                     count, incr, flags);
 435
 436         } else {
 437                 amdgpu_vm_set_pte_pde(adev, vm_update_params->ib, pe, addr,
 438                                       count, incr, flags);
 439         }
 440 }
 441
 442 /**
 443  * amdgpu_vm_clear_bo - initially clear the page dir/table
 444  *
 445  * @adev: amdgpu_device pointer
 446  * @bo: bo to clear
 447  *
 448  * need to reserve bo first before calling it.
 449  */
 450 static int amdgpu_vm_clear_bo(struct amdgpu_device *adev,
 451                               struct amdgpu_vm *vm,
 452                               struct amdgpu_bo *bo)
 453 {
 454         struct amdgpu_ring *ring;
 455         struct fence *fence = NULL;
 456         struct amdgpu_job *job;
 457         struct amdgpu_vm_update_params vm_update_params;
 458         unsigned entries;
 459         uint64_t addr;
 460         int r;
 461
 462         memset(&vm_update_params, 0, sizeof(vm_update_params));
 463         ring = container_of(vm->entity.sched, struct amdgpu_ring, sched);
 464
 465         r = reservation_object_reserve_shared(bo->tbo.resv);
 466         if (r)
 467                 return r;
 468
 469         r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);
 470         if (r)
 471                 goto error;
 472
 473         addr = amdgpu_bo_gpu_offset(bo);
 474         entries = amdgpu_bo_size(bo) / 8;
 475
 476         r = amdgpu_job_alloc_with_ib(adev, 64, &job);
 477         if (r)
 478                 goto error;
 479
 480         vm_update_params.ib = &job->ibs[0];
 481         amdgpu_vm_update_pages(adev, &vm_update_params, addr, 0, entries,
 482                                0, 0);
 483         amdgpu_ring_pad_ib(ring, &job->ibs[0]);
 484
 485         WARN_ON(job->ibs[0].length_dw > 64);
 486         r = amdgpu_job_submit(job, ring, &vm->entity,
 487                               AMDGPU_FENCE_OWNER_VM, &fence);
 488         if (r)
 489                 goto error_free;
 490
 491         amdgpu_bo_fence(bo, fence, true);
 492         fence_put(fence);
 493         return 0;
 494
 495 error_free:
 496         amdgpu_job_free(job);
 497
 498 error:
 499         return r;
 500 }
 501
 502 /**
 503  * amdgpu_vm_map_gart - Resolve gart mapping of addr
 504  *
 505  * @pages_addr: optional DMA address to use for lookup
 506  * @addr: the unmapped addr
 507  *
 508  * Look up the physical address of the page that the pte resolves
 509  * to and return the pointer for the page table entry.
 510  */
 511 uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr)
 512 {
 513         uint64_t result;
 514
 515         if (pages_addr) {
 516                 /* page table offset */
 517                 result = pages_addr[addr >> PAGE_SHIFT];
 518
 519                 /* in case cpu page size != gpu page size*/
 520                 result |= addr & (~PAGE_MASK);
 521
 522         } else {
 523                 /* No mapping required */
 524                 result = addr;
 525         }
 526
 527         result &= 0xFFFFFFFFFFFFF000ULL;
 528
 529         return result;
 530 }
 531
 532 /**
 533  * amdgpu_vm_update_pdes - make sure that page directory is valid
 534  *
 535  * @adev: amdgpu_device pointer
 536  * @vm: requested vm
 537  * @start: start of GPU address range
 538  * @end: end of GPU address range
 539  *
 540  * Allocates new page tables if necessary
 541  * and updates the page directory.
 542  * Returns 0 for success, error for failure.
 543  */
 544 int amdgpu_vm_update_page_directory(struct amdgpu_device *adev,
 545                                     struct amdgpu_vm *vm)
 546 {
 547         struct amdgpu_ring *ring;
 548         struct amdgpu_bo *pd = vm->page_directory;
 549         uint64_t pd_addr = amdgpu_bo_gpu_offset(pd);
 550         uint32_t incr = AMDGPU_VM_PTE_COUNT * 8;
 551         uint64_t last_pde = ~0, last_pt = ~0;
 552         unsigned count = 0, pt_idx, ndw;
 553         struct amdgpu_job *job;
 554         struct amdgpu_vm_update_params vm_update_params;
 555         struct fence *fence = NULL;
 556
 557         int r;
 558
 559         memset(&vm_update_params, 0, sizeof(vm_update_params));
 560         ring = container_of(vm->entity.sched, struct amdgpu_ring, sched);
 561
 562         /* padding, etc. */
 563         ndw = 64;
 564
 565         /* assume the worst case */
 566         ndw += vm->max_pde_used * 6;
 567
 568         r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
 569         if (r)
 570                 return r;
 571
 572         vm_update_params.ib = &job->ibs[0];
 573
 574         /* walk over the address space and update the page directory */
 575         for (pt_idx = 0; pt_idx <= vm->max_pde_used; ++pt_idx) {
 576                 struct amdgpu_bo *bo = vm->page_tables[pt_idx].entry.robj;
 577                 uint64_t pde, pt;
 578
 579                 if (bo == NULL)
 580                         continue;
 581
 582                 pt = amdgpu_bo_gpu_offset(bo);
 583                 if (vm->page_tables[pt_idx].addr == pt)
 584                         continue;
 585                 vm->page_tables[pt_idx].addr = pt;
 586
 587                 pde = pd_addr + pt_idx * 8;
 588                 if (((last_pde + 8 * count) != pde) ||
 589                     ((last_pt + incr * count) != pt)) {
 590
 591                         if (count) {
 592                                 amdgpu_vm_update_pages(adev, &vm_update_params,
 593                                                        last_pde, last_pt,
 594                                                        count, incr,
 595                                                        AMDGPU_PTE_VALID);
 596                         }
 597
 598                         count = 1;
 599                         last_pde = pde;
 600                         last_pt = pt;
 601                 } else {
 602                         ++count;
 603                 }
 604         }
 605
 606         if (count)
 607                 amdgpu_vm_update_pages(adev, &vm_update_params,
 608                                         last_pde, last_pt,
 609                                         count, incr, AMDGPU_PTE_VALID);
 610
 611         if (vm_update_params.ib->length_dw != 0) {
 612                 amdgpu_ring_pad_ib(ring, vm_update_params.ib);
 613                 amdgpu_sync_resv(adev, &job->sync, pd->tbo.resv,
 614                                  AMDGPU_FENCE_OWNER_VM);
 615                 WARN_ON(vm_update_params.ib->length_dw > ndw);
 616                 r = amdgpu_job_submit(job, ring, &vm->entity,
 617                                       AMDGPU_FENCE_OWNER_VM, &fence);
 618                 if (r)
 619                         goto error_free;
 620
 621                 amdgpu_bo_fence(pd, fence, true);
 622                 fence_put(vm->page_directory_fence);
 623                 vm->page_directory_fence = fence_get(fence);
 624                 fence_put(fence);
 625
 626         } else {
 627                 amdgpu_job_free(job);
 628         }
 629
 630         return 0;
 631
 632 error_free:
 633         amdgpu_job_free(job);
 634         return r;
 635 }
 636
 637 /**
 638  * amdgpu_vm_frag_ptes - add fragment information to PTEs
 639  *
 640  * @adev: amdgpu_device pointer
 641  * @vm_update_params: see amdgpu_vm_update_params definition
 642  * @pe_start: first PTE to handle
 643  * @pe_end: last PTE to handle
 644  * @addr: addr those PTEs should point to
 645  * @flags: hw mapping flags
 646  */
 647 static void amdgpu_vm_frag_ptes(struct amdgpu_device *adev,
 648                                 struct amdgpu_vm_update_params
 649                                         *vm_update_params,
 650                                 uint64_t pe_start, uint64_t pe_end,
 651                                 uint64_t addr, uint32_t flags)
 652 {
 653         /**
 654          * The MC L1 TLB supports variable sized pages, based on a fragment
 655          * field in the PTE. When this field is set to a non-zero value, page
 656          * granularity is increased from 4KB to (1 << (12 + frag)). The PTE
 657          * flags are considered valid for all PTEs within the fragment range
 658          * and corresponding mappings are assumed to be physically contiguous.
 659          *
 660          * The L1 TLB can store a single PTE for the whole fragment,
 661          * significantly increasing the space available for translation
 662          * caching. This leads to large improvements in throughput when the
 663          * TLB is under pressure.
 664          *
 665          * The L2 TLB distributes small and large fragments into two
 666          * asymmetric partitions. The large fragment cache is significantly
 667          * larger. Thus, we try to use large fragments wherever possible.
 668          * Userspace can support this by aligning virtual base address and
 669          * allocation size to the fragment size.
 670          */
 671
 672         /* SI and newer are optimized for 64KB */
 673         uint64_t frag_flags = AMDGPU_PTE_FRAG_64KB;
 674         uint64_t frag_align = 0x80;
 675
 676         uint64_t frag_start = ALIGN(pe_start, frag_align);
 677         uint64_t frag_end = pe_end & ~(frag_align - 1);
 678
 679         unsigned count;
 680
 681         /* Abort early if there isn't anything to do */
 682         if (pe_start == pe_end)
 683                 return;
 684
 685         /* system pages are non continuously */
 686         if (vm_update_params->src || vm_update_params->pages_addr ||
 687                 !(flags & AMDGPU_PTE_VALID) || (frag_start >= frag_end)) {
 688
 689                 count = (pe_end - pe_start) / 8;
 690                 amdgpu_vm_update_pages(adev, vm_update_params, pe_start,
 691                                        addr, count, AMDGPU_GPU_PAGE_SIZE,
 692                                        flags);
 693                 return;
 694         }
 695
 696         /* handle the 4K area at the beginning */
 697         if (pe_start != frag_start) {
 698                 count = (frag_start - pe_start) / 8;
 699                 amdgpu_vm_update_pages(adev, vm_update_params, pe_start, addr,
 700                                        count, AMDGPU_GPU_PAGE_SIZE, flags);
 701                 addr += AMDGPU_GPU_PAGE_SIZE * count;
 702         }
 703
 704         /* handle the area in the middle */
 705         count = (frag_end - frag_start) / 8;
 706         amdgpu_vm_update_pages(adev, vm_update_params, frag_start, addr, count,
 707                                AMDGPU_GPU_PAGE_SIZE, flags | frag_flags);
 708
 709         /* handle the 4K area at the end */
 710         if (frag_end != pe_end) {
 711                 addr += AMDGPU_GPU_PAGE_SIZE * count;
 712                 count = (pe_end - frag_end) / 8;
 713                 amdgpu_vm_update_pages(adev, vm_update_params, frag_end, addr,
 714                                        count, AMDGPU_GPU_PAGE_SIZE, flags);
 715         }
 716 }
 717
 718 /**
 719  * amdgpu_vm_update_ptes - make sure that page tables are valid
 720  *
 721  * @adev: amdgpu_device pointer
 722  * @vm_update_params: see amdgpu_vm_update_params definition
 723  * @vm: requested vm
 724  * @start: start of GPU address range
 725  * @end: end of GPU address range
 726  * @dst: destination address to map to
 727  * @flags: mapping flags
 728  *
 729  * Update the page tables in the range @start - @end.
 730  */
 731 static void amdgpu_vm_update_ptes(struct amdgpu_device *adev,
 732                                   struct amdgpu_vm_update_params
 733                                         *vm_update_params,
 734                                   struct amdgpu_vm *vm,
 735                                   uint64_t start, uint64_t end,
 736                                   uint64_t dst, uint32_t flags)
 737 {
 738         const uint64_t mask = AMDGPU_VM_PTE_COUNT - 1;
 739
 740         uint64_t last_pe_start = ~0, last_pe_end = ~0, last_dst = ~0;
 741         uint64_t addr;
 742
 743         /* walk over the address space and update the page tables */
 744         for (addr = start; addr < end; ) {
 745                 uint64_t pt_idx = addr >> amdgpu_vm_block_size;
 746                 struct amdgpu_bo *pt = vm->page_tables[pt_idx].entry.robj;
 747                 unsigned nptes;
 748                 uint64_t pe_start;
 749
 750                 if ((addr & ~mask) == (end & ~mask))
 751                         nptes = end - addr;
 752                 else
 753                         nptes = AMDGPU_VM_PTE_COUNT - (addr & mask);
 754
 755                 pe_start = amdgpu_bo_gpu_offset(pt);
 756                 pe_start += (addr & mask) * 8;
 757
 758                 if (last_pe_end != pe_start) {
 759
 760                         amdgpu_vm_frag_ptes(adev, vm_update_params,
 761                                             last_pe_start, last_pe_end,
 762                                             last_dst, flags);
 763
 764                         last_pe_start = pe_start;
 765                         last_pe_end = pe_start + 8 * nptes;
 766                         last_dst = dst;
 767                 } else {
 768                         last_pe_end += 8 * nptes;
 769                 }
 770
 771                 addr += nptes;
 772                 dst += nptes * AMDGPU_GPU_PAGE_SIZE;
 773         }
 774
 775         amdgpu_vm_frag_ptes(adev, vm_update_params, last_pe_start,
 776                             last_pe_end, last_dst, flags);
 777 }
 778
 779 /**
 780  * amdgpu_vm_bo_update_mapping - update a mapping in the vm page table
 781  *
 782  * @adev: amdgpu_device pointer
 783  * @src: address where to copy page table entries from
 784  * @pages_addr: DMA addresses to use for mapping
 785  * @vm: requested vm
 786  * @start: start of mapped range
 787  * @last: last mapped entry
 788  * @flags: flags for the entries
 789  * @addr: addr to set the area to
 790  * @fence: optional resulting fence
 791  *
 792  * Fill in the page table entries between @start and @last.
 793  * Returns 0 for success, -EINVAL for failure.
 794  */
 795 static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
 796                                        uint64_t src,
 797                                        dma_addr_t *pages_addr,
 798                                        struct amdgpu_vm *vm,
 799                                        uint64_t start, uint64_t last,
 800                                        uint32_t flags, uint64_t addr,
 801                                        struct fence **fence)
 802 {
 803         struct amdgpu_ring *ring;
 804         void *owner = AMDGPU_FENCE_OWNER_VM;
 805         unsigned nptes, ncmds, ndw;
 806         struct amdgpu_job *job;
 807         struct amdgpu_vm_update_params vm_update_params;
 808         struct fence *f = NULL;
 809         int r;
 810
 811         ring = container_of(vm->entity.sched, struct amdgpu_ring, sched);
 812         memset(&vm_update_params, 0, sizeof(vm_update_params));
 813         vm_update_params.src = src;
 814         vm_update_params.pages_addr = pages_addr;
 815
 816         /* sync to everything on unmapping */
 817         if (!(flags & AMDGPU_PTE_VALID))
 818                 owner = AMDGPU_FENCE_OWNER_UNDEFINED;
 819
 820         nptes = last - start + 1;
 821
 822         /*
 823          * reserve space for one command every (1 << BLOCK_SIZE)
 824          *  entries or 2k dwords (whatever is smaller)
 825          */
 826         ncmds = (nptes >> min(amdgpu_vm_block_size, 11)) + 1;
 827
 828         /* padding, etc. */
 829         ndw = 64;
 830
 831         if (vm_update_params.src) {
 832                 /* only copy commands needed */
 833                 ndw += ncmds * 7;
 834
 835         } else if (vm_update_params.pages_addr) {
 836                 /* header for write data commands */
 837                 ndw += ncmds * 4;
 838
 839                 /* body of write data command */
 840                 ndw += nptes * 2;
 841
 842         } else {
 843                 /* set page commands needed */
 844                 ndw += ncmds * 10;
 845
 846                 /* two extra commands for begin/end of fragment */
 847                 ndw += 2 * 10;
 848         }
 849
 850         r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
 851         if (r)
 852                 return r;
 853
 854         vm_update_params.ib = &job->ibs[0];
 855
 856         r = amdgpu_sync_resv(adev, &job->sync, vm->page_directory->tbo.resv,
 857                              owner);
 858         if (r)
 859                 goto error_free;
 860
 861         r = reservation_object_reserve_shared(vm->page_directory->tbo.resv);
 862         if (r)
 863                 goto error_free;
 864
 865         amdgpu_vm_update_ptes(adev, &vm_update_params, vm, start,
 866                               last + 1, addr, flags);
 867
 868         amdgpu_ring_pad_ib(ring, vm_update_params.ib);
 869         WARN_ON(vm_update_params.ib->length_dw > ndw);
 870         r = amdgpu_job_submit(job, ring, &vm->entity,
 871                               AMDGPU_FENCE_OWNER_VM, &f);
 872         if (r)
 873                 goto error_free;
 874
 875         amdgpu_bo_fence(vm->page_directory, f, true);
 876         if (fence) {
 877                 fence_put(*fence);
 878                 *fence = fence_get(f);
 879         }
 880         fence_put(f);
 881         return 0;
 882
 883 error_free:
 884         amdgpu_job_free(job);
 885         return r;
 886 }
 887
 888 /**
 889  * amdgpu_vm_bo_split_mapping - split a mapping into smaller chunks
 890  *
 891  * @adev: amdgpu_device pointer
 892  * @gtt_flags: flags as they are used for GTT
 893  * @pages_addr: DMA addresses to use for mapping
 894  * @vm: requested vm
 895  * @mapping: mapped range and flags to use for the update
 896  * @addr: addr to set the area to
 897  * @flags: HW flags for the mapping
 898  * @fence: optional resulting fence
 899  *
 900  * Split the mapping into smaller chunks so that each update fits
 901  * into a SDMA IB.
 902  * Returns 0 for success, -EINVAL for failure.
 903  */
 904 static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
 905                                       uint32_t gtt_flags,
 906                                       dma_addr_t *pages_addr,
 907                                       struct amdgpu_vm *vm,
 908                                       struct amdgpu_bo_va_mapping *mapping,
 909                                       uint32_t flags, uint64_t addr,
 910                                       struct fence **fence)
 911 {
 912         const uint64_t max_size = 64ULL * 1024ULL * 1024ULL / AMDGPU_GPU_PAGE_SIZE;
 913
 914         uint64_t src = 0, start = mapping->it.start;
 915         int r;
 916
 917         /* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here
 918          * but in case of something, we filter the flags in first place
 919          */
 920         if (!(mapping->flags & AMDGPU_PTE_READABLE))
 921                 flags &= ~AMDGPU_PTE_READABLE;
 922         if (!(mapping->flags & AMDGPU_PTE_WRITEABLE))
 923                 flags &= ~AMDGPU_PTE_WRITEABLE;
 924
 925         trace_amdgpu_vm_bo_update(mapping);
 926
 927         if (pages_addr) {
 928                 if (flags == gtt_flags)
 929                         src = adev->gart.table_addr + (addr >> 12) * 8;
 930                 addr = 0;
 931         }
 932         addr += mapping->offset;
 933
 934         if (!pages_addr || src)
 935                 return amdgpu_vm_bo_update_mapping(adev, src, pages_addr, vm,
 936                                                    start, mapping->it.last,
 937                                                    flags, addr, fence);
 938
 939         while (start != mapping->it.last + 1) {
 940                 uint64_t last;
 941
 942                 last = min((uint64_t)mapping->it.last, start + max_size - 1);
 943                 r = amdgpu_vm_bo_update_mapping(adev, src, pages_addr, vm,
 944                                                 start, last, flags, addr,
 945                                                 fence);
 946                 if (r)
 947                         return r;
 948
 949                 start = last + 1;
 950                 addr += max_size * AMDGPU_GPU_PAGE_SIZE;
 951         }
 952
 953         return 0;
 954 }
 955
 956 /**
 957  * amdgpu_vm_bo_update - update all BO mappings in the vm page table
 958  *
 959  * @adev: amdgpu_device pointer
 960  * @bo_va: requested BO and VM object
 961  * @mem: ttm mem
 962  *
 963  * Fill in the page table entries for @bo_va.
 964  * Returns 0 for success, -EINVAL for failure.
 965  *
 966  * Object have to be reserved and mutex must be locked!
 967  */
 968 int amdgpu_vm_bo_update(struct amdgpu_device *adev,
 969                         struct amdgpu_bo_va *bo_va,
 970                         struct ttm_mem_reg *mem)
 971 {
 972         struct amdgpu_vm *vm = bo_va->vm;
 973         struct amdgpu_bo_va_mapping *mapping;
 974         dma_addr_t *pages_addr = NULL;
 975         uint32_t gtt_flags, flags;
 976         uint64_t addr;
 977         int r;
 978
 979         if (mem) {
 980                 struct ttm_dma_tt *ttm;
 981
 982                 addr = (u64)mem->start << PAGE_SHIFT;
 983                 switch (mem->mem_type) {
 984                 case TTM_PL_TT:
 985                         ttm = container_of(bo_va->bo->tbo.ttm, struct
 986                                            ttm_dma_tt, ttm);
 987                         pages_addr = ttm->dma_address;
 988                         break;
 989
 990                 case TTM_PL_VRAM:
 991                         addr += adev->vm_manager.vram_base_offset;
 992                         break;
 993
 994                 default:
 995                         break;
 996                 }
 997         } else {
 998                 addr = 0;
 999         }
1000
1001         flags = amdgpu_ttm_tt_pte_flags(adev, bo_va->bo->tbo.ttm, mem);
1002         gtt_flags = (adev == bo_va->bo->adev) ? flags : 0;
1003
1004         spin_lock(&vm->status_lock);
1005         if (!list_empty(&bo_va->vm_status))
1006                 list_splice_init(&bo_va->valids, &bo_va->invalids);
1007         spin_unlock(&vm->status_lock);
1008
1009         list_for_each_entry(mapping, &bo_va->invalids, list) {
1010                 r = amdgpu_vm_bo_split_mapping(adev, gtt_flags, pages_addr, vm,
1011                                                mapping, flags, addr,
1012                                                &bo_va->last_pt_update);
1013                 if (r)
1014                         return r;
1015         }
1016
1017         if (trace_amdgpu_vm_bo_mapping_enabled()) {
1018                 list_for_each_entry(mapping, &bo_va->valids, list)
1019                         trace_amdgpu_vm_bo_mapping(mapping);
1020
1021                 list_for_each_entry(mapping, &bo_va->invalids, list)
1022                         trace_amdgpu_vm_bo_mapping(mapping);
1023         }
1024
1025         spin_lock(&vm->status_lock);
1026         list_splice_init(&bo_va->invalids, &bo_va->valids);
1027         list_del_init(&bo_va->vm_status);
1028         if (!mem)
1029                 list_add(&bo_va->vm_status, &vm->cleared);
1030         spin_unlock(&vm->status_lock);
1031
1032         return 0;
1033 }
1034
1035 /**
1036  * amdgpu_vm_clear_freed - clear freed BOs in the PT
1037  *
1038  * @adev: amdgpu_device pointer
1039  * @vm: requested vm
1040  *
1041  * Make sure all freed BOs are cleared in the PT.
1042  * Returns 0 for success.
1043  *
1044  * PTs have to be reserved and mutex must be locked!
1045  */
1046 int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
1047                           struct amdgpu_vm *vm)
1048 {
1049         struct amdgpu_bo_va_mapping *mapping;
1050         int r;
1051
1052         while (!list_empty(&vm->freed)) {
1053                 mapping = list_first_entry(&vm->freed,
1054                         struct amdgpu_bo_va_mapping, list);
1055                 list_del(&mapping->list);
1056
1057                 r = amdgpu_vm_bo_split_mapping(adev, 0, NULL, vm, mapping,
1058                                                0, 0, NULL);
1059                 kfree(mapping);
1060                 if (r)
1061                         return r;
1062
1063         }
1064         return 0;
1065
1066 }
1067
1068 /**
1069  * amdgpu_vm_clear_invalids - clear invalidated BOs in the PT
1070  *
1071  * @adev: amdgpu_device pointer
1072  * @vm: requested vm
1073  *
1074  * Make sure all invalidated BOs are cleared in the PT.
1075  * Returns 0 for success.
1076  *
1077  * PTs have to be reserved and mutex must be locked!
1078  */
1079 int amdgpu_vm_clear_invalids(struct amdgpu_device *adev,
1080                              struct amdgpu_vm *vm, struct amdgpu_sync *sync)
1081 {
1082         struct amdgpu_bo_va *bo_va = NULL;
1083         int r = 0;
1084
1085         spin_lock(&vm->status_lock);
1086         while (!list_empty(&vm->invalidated)) {
1087                 bo_va = list_first_entry(&vm->invalidated,
1088                         struct amdgpu_bo_va, vm_status);
1089                 spin_unlock(&vm->status_lock);
1090
1091                 r = amdgpu_vm_bo_update(adev, bo_va, NULL);
1092                 if (r)
1093                         return r;
1094
1095                 spin_lock(&vm->status_lock);
1096         }
1097         spin_unlock(&vm->status_lock);
1098
1099         if (bo_va)
1100                 r = amdgpu_sync_fence(adev, sync, bo_va->last_pt_update);
1101
1102         return r;
1103 }
1104
1105 /**
1106  * amdgpu_vm_bo_add - add a bo to a specific vm
1107  *
1108  * @adev: amdgpu_device pointer
1109  * @vm: requested vm
1110  * @bo: amdgpu buffer object
1111  *
1112  * Add @bo into the requested vm.
1113  * Add @bo to the list of bos associated with the vm
1114  * Returns newly added bo_va or NULL for failure
1115  *
1116  * Object has to be reserved!
1117  */
1118 struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
1119                                       struct amdgpu_vm *vm,
1120                                       struct amdgpu_bo *bo)
1121 {
1122         struct amdgpu_bo_va *bo_va;
1123
1124         bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL);
1125         if (bo_va == NULL) {
1126                 return NULL;
1127         }
1128         bo_va->vm = vm;
1129         bo_va->bo = bo;
1130         bo_va->ref_count = 1;
1131         INIT_LIST_HEAD(&bo_va->bo_list);
1132         INIT_LIST_HEAD(&bo_va->valids);
1133         INIT_LIST_HEAD(&bo_va->invalids);
1134         INIT_LIST_HEAD(&bo_va->vm_status);
1135
1136         list_add_tail(&bo_va->bo_list, &bo->va);
1137
1138         return bo_va;
1139 }
1140
1141 /**
1142  * amdgpu_vm_bo_map - map bo inside a vm
1143  *
1144  * @adev: amdgpu_device pointer
1145  * @bo_va: bo_va to store the address
1146  * @saddr: where to map the BO
1147  * @offset: requested offset in the BO
1148  * @flags: attributes of pages (read/write/valid/etc.)
1149  *
1150  * Add a mapping of the BO at the specefied addr into the VM.
1151  * Returns 0 for success, error for failure.
1152  *
1153  * Object has to be reserved and unreserved outside!
1154  */
1155 int amdgpu_vm_bo_map(struct amdgpu_device *adev,
1156                      struct amdgpu_bo_va *bo_va,
1157                      uint64_t saddr, uint64_t offset,
1158                      uint64_t size, uint32_t flags)
1159 {
1160         struct amdgpu_bo_va_mapping *mapping;
1161         struct amdgpu_vm *vm = bo_va->vm;
1162         struct interval_tree_node *it;
1163         unsigned last_pfn, pt_idx;
1164         uint64_t eaddr;
1165         int r;
1166
1167         /* validate the parameters */
1168         if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK ||
1169             size == 0 || size & AMDGPU_GPU_PAGE_MASK)
1170                 return -EINVAL;
1171
1172         /* make sure object fit at this offset */
1173         eaddr = saddr + size - 1;
1174         if ((saddr >= eaddr) || (offset + size > amdgpu_bo_size(bo_va->bo)))
1175                 return -EINVAL;
1176
1177         last_pfn = eaddr / AMDGPU_GPU_PAGE_SIZE;
1178         if (last_pfn >= adev->vm_manager.max_pfn) {
1179                 dev_err(adev->dev, "va above limit (0x%08X >= 0x%08X)\n",
1180                         last_pfn, adev->vm_manager.max_pfn);
1181                 return -EINVAL;
1182         }
1183
1184         saddr /= AMDGPU_GPU_PAGE_SIZE;
1185         eaddr /= AMDGPU_GPU_PAGE_SIZE;
1186
1187         it = interval_tree_iter_first(&vm->va, saddr, eaddr);
1188         if (it) {
1189                 struct amdgpu_bo_va_mapping *tmp;
1190                 tmp = container_of(it, struct amdgpu_bo_va_mapping, it);
1191                 /* bo and tmp overlap, invalid addr */
1192                 dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with "
1193                         "0x%010lx-0x%010lx\n", bo_va->bo, saddr, eaddr,
1194                         tmp->it.start, tmp->it.last + 1);
1195                 r = -EINVAL;
1196                 goto error;
1197         }
1198
1199         mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
1200         if (!mapping) {
1201                 r = -ENOMEM;
1202                 goto error;
1203         }
1204
1205         INIT_LIST_HEAD(&mapping->list);
1206         mapping->it.start = saddr;
1207         mapping->it.last = eaddr;
1208         mapping->offset = offset;
1209         mapping->flags = flags;
1210
1211         list_add(&mapping->list, &bo_va->invalids);
1212         interval_tree_insert(&mapping->it, &vm->va);
1213
1214         /* Make sure the page tables are allocated */
1215         saddr >>= amdgpu_vm_block_size;
1216         eaddr >>= amdgpu_vm_block_size;
1217
1218         BUG_ON(eaddr >= amdgpu_vm_num_pdes(adev));
1219
1220         if (eaddr > vm->max_pde_used)
1221                 vm->max_pde_used = eaddr;
1222
1223         /* walk over the address space and allocate the page tables */
1224         for (pt_idx = saddr; pt_idx <= eaddr; ++pt_idx) {
1225                 struct reservation_object *resv = vm->page_directory->tbo.resv;
1226                 struct amdgpu_bo_list_entry *entry;
1227                 struct amdgpu_bo *pt;
1228
1229                 entry = &vm->page_tables[pt_idx].entry;
1230                 if (entry->robj)
1231                         continue;
1232
1233                 r = amdgpu_bo_create(adev, AMDGPU_VM_PTE_COUNT * 8,
1234                                      AMDGPU_GPU_PAGE_SIZE, true,
1235                                      AMDGPU_GEM_DOMAIN_VRAM,
1236                                      AMDGPU_GEM_CREATE_NO_CPU_ACCESS,
1237                                      NULL, resv, &pt);
1238                 if (r)
1239                         goto error_free;
1240
1241                 /* Keep a reference to the page table to avoid freeing
1242                  * them up in the wrong order.
1243                  */
1244                 pt->parent = amdgpu_bo_ref(vm->page_directory);
1245
1246                 r = amdgpu_vm_clear_bo(adev, vm, pt);
1247                 if (r) {
1248                         amdgpu_bo_unref(&pt);
1249                         goto error_free;
1250                 }
1251
1252                 entry->robj = pt;
1253                 entry->priority = 0;
1254                 entry->tv.bo = &entry->robj->tbo;
1255                 entry->tv.shared = true;
1256                 entry->user_pages = NULL;
1257                 vm->page_tables[pt_idx].addr = 0;
1258         }
1259
1260         return 0;
1261
1262 error_free:
1263         list_del(&mapping->list);
1264         interval_tree_remove(&mapping->it, &vm->va);
1265         trace_amdgpu_vm_bo_unmap(bo_va, mapping);
1266         kfree(mapping);
1267
1268 error:
1269         return r;
1270 }
1271
1272 /**
1273  * amdgpu_vm_bo_unmap - remove bo mapping from vm
1274  *
1275  * @adev: amdgpu_device pointer
1276  * @bo_va: bo_va to remove the address from
1277  * @saddr: where to the BO is mapped
1278  *
1279  * Remove a mapping of the BO at the specefied addr from the VM.
1280  * Returns 0 for success, error for failure.
1281  *
1282  * Object has to be reserved and unreserved outside!
1283  */
1284 int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
1285                        struct amdgpu_bo_va *bo_va,
1286                        uint64_t saddr)
1287 {
1288         struct amdgpu_bo_va_mapping *mapping;
1289         struct amdgpu_vm *vm = bo_va->vm;
1290         bool valid = true;
1291
1292         saddr /= AMDGPU_GPU_PAGE_SIZE;
1293
1294         list_for_each_entry(mapping, &bo_va->valids, list) {
1295                 if (mapping->it.start == saddr)
1296                         break;
1297         }
1298
1299         if (&mapping->list == &bo_va->valids) {
1300                 valid = false;
1301
1302                 list_for_each_entry(mapping, &bo_va->invalids, list) {
1303                         if (mapping->it.start == saddr)
1304                                 break;
1305                 }
1306
1307                 if (&mapping->list == &bo_va->invalids)
1308                         return -ENOENT;
1309         }
1310
1311         list_del(&mapping->list);
1312         interval_tree_remove(&mapping->it, &vm->va);
1313         trace_amdgpu_vm_bo_unmap(bo_va, mapping);
1314
1315         if (valid)
1316                 list_add(&mapping->list, &vm->freed);
1317         else
1318                 kfree(mapping);
1319
1320         return 0;
1321 }
1322
1323 /**
1324  * amdgpu_vm_bo_rmv - remove a bo to a specific vm
1325  *
1326  * @adev: amdgpu_device pointer
1327  * @bo_va: requested bo_va
1328  *
1329  * Remove @bo_va->bo from the requested vm.
1330  *
1331  * Object have to be reserved!
1332  */
1333 void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
1334                       struct amdgpu_bo_va *bo_va)
1335 {
1336         struct amdgpu_bo_va_mapping *mapping, *next;
1337         struct amdgpu_vm *vm = bo_va->vm;
1338
1339         list_del(&bo_va->bo_list);
1340
1341         spin_lock(&vm->status_lock);
1342         list_del(&bo_va->vm_status);
1343         spin_unlock(&vm->status_lock);
1344
1345         list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
1346                 list_del(&mapping->list);
1347                 interval_tree_remove(&mapping->it, &vm->va);
1348                 trace_amdgpu_vm_bo_unmap(bo_va, mapping);
1349                 list_add(&mapping->list, &vm->freed);
1350         }
1351         list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) {
1352                 list_del(&mapping->list);
1353                 interval_tree_remove(&mapping->it, &vm->va);
1354                 kfree(mapping);
1355         }
1356
1357         fence_put(bo_va->last_pt_update);
1358         kfree(bo_va);
1359 }
1360
1361 /**
1362  * amdgpu_vm_bo_invalidate - mark the bo as invalid
1363  *
1364  * @adev: amdgpu_device pointer
1365  * @vm: requested vm
1366  * @bo: amdgpu buffer object
1367  *
1368  * Mark @bo as invalid.
1369  */
1370 void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
1371                              struct amdgpu_bo *bo)
1372 {
1373         struct amdgpu_bo_va *bo_va;
1374
1375         list_for_each_entry(bo_va, &bo->va, bo_list) {
1376                 spin_lock(&bo_va->vm->status_lock);
1377                 if (list_empty(&bo_va->vm_status))
1378                         list_add(&bo_va->vm_status, &bo_va->vm->invalidated);
1379                 spin_unlock(&bo_va->vm->status_lock);
1380         }
1381 }
1382
1383 /**
1384  * amdgpu_vm_init - initialize a vm instance
1385  *
1386  * @adev: amdgpu_device pointer
1387  * @vm: requested vm
1388  *
1389  * Init @vm fields.
1390  */
1391 int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm)
1392 {
1393         const unsigned align = min(AMDGPU_VM_PTB_ALIGN_SIZE,
1394                 AMDGPU_VM_PTE_COUNT * 8);
1395         unsigned pd_size, pd_entries;
1396         unsigned ring_instance;
1397         struct amdgpu_ring *ring;
1398         struct amd_sched_rq *rq;
1399         int i, r;
1400
1401         for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
1402                 vm->ids[i] = NULL;
1403         vm->va = RB_ROOT;
1404         vm->client_id = atomic64_inc_return(&adev->vm_manager.client_counter);
1405         spin_lock_init(&vm->status_lock);
1406         INIT_LIST_HEAD(&vm->invalidated);
1407         INIT_LIST_HEAD(&vm->cleared);
1408         INIT_LIST_HEAD(&vm->freed);
1409
1410         pd_size = amdgpu_vm_directory_size(adev);
1411         pd_entries = amdgpu_vm_num_pdes(adev);
1412
1413         /* allocate page table array */
1414         vm->page_tables = drm_calloc_large(pd_entries, sizeof(struct amdgpu_vm_pt));
1415         if (vm->page_tables == NULL) {
1416                 DRM_ERROR("Cannot allocate memory for page table array\n");
1417                 return -ENOMEM;
1418         }
1419
1420         /* create scheduler entity for page table updates */
1421
1422         ring_instance = atomic_inc_return(&adev->vm_manager.vm_pte_next_ring);
1423         ring_instance %= adev->vm_manager.vm_pte_num_rings;
1424         ring = adev->vm_manager.vm_pte_rings[ring_instance];
1425         rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_KERNEL];
1426         r = amd_sched_entity_init(&ring->sched, &vm->entity,
1427                                   rq, amdgpu_sched_jobs);
1428         if (r)
1429                 return r;
1430
1431         vm->page_directory_fence = NULL;
1432
1433         r = amdgpu_bo_create(adev, pd_size, align, true,
1434                              AMDGPU_GEM_DOMAIN_VRAM,
1435                              AMDGPU_GEM_CREATE_NO_CPU_ACCESS,
1436                              NULL, NULL, &vm->page_directory);
1437         if (r)
1438                 goto error_free_sched_entity;
1439
1440         r = amdgpu_bo_reserve(vm->page_directory, false);
1441         if (r)
1442                 goto error_free_page_directory;
1443
1444         r = amdgpu_vm_clear_bo(adev, vm, vm->page_directory);
1445         amdgpu_bo_unreserve(vm->page_directory);
1446         if (r)
1447                 goto error_free_page_directory;
1448
1449         return 0;
1450
1451 error_free_page_directory:
1452         amdgpu_bo_unref(&vm->page_directory);
1453         vm->page_directory = NULL;
1454
1455 error_free_sched_entity:
1456         amd_sched_entity_fini(&ring->sched, &vm->entity);
1457
1458         return r;
1459 }
1460
1461 /**
1462  * amdgpu_vm_fini - tear down a vm instance
1463  *
1464  * @adev: amdgpu_device pointer
1465  * @vm: requested vm
1466  *
1467  * Tear down @vm.
1468  * Unbind the VM and remove all bos from the vm bo list
1469  */
1470 void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
1471 {
1472         struct amdgpu_bo_va_mapping *mapping, *tmp;
1473         int i;
1474
1475         amd_sched_entity_fini(vm->entity.sched, &vm->entity);
1476
1477         if (!RB_EMPTY_ROOT(&vm->va)) {
1478                 dev_err(adev->dev, "still active bo inside vm\n");
1479         }
1480         rbtree_postorder_for_each_entry_safe(mapping, tmp, &vm->va, it.rb) {
1481                 list_del(&mapping->list);
1482                 interval_tree_remove(&mapping->it, &vm->va);
1483                 kfree(mapping);
1484         }
1485         list_for_each_entry_safe(mapping, tmp, &vm->freed, list) {
1486                 list_del(&mapping->list);
1487                 kfree(mapping);
1488         }
1489
1490         for (i = 0; i < amdgpu_vm_num_pdes(adev); i++)
1491                 amdgpu_bo_unref(&vm->page_tables[i].entry.robj);
1492         drm_free_large(vm->page_tables);
1493
1494         amdgpu_bo_unref(&vm->page_directory);
1495         fence_put(vm->page_directory_fence);
1496 }
1497
1498 /**
1499  * amdgpu_vm_manager_init - init the VM manager
1500  *
1501  * @adev: amdgpu_device pointer
1502  *
1503  * Initialize the VM manager structures
1504  */
1505 void amdgpu_vm_manager_init(struct amdgpu_device *adev)
1506 {
1507         unsigned i;
1508
1509         INIT_LIST_HEAD(&adev->vm_manager.ids_lru);
1510
1511         /* skip over VMID 0, since it is the system VM */
1512         for (i = 1; i < adev->vm_manager.num_ids; ++i) {
1513                 amdgpu_vm_reset_id(adev, i);
1514                 amdgpu_sync_create(&adev->vm_manager.ids[i].active);
1515                 list_add_tail(&adev->vm_manager.ids[i].list,
1516                               &adev->vm_manager.ids_lru);
1517         }
1518
1519         atomic_set(&adev->vm_manager.vm_pte_next_ring, 0);
1520         atomic64_set(&adev->vm_manager.client_counter, 0);
1521 }
1522
1523 /**
1524  * amdgpu_vm_manager_fini - cleanup VM manager
1525  *
1526  * @adev: amdgpu_device pointer
1527  *
1528  * Cleanup the VM manager and free resources.
1529  */
1530 void amdgpu_vm_manager_fini(struct amdgpu_device *adev)
1531 {
1532         unsigned i;
1533
1534         for (i = 0; i < AMDGPU_NUM_VM; ++i) {
1535                 struct amdgpu_vm_id *id = &adev->vm_manager.ids[i];
1536
1537                 fence_put(adev->vm_manager.ids[i].first);
1538                 amdgpu_sync_free(&adev->vm_manager.ids[i].active);
1539                 fence_put(id->flushed_updates);
1540         }
1541 }