drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  */
  27
  28 #include <drm/drmP.h>
  29 #include <drm/drm_vma_manager.h>
  30 #include <drm/i915_drm.h>
  31 #include "i915_drv.h"
  32 #include "i915_trace.h"
  33 #include "intel_drv.h"
  34 #include <linux/oom.h>
  35 #include <linux/shmem_fs.h>
  36 #include <linux/slab.h>
  37 #include <linux/swap.h>
  38 #include <linux/pci.h>
  39 #include <linux/dma-buf.h>
  40
  41 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
  42 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj,
  43                                                    bool force);
  44 static __must_check int
  45 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
  46                                bool readonly);
  47 static void
  48 i915_gem_object_retire(struct drm_i915_gem_object *obj);
  49
  50 static void i915_gem_write_fence(struct drm_device *dev, int reg,
  51                                  struct drm_i915_gem_object *obj);
  52 static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
  53                                          struct drm_i915_fence_reg *fence,
  54                                          bool enable);
  55
  56 static unsigned long i915_gem_shrinker_count(struct shrinker *shrinker,
  57                                              struct shrink_control *sc);
  58 static unsigned long i915_gem_shrinker_scan(struct shrinker *shrinker,
  59                                             struct shrink_control *sc);
  60 static int i915_gem_shrinker_oom(struct notifier_block *nb,
  61                                  unsigned long event,
  62                                  void *ptr);
  63 static unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv);
  64
  65 static bool cpu_cache_is_coherent(struct drm_device *dev,
  66                                   enum i915_cache_level level)
  67 {
  68         return HAS_LLC(dev) || level != I915_CACHE_NONE;
  69 }
  70
  71 static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
  72 {
  73         if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
  74                 return true;
  75
  76         return obj->pin_display;
  77 }
  78
  79 static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
  80 {
  81         if (obj->tiling_mode)
  82                 i915_gem_release_mmap(obj);
  83
  84         /* As we do not have an associated fence register, we will force
  85          * a tiling change if we ever need to acquire one.
  86          */
  87         obj->fence_dirty = false;
  88         obj->fence_reg = I915_FENCE_REG_NONE;
  89 }
  90
  91 /* some bookkeeping */
  92 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
  93                                   size_t size)
  94 {
  95         spin_lock(&dev_priv->mm.object_stat_lock);
  96         dev_priv->mm.object_count++;
  97         dev_priv->mm.object_memory += size;
  98         spin_unlock(&dev_priv->mm.object_stat_lock);
  99 }
 100
 101 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
 102                                      size_t size)
 103 {
 104         spin_lock(&dev_priv->mm.object_stat_lock);
 105         dev_priv->mm.object_count--;
 106         dev_priv->mm.object_memory -= size;
 107         spin_unlock(&dev_priv->mm.object_stat_lock);
 108 }
 109
 110 static int
 111 i915_gem_wait_for_error(struct i915_gpu_error *error)
 112 {
 113         int ret;
 114
 115 #define EXIT_COND (!i915_reset_in_progress(error) || \
 116                    i915_terminally_wedged(error))
 117         if (EXIT_COND)
 118                 return 0;
 119
 120         /*
 121          * Only wait 10 seconds for the gpu reset to complete to avoid hanging
 122          * userspace. If it takes that long something really bad is going on and
 123          * we should simply try to bail out and fail as gracefully as possible.
 124          */
 125         ret = wait_event_interruptible_timeout(error->reset_queue,
 126                                                EXIT_COND,
 127                                                10*HZ);
 128         if (ret == 0) {
 129                 DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
 130                 return -EIO;
 131         } else if (ret < 0) {
 132                 return ret;
 133         }
 134 #undef EXIT_COND
 135
 136         return 0;
 137 }
 138
 139 int i915_mutex_lock_interruptible(struct drm_device *dev)
 140 {
 141         struct drm_i915_private *dev_priv = dev->dev_private;
 142         int ret;
 143
 144         ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
 145         if (ret)
 146                 return ret;
 147
 148         ret = mutex_lock_interruptible(&dev->struct_mutex);
 149         if (ret)
 150                 return ret;
 151
 152         WARN_ON(i915_verify_lists(dev));
 153         return 0;
 154 }
 155
 156 static inline bool
 157 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
 158 {
 159         return i915_gem_obj_bound_any(obj) && !obj->active;
 160 }
 161
 162 int
 163 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 164                             struct drm_file *file)
 165 {
 166         struct drm_i915_private *dev_priv = dev->dev_private;
 167         struct drm_i915_gem_get_aperture *args = data;
 168         struct drm_i915_gem_object *obj;
 169         size_t pinned;
 170
 171         pinned = 0;
 172         mutex_lock(&dev->struct_mutex);
 173         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
 174                 if (i915_gem_obj_is_pinned(obj))
 175                         pinned += i915_gem_obj_ggtt_size(obj);
 176         mutex_unlock(&dev->struct_mutex);
 177
 178         args->aper_size = dev_priv->gtt.base.total;
 179         args->aper_available_size = args->aper_size - pinned;
 180
 181         return 0;
 182 }
 183
 184 static int
 185 i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
 186 {
 187         struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
 188         char *vaddr = obj->phys_handle->vaddr;
 189         struct sg_table *st;
 190         struct scatterlist *sg;
 191         int i;
 192
 193         if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj)))
 194                 return -EINVAL;
 195
 196         for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
 197                 struct page *page;
 198                 char *src;
 199
 200                 page = shmem_read_mapping_page(mapping, i);
 201                 if (IS_ERR(page))
 202                         return PTR_ERR(page);
 203
 204                 src = kmap_atomic(page);
 205                 memcpy(vaddr, src, PAGE_SIZE);
 206                 drm_clflush_virt_range(vaddr, PAGE_SIZE);
 207                 kunmap_atomic(src);
 208
 209                 page_cache_release(page);
 210                 vaddr += PAGE_SIZE;
 211         }
 212
 213         i915_gem_chipset_flush(obj->base.dev);
 214
 215         st = kmalloc(sizeof(*st), GFP_KERNEL);
 216         if (st == NULL)
 217                 return -ENOMEM;
 218
 219         if (sg_alloc_table(st, 1, GFP_KERNEL)) {
 220                 kfree(st);
 221                 return -ENOMEM;
 222         }
 223
 224         sg = st->sgl;
 225         sg->offset = 0;
 226         sg->length = obj->base.size;
 227
 228         sg_dma_address(sg) = obj->phys_handle->busaddr;
 229         sg_dma_len(sg) = obj->base.size;
 230
 231         obj->pages = st;
 232         obj->has_dma_mapping = true;
 233         return 0;
 234 }
 235
 236 static void
 237 i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj)
 238 {
 239         int ret;
 240
 241         BUG_ON(obj->madv == __I915_MADV_PURGED);
 242
 243         ret = i915_gem_object_set_to_cpu_domain(obj, true);
 244         if (ret) {
 245                 /* In the event of a disaster, abandon all caches and
 246                  * hope for the best.
 247                  */
 248                 WARN_ON(ret != -EIO);
 249                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
 250         }
 251
 252         if (obj->madv == I915_MADV_DONTNEED)
 253                 obj->dirty = 0;
 254
 255         if (obj->dirty) {
 256                 struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
 257                 char *vaddr = obj->phys_handle->vaddr;
 258                 int i;
 259
 260                 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
 261                         struct page *page;
 262                         char *dst;
 263
 264                         page = shmem_read_mapping_page(mapping, i);
 265                         if (IS_ERR(page))
 266                                 continue;
 267
 268                         dst = kmap_atomic(page);
 269                         drm_clflush_virt_range(vaddr, PAGE_SIZE);
 270                         memcpy(dst, vaddr, PAGE_SIZE);
 271                         kunmap_atomic(dst);
 272
 273                         set_page_dirty(page);
 274                         if (obj->madv == I915_MADV_WILLNEED)
 275                                 mark_page_accessed(page);
 276                         page_cache_release(page);
 277                         vaddr += PAGE_SIZE;
 278                 }
 279                 obj->dirty = 0;
 280         }
 281
 282         sg_free_table(obj->pages);
 283         kfree(obj->pages);
 284
 285         obj->has_dma_mapping = false;
 286 }
 287
 288 static void
 289 i915_gem_object_release_phys(struct drm_i915_gem_object *obj)
 290 {
 291         drm_pci_free(obj->base.dev, obj->phys_handle);
 292 }
 293
 294 static const struct drm_i915_gem_object_ops i915_gem_phys_ops = {
 295         .get_pages = i915_gem_object_get_pages_phys,
 296         .put_pages = i915_gem_object_put_pages_phys,
 297         .release = i915_gem_object_release_phys,
 298 };
 299
 300 static int
 301 drop_pages(struct drm_i915_gem_object *obj)
 302 {
 303         struct i915_vma *vma, *next;
 304         int ret;
 305
 306         drm_gem_object_reference(&obj->base);
 307         list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link)
 308                 if (i915_vma_unbind(vma))
 309                         break;
 310
 311         ret = i915_gem_object_put_pages(obj);
 312         drm_gem_object_unreference(&obj->base);
 313
 314         return ret;
 315 }
 316
 317 int
 318 i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
 319                             int align)
 320 {
 321         drm_dma_handle_t *phys;
 322         int ret;
 323
 324         if (obj->phys_handle) {
 325                 if ((unsigned long)obj->phys_handle->vaddr & (align -1))
 326                         return -EBUSY;
 327
 328                 return 0;
 329         }
 330
 331         if (obj->madv != I915_MADV_WILLNEED)
 332                 return -EFAULT;
 333
 334         if (obj->base.filp == NULL)
 335                 return -EINVAL;
 336
 337         ret = drop_pages(obj);
 338         if (ret)
 339                 return ret;
 340
 341         /* create a new object */
 342         phys = drm_pci_alloc(obj->base.dev, obj->base.size, align);
 343         if (!phys)
 344                 return -ENOMEM;
 345
 346         obj->phys_handle = phys;
 347         obj->ops = &i915_gem_phys_ops;
 348
 349         return i915_gem_object_get_pages(obj);
 350 }
 351
 352 static int
 353 i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
 354                      struct drm_i915_gem_pwrite *args,
 355                      struct drm_file *file_priv)
 356 {
 357         struct drm_device *dev = obj->base.dev;
 358         void *vaddr = obj->phys_handle->vaddr + args->offset;
 359         char __user *user_data = to_user_ptr(args->data_ptr);
 360         int ret;
 361
 362         /* We manually control the domain here and pretend that it
 363          * remains coherent i.e. in the GTT domain, like shmem_pwrite.
 364          */
 365         ret = i915_gem_object_wait_rendering(obj, false);
 366         if (ret)
 367                 return ret;
 368
 369         if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
 370                 unsigned long unwritten;
 371
 372                 /* The physical object once assigned is fixed for the lifetime
 373                  * of the obj, so we can safely drop the lock and continue
 374                  * to access vaddr.
 375                  */
 376                 mutex_unlock(&dev->struct_mutex);
 377                 unwritten = copy_from_user(vaddr, user_data, args->size);
 378                 mutex_lock(&dev->struct_mutex);
 379                 if (unwritten)
 380                         return -EFAULT;
 381         }
 382
 383         drm_clflush_virt_range(vaddr, args->size);
 384         i915_gem_chipset_flush(dev);
 385         return 0;
 386 }
 387
 388 void *i915_gem_object_alloc(struct drm_device *dev)
 389 {
 390         struct drm_i915_private *dev_priv = dev->dev_private;
 391         return kmem_cache_zalloc(dev_priv->slab, GFP_KERNEL);
 392 }
 393
 394 void i915_gem_object_free(struct drm_i915_gem_object *obj)
 395 {
 396         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
 397         kmem_cache_free(dev_priv->slab, obj);
 398 }
 399
 400 static int
 401 i915_gem_create(struct drm_file *file,
 402                 struct drm_device *dev,
 403                 uint64_t size,
 404                 uint32_t *handle_p)
 405 {
 406         struct drm_i915_gem_object *obj;
 407         int ret;
 408         u32 handle;
 409
 410         size = roundup(size, PAGE_SIZE);
 411         if (size == 0)
 412                 return -EINVAL;
 413
 414         /* Allocate the new object */
 415         obj = i915_gem_alloc_object(dev, size);
 416         if (obj == NULL)
 417                 return -ENOMEM;
 418
 419         ret = drm_gem_handle_create(file, &obj->base, &handle);
 420         /* drop reference from allocate - handle holds it now */
 421         drm_gem_object_unreference_unlocked(&obj->base);
 422         if (ret)
 423                 return ret;
 424
 425         *handle_p = handle;
 426         return 0;
 427 }
 428
 429 int
 430 i915_gem_dumb_create(struct drm_file *file,
 431                      struct drm_device *dev,
 432                      struct drm_mode_create_dumb *args)
 433 {
 434         /* have to work out size/pitch and return them */
 435         args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64);
 436         args->size = args->pitch * args->height;
 437         return i915_gem_create(file, dev,
 438                                args->size, &args->handle);
 439 }
 440
 441 /**
 442  * Creates a new mm object and returns a handle to it.
 443  */
 444 int
 445 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 446                       struct drm_file *file)
 447 {
 448         struct drm_i915_gem_create *args = data;
 449
 450         return i915_gem_create(file, dev,
 451                                args->size, &args->handle);
 452 }
 453
 454 static inline int
 455 __copy_to_user_swizzled(char __user *cpu_vaddr,
 456                         const char *gpu_vaddr, int gpu_offset,
 457                         int length)
 458 {
 459         int ret, cpu_offset = 0;
 460
 461         while (length > 0) {
 462                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 463                 int this_length = min(cacheline_end - gpu_offset, length);
 464                 int swizzled_gpu_offset = gpu_offset ^ 64;
 465
 466                 ret = __copy_to_user(cpu_vaddr + cpu_offset,
 467                                      gpu_vaddr + swizzled_gpu_offset,
 468                                      this_length);
 469                 if (ret)
 470                         return ret + length;
 471
 472                 cpu_offset += this_length;
 473                 gpu_offset += this_length;
 474                 length -= this_length;
 475         }
 476
 477         return 0;
 478 }
 479
 480 static inline int
 481 __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
 482                           const char __user *cpu_vaddr,
 483                           int length)
 484 {
 485         int ret, cpu_offset = 0;
 486
 487         while (length > 0) {
 488                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 489                 int this_length = min(cacheline_end - gpu_offset, length);
 490                 int swizzled_gpu_offset = gpu_offset ^ 64;
 491
 492                 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
 493                                        cpu_vaddr + cpu_offset,
 494                                        this_length);
 495                 if (ret)
 496                         return ret + length;
 497
 498                 cpu_offset += this_length;
 499                 gpu_offset += this_length;
 500                 length -= this_length;
 501         }
 502
 503         return 0;
 504 }
 505
 506 /*
 507  * Pins the specified object's pages and synchronizes the object with
 508  * GPU accesses. Sets needs_clflush to non-zero if the caller should
 509  * flush the object from the CPU cache.
 510  */
 511 int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
 512                                     int *needs_clflush)
 513 {
 514         int ret;
 515
 516         *needs_clflush = 0;
 517
 518         if (!obj->base.filp)
 519                 return -EINVAL;
 520
 521         if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
 522                 /* If we're not in the cpu read domain, set ourself into the gtt
 523                  * read domain and manually flush cachelines (if required). This
 524                  * optimizes for the case when the gpu will dirty the data
 525                  * anyway again before the next pread happens. */
 526                 *needs_clflush = !cpu_cache_is_coherent(obj->base.dev,
 527                                                         obj->cache_level);
 528                 ret = i915_gem_object_wait_rendering(obj, true);
 529                 if (ret)
 530                         return ret;
 531
 532                 i915_gem_object_retire(obj);
 533         }
 534
 535         ret = i915_gem_object_get_pages(obj);
 536         if (ret)
 537                 return ret;
 538
 539         i915_gem_object_pin_pages(obj);
 540
 541         return ret;
 542 }
 543
 544 /* Per-page copy function for the shmem pread fastpath.
 545  * Flushes invalid cachelines before reading the target if
 546  * needs_clflush is set. */
 547 static int
 548 shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
 549                  char __user *user_data,
 550                  bool page_do_bit17_swizzling, bool needs_clflush)
 551 {
 552         char *vaddr;
 553         int ret;
 554
 555         if (unlikely(page_do_bit17_swizzling))
 556                 return -EINVAL;
 557
 558         vaddr = kmap_atomic(page);
 559         if (needs_clflush)
 560                 drm_clflush_virt_range(vaddr + shmem_page_offset,
 561                                        page_length);
 562         ret = __copy_to_user_inatomic(user_data,
 563                                       vaddr + shmem_page_offset,
 564                                       page_length);
 565         kunmap_atomic(vaddr);
 566
 567         return ret ? -EFAULT : 0;
 568 }
 569
 570 static void
 571 shmem_clflush_swizzled_range(char *addr, unsigned long length,
 572                              bool swizzled)
 573 {
 574         if (unlikely(swizzled)) {
 575                 unsigned long start = (unsigned long) addr;
 576                 unsigned long end = (unsigned long) addr + length;
 577
 578                 /* For swizzling simply ensure that we always flush both
 579                  * channels. Lame, but simple and it works. Swizzled
 580                  * pwrite/pread is far from a hotpath - current userspace
 581                  * doesn't use it at all. */
 582                 start = round_down(start, 128);
 583                 end = round_up(end, 128);
 584
 585                 drm_clflush_virt_range((void *)start, end - start);
 586         } else {
 587                 drm_clflush_virt_range(addr, length);
 588         }
 589
 590 }
 591
 592 /* Only difference to the fast-path function is that this can handle bit17
 593  * and uses non-atomic copy and kmap functions. */
 594 static int
 595 shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
 596                  char __user *user_data,
 597                  bool page_do_bit17_swizzling, bool needs_clflush)
 598 {
 599         char *vaddr;
 600         int ret;
 601
 602         vaddr = kmap(page);
 603         if (needs_clflush)
 604                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
 605                                              page_length,
 606                                              page_do_bit17_swizzling);
 607
 608         if (page_do_bit17_swizzling)
 609                 ret = __copy_to_user_swizzled(user_data,
 610                                               vaddr, shmem_page_offset,
 611                                               page_length);
 612         else
 613                 ret = __copy_to_user(user_data,
 614                                      vaddr + shmem_page_offset,
 615                                      page_length);
 616         kunmap(page);
 617
 618         return ret ? - EFAULT : 0;
 619 }
 620
 621 static int
 622 i915_gem_shmem_pread(struct drm_device *dev,
 623                      struct drm_i915_gem_object *obj,
 624                      struct drm_i915_gem_pread *args,
 625                      struct drm_file *file)
 626 {
 627         char __user *user_data;
 628         ssize_t remain;
 629         loff_t offset;
 630         int shmem_page_offset, page_length, ret = 0;
 631         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
 632         int prefaulted = 0;
 633         int needs_clflush = 0;
 634         struct sg_page_iter sg_iter;
 635
 636         user_data = to_user_ptr(args->data_ptr);
 637         remain = args->size;
 638
 639         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 640
 641         ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
 642         if (ret)
 643                 return ret;
 644
 645         offset = args->offset;
 646
 647         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
 648                          offset >> PAGE_SHIFT) {
 649                 struct page *page = sg_page_iter_page(&sg_iter);
 650
 651                 if (remain <= 0)
 652                         break;
 653
 654                 /* Operation in this page
 655                  *
 656                  * shmem_page_offset = offset within page in shmem file
 657                  * page_length = bytes to copy for this page
 658                  */
 659                 shmem_page_offset = offset_in_page(offset);
 660                 page_length = remain;
 661                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 662                         page_length = PAGE_SIZE - shmem_page_offset;
 663
 664                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
 665                         (page_to_phys(page) & (1 << 17)) != 0;
 666
 667                 ret = shmem_pread_fast(page, shmem_page_offset, page_length,
 668                                        user_data, page_do_bit17_swizzling,
 669                                        needs_clflush);
 670                 if (ret == 0)
 671                         goto next_page;
 672
 673                 mutex_unlock(&dev->struct_mutex);
 674
 675                 if (likely(!i915.prefault_disable) && !prefaulted) {
 676                         ret = fault_in_multipages_writeable(user_data, remain);
 677                         /* Userspace is tricking us, but we've already clobbered
 678                          * its pages with the prefault and promised to write the
 679                          * data up to the first fault. Hence ignore any errors
 680                          * and just continue. */
 681                         (void)ret;
 682                         prefaulted = 1;
 683                 }
 684
 685                 ret = shmem_pread_slow(page, shmem_page_offset, page_length,
 686                                        user_data, page_do_bit17_swizzling,
 687                                        needs_clflush);
 688
 689                 mutex_lock(&dev->struct_mutex);
 690
 691                 if (ret)
 692                         goto out;
 693
 694 next_page:
 695                 remain -= page_length;
 696                 user_data += page_length;
 697                 offset += page_length;
 698         }
 699
 700 out:
 701         i915_gem_object_unpin_pages(obj);
 702
 703         return ret;
 704 }
 705
 706 /**
 707  * Reads data from the object referenced by handle.
 708  *
 709  * On error, the contents of *data are undefined.
 710  */
 711 int
 712 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 713                      struct drm_file *file)
 714 {
 715         struct drm_i915_gem_pread *args = data;
 716         struct drm_i915_gem_object *obj;
 717         int ret = 0;
 718
 719         if (args->size == 0)
 720                 return 0;
 721
 722         if (!access_ok(VERIFY_WRITE,
 723                        to_user_ptr(args->data_ptr),
 724                        args->size))
 725                 return -EFAULT;
 726
 727         ret = i915_mutex_lock_interruptible(dev);
 728         if (ret)
 729                 return ret;
 730
 731         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
 732         if (&obj->base == NULL) {
 733                 ret = -ENOENT;
 734                 goto unlock;
 735         }
 736
 737         /* Bounds check source.  */
 738         if (args->offset > obj->base.size ||
 739             args->size > obj->base.size - args->offset) {
 740                 ret = -EINVAL;
 741                 goto out;
 742         }
 743
 744         /* prime objects have no backing filp to GEM pread/pwrite
 745          * pages from.
 746          */
 747         if (!obj->base.filp) {
 748                 ret = -EINVAL;
 749                 goto out;
 750         }
 751
 752         trace_i915_gem_object_pread(obj, args->offset, args->size);
 753
 754         ret = i915_gem_shmem_pread(dev, obj, args, file);
 755
 756 out:
 757         drm_gem_object_unreference(&obj->base);
 758 unlock:
 759         mutex_unlock(&dev->struct_mutex);
 760         return ret;
 761 }
 762
 763 /* This is the fast write path which cannot handle
 764  * page faults in the source data
 765  */
 766
 767 static inline int
 768 fast_user_write(struct io_mapping *mapping,
 769                 loff_t page_base, int page_offset,
 770                 char __user *user_data,
 771                 int length)
 772 {
 773         void __iomem *vaddr_atomic;
 774         void *vaddr;
 775         unsigned long unwritten;
 776
 777         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
 778         /* We can use the cpu mem copy function because this is X86. */
 779         vaddr = (void __force*)vaddr_atomic + page_offset;
 780         unwritten = __copy_from_user_inatomic_nocache(vaddr,
 781                                                       user_data, length);
 782         io_mapping_unmap_atomic(vaddr_atomic);
 783         return unwritten;
 784 }
 785
 786 /**
 787  * This is the fast pwrite path, where we copy the data directly from the
 788  * user into the GTT, uncached.
 789  */
 790 static int
 791 i915_gem_gtt_pwrite_fast(struct drm_device *dev,
 792                          struct drm_i915_gem_object *obj,
 793                          struct drm_i915_gem_pwrite *args,
 794                          struct drm_file *file)
 795 {
 796         struct drm_i915_private *dev_priv = dev->dev_private;
 797         ssize_t remain;
 798         loff_t offset, page_base;
 799         char __user *user_data;
 800         int page_offset, page_length, ret;
 801
 802         ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE | PIN_NONBLOCK);
 803         if (ret)
 804                 goto out;
 805
 806         ret = i915_gem_object_set_to_gtt_domain(obj, true);
 807         if (ret)
 808                 goto out_unpin;
 809
 810         ret = i915_gem_object_put_fence(obj);
 811         if (ret)
 812                 goto out_unpin;
 813
 814         user_data = to_user_ptr(args->data_ptr);
 815         remain = args->size;
 816
 817         offset = i915_gem_obj_ggtt_offset(obj) + args->offset;
 818
 819         while (remain > 0) {
 820                 /* Operation in this page
 821                  *
 822                  * page_base = page offset within aperture
 823                  * page_offset = offset within page
 824                  * page_length = bytes to copy for this page
 825                  */
 826                 page_base = offset & PAGE_MASK;
 827                 page_offset = offset_in_page(offset);
 828                 page_length = remain;
 829                 if ((page_offset + remain) > PAGE_SIZE)
 830                         page_length = PAGE_SIZE - page_offset;
 831
 832                 /* If we get a fault while copying data, then (presumably) our
 833                  * source page isn't available.  Return the error and we'll
 834                  * retry in the slow path.
 835                  */
 836                 if (fast_user_write(dev_priv->gtt.mappable, page_base,
 837                                     page_offset, user_data, page_length)) {
 838                         ret = -EFAULT;
 839                         goto out_unpin;
 840                 }
 841
 842                 remain -= page_length;
 843                 user_data += page_length;
 844                 offset += page_length;
 845         }
 846
 847 out_unpin:
 848         i915_gem_object_ggtt_unpin(obj);
 849 out:
 850         return ret;
 851 }
 852
 853 /* Per-page copy function for the shmem pwrite fastpath.
 854  * Flushes invalid cachelines before writing to the target if
 855  * needs_clflush_before is set and flushes out any written cachelines after
 856  * writing if needs_clflush is set. */
 857 static int
 858 shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
 859                   char __user *user_data,
 860                   bool page_do_bit17_swizzling,
 861                   bool needs_clflush_before,
 862                   bool needs_clflush_after)
 863 {
 864         char *vaddr;
 865         int ret;
 866
 867         if (unlikely(page_do_bit17_swizzling))
 868                 return -EINVAL;
 869
 870         vaddr = kmap_atomic(page);
 871         if (needs_clflush_before)
 872                 drm_clflush_virt_range(vaddr + shmem_page_offset,
 873                                        page_length);
 874         ret = __copy_from_user_inatomic(vaddr + shmem_page_offset,
 875                                         user_data, page_length);
 876         if (needs_clflush_after)
 877                 drm_clflush_virt_range(vaddr + shmem_page_offset,
 878                                        page_length);
 879         kunmap_atomic(vaddr);
 880
 881         return ret ? -EFAULT : 0;
 882 }
 883
 884 /* Only difference to the fast-path function is that this can handle bit17
 885  * and uses non-atomic copy and kmap functions. */
 886 static int
 887 shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
 888                   char __user *user_data,
 889                   bool page_do_bit17_swizzling,
 890                   bool needs_clflush_before,
 891                   bool needs_clflush_after)
 892 {
 893         char *vaddr;
 894         int ret;
 895
 896         vaddr = kmap(page);
 897         if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
 898                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
 899                                              page_length,
 900                                              page_do_bit17_swizzling);
 901         if (page_do_bit17_swizzling)
 902                 ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
 903                                                 user_data,
 904                                                 page_length);
 905         else
 906                 ret = __copy_from_user(vaddr + shmem_page_offset,
 907                                        user_data,
 908                                        page_length);
 909         if (needs_clflush_after)
 910                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
 911                                              page_length,
 912                                              page_do_bit17_swizzling);
 913         kunmap(page);
 914
 915         return ret ? -EFAULT : 0;
 916 }
 917
 918 static int
 919 i915_gem_shmem_pwrite(struct drm_device *dev,
 920                       struct drm_i915_gem_object *obj,
 921                       struct drm_i915_gem_pwrite *args,
 922                       struct drm_file *file)
 923 {
 924         ssize_t remain;
 925         loff_t offset;
 926         char __user *user_data;
 927         int shmem_page_offset, page_length, ret = 0;
 928         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
 929         int hit_slowpath = 0;
 930         int needs_clflush_after = 0;
 931         int needs_clflush_before = 0;
 932         struct sg_page_iter sg_iter;
 933
 934         user_data = to_user_ptr(args->data_ptr);
 935         remain = args->size;
 936
 937         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 938
 939         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
 940                 /* If we're not in the cpu write domain, set ourself into the gtt
 941                  * write domain and manually flush cachelines (if required). This
 942                  * optimizes for the case when the gpu will use the data
 943                  * right away and we therefore have to clflush anyway. */
 944                 needs_clflush_after = cpu_write_needs_clflush(obj);
 945                 ret = i915_gem_object_wait_rendering(obj, false);
 946                 if (ret)
 947                         return ret;
 948
 949                 i915_gem_object_retire(obj);
 950         }
 951         /* Same trick applies to invalidate partially written cachelines read
 952          * before writing. */
 953         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
 954                 needs_clflush_before =
 955                         !cpu_cache_is_coherent(dev, obj->cache_level);
 956
 957         ret = i915_gem_object_get_pages(obj);
 958         if (ret)
 959                 return ret;
 960
 961         i915_gem_object_pin_pages(obj);
 962
 963         offset = args->offset;
 964         obj->dirty = 1;
 965
 966         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
 967                          offset >> PAGE_SHIFT) {
 968                 struct page *page = sg_page_iter_page(&sg_iter);
 969                 int partial_cacheline_write;
 970
 971                 if (remain <= 0)
 972                         break;
 973
 974                 /* Operation in this page
 975                  *
 976                  * shmem_page_offset = offset within page in shmem file
 977                  * page_length = bytes to copy for this page
 978                  */
 979                 shmem_page_offset = offset_in_page(offset);
 980
 981                 page_length = remain;
 982                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 983                         page_length = PAGE_SIZE - shmem_page_offset;
 984
 985                 /* If we don't overwrite a cacheline completely we need to be
 986                  * careful to have up-to-date data by first clflushing. Don't
 987                  * overcomplicate things and flush the entire patch. */
 988                 partial_cacheline_write = needs_clflush_before &&
 989                         ((shmem_page_offset | page_length)
 990                                 & (boot_cpu_data.x86_clflush_size - 1));
 991
 992                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
 993                         (page_to_phys(page) & (1 << 17)) != 0;
 994
 995                 ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
 996                                         user_data, page_do_bit17_swizzling,
 997                                         partial_cacheline_write,
 998                                         needs_clflush_after);
 999                 if (ret == 0)
1000                         goto next_page;
1001
1002                 hit_slowpath = 1;
1003                 mutex_unlock(&dev->struct_mutex);
1004                 ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
1005                                         user_data, page_do_bit17_swizzling,
1006                                         partial_cacheline_write,
1007                                         needs_clflush_after);
1008
1009                 mutex_lock(&dev->struct_mutex);
1010
1011                 if (ret)
1012                         goto out;
1013
1014 next_page:
1015                 remain -= page_length;
1016                 user_data += page_length;
1017                 offset += page_length;
1018         }
1019
1020 out:
1021         i915_gem_object_unpin_pages(obj);
1022
1023         if (hit_slowpath) {
1024                 /*
1025                  * Fixup: Flush cpu caches in case we didn't flush the dirty
1026                  * cachelines in-line while writing and the object moved
1027                  * out of the cpu write domain while we've dropped the lock.
1028                  */
1029                 if (!needs_clflush_after &&
1030                     obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
1031                         if (i915_gem_clflush_object(obj, obj->pin_display))
1032                                 i915_gem_chipset_flush(dev);
1033                 }
1034         }
1035
1036         if (needs_clflush_after)
1037                 i915_gem_chipset_flush(dev);
1038
1039         return ret;
1040 }
1041
1042 /**
1043  * Writes data to the object referenced by handle.
1044  *
1045  * On error, the contents of the buffer that were to be modified are undefined.
1046  */
1047 int
1048 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
1049                       struct drm_file *file)
1050 {
1051         struct drm_i915_private *dev_priv = dev->dev_private;
1052         struct drm_i915_gem_pwrite *args = data;
1053         struct drm_i915_gem_object *obj;
1054         int ret;
1055
1056         if (args->size == 0)
1057                 return 0;
1058
1059         if (!access_ok(VERIFY_READ,
1060                        to_user_ptr(args->data_ptr),
1061                        args->size))
1062                 return -EFAULT;
1063
1064         if (likely(!i915.prefault_disable)) {
1065                 ret = fault_in_multipages_readable(to_user_ptr(args->data_ptr),
1066                                                    args->size);
1067                 if (ret)
1068                         return -EFAULT;
1069         }
1070
1071         intel_runtime_pm_get(dev_priv);
1072
1073         ret = i915_mutex_lock_interruptible(dev);
1074         if (ret)
1075                 goto put_rpm;
1076
1077         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1078         if (&obj->base == NULL) {
1079                 ret = -ENOENT;
1080                 goto unlock;
1081         }
1082
1083         /* Bounds check destination. */
1084         if (args->offset > obj->base.size ||
1085             args->size > obj->base.size - args->offset) {
1086                 ret = -EINVAL;
1087                 goto out;
1088         }
1089
1090         /* prime objects have no backing filp to GEM pread/pwrite
1091          * pages from.
1092          */
1093         if (!obj->base.filp) {
1094                 ret = -EINVAL;
1095                 goto out;
1096         }
1097
1098         trace_i915_gem_object_pwrite(obj, args->offset, args->size);
1099
1100         ret = -EFAULT;
1101         /* We can only do the GTT pwrite on untiled buffers, as otherwise
1102          * it would end up going through the fenced access, and we'll get
1103          * different detiling behavior between reading and writing.
1104          * pread/pwrite currently are reading and writing from the CPU
1105          * perspective, requiring manual detiling by the client.
1106          */
1107         if (obj->tiling_mode == I915_TILING_NONE &&
1108             obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
1109             cpu_write_needs_clflush(obj)) {
1110                 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
1111                 /* Note that the gtt paths might fail with non-page-backed user
1112                  * pointers (e.g. gtt mappings when moving data between
1113                  * textures). Fallback to the shmem path in that case. */
1114         }
1115
1116         if (ret == -EFAULT || ret == -ENOSPC) {
1117                 if (obj->phys_handle)
1118                         ret = i915_gem_phys_pwrite(obj, args, file);
1119                 else
1120                         ret = i915_gem_shmem_pwrite(dev, obj, args, file);
1121         }
1122
1123 out:
1124         drm_gem_object_unreference(&obj->base);
1125 unlock:
1126         mutex_unlock(&dev->struct_mutex);
1127 put_rpm:
1128         intel_runtime_pm_put(dev_priv);
1129
1130         return ret;
1131 }
1132
1133 int
1134 i915_gem_check_wedge(struct i915_gpu_error *error,
1135                      bool interruptible)
1136 {
1137         if (i915_reset_in_progress(error)) {
1138                 /* Non-interruptible callers can't handle -EAGAIN, hence return
1139                  * -EIO unconditionally for these. */
1140                 if (!interruptible)
1141                         return -EIO;
1142
1143                 /* Recovery complete, but the reset failed ... */
1144                 if (i915_terminally_wedged(error))
1145                         return -EIO;
1146
1147                 /*
1148                  * Check if GPU Reset is in progress - we need intel_ring_begin
1149                  * to work properly to reinit the hw state while the gpu is
1150                  * still marked as reset-in-progress. Handle this with a flag.
1151                  */
1152                 if (!error->reload_in_reset)
1153                         return -EAGAIN;
1154         }
1155
1156         return 0;
1157 }
1158
1159 /*
1160  * Compare seqno against outstanding lazy request. Emit a request if they are
1161  * equal.
1162  */
1163 int
1164 i915_gem_check_olr(struct intel_engine_cs *ring, u32 seqno)
1165 {
1166         int ret;
1167
1168         BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));
1169
1170         ret = 0;
1171         if (seqno == ring->outstanding_lazy_seqno)
1172                 ret = i915_add_request(ring, NULL);
1173
1174         return ret;
1175 }
1176
1177 static void fake_irq(unsigned long data)
1178 {
1179         wake_up_process((struct task_struct *)data);
1180 }
1181
1182 static bool missed_irq(struct drm_i915_private *dev_priv,
1183                        struct intel_engine_cs *ring)
1184 {
1185         return test_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings);
1186 }
1187
1188 static bool can_wait_boost(struct drm_i915_file_private *file_priv)
1189 {
1190         if (file_priv == NULL)
1191                 return true;
1192
1193         return !atomic_xchg(&file_priv->rps_wait_boost, true);
1194 }
1195
1196 /**
1197  * __i915_wait_seqno - wait until execution of seqno has finished
1198  * @ring: the ring expected to report seqno
1199  * @seqno: duh!
1200  * @reset_counter: reset sequence associated with the given seqno
1201  * @interruptible: do an interruptible wait (normally yes)
1202  * @timeout: in - how long to wait (NULL forever); out - how much time remaining
1203  *
1204  * Note: It is of utmost importance that the passed in seqno and reset_counter
1205  * values have been read by the caller in an smp safe manner. Where read-side
1206  * locks are involved, it is sufficient to read the reset_counter before
1207  * unlocking the lock that protects the seqno. For lockless tricks, the
1208  * reset_counter _must_ be read before, and an appropriate smp_rmb must be
1209  * inserted.
1210  *
1211  * Returns 0 if the seqno was found within the alloted time. Else returns the
1212  * errno with remaining time filled in timeout argument.
1213  */
1214 int __i915_wait_seqno(struct intel_engine_cs *ring, u32 seqno,
1215                         unsigned reset_counter,
1216                         bool interruptible,
1217                         s64 *timeout,
1218                         struct drm_i915_file_private *file_priv)
1219 {
1220         struct drm_device *dev = ring->dev;
1221         struct drm_i915_private *dev_priv = dev->dev_private;
1222         const bool irq_test_in_progress =
1223                 ACCESS_ONCE(dev_priv->gpu_error.test_irq_rings) & intel_ring_flag(ring);
1224         DEFINE_WAIT(wait);
1225         unsigned long timeout_expire;
1226         s64 before, now;
1227         int ret;
1228
1229         WARN(!intel_irqs_enabled(dev_priv), "IRQs disabled");
1230
1231         if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
1232                 return 0;
1233
1234         timeout_expire = timeout ?
1235                 jiffies + nsecs_to_jiffies_timeout((u64)*timeout) : 0;
1236
1237         if (INTEL_INFO(dev)->gen >= 6 && ring->id == RCS && can_wait_boost(file_priv)) {
1238                 gen6_rps_boost(dev_priv);
1239                 if (file_priv)
1240                         mod_delayed_work(dev_priv->wq,
1241                                          &file_priv->mm.idle_work,
1242                                          msecs_to_jiffies(100));
1243         }
1244
1245         if (!irq_test_in_progress && WARN_ON(!ring->irq_get(ring)))
1246                 return -ENODEV;
1247
1248         /* Record current time in case interrupted by signal, or wedged */
1249         trace_i915_gem_request_wait_begin(ring, seqno);
1250         before = ktime_get_raw_ns();
1251         for (;;) {
1252                 struct timer_list timer;
1253
1254                 prepare_to_wait(&ring->irq_queue, &wait,
1255                                 interruptible ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE);
1256
1257                 /* We need to check whether any gpu reset happened in between
1258                  * the caller grabbing the seqno and now ... */
1259                 if (reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter)) {
1260                         /* ... but upgrade the -EAGAIN to an -EIO if the gpu
1261                          * is truely gone. */
1262                         ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
1263                         if (ret == 0)
1264                                 ret = -EAGAIN;
1265                         break;
1266                 }
1267
1268                 if (i915_seqno_passed(ring->get_seqno(ring, false), seqno)) {
1269                         ret = 0;
1270                         break;
1271                 }
1272
1273                 if (interruptible && signal_pending(current)) {
1274                         ret = -ERESTARTSYS;
1275                         break;
1276                 }
1277
1278                 if (timeout && time_after_eq(jiffies, timeout_expire)) {
1279                         ret = -ETIME;
1280                         break;
1281                 }
1282
1283                 timer.function = NULL;
1284                 if (timeout || missed_irq(dev_priv, ring)) {
1285                         unsigned long expire;
1286
1287                         setup_timer_on_stack(&timer, fake_irq, (unsigned long)current);
1288                         expire = missed_irq(dev_priv, ring) ? jiffies + 1 : timeout_expire;
1289                         mod_timer(&timer, expire);
1290                 }
1291
1292                 io_schedule();
1293
1294                 if (timer.function) {
1295                         del_singleshot_timer_sync(&timer);
1296                         destroy_timer_on_stack(&timer);
1297                 }
1298         }
1299         now = ktime_get_raw_ns();
1300         trace_i915_gem_request_wait_end(ring, seqno);
1301
1302         if (!irq_test_in_progress)
1303                 ring->irq_put(ring);
1304
1305         finish_wait(&ring->irq_queue, &wait);
1306
1307         if (timeout) {
1308                 s64 tres = *timeout - (now - before);
1309
1310                 *timeout = tres < 0 ? 0 : tres;
1311
1312                 /*
1313                  * Apparently ktime isn't accurate enough and occasionally has a
1314                  * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch
1315                  * things up to make the test happy. We allow up to 1 jiffy.
1316                  *
1317                  * This is a regrssion from the timespec->ktime conversion.
1318                  */
1319                 if (ret == -ETIME && *timeout < jiffies_to_usecs(1)*1000)
1320                         *timeout = 0;
1321         }
1322
1323         return ret;
1324 }
1325
1326 /**
1327  * Waits for a sequence number to be signaled, and cleans up the
1328  * request and object lists appropriately for that event.
1329  */
1330 int
1331 i915_wait_seqno(struct intel_engine_cs *ring, uint32_t seqno)
1332 {
1333         struct drm_device *dev = ring->dev;
1334         struct drm_i915_private *dev_priv = dev->dev_private;
1335         bool interruptible = dev_priv->mm.interruptible;
1336         unsigned reset_counter;
1337         int ret;
1338
1339         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1340         BUG_ON(seqno == 0);
1341
1342         ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
1343         if (ret)
1344                 return ret;
1345
1346         ret = i915_gem_check_olr(ring, seqno);
1347         if (ret)
1348                 return ret;
1349
1350         reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
1351         return __i915_wait_seqno(ring, seqno, reset_counter, interruptible,
1352                                  NULL, NULL);
1353 }
1354
1355 static int
1356 i915_gem_object_wait_rendering__tail(struct drm_i915_gem_object *obj)
1357 {
1358         if (!obj->active)
1359                 return 0;
1360
1361         /* Manually manage the write flush as we may have not yet
1362          * retired the buffer.
1363          *
1364          * Note that the last_write_seqno is always the earlier of
1365          * the two (read/write) seqno, so if we haved successfully waited,
1366          * we know we have passed the last write.
1367          */
1368         obj->last_write_seqno = 0;
1369
1370         return 0;
1371 }
1372
1373 /**
1374  * Ensures that all rendering to the object has completed and the object is
1375  * safe to unbind from the GTT or access from the CPU.
1376  */
1377 static __must_check int
1378 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
1379                                bool readonly)
1380 {
1381         struct intel_engine_cs *ring = obj->ring;
1382         u32 seqno;
1383         int ret;
1384
1385         seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
1386         if (seqno == 0)
1387                 return 0;
1388
1389         ret = i915_wait_seqno(ring, seqno);
1390         if (ret)
1391                 return ret;
1392
1393         return i915_gem_object_wait_rendering__tail(obj);
1394 }
1395
1396 /* A nonblocking variant of the above wait. This is a highly dangerous routine
1397  * as the object state may change during this call.
1398  */
1399 static __must_check int
1400 i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
1401                                             struct drm_i915_file_private *file_priv,
1402                                             bool readonly)
1403 {
1404         struct drm_device *dev = obj->base.dev;
1405         struct drm_i915_private *dev_priv = dev->dev_private;
1406         struct intel_engine_cs *ring = obj->ring;
1407         unsigned reset_counter;
1408         u32 seqno;
1409         int ret;
1410
1411         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1412         BUG_ON(!dev_priv->mm.interruptible);
1413
1414         seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
1415         if (seqno == 0)
1416                 return 0;
1417
1418         ret = i915_gem_check_wedge(&dev_priv->gpu_error, true);
1419         if (ret)
1420                 return ret;
1421
1422         ret = i915_gem_check_olr(ring, seqno);
1423         if (ret)
1424                 return ret;
1425
1426         reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
1427         mutex_unlock(&dev->struct_mutex);
1428         ret = __i915_wait_seqno(ring, seqno, reset_counter, true, NULL,
1429                                 file_priv);
1430         mutex_lock(&dev->struct_mutex);
1431         if (ret)
1432                 return ret;
1433
1434         return i915_gem_object_wait_rendering__tail(obj);
1435 }
1436
1437 /**
1438  * Called when user space prepares to use an object with the CPU, either
1439  * through the mmap ioctl's mapping or a GTT mapping.
1440  */
1441 int
1442 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1443                           struct drm_file *file)
1444 {
1445         struct drm_i915_gem_set_domain *args = data;
1446         struct drm_i915_gem_object *obj;
1447         uint32_t read_domains = args->read_domains;
1448         uint32_t write_domain = args->write_domain;
1449         int ret;
1450
1451         /* Only handle setting domains to types used by the CPU. */
1452         if (write_domain & I915_GEM_GPU_DOMAINS)
1453                 return -EINVAL;
1454
1455         if (read_domains & I915_GEM_GPU_DOMAINS)
1456                 return -EINVAL;
1457
1458         /* Having something in the write domain implies it's in the read
1459          * domain, and only that read domain.  Enforce that in the request.
1460          */
1461         if (write_domain != 0 && read_domains != write_domain)
1462                 return -EINVAL;
1463
1464         ret = i915_mutex_lock_interruptible(dev);
1465         if (ret)
1466                 return ret;
1467
1468         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1469         if (&obj->base == NULL) {
1470                 ret = -ENOENT;
1471                 goto unlock;
1472         }
1473
1474         /* Try to flush the object off the GPU without holding the lock.
1475          * We will repeat the flush holding the lock in the normal manner
1476          * to catch cases where we are gazumped.
1477          */
1478         ret = i915_gem_object_wait_rendering__nonblocking(obj,
1479                                                           file->driver_priv,
1480                                                           !write_domain);
1481         if (ret)
1482                 goto unref;
1483
1484         if (read_domains & I915_GEM_DOMAIN_GTT) {
1485                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1486
1487                 /* Silently promote "you're not bound, there was nothing to do"
1488                  * to success, since the client was just asking us to
1489                  * make sure everything was done.
1490                  */
1491                 if (ret == -EINVAL)
1492                         ret = 0;
1493         } else {
1494                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1495         }
1496
1497 unref:
1498         drm_gem_object_unreference(&obj->base);
1499 unlock:
1500         mutex_unlock(&dev->struct_mutex);
1501         return ret;
1502 }
1503
1504 /**
1505  * Called when user space has done writes to this buffer
1506  */
1507 int
1508 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1509                          struct drm_file *file)
1510 {
1511         struct drm_i915_gem_sw_finish *args = data;
1512         struct drm_i915_gem_object *obj;
1513         int ret = 0;
1514
1515         ret = i915_mutex_lock_interruptible(dev);
1516         if (ret)
1517                 return ret;
1518
1519         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1520         if (&obj->base == NULL) {
1521                 ret = -ENOENT;
1522                 goto unlock;
1523         }
1524
1525         /* Pinned buffers may be scanout, so flush the cache */
1526         if (obj->pin_display)
1527                 i915_gem_object_flush_cpu_write_domain(obj, true);
1528
1529         drm_gem_object_unreference(&obj->base);
1530 unlock:
1531         mutex_unlock(&dev->struct_mutex);
1532         return ret;
1533 }
1534
1535 /**
1536  * Maps the contents of an object, returning the address it is mapped
1537  * into.
1538  *
1539  * While the mapping holds a reference on the contents of the object, it doesn't
1540  * imply a ref on the object itself.
1541  *
1542  * IMPORTANT:
1543  *
1544  * DRM driver writers who look a this function as an example for how to do GEM
1545  * mmap support, please don't implement mmap support like here. The modern way
1546  * to implement DRM mmap support is with an mmap offset ioctl (like
1547  * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly.
1548  * That way debug tooling like valgrind will understand what's going on, hiding
1549  * the mmap call in a driver private ioctl will break that. The i915 driver only
1550  * does cpu mmaps this way because we didn't know better.
1551  */
1552 int
1553 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1554                     struct drm_file *file)
1555 {
1556         struct drm_i915_gem_mmap *args = data;
1557         struct drm_gem_object *obj;
1558         unsigned long addr;
1559
1560         obj = drm_gem_object_lookup(dev, file, args->handle);
1561         if (obj == NULL)
1562                 return -ENOENT;
1563
1564         /* prime objects have no backing filp to GEM mmap
1565          * pages from.
1566          */
1567         if (!obj->filp) {
1568                 drm_gem_object_unreference_unlocked(obj);
1569                 return -EINVAL;
1570         }
1571
1572         addr = vm_mmap(obj->filp, 0, args->size,
1573                        PROT_READ | PROT_WRITE, MAP_SHARED,
1574                        args->offset);
1575         drm_gem_object_unreference_unlocked(obj);
1576         if (IS_ERR((void *)addr))
1577                 return addr;
1578
1579         args->addr_ptr = (uint64_t) addr;
1580
1581         return 0;
1582 }
1583
1584 /**
1585  * i915_gem_fault - fault a page into the GTT
1586  * vma: VMA in question
1587  * vmf: fault info
1588  *
1589  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1590  * from userspace.  The fault handler takes care of binding the object to
1591  * the GTT (if needed), allocating and programming a fence register (again,
1592  * only if needed based on whether the old reg is still valid or the object
1593  * is tiled) and inserting a new PTE into the faulting process.
1594  *
1595  * Note that the faulting process may involve evicting existing objects
1596  * from the GTT and/or fence registers to make room.  So performance may
1597  * suffer if the GTT working set is large or there are few fence registers
1598  * left.
1599  */
1600 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1601 {
1602         struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
1603         struct drm_device *dev = obj->base.dev;
1604         struct drm_i915_private *dev_priv = dev->dev_private;
1605         pgoff_t page_offset;
1606         unsigned long pfn;
1607         int ret = 0;
1608         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1609
1610         intel_runtime_pm_get(dev_priv);
1611
1612         /* We don't use vmf->pgoff since that has the fake offset */
1613         page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1614                 PAGE_SHIFT;
1615
1616         ret = i915_mutex_lock_interruptible(dev);
1617         if (ret)
1618                 goto out;
1619
1620         trace_i915_gem_object_fault(obj, page_offset, true, write);
1621
1622         /* Try to flush the object off the GPU first without holding the lock.
1623          * Upon reacquiring the lock, we will perform our sanity checks and then
1624          * repeat the flush holding the lock in the normal manner to catch cases
1625          * where we are gazumped.
1626          */
1627         ret = i915_gem_object_wait_rendering__nonblocking(obj, NULL, !write);
1628         if (ret)
1629                 goto unlock;
1630
1631         /* Access to snoopable pages through the GTT is incoherent. */
1632         if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) {
1633                 ret = -EFAULT;
1634                 goto unlock;
1635         }
1636
1637         /* Now bind it into the GTT if needed */
1638         ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE);
1639         if (ret)
1640                 goto unlock;
1641
1642         ret = i915_gem_object_set_to_gtt_domain(obj, write);
1643         if (ret)
1644                 goto unpin;
1645
1646         ret = i915_gem_object_get_fence(obj);
1647         if (ret)
1648                 goto unpin;
1649
1650         /* Finally, remap it using the new GTT offset */
1651         pfn = dev_priv->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj);
1652         pfn >>= PAGE_SHIFT;
1653
1654         if (!obj->fault_mappable) {
1655                 unsigned long size = min_t(unsigned long,
1656                                            vma->vm_end - vma->vm_start,
1657                                            obj->base.size);
1658                 int i;
1659
1660                 for (i = 0; i < size >> PAGE_SHIFT; i++) {
1661                         ret = vm_insert_pfn(vma,
1662                                             (unsigned long)vma->vm_start + i * PAGE_SIZE,
1663                                             pfn + i);
1664                         if (ret)
1665                                 break;
1666                 }
1667
1668                 obj->fault_mappable = true;
1669         } else
1670                 ret = vm_insert_pfn(vma,
1671                                     (unsigned long)vmf->virtual_address,
1672                                     pfn + page_offset);
1673 unpin:
1674         i915_gem_object_ggtt_unpin(obj);
1675 unlock:
1676         mutex_unlock(&dev->struct_mutex);
1677 out:
1678         switch (ret) {
1679         case -EIO:
1680                 /*
1681                  * We eat errors when the gpu is terminally wedged to avoid
1682                  * userspace unduly crashing (gl has no provisions for mmaps to
1683                  * fail). But any other -EIO isn't ours (e.g. swap in failure)
1684                  * and so needs to be reported.
1685                  */
1686                 if (!i915_terminally_wedged(&dev_priv->gpu_error)) {
1687                         ret = VM_FAULT_SIGBUS;
1688                         break;
1689                 }
1690         case -EAGAIN:
1691                 /*
1692                  * EAGAIN means the gpu is hung and we'll wait for the error
1693                  * handler to reset everything when re-faulting in
1694                  * i915_mutex_lock_interruptible.
1695                  */
1696         case 0:
1697         case -ERESTARTSYS:
1698         case -EINTR:
1699         case -EBUSY:
1700                 /*
1701                  * EBUSY is ok: this just means that another thread
1702                  * already did the job.
1703                  */
1704                 ret = VM_FAULT_NOPAGE;
1705                 break;
1706         case -ENOMEM:
1707                 ret = VM_FAULT_OOM;
1708                 break;
1709         case -ENOSPC:
1710         case -EFAULT:
1711                 ret = VM_FAULT_SIGBUS;
1712                 break;
1713         default:
1714                 WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret);
1715                 ret = VM_FAULT_SIGBUS;
1716                 break;
1717         }
1718
1719         intel_runtime_pm_put(dev_priv);
1720         return ret;
1721 }
1722
1723 /**
1724  * i915_gem_release_mmap - remove physical page mappings
1725  * @obj: obj in question
1726  *
1727  * Preserve the reservation of the mmapping with the DRM core code, but
1728  * relinquish ownership of the pages back to the system.
1729  *
1730  * It is vital that we remove the page mapping if we have mapped a tiled
1731  * object through the GTT and then lose the fence register due to
1732  * resource pressure. Similarly if the object has been moved out of the
1733  * aperture, than pages mapped into userspace must be revoked. Removing the
1734  * mapping will then trigger a page fault on the next user access, allowing
1735  * fixup by i915_gem_fault().
1736  */
1737 void
1738 i915_gem_release_mmap(struct drm_i915_gem_object *obj)
1739 {
1740         if (!obj->fault_mappable)
1741                 return;
1742
1743         drm_vma_node_unmap(&obj->base.vma_node,
1744                            obj->base.dev->anon_inode->i_mapping);
1745         obj->fault_mappable = false;
1746 }
1747
1748 void
1749 i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv)
1750 {
1751         struct drm_i915_gem_object *obj;
1752
1753         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
1754                 i915_gem_release_mmap(obj);
1755 }
1756
1757 uint32_t
1758 i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
1759 {
1760         uint32_t gtt_size;
1761
1762         if (INTEL_INFO(dev)->gen >= 4 ||
1763             tiling_mode == I915_TILING_NONE)
1764                 return size;
1765
1766         /* Previous chips need a power-of-two fence region when tiling */
1767         if (INTEL_INFO(dev)->gen == 3)
1768                 gtt_size = 1024*1024;
1769         else
1770                 gtt_size = 512*1024;
1771
1772         while (gtt_size < size)
1773                 gtt_size <<= 1;
1774
1775         return gtt_size;
1776 }
1777
1778 /**
1779  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1780  * @obj: object to check
1781  *
1782  * Return the required GTT alignment for an object, taking into account
1783  * potential fence register mapping.
1784  */
1785 uint32_t
1786 i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
1787                            int tiling_mode, bool fenced)
1788 {
1789         /*
1790          * Minimum alignment is 4k (GTT page size), but might be greater
1791          * if a fence register is needed for the object.
1792          */
1793         if (INTEL_INFO(dev)->gen >= 4 || (!fenced && IS_G33(dev)) ||
1794             tiling_mode == I915_TILING_NONE)
1795                 return 4096;
1796
1797         /*
1798          * Previous chips need to be aligned to the size of the smallest
1799          * fence register that can contain the object.
1800          */
1801         return i915_gem_get_gtt_size(dev, size, tiling_mode);
1802 }
1803
1804 static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
1805 {
1806         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
1807         int ret;
1808
1809         if (drm_vma_node_has_offset(&obj->base.vma_node))
1810                 return 0;
1811
1812         dev_priv->mm.shrinker_no_lock_stealing = true;
1813
1814         ret = drm_gem_create_mmap_offset(&obj->base);
1815         if (ret != -ENOSPC)
1816                 goto out;
1817
1818         /* Badly fragmented mmap space? The only way we can recover
1819          * space is by destroying unwanted objects. We can't randomly release
1820          * mmap_offsets as userspace expects them to be persistent for the
1821          * lifetime of the objects. The closest we can is to release the
1822          * offsets on purgeable objects by truncating it and marking it purged,
1823          * which prevents userspace from ever using that object again.
1824          */
1825         i915_gem_shrink(dev_priv,
1826                         obj->base.size >> PAGE_SHIFT,
1827                         I915_SHRINK_BOUND |
1828                         I915_SHRINK_UNBOUND |
1829                         I915_SHRINK_PURGEABLE);
1830         ret = drm_gem_create_mmap_offset(&obj->base);
1831         if (ret != -ENOSPC)
1832                 goto out;
1833
1834         i915_gem_shrink_all(dev_priv);
1835         ret = drm_gem_create_mmap_offset(&obj->base);
1836 out:
1837         dev_priv->mm.shrinker_no_lock_stealing = false;
1838
1839         return ret;
1840 }
1841
1842 static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj)
1843 {
1844         drm_gem_free_mmap_offset(&obj->base);
1845 }
1846
1847 int
1848 i915_gem_mmap_gtt(struct drm_file *file,
1849                   struct drm_device *dev,
1850                   uint32_t handle,
1851                   uint64_t *offset)
1852 {
1853         struct drm_i915_private *dev_priv = dev->dev_private;
1854         struct drm_i915_gem_object *obj;
1855         int ret;
1856
1857         ret = i915_mutex_lock_interruptible(dev);
1858         if (ret)
1859                 return ret;
1860
1861         obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
1862         if (&obj->base == NULL) {
1863                 ret = -ENOENT;
1864                 goto unlock;
1865         }
1866
1867         if (obj->base.size > dev_priv->gtt.mappable_end) {
1868                 ret = -E2BIG;
1869                 goto out;
1870         }
1871
1872         if (obj->madv != I915_MADV_WILLNEED) {
1873                 DRM_DEBUG("Attempting to mmap a purgeable buffer\n");
1874                 ret = -EFAULT;
1875                 goto out;
1876         }
1877
1878         ret = i915_gem_object_create_mmap_offset(obj);
1879         if (ret)
1880                 goto out;
1881
1882         *offset = drm_vma_node_offset_addr(&obj->base.vma_node);
1883
1884 out:
1885         drm_gem_object_unreference(&obj->base);
1886 unlock:
1887         mutex_unlock(&dev->struct_mutex);
1888         return ret;
1889 }
1890
1891 /**
1892  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1893  * @dev: DRM device
1894  * @data: GTT mapping ioctl data
1895  * @file: GEM object info
1896  *
1897  * Simply returns the fake offset to userspace so it can mmap it.
1898  * The mmap call will end up in drm_gem_mmap(), which will set things
1899  * up so we can get faults in the handler above.
1900  *
1901  * The fault handler will take care of binding the object into the GTT
1902  * (since it may have been evicted to make room for something), allocating
1903  * a fence register, and mapping the appropriate aperture address into
1904  * userspace.
1905  */
1906 int
1907 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1908                         struct drm_file *file)
1909 {
1910         struct drm_i915_gem_mmap_gtt *args = data;
1911
1912         return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
1913 }
1914
1915 static inline int
1916 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
1917 {
1918         return obj->madv == I915_MADV_DONTNEED;
1919 }
1920
1921 /* Immediately discard the backing storage */
1922 static void
1923 i915_gem_object_truncate(struct drm_i915_gem_object *obj)
1924 {
1925         i915_gem_object_free_mmap_offset(obj);
1926
1927         if (obj->base.filp == NULL)
1928                 return;
1929
1930         /* Our goal here is to return as much of the memory as
1931          * is possible back to the system as we are called from OOM.
1932          * To do this we must instruct the shmfs to drop all of its
1933          * backing pages, *now*.
1934          */
1935         shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);
1936         obj->madv = __I915_MADV_PURGED;
1937 }
1938
1939 /* Try to discard unwanted pages */
1940 static void
1941 i915_gem_object_invalidate(struct drm_i915_gem_object *obj)
1942 {
1943         struct address_space *mapping;
1944
1945         switch (obj->madv) {
1946         case I915_MADV_DONTNEED:
1947                 i915_gem_object_truncate(obj);
1948         case __I915_MADV_PURGED:
1949                 return;
1950         }
1951
1952         if (obj->base.filp == NULL)
1953                 return;
1954
1955         mapping = file_inode(obj->base.filp)->i_mapping,
1956         invalidate_mapping_pages(mapping, 0, (loff_t)-1);
1957 }
1958
1959 static void
1960 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
1961 {
1962         struct sg_page_iter sg_iter;
1963         int ret;
1964
1965         BUG_ON(obj->madv == __I915_MADV_PURGED);
1966
1967         ret = i915_gem_object_set_to_cpu_domain(obj, true);
1968         if (ret) {
1969                 /* In the event of a disaster, abandon all caches and
1970                  * hope for the best.
1971                  */
1972                 WARN_ON(ret != -EIO);
1973                 i915_gem_clflush_object(obj, true);
1974                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
1975         }
1976
1977         if (i915_gem_object_needs_bit17_swizzle(obj))
1978                 i915_gem_object_save_bit_17_swizzle(obj);
1979
1980         if (obj->madv == I915_MADV_DONTNEED)
1981                 obj->dirty = 0;
1982
1983         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
1984                 struct page *page = sg_page_iter_page(&sg_iter);
1985
1986                 if (obj->dirty)
1987                         set_page_dirty(page);
1988
1989                 if (obj->madv == I915_MADV_WILLNEED)
1990                         mark_page_accessed(page);
1991
1992                 page_cache_release(page);
1993         }
1994         obj->dirty = 0;
1995
1996         sg_free_table(obj->pages);
1997         kfree(obj->pages);
1998 }
1999
2000 int
2001 i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
2002 {
2003         const struct drm_i915_gem_object_ops *ops = obj->ops;
2004
2005         if (obj->pages == NULL)
2006                 return 0;
2007
2008         if (obj->pages_pin_count)
2009                 return -EBUSY;
2010
2011         BUG_ON(i915_gem_obj_bound_any(obj));
2012
2013         /* ->put_pages might need to allocate memory for the bit17 swizzle
2014          * array, hence protect them from being reaped by removing them from gtt
2015          * lists early. */
2016         list_del(&obj->global_list);
2017
2018         ops->put_pages(obj);
2019         obj->pages = NULL;
2020
2021         i915_gem_object_invalidate(obj);
2022
2023         return 0;
2024 }
2025
2026 unsigned long
2027 i915_gem_shrink(struct drm_i915_private *dev_priv,
2028                 long target, unsigned flags)
2029 {
2030         const struct {
2031                 struct list_head *list;
2032                 unsigned int bit;
2033         } phases[] = {
2034                 { &dev_priv->mm.unbound_list, I915_SHRINK_UNBOUND },
2035                 { &dev_priv->mm.bound_list, I915_SHRINK_BOUND },
2036                 { NULL, 0 },
2037         }, *phase;
2038         unsigned long count = 0;
2039
2040         /*
2041          * As we may completely rewrite the (un)bound list whilst unbinding
2042          * (due to retiring requests) we have to strictly process only
2043          * one element of the list at the time, and recheck the list
2044          * on every iteration.
2045          *
2046          * In particular, we must hold a reference whilst removing the
2047          * object as we may end up waiting for and/or retiring the objects.
2048          * This might release the final reference (held by the active list)
2049          * and result in the object being freed from under us. This is
2050          * similar to the precautions the eviction code must take whilst
2051          * removing objects.
2052          *
2053          * Also note that although these lists do not hold a reference to
2054          * the object we can safely grab one here: The final object
2055          * unreferencing and the bound_list are both protected by the
2056          * dev->struct_mutex and so we won't ever be able to observe an
2057          * object on the bound_list with a reference count equals 0.
2058          */
2059         for (phase = phases; phase->list; phase++) {
2060                 struct list_head still_in_list;
2061
2062                 if ((flags & phase->bit) == 0)
2063                         continue;
2064
2065                 INIT_LIST_HEAD(&still_in_list);
2066                 while (count < target && !list_empty(phase->list)) {
2067                         struct drm_i915_gem_object *obj;
2068                         struct i915_vma *vma, *v;
2069
2070                         obj = list_first_entry(phase->list,
2071                                                typeof(*obj), global_list);
2072                         list_move_tail(&obj->global_list, &still_in_list);
2073
2074                         if (flags & I915_SHRINK_PURGEABLE &&
2075                             !i915_gem_object_is_purgeable(obj))
2076                                 continue;
2077
2078                         drm_gem_object_reference(&obj->base);
2079
2080                         /* For the unbound phase, this should be a no-op! */
2081                         list_for_each_entry_safe(vma, v,
2082                                                  &obj->vma_list, vma_link)
2083                                 if (i915_vma_unbind(vma))
2084                                         break;
2085
2086                         if (i915_gem_object_put_pages(obj) == 0)
2087                                 count += obj->base.size >> PAGE_SHIFT;
2088
2089                         drm_gem_object_unreference(&obj->base);
2090                 }
2091                 list_splice(&still_in_list, phase->list);
2092         }
2093
2094         return count;
2095 }
2096
2097 static unsigned long
2098 i915_gem_shrink_all(struct drm_i915_private *dev_priv)
2099 {
2100         i915_gem_evict_everything(dev_priv->dev);
2101         return i915_gem_shrink(dev_priv, LONG_MAX,
2102                                I915_SHRINK_BOUND | I915_SHRINK_UNBOUND);
2103 }
2104
2105 static int
2106 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2107 {
2108         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2109         int page_count, i;
2110         struct address_space *mapping;
2111         struct sg_table *st;
2112         struct scatterlist *sg;
2113         struct sg_page_iter sg_iter;
2114         struct page *page;
2115         unsigned long last_pfn = 0;     /* suppress gcc warning */
2116         gfp_t gfp;
2117
2118         /* Assert that the object is not currently in any GPU domain. As it
2119          * wasn't in the GTT, there shouldn't be any way it could have been in
2120          * a GPU cache
2121          */
2122         BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
2123         BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
2124
2125         st = kmalloc(sizeof(*st), GFP_KERNEL);
2126         if (st == NULL)
2127                 return -ENOMEM;
2128
2129         page_count = obj->base.size / PAGE_SIZE;
2130         if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
2131                 kfree(st);
2132                 return -ENOMEM;
2133         }
2134
2135         /* Get the list of pages out of our struct file.  They'll be pinned
2136          * at this point until we release them.
2137          *
2138          * Fail silently without starting the shrinker
2139          */
2140         mapping = file_inode(obj->base.filp)->i_mapping;
2141         gfp = mapping_gfp_mask(mapping);
2142         gfp |= __GFP_NORETRY | __GFP_NOWARN | __GFP_NO_KSWAPD;
2143         gfp &= ~(__GFP_IO | __GFP_WAIT);
2144         sg = st->sgl;
2145         st->nents = 0;
2146         for (i = 0; i < page_count; i++) {
2147                 page = shmem_read_mapping_page_gfp(mapping, i, gfp);
2148                 if (IS_ERR(page)) {
2149                         i915_gem_shrink(dev_priv,
2150                                         page_count,
2151                                         I915_SHRINK_BOUND |
2152                                         I915_SHRINK_UNBOUND |
2153                                         I915_SHRINK_PURGEABLE);
2154                         page = shmem_read_mapping_page_gfp(mapping, i, gfp);
2155                 }
2156                 if (IS_ERR(page)) {
2157                         /* We've tried hard to allocate the memory by reaping
2158                          * our own buffer, now let the real VM do its job and
2159                          * go down in flames if truly OOM.
2160                          */
2161                         i915_gem_shrink_all(dev_priv);
2162                         page = shmem_read_mapping_page(mapping, i);
2163                         if (IS_ERR(page))
2164                                 goto err_pages;
2165                 }
2166 #ifdef CONFIG_SWIOTLB
2167                 if (swiotlb_nr_tbl()) {
2168                         st->nents++;
2169                         sg_set_page(sg, page, PAGE_SIZE, 0);
2170                         sg = sg_next(sg);
2171                         continue;
2172                 }
2173 #endif
2174                 if (!i || page_to_pfn(page) != last_pfn + 1) {
2175                         if (i)
2176                                 sg = sg_next(sg);
2177                         st->nents++;
2178                         sg_set_page(sg, page, PAGE_SIZE, 0);
2179                 } else {
2180                         sg->length += PAGE_SIZE;
2181                 }
2182                 last_pfn = page_to_pfn(page);
2183
2184                 /* Check that the i965g/gm workaround works. */
2185                 WARN_ON((gfp & __GFP_DMA32) && (last_pfn >= 0x00100000UL));
2186         }
2187 #ifdef CONFIG_SWIOTLB
2188         if (!swiotlb_nr_tbl())
2189 #endif
2190                 sg_mark_end(sg);
2191         obj->pages = st;
2192
2193         if (i915_gem_object_needs_bit17_swizzle(obj))
2194                 i915_gem_object_do_bit_17_swizzle(obj);
2195
2196         if (obj->tiling_mode != I915_TILING_NONE &&
2197             dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES)
2198                 i915_gem_object_pin_pages(obj);
2199
2200         return 0;
2201
2202 err_pages:
2203         sg_mark_end(sg);
2204         for_each_sg_page(st->sgl, &sg_iter, st->nents, 0)
2205                 page_cache_release(sg_page_iter_page(&sg_iter));
2206         sg_free_table(st);
2207         kfree(st);
2208
2209         /* shmemfs first checks if there is enough memory to allocate the page
2210          * and reports ENOSPC should there be insufficient, along with the usual
2211          * ENOMEM for a genuine allocation failure.
2212          *
2213          * We use ENOSPC in our driver to mean that we have run out of aperture
2214          * space and so want to translate the error from shmemfs back to our
2215          * usual understanding of ENOMEM.
2216          */
2217         if (PTR_ERR(page) == -ENOSPC)
2218                 return -ENOMEM;
2219         else
2220                 return PTR_ERR(page);
2221 }
2222
2223 /* Ensure that the associated pages are gathered from the backing storage
2224  * and pinned into our object. i915_gem_object_get_pages() may be called
2225  * multiple times before they are released by a single call to
2226  * i915_gem_object_put_pages() - once the pages are no longer referenced
2227  * either as a result of memory pressure (reaping pages under the shrinker)
2228  * or as the object is itself released.
2229  */
2230 int
2231 i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2232 {
2233         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2234         const struct drm_i915_gem_object_ops *ops = obj->ops;
2235         int ret;
2236
2237         if (obj->pages)
2238                 return 0;
2239
2240         if (obj->madv != I915_MADV_WILLNEED) {
2241                 DRM_DEBUG("Attempting to obtain a purgeable object\n");
2242                 return -EFAULT;
2243         }
2244
2245         BUG_ON(obj->pages_pin_count);
2246
2247         ret = ops->get_pages(obj);
2248         if (ret)
2249                 return ret;
2250
2251         list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
2252         return 0;
2253 }
2254
2255 static void
2256 i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
2257                                struct intel_engine_cs *ring)
2258 {
2259         u32 seqno = intel_ring_get_seqno(ring);
2260
2261         BUG_ON(ring == NULL);
2262         if (obj->ring != ring && obj->last_write_seqno) {
2263                 /* Keep the seqno relative to the current ring */
2264                 obj->last_write_seqno = seqno;
2265         }
2266         obj->ring = ring;
2267
2268         /* Add a reference if we're newly entering the active list. */
2269         if (!obj->active) {
2270                 drm_gem_object_reference(&obj->base);
2271                 obj->active = 1;
2272         }
2273
2274         list_move_tail(&obj->ring_list, &ring->active_list);
2275
2276         obj->last_read_seqno = seqno;
2277 }
2278
2279 void i915_vma_move_to_active(struct i915_vma *vma,
2280                              struct intel_engine_cs *ring)
2281 {
2282         list_move_tail(&vma->mm_list, &vma->vm->active_list);
2283         return i915_gem_object_move_to_active(vma->obj, ring);
2284 }
2285
2286 static void
2287 i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
2288 {
2289         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2290         struct i915_address_space *vm;
2291         struct i915_vma *vma;
2292
2293         BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
2294         BUG_ON(!obj->active);
2295
2296         list_for_each_entry(vm, &dev_priv->vm_list, global_link) {
2297                 vma = i915_gem_obj_to_vma(obj, vm);
2298                 if (vma && !list_empty(&vma->mm_list))
2299                         list_move_tail(&vma->mm_list, &vm->inactive_list);
2300         }
2301
2302         intel_fb_obj_flush(obj, true);
2303
2304         list_del_init(&obj->ring_list);
2305         obj->ring = NULL;
2306
2307         obj->last_read_seqno = 0;
2308         obj->last_write_seqno = 0;
2309         obj->base.write_domain = 0;
2310
2311         obj->last_fenced_seqno = 0;
2312
2313         obj->active = 0;
2314         drm_gem_object_unreference(&obj->base);
2315
2316         WARN_ON(i915_verify_lists(dev));
2317 }
2318
2319 static void
2320 i915_gem_object_retire(struct drm_i915_gem_object *obj)
2321 {
2322         struct intel_engine_cs *ring = obj->ring;
2323
2324         if (ring == NULL)
2325                 return;
2326
2327         if (i915_seqno_passed(ring->get_seqno(ring, true),
2328                               obj->last_read_seqno))
2329                 i915_gem_object_move_to_inactive(obj);
2330 }
2331
2332 static int
2333 i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
2334 {
2335         struct drm_i915_private *dev_priv = dev->dev_private;
2336         struct intel_engine_cs *ring;
2337         int ret, i, j;
2338
2339         /* Carefully retire all requests without writing to the rings */
2340         for_each_ring(ring, dev_priv, i) {
2341                 ret = intel_ring_idle(ring);
2342                 if (ret)
2343                         return ret;
2344         }
2345         i915_gem_retire_requests(dev);
2346
2347         /* Finally reset hw state */
2348         for_each_ring(ring, dev_priv, i) {
2349                 intel_ring_init_seqno(ring, seqno);
2350
2351                 for (j = 0; j < ARRAY_SIZE(ring->semaphore.sync_seqno); j++)
2352                         ring->semaphore.sync_seqno[j] = 0;
2353         }
2354
2355         return 0;
2356 }
2357
2358 int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
2359 {
2360         struct drm_i915_private *dev_priv = dev->dev_private;
2361         int ret;
2362
2363         if (seqno == 0)
2364                 return -EINVAL;
2365
2366         /* HWS page needs to be set less than what we
2367          * will inject to ring
2368          */
2369         ret = i915_gem_init_seqno(dev, seqno - 1);
2370         if (ret)
2371                 return ret;
2372
2373         /* Carefully set the last_seqno value so that wrap
2374          * detection still works
2375          */
2376         dev_priv->next_seqno = seqno;
2377         dev_priv->last_seqno = seqno - 1;
2378         if (dev_priv->last_seqno == 0)
2379                 dev_priv->last_seqno--;
2380
2381         return 0;
2382 }
2383
2384 int
2385 i915_gem_get_seqno(struct drm_device *dev, u32 *seqno)
2386 {
2387         struct drm_i915_private *dev_priv = dev->dev_private;
2388
2389         /* reserve 0 for non-seqno */
2390         if (dev_priv->next_seqno == 0) {
2391                 int ret = i915_gem_init_seqno(dev, 0);
2392                 if (ret)
2393                         return ret;
2394
2395                 dev_priv->next_seqno = 1;
2396         }
2397
2398         *seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
2399         return 0;
2400 }
2401
2402 int __i915_add_request(struct intel_engine_cs *ring,
2403                        struct drm_file *file,
2404                        struct drm_i915_gem_object *obj,
2405                        u32 *out_seqno)
2406 {
2407         struct drm_i915_private *dev_priv = ring->dev->dev_private;
2408         struct drm_i915_gem_request *request;
2409         struct intel_ringbuffer *ringbuf;
2410         u32 request_ring_position, request_start;
2411         int ret;
2412
2413         request = ring->preallocated_lazy_request;
2414         if (WARN_ON(request == NULL))
2415                 return -ENOMEM;
2416
2417         if (i915.enable_execlists) {
2418                 struct intel_context *ctx = request->ctx;
2419                 ringbuf = ctx->engine[ring->id].ringbuf;
2420         } else
2421                 ringbuf = ring->buffer;
2422
2423         request_start = intel_ring_get_tail(ringbuf);
2424         /*
2425          * Emit any outstanding flushes - execbuf can fail to emit the flush
2426          * after having emitted the batchbuffer command. Hence we need to fix
2427          * things up similar to emitting the lazy request. The difference here
2428          * is that the flush _must_ happen before the next request, no matter
2429          * what.
2430          */
2431         if (i915.enable_execlists) {
2432                 ret = logical_ring_flush_all_caches(ringbuf);
2433                 if (ret)
2434                         return ret;
2435         } else {
2436                 ret = intel_ring_flush_all_caches(ring);
2437                 if (ret)
2438                         return ret;
2439         }
2440
2441         /* Record the position of the start of the request so that
2442          * should we detect the updated seqno part-way through the
2443          * GPU processing the request, we never over-estimate the
2444          * position of the head.
2445          */
2446         request_ring_position = intel_ring_get_tail(ringbuf);
2447
2448         if (i915.enable_execlists) {
2449                 ret = ring->emit_request(ringbuf);
2450                 if (ret)
2451                         return ret;
2452         } else {
2453                 ret = ring->add_request(ring);
2454                 if (ret)
2455                         return ret;
2456         }
2457
2458         request->seqno = intel_ring_get_seqno(ring);
2459         request->ring = ring;
2460         request->head = request_start;
2461         request->tail = request_ring_position;
2462
2463         /* Whilst this request exists, batch_obj will be on the
2464          * active_list, and so will hold the active reference. Only when this
2465          * request is retired will the the batch_obj be moved onto the
2466          * inactive_list and lose its active reference. Hence we do not need
2467          * to explicitly hold another reference here.
2468          */
2469         request->batch_obj = obj;
2470
2471         if (!i915.enable_execlists) {
2472                 /* Hold a reference to the current context so that we can inspect
2473                  * it later in case a hangcheck error event fires.
2474                  */
2475                 request->ctx = ring->last_context;
2476                 if (request->ctx)
2477                         i915_gem_context_reference(request->ctx);
2478         }
2479
2480         request->emitted_jiffies = jiffies;
2481         list_add_tail(&request->list, &ring->request_list);
2482         request->file_priv = NULL;
2483
2484         if (file) {
2485                 struct drm_i915_file_private *file_priv = file->driver_priv;
2486
2487                 spin_lock(&file_priv->mm.lock);
2488                 request->file_priv = file_priv;
2489                 list_add_tail(&request->client_list,
2490                               &file_priv->mm.request_list);
2491                 spin_unlock(&file_priv->mm.lock);
2492         }
2493
2494         trace_i915_gem_request_add(ring, request->seqno);
2495         ring->outstanding_lazy_seqno = 0;
2496         ring->preallocated_lazy_request = NULL;
2497
2498         i915_queue_hangcheck(ring->dev);
2499
2500         cancel_delayed_work_sync(&dev_priv->mm.idle_work);
2501         queue_delayed_work(dev_priv->wq,
2502                            &dev_priv->mm.retire_work,
2503                            round_jiffies_up_relative(HZ));
2504         intel_mark_busy(dev_priv->dev);
2505
2506         if (out_seqno)
2507                 *out_seqno = request->seqno;
2508         return 0;
2509 }
2510
2511 static inline void
2512 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
2513 {
2514         struct drm_i915_file_private *file_priv = request->file_priv;
2515
2516         if (!file_priv)
2517                 return;
2518
2519         spin_lock(&file_priv->mm.lock);
2520         list_del(&request->client_list);
2521         request->file_priv = NULL;
2522         spin_unlock(&file_priv->mm.lock);
2523 }
2524
2525 static bool i915_context_is_banned(struct drm_i915_private *dev_priv,
2526                                    const struct intel_context *ctx)
2527 {
2528         unsigned long elapsed;
2529
2530         elapsed = get_seconds() - ctx->hang_stats.guilty_ts;
2531
2532         if (ctx->hang_stats.banned)
2533                 return true;
2534
2535         if (elapsed <= DRM_I915_CTX_BAN_PERIOD) {
2536                 if (!i915_gem_context_is_default(ctx)) {
2537                         DRM_DEBUG("context hanging too fast, banning!\n");
2538                         return true;
2539                 } else if (i915_stop_ring_allow_ban(dev_priv)) {
2540                         if (i915_stop_ring_allow_warn(dev_priv))
2541                                 DRM_ERROR("gpu hanging too fast, banning!\n");
2542                         return true;
2543                 }
2544         }
2545
2546         return false;
2547 }
2548
2549 static void i915_set_reset_status(struct drm_i915_private *dev_priv,
2550                                   struct intel_context *ctx,
2551                                   const bool guilty)
2552 {
2553         struct i915_ctx_hang_stats *hs;
2554
2555         if (WARN_ON(!ctx))
2556                 return;
2557
2558         hs = &ctx->hang_stats;
2559
2560         if (guilty) {
2561                 hs->banned = i915_context_is_banned(dev_priv, ctx);
2562                 hs->batch_active++;
2563                 hs->guilty_ts = get_seconds();
2564         } else {
2565                 hs->batch_pending++;
2566         }
2567 }
2568
2569 static void i915_gem_free_request(struct drm_i915_gem_request *request)
2570 {
2571         struct intel_context *ctx = request->ctx;
2572
2573         list_del(&request->list);
2574         i915_gem_request_remove_from_client(request);
2575
2576         if (ctx) {
2577                 if (i915.enable_execlists) {
2578                         struct intel_engine_cs *ring = request->ring;
2579
2580                         if (ctx != ring->default_context)
2581                                 intel_lr_context_unpin(ring, ctx);
2582                 }
2583                 i915_gem_context_unreference(ctx);
2584         }
2585         kfree(request);
2586 }
2587
2588 struct drm_i915_gem_request *
2589 i915_gem_find_active_request(struct intel_engine_cs *ring)
2590 {
2591         struct drm_i915_gem_request *request;
2592         u32 completed_seqno;
2593
2594         completed_seqno = ring->get_seqno(ring, false);
2595
2596         list_for_each_entry(request, &ring->request_list, list) {
2597                 if (i915_seqno_passed(completed_seqno, request->seqno))
2598                         continue;
2599
2600                 return request;
2601         }
2602
2603         return NULL;
2604 }
2605
2606 static void i915_gem_reset_ring_status(struct drm_i915_private *dev_priv,
2607                                        struct intel_engine_cs *ring)
2608 {
2609         struct drm_i915_gem_request *request;
2610         bool ring_hung;
2611
2612         request = i915_gem_find_active_request(ring);
2613
2614         if (request == NULL)
2615                 return;
2616
2617         ring_hung = ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG;
2618
2619         i915_set_reset_status(dev_priv, request->ctx, ring_hung);
2620
2621         list_for_each_entry_continue(request, &ring->request_list, list)
2622                 i915_set_reset_status(dev_priv, request->ctx, false);
2623 }
2624
2625 static void i915_gem_reset_ring_cleanup(struct drm_i915_private *dev_priv,
2626                                         struct intel_engine_cs *ring)
2627 {
2628         while (!list_empty(&ring->active_list)) {
2629                 struct drm_i915_gem_object *obj;
2630
2631                 obj = list_first_entry(&ring->active_list,
2632                                        struct drm_i915_gem_object,
2633                                        ring_list);
2634
2635                 i915_gem_object_move_to_inactive(obj);
2636         }
2637
2638         /*
2639          * Clear the execlists queue up before freeing the requests, as those
2640          * are the ones that keep the context and ringbuffer backing objects
2641          * pinned in place.
2642          */
2643         while (!list_empty(&ring->execlist_queue)) {
2644                 struct intel_ctx_submit_request *submit_req;
2645
2646                 submit_req = list_first_entry(&ring->execlist_queue,
2647                                 struct intel_ctx_submit_request,
2648                                 execlist_link);
2649                 list_del(&submit_req->execlist_link);
2650                 intel_runtime_pm_put(dev_priv);
2651                 i915_gem_context_unreference(submit_req->ctx);
2652                 kfree(submit_req);
2653         }
2654
2655         /*
2656          * We must free the requests after all the corresponding objects have
2657          * been moved off active lists. Which is the same order as the normal
2658          * retire_requests function does. This is important if object hold
2659          * implicit references on things like e.g. ppgtt address spaces through
2660          * the request.
2661          */
2662         while (!list_empty(&ring->request_list)) {
2663                 struct drm_i915_gem_request *request;
2664
2665                 request = list_first_entry(&ring->request_list,
2666                                            struct drm_i915_gem_request,
2667                                            list);
2668
2669                 i915_gem_free_request(request);
2670         }
2671
2672         /* These may not have been flush before the reset, do so now */
2673         kfree(ring->preallocated_lazy_request);
2674         ring->preallocated_lazy_request = NULL;
2675         ring->outstanding_lazy_seqno = 0;
2676 }
2677
2678 void i915_gem_restore_fences(struct drm_device *dev)
2679 {
2680         struct drm_i915_private *dev_priv = dev->dev_private;
2681         int i;
2682
2683         for (i = 0; i < dev_priv->num_fence_regs; i++) {
2684                 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
2685
2686                 /*
2687                  * Commit delayed tiling changes if we have an object still
2688                  * attached to the fence, otherwise just clear the fence.
2689                  */
2690                 if (reg->obj) {
2691                         i915_gem_object_update_fence(reg->obj, reg,
2692                                                      reg->obj->tiling_mode);
2693                 } else {
2694                         i915_gem_write_fence(dev, i, NULL);
2695                 }
2696         }
2697 }
2698
2699 void i915_gem_reset(struct drm_device *dev)
2700 {
2701         struct drm_i915_private *dev_priv = dev->dev_private;
2702         struct intel_engine_cs *ring;
2703         int i;
2704
2705         /*
2706          * Before we free the objects from the requests, we need to inspect
2707          * them for finding the guilty party. As the requests only borrow
2708          * their reference to the objects, the inspection must be done first.
2709          */
2710         for_each_ring(ring, dev_priv, i)
2711                 i915_gem_reset_ring_status(dev_priv, ring);
2712
2713         for_each_ring(ring, dev_priv, i)
2714                 i915_gem_reset_ring_cleanup(dev_priv, ring);
2715
2716         i915_gem_context_reset(dev);
2717
2718         i915_gem_restore_fences(dev);
2719 }
2720
2721 /**
2722  * This function clears the request list as sequence numbers are passed.
2723  */
2724 void
2725 i915_gem_retire_requests_ring(struct intel_engine_cs *ring)
2726 {
2727         uint32_t seqno;
2728
2729         if (list_empty(&ring->request_list))
2730                 return;
2731
2732         WARN_ON(i915_verify_lists(ring->dev));
2733
2734         seqno = ring->get_seqno(ring, true);
2735
2736         /* Move any buffers on the active list that are no longer referenced
2737          * by the ringbuffer to the flushing/inactive lists as appropriate,
2738          * before we free the context associated with the requests.
2739          */
2740         while (!list_empty(&ring->active_list)) {
2741                 struct drm_i915_gem_object *obj;
2742
2743                 obj = list_first_entry(&ring->active_list,
2744                                       struct drm_i915_gem_object,
2745                                       ring_list);
2746
2747                 if (!i915_seqno_passed(seqno, obj->last_read_seqno))
2748                         break;
2749
2750                 i915_gem_object_move_to_inactive(obj);
2751         }
2752
2753
2754         while (!list_empty(&ring->request_list)) {
2755                 struct drm_i915_gem_request *request;
2756                 struct intel_ringbuffer *ringbuf;
2757
2758                 request = list_first_entry(&ring->request_list,
2759                                            struct drm_i915_gem_request,
2760                                            list);
2761
2762                 if (!i915_seqno_passed(seqno, request->seqno))
2763                         break;
2764
2765                 trace_i915_gem_request_retire(ring, request->seqno);
2766
2767                 /* This is one of the few common intersection points
2768                  * between legacy ringbuffer submission and execlists:
2769                  * we need to tell them apart in order to find the correct
2770                  * ringbuffer to which the request belongs to.
2771                  */
2772                 if (i915.enable_execlists) {
2773                         struct intel_context *ctx = request->ctx;
2774                         ringbuf = ctx->engine[ring->id].ringbuf;
2775                 } else
2776                         ringbuf = ring->buffer;
2777
2778                 /* We know the GPU must have read the request to have
2779                  * sent us the seqno + interrupt, so use the position
2780                  * of tail of the request to update the last known position
2781                  * of the GPU head.
2782                  */
2783                 ringbuf->last_retired_head = request->tail;
2784
2785                 i915_gem_free_request(request);
2786         }
2787
2788         if (unlikely(ring->trace_irq_seqno &&
2789                      i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
2790                 ring->irq_put(ring);
2791                 ring->trace_irq_seqno = 0;
2792         }
2793
2794         WARN_ON(i915_verify_lists(ring->dev));
2795 }
2796
2797 bool
2798 i915_gem_retire_requests(struct drm_device *dev)
2799 {
2800         struct drm_i915_private *dev_priv = dev->dev_private;
2801         struct intel_engine_cs *ring;
2802         bool idle = true;
2803         int i;
2804
2805         for_each_ring(ring, dev_priv, i) {
2806                 i915_gem_retire_requests_ring(ring);
2807                 idle &= list_empty(&ring->request_list);
2808                 if (i915.enable_execlists) {
2809                         unsigned long flags;
2810
2811                         spin_lock_irqsave(&ring->execlist_lock, flags);
2812                         idle &= list_empty(&ring->execlist_queue);
2813                         spin_unlock_irqrestore(&ring->execlist_lock, flags);
2814
2815                         intel_execlists_retire_requests(ring);
2816                 }
2817         }
2818
2819         if (idle)
2820                 mod_delayed_work(dev_priv->wq,
2821                                    &dev_priv->mm.idle_work,
2822                                    msecs_to_jiffies(100));
2823
2824         return idle;
2825 }
2826
2827 static void
2828 i915_gem_retire_work_handler(struct work_struct *work)
2829 {
2830         struct drm_i915_private *dev_priv =
2831                 container_of(work, typeof(*dev_priv), mm.retire_work.work);
2832         struct drm_device *dev = dev_priv->dev;
2833         bool idle;
2834
2835         /* Come back later if the device is busy... */
2836         idle = false;
2837         if (mutex_trylock(&dev->struct_mutex)) {
2838                 idle = i915_gem_retire_requests(dev);
2839                 mutex_unlock(&dev->struct_mutex);
2840         }
2841         if (!idle)
2842                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
2843                                    round_jiffies_up_relative(HZ));
2844 }
2845
2846 static void
2847 i915_gem_idle_work_handler(struct work_struct *work)
2848 {
2849         struct drm_i915_private *dev_priv =
2850                 container_of(work, typeof(*dev_priv), mm.idle_work.work);
2851
2852         intel_mark_idle(dev_priv->dev);
2853 }
2854
2855 /**
2856  * Ensures that an object will eventually get non-busy by flushing any required
2857  * write domains, emitting any outstanding lazy request and retiring and
2858  * completed requests.
2859  */
2860 static int
2861 i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
2862 {
2863         int ret;
2864
2865         if (obj->active) {
2866                 ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
2867                 if (ret)
2868                         return ret;
2869
2870                 i915_gem_retire_requests_ring(obj->ring);
2871         }
2872
2873         return 0;
2874 }
2875
2876 /**
2877  * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
2878  * @DRM_IOCTL_ARGS: standard ioctl arguments
2879  *
2880  * Returns 0 if successful, else an error is returned with the remaining time in
2881  * the timeout parameter.
2882  *  -ETIME: object is still busy after timeout
2883  *  -ERESTARTSYS: signal interrupted the wait
2884  *  -ENONENT: object doesn't exist
2885  * Also possible, but rare:
2886  *  -EAGAIN: GPU wedged
2887  *  -ENOMEM: damn
2888  *  -ENODEV: Internal IRQ fail
2889  *  -E?: The add request failed
2890  *
2891  * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
2892  * non-zero timeout parameter the wait ioctl will wait for the given number of
2893  * nanoseconds on an object becoming unbusy. Since the wait itself does so
2894  * without holding struct_mutex the object may become re-busied before this
2895  * function completes. A similar but shorter * race condition exists in the busy
2896  * ioctl
2897  */
2898 int
2899 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
2900 {
2901         struct drm_i915_private *dev_priv = dev->dev_private;
2902         struct drm_i915_gem_wait *args = data;
2903         struct drm_i915_gem_object *obj;
2904         struct intel_engine_cs *ring = NULL;
2905         unsigned reset_counter;
2906         u32 seqno = 0;
2907         int ret = 0;
2908
2909         if (args->flags != 0)
2910                 return -EINVAL;
2911
2912         ret = i915_mutex_lock_interruptible(dev);
2913         if (ret)
2914                 return ret;
2915
2916         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->bo_handle));
2917         if (&obj->base == NULL) {
2918                 mutex_unlock(&dev->struct_mutex);
2919                 return -ENOENT;
2920         }
2921
2922         /* Need to make sure the object gets inactive eventually. */
2923         ret = i915_gem_object_flush_active(obj);
2924         if (ret)
2925                 goto out;
2926
2927         if (obj->active) {
2928                 seqno = obj->last_read_seqno;
2929                 ring = obj->ring;
2930         }
2931
2932         if (seqno == 0)
2933                  goto out;
2934
2935         /* Do this after OLR check to make sure we make forward progress polling
2936          * on this IOCTL with a timeout <=0 (like busy ioctl)
2937          */
2938         if (args->timeout_ns <= 0) {
2939                 ret = -ETIME;
2940                 goto out;
2941         }
2942
2943         drm_gem_object_unreference(&obj->base);
2944         reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
2945         mutex_unlock(&dev->struct_mutex);
2946
2947         return __i915_wait_seqno(ring, seqno, reset_counter, true,
2948                                  &args->timeout_ns, file->driver_priv);
2949
2950 out:
2951         drm_gem_object_unreference(&obj->base);
2952         mutex_unlock(&dev->struct_mutex);
2953         return ret;
2954 }
2955
2956 /**
2957  * i915_gem_object_sync - sync an object to a ring.
2958  *
2959  * @obj: object which may be in use on another ring.
2960  * @to: ring we wish to use the object on. May be NULL.
2961  *
2962  * This code is meant to abstract object synchronization with the GPU.
2963  * Calling with NULL implies synchronizing the object with the CPU
2964  * rather than a particular GPU ring.
2965  *
2966  * Returns 0 if successful, else propagates up the lower layer error.
2967  */
2968 int
2969 i915_gem_object_sync(struct drm_i915_gem_object *obj,
2970                      struct intel_engine_cs *to)
2971 {
2972         struct intel_engine_cs *from = obj->ring;
2973         u32 seqno;
2974         int ret, idx;
2975
2976         if (from == NULL || to == from)
2977                 return 0;
2978
2979         if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev))
2980                 return i915_gem_object_wait_rendering(obj, false);
2981
2982         idx = intel_ring_sync_index(from, to);
2983
2984         seqno = obj->last_read_seqno;
2985         /* Optimization: Avoid semaphore sync when we are sure we already
2986          * waited for an object with higher seqno */
2987         if (seqno <= from->semaphore.sync_seqno[idx])
2988                 return 0;
2989
2990         ret = i915_gem_check_olr(obj->ring, seqno);
2991         if (ret)
2992                 return ret;
2993
2994         trace_i915_gem_ring_sync_to(from, to, seqno);
2995         ret = to->semaphore.sync_to(to, from, seqno);
2996         if (!ret)
2997                 /* We use last_read_seqno because sync_to()
2998                  * might have just caused seqno wrap under
2999                  * the radar.
3000                  */
3001                 from->semaphore.sync_seqno[idx] = obj->last_read_seqno;
3002
3003         return ret;
3004 }
3005
3006 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
3007 {
3008         u32 old_write_domain, old_read_domains;
3009
3010         /* Force a pagefault for domain tracking on next user access */
3011         i915_gem_release_mmap(obj);
3012
3013         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
3014                 return;
3015
3016         /* Wait for any direct GTT access to complete */
3017         mb();
3018
3019         old_read_domains = obj->base.read_domains;
3020         old_write_domain = obj->base.write_domain;
3021
3022         obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
3023         obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
3024
3025         trace_i915_gem_object_change_domain(obj,
3026                                             old_read_domains,
3027                                             old_write_domain);
3028 }
3029
3030 int i915_vma_unbind(struct i915_vma *vma)
3031 {
3032         struct drm_i915_gem_object *obj = vma->obj;
3033         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3034         int ret;
3035
3036         if (list_empty(&vma->vma_link))
3037                 return 0;
3038
3039         if (!drm_mm_node_allocated(&vma->node)) {
3040                 i915_gem_vma_destroy(vma);
3041                 return 0;
3042         }
3043
3044         if (vma->pin_count)
3045                 return -EBUSY;
3046
3047         BUG_ON(obj->pages == NULL);
3048
3049         ret = i915_gem_object_finish_gpu(obj);
3050         if (ret)
3051                 return ret;
3052         /* Continue on if we fail due to EIO, the GPU is hung so we
3053          * should be safe and we need to cleanup or else we might
3054          * cause memory corruption through use-after-free.
3055          */
3056
3057         /* Throw away the active reference before moving to the unbound list */
3058         i915_gem_object_retire(obj);
3059
3060         if (i915_is_ggtt(vma->vm)) {
3061                 i915_gem_object_finish_gtt(obj);
3062
3063                 /* release the fence reg _after_ flushing */
3064                 ret = i915_gem_object_put_fence(obj);
3065                 if (ret)
3066                         return ret;
3067         }
3068
3069         trace_i915_vma_unbind(vma);
3070
3071         vma->unbind_vma(vma);
3072
3073         list_del_init(&vma->mm_list);
3074         if (i915_is_ggtt(vma->vm))
3075                 obj->map_and_fenceable = false;
3076
3077         drm_mm_remove_node(&vma->node);
3078         i915_gem_vma_destroy(vma);
3079
3080         /* Since the unbound list is global, only move to that list if
3081          * no more VMAs exist. */
3082         if (list_empty(&obj->vma_list)) {
3083                 i915_gem_gtt_finish_object(obj);
3084                 list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list);
3085         }
3086
3087         /* And finally now the object is completely decoupled from this vma,
3088          * we can drop its hold on the backing storage and allow it to be
3089          * reaped by the shrinker.
3090          */
3091         i915_gem_object_unpin_pages(obj);
3092
3093         return 0;
3094 }
3095
3096 int i915_gpu_idle(struct drm_device *dev)
3097 {
3098         struct drm_i915_private *dev_priv = dev->dev_private;
3099         struct intel_engine_cs *ring;
3100         int ret, i;
3101
3102         /* Flush everything onto the inactive list. */
3103         for_each_ring(ring, dev_priv, i) {
3104                 if (!i915.enable_execlists) {
3105                         ret = i915_switch_context(ring, ring->default_context);
3106                         if (ret)
3107                                 return ret;
3108                 }
3109
3110                 ret = intel_ring_idle(ring);
3111                 if (ret)
3112                         return ret;
3113         }
3114
3115         return 0;
3116 }
3117
3118 static void i965_write_fence_reg(struct drm_device *dev, int reg,
3119                                  struct drm_i915_gem_object *obj)
3120 {
3121         struct drm_i915_private *dev_priv = dev->dev_private;
3122         int fence_reg;
3123         int fence_pitch_shift;
3124
3125         if (INTEL_INFO(dev)->gen >= 6) {
3126                 fence_reg = FENCE_REG_SANDYBRIDGE_0;
3127                 fence_pitch_shift = SANDYBRIDGE_FENCE_PITCH_SHIFT;
3128         } else {
3129                 fence_reg = FENCE_REG_965_0;
3130                 fence_pitch_shift = I965_FENCE_PITCH_SHIFT;
3131         }
3132
3133         fence_reg += reg * 8;
3134
3135         /* To w/a incoherency with non-atomic 64-bit register updates,
3136          * we split the 64-bit update into two 32-bit writes. In order
3137          * for a partial fence not to be evaluated between writes, we
3138          * precede the update with write to turn off the fence register,
3139          * and only enable the fence as the last step.
3140          *
3141          * For extra levels of paranoia, we make sure each step lands
3142          * before applying the next step.
3143          */
3144         I915_WRITE(fence_reg, 0);
3145         POSTING_READ(fence_reg);
3146
3147         if (obj) {
3148                 u32 size = i915_gem_obj_ggtt_size(obj);
3149                 uint64_t val;
3150
3151                 val = (uint64_t)((i915_gem_obj_ggtt_offset(obj) + size - 4096) &
3152                                  0xfffff000) << 32;
3153                 val |= i915_gem_obj_ggtt_offset(obj) & 0xfffff000;
3154                 val |= (uint64_t)((obj->stride / 128) - 1) << fence_pitch_shift;
3155                 if (obj->tiling_mode == I915_TILING_Y)
3156                         val |= 1 << I965_FENCE_TILING_Y_SHIFT;
3157                 val |= I965_FENCE_REG_VALID;
3158
3159                 I915_WRITE(fence_reg + 4, val >> 32);
3160                 POSTING_READ(fence_reg + 4);
3161
3162                 I915_WRITE(fence_reg + 0, val);
3163                 POSTING_READ(fence_reg);
3164         } else {
3165                 I915_WRITE(fence_reg + 4, 0);
3166                 POSTING_READ(fence_reg + 4);
3167         }
3168 }
3169
3170 static void i915_write_fence_reg(struct drm_device *dev, int reg,
3171                                  struct drm_i915_gem_object *obj)
3172 {
3173         struct drm_i915_private *dev_priv = dev->dev_private;
3174         u32 val;
3175
3176         if (obj) {
3177                 u32 size = i915_gem_obj_ggtt_size(obj);
3178                 int pitch_val;
3179                 int tile_width;
3180
3181                 WARN((i915_gem_obj_ggtt_offset(obj) & ~I915_FENCE_START_MASK) ||
3182                      (size & -size) != size ||
3183                      (i915_gem_obj_ggtt_offset(obj) & (size - 1)),
3184                      "object 0x%08lx [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
3185                      i915_gem_obj_ggtt_offset(obj), obj->map_and_fenceable, size);
3186
3187                 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
3188                         tile_width = 128;
3189                 else
3190                         tile_width = 512;
3191
3192                 /* Note: pitch better be a power of two tile widths */
3193                 pitch_val = obj->stride / tile_width;
3194                 pitch_val = ffs(pitch_val) - 1;
3195
3196                 val = i915_gem_obj_ggtt_offset(obj);
3197                 if (obj->tiling_mode == I915_TILING_Y)
3198                         val |= 1 << I830_FENCE_TILING_Y_SHIFT;
3199                 val |= I915_FENCE_SIZE_BITS(size);
3200                 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
3201                 val |= I830_FENCE_REG_VALID;
3202         } else
3203                 val = 0;
3204
3205         if (reg < 8)
3206                 reg = FENCE_REG_830_0 + reg * 4;
3207         else
3208                 reg = FENCE_REG_945_8 + (reg - 8) * 4;
3209
3210         I915_WRITE(reg, val);
3211         POSTING_READ(reg);
3212 }
3213
3214 static void i830_write_fence_reg(struct drm_device *dev, int reg,
3215                                 struct drm_i915_gem_object *obj)
3216 {
3217         struct drm_i915_private *dev_priv = dev->dev_private;
3218         uint32_t val;
3219
3220         if (obj) {
3221                 u32 size = i915_gem_obj_ggtt_size(obj);
3222                 uint32_t pitch_val;
3223
3224                 WARN((i915_gem_obj_ggtt_offset(obj) & ~I830_FENCE_START_MASK) ||
3225                      (size & -size) != size ||
3226                      (i915_gem_obj_ggtt_offset(obj) & (size - 1)),
3227                      "object 0x%08lx not 512K or pot-size 0x%08x aligned\n",
3228                      i915_gem_obj_ggtt_offset(obj), size);
3229
3230                 pitch_val = obj->stride / 128;
3231                 pitch_val = ffs(pitch_val) - 1;
3232
3233                 val = i915_gem_obj_ggtt_offset(obj);
3234                 if (obj->tiling_mode == I915_TILING_Y)
3235                         val |= 1 << I830_FENCE_TILING_Y_SHIFT;
3236                 val |= I830_FENCE_SIZE_BITS(size);
3237                 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
3238                 val |= I830_FENCE_REG_VALID;
3239         } else
3240                 val = 0;
3241
3242         I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
3243         POSTING_READ(FENCE_REG_830_0 + reg * 4);
3244 }
3245
3246 inline static bool i915_gem_object_needs_mb(struct drm_i915_gem_object *obj)
3247 {
3248         return obj && obj->base.read_domains & I915_GEM_DOMAIN_GTT;
3249 }
3250
3251 static void i915_gem_write_fence(struct drm_device *dev, int reg,
3252                                  struct drm_i915_gem_object *obj)
3253 {
3254         struct drm_i915_private *dev_priv = dev->dev_private;
3255
3256         /* Ensure that all CPU reads are completed before installing a fence
3257          * and all writes before removing the fence.
3258          */
3259         if (i915_gem_object_needs_mb(dev_priv->fence_regs[reg].obj))
3260                 mb();
3261
3262         WARN(obj && (!obj->stride || !obj->tiling_mode),
3263              "bogus fence setup with stride: 0x%x, tiling mode: %i\n",
3264              obj->stride, obj->tiling_mode);
3265
3266         switch (INTEL_INFO(dev)->gen) {
3267         case 9:
3268         case 8:
3269         case 7:
3270         case 6:
3271         case 5:
3272         case 4: i965_write_fence_reg(dev, reg, obj); break;
3273         case 3: i915_write_fence_reg(dev, reg, obj); break;
3274         case 2: i830_write_fence_reg(dev, reg, obj); break;
3275         default: BUG();
3276         }
3277
3278         /* And similarly be paranoid that no direct access to this region
3279          * is reordered to before the fence is installed.
3280          */
3281         if (i915_gem_object_needs_mb(obj))
3282                 mb();
3283 }
3284
3285 static inline int fence_number(struct drm_i915_private *dev_priv,
3286                                struct drm_i915_fence_reg *fence)
3287 {
3288         return fence - dev_priv->fence_regs;
3289 }
3290
3291 static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
3292                                          struct drm_i915_fence_reg *fence,
3293                                          bool enable)
3294 {
3295         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3296         int reg = fence_number(dev_priv, fence);
3297
3298         i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
3299
3300         if (enable) {
3301                 obj->fence_reg = reg;
3302                 fence->obj = obj;
3303                 list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
3304         } else {
3305                 obj->fence_reg = I915_FENCE_REG_NONE;
3306                 fence->obj = NULL;
3307                 list_del_init(&fence->lru_list);
3308         }
3309         obj->fence_dirty = false;
3310 }
3311
3312 static int
3313 i915_gem_object_wait_fence(struct drm_i915_gem_object *obj)
3314 {
3315         if (obj->last_fenced_seqno) {
3316                 int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
3317                 if (ret)
3318                         return ret;
3319
3320                 obj->last_fenced_seqno = 0;
3321         }
3322
3323         return 0;
3324 }
3325
3326 int
3327 i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
3328 {
3329         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3330         struct drm_i915_fence_reg *fence;
3331         int ret;
3332
3333         ret = i915_gem_object_wait_fence(obj);
3334         if (ret)
3335                 return ret;
3336
3337         if (obj->fence_reg == I915_FENCE_REG_NONE)
3338                 return 0;
3339
3340         fence = &dev_priv->fence_regs[obj->fence_reg];
3341
3342         if (WARN_ON(fence->pin_count))
3343                 return -EBUSY;
3344
3345         i915_gem_object_fence_lost(obj);
3346         i915_gem_object_update_fence(obj, fence, false);
3347
3348         return 0;
3349 }
3350
3351 static struct drm_i915_fence_reg *
3352 i915_find_fence_reg(struct drm_device *dev)
3353 {
3354         struct drm_i915_private *dev_priv = dev->dev_private;
3355         struct drm_i915_fence_reg *reg, *avail;
3356         int i;
3357
3358         /* First try to find a free reg */
3359         avail = NULL;
3360         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
3361                 reg = &dev_priv->fence_regs[i];
3362                 if (!reg->obj)
3363                         return reg;
3364
3365                 if (!reg->pin_count)
3366                         avail = reg;
3367         }
3368
3369         if (avail == NULL)
3370                 goto deadlock;
3371
3372         /* None available, try to steal one or wait for a user to finish */
3373         list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
3374                 if (reg->pin_count)
3375                         continue;
3376
3377                 return reg;
3378         }
3379
3380 deadlock:
3381         /* Wait for completion of pending flips which consume fences */
3382         if (intel_has_pending_fb_unpin(dev))
3383                 return ERR_PTR(-EAGAIN);
3384
3385         return ERR_PTR(-EDEADLK);
3386 }
3387
3388 /**
3389  * i915_gem_object_get_fence - set up fencing for an object
3390  * @obj: object to map through a fence reg
3391  *
3392  * When mapping objects through the GTT, userspace wants to be able to write
3393  * to them without having to worry about swizzling if the object is tiled.
3394  * This function walks the fence regs looking for a free one for @obj,
3395  * stealing one if it can't find any.
3396  *
3397  * It then sets up the reg based on the object's properties: address, pitch
3398  * and tiling format.
3399  *
3400  * For an untiled surface, this removes any existing fence.
3401  */
3402 int
3403 i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
3404 {
3405         struct drm_device *dev = obj->base.dev;
3406         struct drm_i915_private *dev_priv = dev->dev_private;
3407         bool enable = obj->tiling_mode != I915_TILING_NONE;
3408         struct drm_i915_fence_reg *reg;
3409         int ret;
3410
3411         /* Have we updated the tiling parameters upon the object and so
3412          * will need to serialise the write to the associated fence register?
3413          */
3414         if (obj->fence_dirty) {
3415                 ret = i915_gem_object_wait_fence(obj);
3416                 if (ret)
3417                         return ret;
3418         }
3419
3420         /* Just update our place in the LRU if our fence is getting reused. */
3421         if (obj->fence_reg != I915_FENCE_REG_NONE) {
3422                 reg = &dev_priv->fence_regs[obj->fence_reg];
3423                 if (!obj->fence_dirty) {
3424                         list_move_tail(&reg->lru_list,
3425                                        &dev_priv->mm.fence_list);
3426                         return 0;
3427                 }
3428         } else if (enable) {
3429                 if (WARN_ON(!obj->map_and_fenceable))
3430                         return -EINVAL;
3431
3432                 reg = i915_find_fence_reg(dev);
3433                 if (IS_ERR(reg))
3434                         return PTR_ERR(reg);
3435
3436                 if (reg->obj) {
3437                         struct drm_i915_gem_object *old = reg->obj;
3438
3439                         ret = i915_gem_object_wait_fence(old);
3440                         if (ret)
3441                                 return ret;
3442
3443                         i915_gem_object_fence_lost(old);
3444                 }
3445         } else
3446                 return 0;
3447
3448         i915_gem_object_update_fence(obj, reg, enable);
3449
3450         return 0;
3451 }
3452
3453 static bool i915_gem_valid_gtt_space(struct i915_vma *vma,
3454                                      unsigned long cache_level)
3455 {
3456         struct drm_mm_node *gtt_space = &vma->node;
3457         struct drm_mm_node *other;
3458
3459         /*
3460          * On some machines we have to be careful when putting differing types
3461          * of snoopable memory together to avoid the prefetcher crossing memory
3462          * domains and dying. During vm initialisation, we decide whether or not
3463          * these constraints apply and set the drm_mm.color_adjust
3464          * appropriately.
3465          */
3466         if (vma->vm->mm.color_adjust == NULL)
3467                 return true;
3468
3469         if (!drm_mm_node_allocated(gtt_space))
3470                 return true;
3471
3472         if (list_empty(&gtt_space->node_list))
3473                 return true;
3474
3475         other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
3476         if (other->allocated && !other->hole_follows && other->color != cache_level)
3477                 return false;
3478
3479         other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
3480         if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
3481                 return false;
3482
3483         return true;
3484 }
3485
3486 /**
3487  * Finds free space in the GTT aperture and binds the object there.
3488  */
3489 static struct i915_vma *
3490 i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
3491                            struct i915_address_space *vm,
3492                            unsigned alignment,
3493                            uint64_t flags)
3494 {
3495         struct drm_device *dev = obj->base.dev;
3496         struct drm_i915_private *dev_priv = dev->dev_private;
3497         u32 size, fence_size, fence_alignment, unfenced_alignment;
3498         unsigned long start =
3499                 flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0;
3500         unsigned long end =
3501                 flags & PIN_MAPPABLE ? dev_priv->gtt.mappable_end : vm->total;
3502         struct i915_vma *vma;
3503         int ret;
3504
3505         fence_size = i915_gem_get_gtt_size(dev,
3506                                            obj->base.size,
3507                                            obj->tiling_mode);
3508         fence_alignment = i915_gem_get_gtt_alignment(dev,
3509                                                      obj->base.size,
3510                                                      obj->tiling_mode, true);
3511         unfenced_alignment =
3512                 i915_gem_get_gtt_alignment(dev,
3513                                            obj->base.size,
3514                                            obj->tiling_mode, false);
3515
3516         if (alignment == 0)
3517                 alignment = flags & PIN_MAPPABLE ? fence_alignment :
3518                                                 unfenced_alignment;
3519         if (flags & PIN_MAPPABLE && alignment & (fence_alignment - 1)) {
3520                 DRM_DEBUG("Invalid object alignment requested %u\n", alignment);
3521                 return ERR_PTR(-EINVAL);
3522         }
3523
3524         size = flags & PIN_MAPPABLE ? fence_size : obj->base.size;
3525
3526         /* If the object is bigger than the entire aperture, reject it early
3527          * before evicting everything in a vain attempt to find space.
3528          */
3529         if (obj->base.size > end) {
3530                 DRM_DEBUG("Attempting to bind an object larger than the aperture: object=%zd > %s aperture=%lu\n",
3531                           obj->base.size,
3532                           flags & PIN_MAPPABLE ? "mappable" : "total",
3533                           end);
3534                 return ERR_PTR(-E2BIG);
3535         }
3536
3537         ret = i915_gem_object_get_pages(obj);
3538         if (ret)
3539                 return ERR_PTR(ret);
3540
3541         i915_gem_object_pin_pages(obj);
3542
3543         vma = i915_gem_obj_lookup_or_create_vma(obj, vm);
3544         if (IS_ERR(vma))
3545                 goto err_unpin;
3546
3547 search_free:
3548         ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node,
3549                                                   size, alignment,
3550                                                   obj->cache_level,
3551                                                   start, end,
3552                                                   DRM_MM_SEARCH_DEFAULT,
3553                                                   DRM_MM_CREATE_DEFAULT);
3554         if (ret) {
3555                 ret = i915_gem_evict_something(dev, vm, size, alignment,
3556                                                obj->cache_level,
3557                                                start, end,
3558                                                flags);
3559                 if (ret == 0)
3560                         goto search_free;
3561
3562                 goto err_free_vma;
3563         }
3564         if (WARN_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level))) {
3565                 ret = -EINVAL;
3566                 goto err_remove_node;
3567         }
3568
3569         ret = i915_gem_gtt_prepare_object(obj);
3570         if (ret)
3571                 goto err_remove_node;
3572
3573         list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
3574         list_add_tail(&vma->mm_list, &vm->inactive_list);
3575
3576         trace_i915_vma_bind(vma, flags);
3577         vma->bind_vma(vma, obj->cache_level,
3578                       flags & PIN_GLOBAL ? GLOBAL_BIND : 0);
3579
3580         return vma;
3581
3582 err_remove_node:
3583         drm_mm_remove_node(&vma->node);
3584 err_free_vma:
3585         i915_gem_vma_destroy(vma);
3586         vma = ERR_PTR(ret);
3587 err_unpin:
3588         i915_gem_object_unpin_pages(obj);
3589         return vma;
3590 }
3591
3592 bool
3593 i915_gem_clflush_object(struct drm_i915_gem_object *obj,
3594                         bool force)
3595 {
3596         /* If we don't have a page list set up, then we're not pinned
3597          * to GPU, and we can ignore the cache flush because it'll happen
3598          * again at bind time.
3599          */
3600         if (obj->pages == NULL)
3601                 return false;
3602
3603         /*
3604          * Stolen memory is always coherent with the GPU as it is explicitly
3605          * marked as wc by the system, or the system is cache-coherent.
3606          */
3607         if (obj->stolen || obj->phys_handle)
3608                 return false;
3609
3610         /* If the GPU is snooping the contents of the CPU cache,
3611          * we do not need to manually clear the CPU cache lines.  However,
3612          * the caches are only snooped when the render cache is
3613          * flushed/invalidated.  As we always have to emit invalidations
3614          * and flushes when moving into and out of the RENDER domain, correct
3615          * snooping behaviour occurs naturally as the result of our domain
3616          * tracking.
3617          */
3618         if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
3619                 return false;
3620
3621         trace_i915_gem_object_clflush(obj);
3622         drm_clflush_sg(obj->pages);
3623
3624         return true;
3625 }
3626
3627 /** Flushes the GTT write domain for the object if it's dirty. */
3628 static void
3629 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
3630 {
3631         uint32_t old_write_domain;
3632
3633         if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
3634                 return;
3635
3636         /* No actual flushing is required for the GTT write domain.  Writes
3637          * to it immediately go to main memory as far as we know, so there's
3638          * no chipset flush.  It also doesn't land in render cache.
3639          *
3640          * However, we do have to enforce the order so that all writes through
3641          * the GTT land before any writes to the device, such as updates to
3642          * the GATT itself.
3643          */
3644         wmb();
3645
3646         old_write_domain = obj->base.write_domain;
3647         obj->base.write_domain = 0;
3648
3649         intel_fb_obj_flush(obj, false);
3650
3651         trace_i915_gem_object_change_domain(obj,
3652                                             obj->base.read_domains,
3653                                             old_write_domain);
3654 }
3655
3656 /** Flushes the CPU write domain for the object if it's dirty. */
3657 static void
3658 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj,
3659                                        bool force)
3660 {
3661         uint32_t old_write_domain;
3662
3663         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
3664                 return;
3665
3666         if (i915_gem_clflush_object(obj, force))
3667                 i915_gem_chipset_flush(obj->base.dev);
3668
3669         old_write_domain = obj->base.write_domain;
3670         obj->base.write_domain = 0;
3671
3672         intel_fb_obj_flush(obj, false);
3673
3674         trace_i915_gem_object_change_domain(obj,
3675                                             obj->base.read_domains,
3676                                             old_write_domain);
3677 }
3678
3679 /**
3680  * Moves a single object to the GTT read, and possibly write domain.
3681  *
3682  * This function returns when the move is complete, including waiting on
3683  * flushes to occur.
3684  */
3685 int
3686 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
3687 {
3688         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3689         struct i915_vma *vma = i915_gem_obj_to_ggtt(obj);
3690         uint32_t old_write_domain, old_read_domains;
3691         int ret;
3692
3693         /* Not valid to be called on unbound objects. */
3694         if (vma == NULL)
3695                 return -EINVAL;
3696
3697         if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
3698                 return 0;
3699
3700         ret = i915_gem_object_wait_rendering(obj, !write);
3701         if (ret)
3702                 return ret;
3703
3704         i915_gem_object_retire(obj);
3705         i915_gem_object_flush_cpu_write_domain(obj, false);
3706
3707         /* Serialise direct access to this object with the barriers for
3708          * coherent writes from the GPU, by effectively invalidating the
3709          * GTT domain upon first access.
3710          */
3711         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
3712                 mb();
3713
3714         old_write_domain = obj->base.write_domain;
3715         old_read_domains = obj->base.read_domains;
3716
3717         /* It should now be out of any other write domains, and we can update
3718          * the domain values for our changes.
3719          */
3720         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
3721         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3722         if (write) {
3723                 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
3724                 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
3725                 obj->dirty = 1;
3726         }
3727
3728         if (write)
3729                 intel_fb_obj_invalidate(obj, NULL);
3730
3731         trace_i915_gem_object_change_domain(obj,
3732                                             old_read_domains,
3733                                             old_write_domain);
3734
3735         /* And bump the LRU for this access */
3736         if (i915_gem_object_is_inactive(obj))
3737                 list_move_tail(&vma->mm_list,
3738                                &dev_priv->gtt.base.inactive_list);
3739
3740         return 0;
3741 }
3742
3743 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
3744                                     enum i915_cache_level cache_level)
3745 {
3746         struct drm_device *dev = obj->base.dev;
3747         struct i915_vma *vma, *next;
3748         int ret;
3749
3750         if (obj->cache_level == cache_level)
3751                 return 0;
3752
3753         if (i915_gem_obj_is_pinned(obj)) {
3754                 DRM_DEBUG("can not change the cache level of pinned objects\n");
3755                 return -EBUSY;
3756         }
3757
3758         list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) {
3759                 if (!i915_gem_valid_gtt_space(vma, cache_level)) {
3760                         ret = i915_vma_unbind(vma);
3761                         if (ret)
3762                                 return ret;
3763                 }
3764         }
3765
3766         if (i915_gem_obj_bound_any(obj)) {
3767                 ret = i915_gem_object_finish_gpu(obj);
3768                 if (ret)
3769                         return ret;
3770
3771                 i915_gem_object_finish_gtt(obj);
3772
3773                 /* Before SandyBridge, you could not use tiling or fence
3774                  * registers with snooped memory, so relinquish any fences
3775                  * currently pointing to our region in the aperture.
3776                  */
3777                 if (INTEL_INFO(dev)->gen < 6) {
3778                         ret = i915_gem_object_put_fence(obj);
3779                         if (ret)
3780                                 return ret;
3781                 }
3782
3783                 list_for_each_entry(vma, &obj->vma_list, vma_link)
3784                         if (drm_mm_node_allocated(&vma->node))
3785                                 vma->bind_vma(vma, cache_level,
3786                                                 vma->bound & GLOBAL_BIND);
3787         }
3788
3789         list_for_each_entry(vma, &obj->vma_list, vma_link)
3790                 vma->node.color = cache_level;
3791         obj->cache_level = cache_level;
3792
3793         if (cpu_write_needs_clflush(obj)) {
3794                 u32 old_read_domains, old_write_domain;
3795
3796                 /* If we're coming from LLC cached, then we haven't
3797                  * actually been tracking whether the data is in the
3798                  * CPU cache or not, since we only allow one bit set
3799                  * in obj->write_domain and have been skipping the clflushes.
3800                  * Just set it to the CPU cache for now.
3801                  */
3802                 i915_gem_object_retire(obj);
3803                 WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
3804
3805                 old_read_domains = obj->base.read_domains;
3806                 old_write_domain = obj->base.write_domain;
3807
3808                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3809                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3810
3811                 trace_i915_gem_object_change_domain(obj,
3812                                                     old_read_domains,
3813                                                     old_write_domain);
3814         }
3815
3816         return 0;
3817 }
3818
3819 int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
3820                                struct drm_file *file)
3821 {
3822         struct drm_i915_gem_caching *args = data;
3823         struct drm_i915_gem_object *obj;
3824         int ret;
3825
3826         ret = i915_mutex_lock_interruptible(dev);
3827         if (ret)
3828                 return ret;
3829
3830         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3831         if (&obj->base == NULL) {
3832                 ret = -ENOENT;
3833                 goto unlock;
3834         }
3835
3836         switch (obj->cache_level) {
3837         case I915_CACHE_LLC:
3838         case I915_CACHE_L3_LLC:
3839                 args->caching = I915_CACHING_CACHED;
3840                 break;
3841
3842         case I915_CACHE_WT:
3843                 args->caching = I915_CACHING_DISPLAY;
3844                 break;
3845
3846         default:
3847                 args->caching = I915_CACHING_NONE;
3848                 break;
3849         }
3850
3851         drm_gem_object_unreference(&obj->base);
3852 unlock:
3853         mutex_unlock(&dev->struct_mutex);
3854         return ret;
3855 }
3856
3857 int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
3858                                struct drm_file *file)
3859 {
3860         struct drm_i915_gem_caching *args = data;
3861         struct drm_i915_gem_object *obj;
3862         enum i915_cache_level level;
3863         int ret;
3864
3865         switch (args->caching) {
3866         case I915_CACHING_NONE:
3867                 level = I915_CACHE_NONE;
3868                 break;
3869         case I915_CACHING_CACHED:
3870                 level = I915_CACHE_LLC;
3871                 break;
3872         case I915_CACHING_DISPLAY:
3873                 level = HAS_WT(dev) ? I915_CACHE_WT : I915_CACHE_NONE;
3874                 break;
3875         default:
3876                 return -EINVAL;
3877         }
3878
3879         ret = i915_mutex_lock_interruptible(dev);
3880         if (ret)
3881                 return ret;
3882
3883         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3884         if (&obj->base == NULL) {
3885                 ret = -ENOENT;
3886                 goto unlock;
3887         }
3888
3889         ret = i915_gem_object_set_cache_level(obj, level);
3890
3891         drm_gem_object_unreference(&obj->base);
3892 unlock:
3893         mutex_unlock(&dev->struct_mutex);
3894         return ret;
3895 }
3896
3897 static bool is_pin_display(struct drm_i915_gem_object *obj)
3898 {
3899         struct i915_vma *vma;
3900
3901         vma = i915_gem_obj_to_ggtt(obj);
3902         if (!vma)
3903                 return false;
3904
3905         /* There are 3 sources that pin objects:
3906          *   1. The display engine (scanouts, sprites, cursors);
3907          *   2. Reservations for execbuffer;
3908          *   3. The user.
3909          *
3910          * We can ignore reservations as we hold the struct_mutex and
3911          * are only called outside of the reservation path.  The user
3912          * can only increment pin_count once, and so if after
3913          * subtracting the potential reference by the user, any pin_count
3914          * remains, it must be due to another use by the display engine.
3915          */
3916         return vma->pin_count - !!obj->user_pin_count;
3917 }
3918
3919 /*
3920  * Prepare buffer for display plane (scanout, cursors, etc).
3921  * Can be called from an uninterruptible phase (modesetting) and allows
3922  * any flushes to be pipelined (for pageflips).
3923  */
3924 int
3925 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
3926                                      u32 alignment,
3927                                      struct intel_engine_cs *pipelined)
3928 {
3929         u32 old_read_domains, old_write_domain;
3930         bool was_pin_display;
3931         int ret;
3932
3933         if (pipelined != obj->ring) {
3934                 ret = i915_gem_object_sync(obj, pipelined);
3935                 if (ret)
3936                         return ret;
3937         }
3938
3939         /* Mark the pin_display early so that we account for the
3940          * display coherency whilst setting up the cache domains.
3941          */
3942         was_pin_display = obj->pin_display;
3943         obj->pin_display = true;
3944
3945         /* The display engine is not coherent with the LLC cache on gen6.  As
3946          * a result, we make sure that the pinning that is about to occur is
3947          * done with uncached PTEs. This is lowest common denominator for all
3948          * chipsets.
3949          *
3950          * However for gen6+, we could do better by using the GFDT bit instead
3951          * of uncaching, which would allow us to flush all the LLC-cached data
3952          * with that bit in the PTE to main memory with just one PIPE_CONTROL.
3953          */
3954         ret = i915_gem_object_set_cache_level(obj,
3955                                               HAS_WT(obj->base.dev) ? I915_CACHE_WT : I915_CACHE_NONE);
3956         if (ret)
3957                 goto err_unpin_display;
3958
3959         /* As the user may map the buffer once pinned in the display plane
3960          * (e.g. libkms for the bootup splash), we have to ensure that we
3961          * always use map_and_fenceable for all scanout buffers.
3962          */
3963         ret = i915_gem_obj_ggtt_pin(obj, alignment, PIN_MAPPABLE);
3964         if (ret)
3965                 goto err_unpin_display;
3966
3967         i915_gem_object_flush_cpu_write_domain(obj, true);
3968
3969         old_write_domain = obj->base.write_domain;
3970         old_read_domains = obj->base.read_domains;
3971
3972         /* It should now be out of any other write domains, and we can update
3973          * the domain values for our changes.
3974          */
3975         obj->base.write_domain = 0;
3976         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3977
3978         trace_i915_gem_object_change_domain(obj,
3979                                             old_read_domains,
3980                                             old_write_domain);
3981
3982         return 0;
3983
3984 err_unpin_display:
3985         WARN_ON(was_pin_display != is_pin_display(obj));
3986         obj->pin_display = was_pin_display;
3987         return ret;
3988 }
3989
3990 void
3991 i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj)
3992 {
3993         i915_gem_object_ggtt_unpin(obj);
3994         obj->pin_display = is_pin_display(obj);
3995 }
3996
3997 int
3998 i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
3999 {
4000         int ret;
4001
4002         if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
4003                 return 0;
4004
4005         ret = i915_gem_object_wait_rendering(obj, false);
4006         if (ret)
4007                 return ret;
4008
4009         /* Ensure that we invalidate the GPU's caches and TLBs. */
4010         obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
4011         return 0;
4012 }
4013
4014 /**
4015  * Moves a single object to the CPU read, and possibly write domain.
4016  *
4017  * This function returns when the move is complete, including waiting on
4018  * flushes to occur.
4019  */
4020 int
4021 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
4022 {
4023         uint32_t old_write_domain, old_read_domains;
4024         int ret;
4025
4026         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
4027                 return 0;
4028
4029         ret = i915_gem_object_wait_rendering(obj, !write);
4030         if (ret)
4031                 return ret;
4032
4033         i915_gem_object_retire(obj);
4034         i915_gem_object_flush_gtt_write_domain(obj);
4035
4036         old_write_domain = obj->base.write_domain;
4037         old_read_domains = obj->base.read_domains;
4038
4039         /* Flush the CPU cache if it's still invalid. */
4040         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
4041                 i915_gem_clflush_object(obj, false);
4042
4043                 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
4044         }
4045
4046         /* It should now be out of any other write domains, and we can update
4047          * the domain values for our changes.
4048          */
4049         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
4050
4051         /* If we're writing through the CPU, then the GPU read domains will
4052          * need to be invalidated at next use.
4053          */
4054         if (write) {
4055                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4056                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4057         }
4058
4059         if (write)
4060                 intel_fb_obj_invalidate(obj, NULL);
4061
4062         trace_i915_gem_object_change_domain(obj,
4063                                             old_read_domains,
4064                                             old_write_domain);
4065
4066         return 0;
4067 }
4068
4069 /* Throttle our rendering by waiting until the ring has completed our requests
4070  * emitted over 20 msec ago.
4071  *
4072  * Note that if we were to use the current jiffies each time around the loop,
4073  * we wouldn't escape the function with any frames outstanding if the time to
4074  * render a frame was over 20ms.
4075  *
4076  * This should get us reasonable parallelism between CPU and GPU but also
4077  * relatively low latency when blocking on a particular request to finish.
4078  */
4079 static int
4080 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
4081 {
4082         struct drm_i915_private *dev_priv = dev->dev_private;
4083         struct drm_i915_file_private *file_priv = file->driver_priv;
4084         unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
4085         struct drm_i915_gem_request *request;
4086         struct intel_engine_cs *ring = NULL;
4087         unsigned reset_counter;
4088         u32 seqno = 0;
4089         int ret;
4090
4091         ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
4092         if (ret)
4093                 return ret;
4094
4095         ret = i915_gem_check_wedge(&dev_priv->gpu_error, false);
4096         if (ret)
4097                 return ret;
4098
4099         spin_lock(&file_priv->mm.lock);
4100         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
4101                 if (time_after_eq(request->emitted_jiffies, recent_enough))
4102                         break;
4103
4104                 ring = request->ring;
4105                 seqno = request->seqno;
4106         }
4107         reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
4108         spin_unlock(&file_priv->mm.lock);
4109
4110         if (seqno == 0)
4111                 return 0;
4112
4113         ret = __i915_wait_seqno(ring, seqno, reset_counter, true, NULL, NULL);
4114         if (ret == 0)
4115                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
4116
4117         return ret;
4118 }
4119
4120 static bool
4121 i915_vma_misplaced(struct i915_vma *vma, uint32_t alignment, uint64_t flags)
4122 {
4123         struct drm_i915_gem_object *obj = vma->obj;
4124
4125         if (alignment &&
4126             vma->node.start & (alignment - 1))
4127                 return true;
4128
4129         if (flags & PIN_MAPPABLE && !obj->map_and_fenceable)
4130                 return true;
4131
4132         if (flags & PIN_OFFSET_BIAS &&
4133             vma->node.start < (flags & PIN_OFFSET_MASK))
4134                 return true;
4135
4136         return false;
4137 }
4138
4139 int
4140 i915_gem_object_pin(struct drm_i915_gem_object *obj,
4141                     struct i915_address_space *vm,
4142                     uint32_t alignment,
4143                     uint64_t flags)
4144 {
4145         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
4146         struct i915_vma *vma;
4147         unsigned bound;
4148         int ret;
4149
4150         if (WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base))
4151                 return -ENODEV;
4152
4153         if (WARN_ON(flags & (PIN_GLOBAL | PIN_MAPPABLE) && !i915_is_ggtt(vm)))
4154                 return -EINVAL;
4155
4156         if (WARN_ON((flags & (PIN_MAPPABLE | PIN_GLOBAL)) == PIN_MAPPABLE))
4157                 return -EINVAL;
4158
4159         vma = i915_gem_obj_to_vma(obj, vm);
4160         if (vma) {
4161                 if (WARN_ON(vma->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
4162                         return -EBUSY;
4163
4164                 if (i915_vma_misplaced(vma, alignment, flags)) {
4165                         WARN(vma->pin_count,
4166                              "bo is already pinned with incorrect alignment:"
4167                              " offset=%lx, req.alignment=%x, req.map_and_fenceable=%d,"
4168                              " obj->map_and_fenceable=%d\n",
4169                              i915_gem_obj_offset(obj, vm), alignment,
4170                              !!(flags & PIN_MAPPABLE),
4171                              obj->map_and_fenceable);
4172                         ret = i915_vma_unbind(vma);
4173                         if (ret)
4174                                 return ret;
4175
4176                         vma = NULL;
4177                 }
4178         }
4179
4180         bound = vma ? vma->bound : 0;
4181         if (vma == NULL || !drm_mm_node_allocated(&vma->node)) {
4182                 vma = i915_gem_object_bind_to_vm(obj, vm, alignment, flags);
4183                 if (IS_ERR(vma))
4184                         return PTR_ERR(vma);
4185         }
4186
4187         if (flags & PIN_GLOBAL && !(vma->bound & GLOBAL_BIND))
4188                 vma->bind_vma(vma, obj->cache_level, GLOBAL_BIND);
4189
4190         if ((bound ^ vma->bound) & GLOBAL_BIND) {
4191                 bool mappable, fenceable;
4192                 u32 fence_size, fence_alignment;
4193
4194                 fence_size = i915_gem_get_gtt_size(obj->base.dev,
4195                                                    obj->base.size,
4196                                                    obj->tiling_mode);
4197                 fence_alignment = i915_gem_get_gtt_alignment(obj->base.dev,
4198                                                              obj->base.size,
4199                                                              obj->tiling_mode,
4200                                                              true);
4201
4202                 fenceable = (vma->node.size == fence_size &&
4203                              (vma->node.start & (fence_alignment - 1)) == 0);
4204
4205                 mappable = (vma->node.start + obj->base.size <=
4206                             dev_priv->gtt.mappable_end);
4207
4208                 obj->map_and_fenceable = mappable && fenceable;
4209         }
4210
4211         WARN_ON(flags & PIN_MAPPABLE && !obj->map_and_fenceable);
4212
4213         vma->pin_count++;
4214         if (flags & PIN_MAPPABLE)
4215                 obj->pin_mappable |= true;
4216
4217         return 0;
4218 }
4219
4220 void
4221 i915_gem_object_ggtt_unpin(struct drm_i915_gem_object *obj)
4222 {
4223         struct i915_vma *vma = i915_gem_obj_to_ggtt(obj);
4224
4225         BUG_ON(!vma);
4226         BUG_ON(vma->pin_count == 0);
4227         BUG_ON(!i915_gem_obj_ggtt_bound(obj));
4228
4229         if (--vma->pin_count == 0)
4230                 obj->pin_mappable = false;
4231 }
4232
4233 bool
4234 i915_gem_object_pin_fence(struct drm_i915_gem_object *obj)
4235 {
4236         if (obj->fence_reg != I915_FENCE_REG_NONE) {
4237                 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
4238                 struct i915_vma *ggtt_vma = i915_gem_obj_to_ggtt(obj);
4239
4240                 WARN_ON(!ggtt_vma ||
4241                         dev_priv->fence_regs[obj->fence_reg].pin_count >
4242                         ggtt_vma->pin_count);
4243                 dev_priv->fence_regs[obj->fence_reg].pin_count++;
4244                 return true;
4245         } else
4246                 return false;
4247 }
4248
4249 void
4250 i915_gem_object_unpin_fence(struct drm_i915_gem_object *obj)
4251 {
4252         if (obj->fence_reg != I915_FENCE_REG_NONE) {
4253                 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
4254                 WARN_ON(dev_priv->fence_regs[obj->fence_reg].pin_count <= 0);
4255                 dev_priv->fence_regs[obj->fence_reg].pin_count--;
4256         }
4257 }
4258
4259 int
4260 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
4261                    struct drm_file *file)
4262 {
4263         struct drm_i915_gem_pin *args = data;
4264         struct drm_i915_gem_object *obj;
4265         int ret;
4266
4267         if (drm_core_check_feature(dev, DRIVER_MODESET))
4268                 return -ENODEV;
4269
4270         ret = i915_mutex_lock_interruptible(dev);
4271         if (ret)
4272                 return ret;
4273
4274         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
4275         if (&obj->base == NULL) {
4276                 ret = -ENOENT;
4277                 goto unlock;
4278         }
4279
4280         if (obj->madv != I915_MADV_WILLNEED) {
4281                 DRM_DEBUG("Attempting to pin a purgeable buffer\n");
4282                 ret = -EFAULT;
4283                 goto out;
4284         }
4285
4286         if (obj->pin_filp != NULL && obj->pin_filp != file) {
4287                 DRM_DEBUG("Already pinned in i915_gem_pin_ioctl(): %d\n",
4288                           args->handle);
4289                 ret = -EINVAL;
4290                 goto out;
4291         }
4292
4293         if (obj->user_pin_count == ULONG_MAX) {
4294                 ret = -EBUSY;
4295                 goto out;
4296         }
4297
4298         if (obj->user_pin_count == 0) {
4299                 ret = i915_gem_obj_ggtt_pin(obj, args->alignment, PIN_MAPPABLE);
4300                 if (ret)
4301                         goto out;
4302         }
4303
4304         obj->user_pin_count++;
4305         obj->pin_filp = file;
4306
4307         args->offset = i915_gem_obj_ggtt_offset(obj);
4308 out:
4309         drm_gem_object_unreference(&obj->base);
4310 unlock:
4311         mutex_unlock(&dev->struct_mutex);
4312         return ret;
4313 }
4314
4315 int
4316 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
4317                      struct drm_file *file)
4318 {
4319         struct drm_i915_gem_pin *args = data;
4320         struct drm_i915_gem_object *obj;
4321         int ret;
4322
4323         if (drm_core_check_feature(dev, DRIVER_MODESET))
4324                 return -ENODEV;
4325
4326         ret = i915_mutex_lock_interruptible(dev);
4327         if (ret)
4328                 return ret;
4329
4330         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
4331         if (&obj->base == NULL) {
4332                 ret = -ENOENT;
4333                 goto unlock;
4334         }
4335
4336         if (obj->pin_filp != file) {
4337                 DRM_DEBUG("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
4338                           args->handle);
4339                 ret = -EINVAL;
4340                 goto out;
4341         }
4342         obj->user_pin_count--;
4343         if (obj->user_pin_count == 0) {
4344                 obj->pin_filp = NULL;
4345                 i915_gem_object_ggtt_unpin(obj);
4346         }
4347
4348 out:
4349         drm_gem_object_unreference(&obj->base);
4350 unlock:
4351         mutex_unlock(&dev->struct_mutex);
4352         return ret;
4353 }
4354
4355 int
4356 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4357                     struct drm_file *file)
4358 {
4359         struct drm_i915_gem_busy *args = data;
4360         struct drm_i915_gem_object *obj;
4361         int ret;
4362
4363         ret = i915_mutex_lock_interruptible(dev);
4364         if (ret)
4365                 return ret;
4366
4367         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
4368         if (&obj->base == NULL) {
4369                 ret = -ENOENT;
4370                 goto unlock;
4371         }
4372
4373         /* Count all active objects as busy, even if they are currently not used
4374          * by the gpu. Users of this interface expect objects to eventually
4375          * become non-busy without any further actions, therefore emit any
4376          * necessary flushes here.
4377          */
4378         ret = i915_gem_object_flush_active(obj);
4379
4380         args->busy = obj->active;
4381         if (obj->ring) {
4382                 BUILD_BUG_ON(I915_NUM_RINGS > 16);
4383                 args->busy |= intel_ring_flag(obj->ring) << 16;
4384         }
4385
4386         drm_gem_object_unreference(&obj->base);
4387 unlock:
4388         mutex_unlock(&dev->struct_mutex);
4389         return ret;
4390 }
4391
4392 int
4393 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4394                         struct drm_file *file_priv)
4395 {
4396         return i915_gem_ring_throttle(dev, file_priv);
4397 }
4398
4399 int
4400 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4401                        struct drm_file *file_priv)
4402 {
4403         struct drm_i915_private *dev_priv = dev->dev_private;
4404         struct drm_i915_gem_madvise *args = data;
4405         struct drm_i915_gem_object *obj;
4406         int ret;
4407
4408         switch (args->madv) {
4409         case I915_MADV_DONTNEED:
4410         case I915_MADV_WILLNEED:
4411             break;
4412         default:
4413             return -EINVAL;
4414         }
4415
4416         ret = i915_mutex_lock_interruptible(dev);
4417         if (ret)
4418                 return ret;
4419
4420         obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
4421         if (&obj->base == NULL) {
4422                 ret = -ENOENT;
4423                 goto unlock;
4424         }
4425
4426         if (i915_gem_obj_is_pinned(obj)) {
4427                 ret = -EINVAL;
4428                 goto out;
4429         }
4430
4431         if (obj->pages &&
4432             obj->tiling_mode != I915_TILING_NONE &&
4433             dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
4434                 if (obj->madv == I915_MADV_WILLNEED)
4435                         i915_gem_object_unpin_pages(obj);
4436                 if (args->madv == I915_MADV_WILLNEED)
4437                         i915_gem_object_pin_pages(obj);
4438         }
4439
4440         if (obj->madv != __I915_MADV_PURGED)
4441                 obj->madv = args->madv;
4442
4443         /* if the object is no longer attached, discard its backing storage */
4444         if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
4445                 i915_gem_object_truncate(obj);
4446
4447         args->retained = obj->madv != __I915_MADV_PURGED;
4448
4449 out:
4450         drm_gem_object_unreference(&obj->base);
4451 unlock:
4452         mutex_unlock(&dev->struct_mutex);
4453         return ret;
4454 }
4455
4456 void i915_gem_object_init(struct drm_i915_gem_object *obj,
4457                           const struct drm_i915_gem_object_ops *ops)
4458 {
4459         INIT_LIST_HEAD(&obj->global_list);
4460         INIT_LIST_HEAD(&obj->ring_list);
4461         INIT_LIST_HEAD(&obj->obj_exec_link);
4462         INIT_LIST_HEAD(&obj->vma_list);
4463
4464         obj->ops = ops;
4465
4466         obj->fence_reg = I915_FENCE_REG_NONE;
4467         obj->madv = I915_MADV_WILLNEED;
4468
4469         i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
4470 }
4471
4472 static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
4473         .get_pages = i915_gem_object_get_pages_gtt,
4474         .put_pages = i915_gem_object_put_pages_gtt,
4475 };
4476
4477 struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
4478                                                   size_t size)
4479 {
4480         struct drm_i915_gem_object *obj;
4481         struct address_space *mapping;
4482         gfp_t mask;
4483
4484         obj = i915_gem_object_alloc(dev);
4485         if (obj == NULL)
4486                 return NULL;
4487
4488         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4489                 i915_gem_object_free(obj);
4490                 return NULL;
4491         }
4492
4493         mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
4494         if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) {
4495                 /* 965gm cannot relocate objects above 4GiB. */
4496                 mask &= ~__GFP_HIGHMEM;
4497                 mask |= __GFP_DMA32;
4498         }
4499
4500         mapping = file_inode(obj->base.filp)->i_mapping;
4501         mapping_set_gfp_mask(mapping, mask);
4502
4503         i915_gem_object_init(obj, &i915_gem_object_ops);
4504
4505         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4506         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4507
4508         if (HAS_LLC(dev)) {
4509                 /* On some devices, we can have the GPU use the LLC (the CPU
4510                  * cache) for about a 10% performance improvement
4511                  * compared to uncached.  Graphics requests other than
4512                  * display scanout are coherent with the CPU in
4513                  * accessing this cache.  This means in this mode we
4514                  * don't need to clflush on the CPU side, and on the
4515                  * GPU side we only need to flush internal caches to
4516                  * get data visible to the CPU.
4517                  *
4518                  * However, we maintain the display planes as UC, and so
4519                  * need to rebind when first used as such.
4520                  */
4521                 obj->cache_level = I915_CACHE_LLC;
4522         } else
4523                 obj->cache_level = I915_CACHE_NONE;
4524
4525         trace_i915_gem_object_create(obj);
4526
4527         return obj;
4528 }
4529
4530 static bool discard_backing_storage(struct drm_i915_gem_object *obj)
4531 {
4532         /* If we are the last user of the backing storage (be it shmemfs
4533          * pages or stolen etc), we know that the pages are going to be
4534          * immediately released. In this case, we can then skip copying
4535          * back the contents from the GPU.
4536          */
4537
4538         if (obj->madv != I915_MADV_WILLNEED)
4539                 return false;
4540
4541         if (obj->base.filp == NULL)
4542                 return true;
4543
4544         /* At first glance, this looks racy, but then again so would be
4545          * userspace racing mmap against close. However, the first external
4546          * reference to the filp can only be obtained through the
4547          * i915_gem_mmap_ioctl() which safeguards us against the user
4548          * acquiring such a reference whilst we are in the middle of
4549          * freeing the object.
4550          */
4551         return atomic_long_read(&obj->base.filp->f_count) == 1;
4552 }
4553
4554 void i915_gem_free_object(struct drm_gem_object *gem_obj)
4555 {
4556         struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
4557         struct drm_device *dev = obj->base.dev;
4558         struct drm_i915_private *dev_priv = dev->dev_private;
4559         struct i915_vma *vma, *next;
4560
4561         intel_runtime_pm_get(dev_priv);
4562
4563         trace_i915_gem_object_destroy(obj);
4564
4565         list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) {
4566                 int ret;
4567
4568                 vma->pin_count = 0;
4569                 ret = i915_vma_unbind(vma);
4570                 if (WARN_ON(ret == -ERESTARTSYS)) {
4571                         bool was_interruptible;
4572
4573                         was_interruptible = dev_priv->mm.interruptible;
4574                         dev_priv->mm.interruptible = false;
4575
4576                         WARN_ON(i915_vma_unbind(vma));
4577
4578                         dev_priv->mm.interruptible = was_interruptible;
4579                 }
4580         }
4581
4582         /* Stolen objects don't hold a ref, but do hold pin count. Fix that up
4583          * before progressing. */
4584         if (obj->stolen)
4585                 i915_gem_object_unpin_pages(obj);
4586
4587         WARN_ON(obj->frontbuffer_bits);
4588
4589         if (obj->pages && obj->madv == I915_MADV_WILLNEED &&
4590             dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES &&
4591             obj->tiling_mode != I915_TILING_NONE)
4592                 i915_gem_object_unpin_pages(obj);
4593
4594         if (WARN_ON(obj->pages_pin_count))
4595                 obj->pages_pin_count = 0;
4596         if (discard_backing_storage(obj))
4597                 obj->madv = I915_MADV_DONTNEED;
4598         i915_gem_object_put_pages(obj);
4599         i915_gem_object_free_mmap_offset(obj);
4600
4601         BUG_ON(obj->pages);
4602
4603         if (obj->base.import_attach)
4604                 drm_prime_gem_destroy(&obj->base, NULL);
4605
4606         if (obj->ops->release)
4607                 obj->ops->release(obj);
4608
4609         drm_gem_object_release(&obj->base);
4610         i915_gem_info_remove_obj(dev_priv, obj->base.size);
4611
4612         kfree(obj->bit_17);
4613         i915_gem_object_free(obj);
4614
4615         intel_runtime_pm_put(dev_priv);
4616 }
4617
4618 struct i915_vma *i915_gem_obj_to_vma(struct drm_i915_gem_object *obj,
4619                                      struct i915_address_space *vm)
4620 {
4621         struct i915_vma *vma;
4622         list_for_each_entry(vma, &obj->vma_list, vma_link)
4623                 if (vma->vm == vm)
4624                         return vma;
4625
4626         return NULL;
4627 }
4628
4629 void i915_gem_vma_destroy(struct i915_vma *vma)
4630 {
4631         struct i915_address_space *vm = NULL;
4632         WARN_ON(vma->node.allocated);
4633
4634         /* Keep the vma as a placeholder in the execbuffer reservation lists */
4635         if (!list_empty(&vma->exec_list))
4636                 return;
4637
4638         vm = vma->vm;
4639
4640         if (!i915_is_ggtt(vm))
4641                 i915_ppgtt_put(i915_vm_to_ppgtt(vm));
4642
4643         list_del(&vma->vma_link);
4644
4645         kfree(vma);
4646 }
4647
4648 static void
4649 i915_gem_stop_ringbuffers(struct drm_device *dev)
4650 {
4651         struct drm_i915_private *dev_priv = dev->dev_private;
4652         struct intel_engine_cs *ring;
4653         int i;
4654
4655         for_each_ring(ring, dev_priv, i)
4656                 dev_priv->gt.stop_ring(ring);
4657 }
4658
4659 int
4660 i915_gem_suspend(struct drm_device *dev)
4661 {
4662         struct drm_i915_private *dev_priv = dev->dev_private;
4663         int ret = 0;
4664
4665         mutex_lock(&dev->struct_mutex);
4666         ret = i915_gpu_idle(dev);
4667         if (ret)
4668                 goto err;
4669
4670         i915_gem_retire_requests(dev);
4671
4672         /* Under UMS, be paranoid and evict. */
4673         if (!drm_core_check_feature(dev, DRIVER_MODESET))
4674                 i915_gem_evict_everything(dev);
4675
4676         i915_gem_stop_ringbuffers(dev);
4677         mutex_unlock(&dev->struct_mutex);
4678
4679         del_timer_sync(&dev_priv->gpu_error.hangcheck_timer);
4680         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4681         flush_delayed_work(&dev_priv->mm.idle_work);
4682
4683         return 0;
4684
4685 err:
4686         mutex_unlock(&dev->struct_mutex);
4687         return ret;
4688 }
4689
4690 int i915_gem_l3_remap(struct intel_engine_cs *ring, int slice)
4691 {
4692         struct drm_device *dev = ring->dev;
4693         struct drm_i915_private *dev_priv = dev->dev_private;
4694         u32 reg_base = GEN7_L3LOG_BASE + (slice * 0x200);
4695         u32 *remap_info = dev_priv->l3_parity.remap_info[slice];
4696         int i, ret;
4697
4698         if (!HAS_L3_DPF(dev) || !remap_info)
4699                 return 0;
4700
4701         ret = intel_ring_begin(ring, GEN7_L3LOG_SIZE / 4 * 3);
4702         if (ret)
4703                 return ret;
4704
4705         /*
4706          * Note: We do not worry about the concurrent register cacheline hang
4707          * here because no other code should access these registers other than
4708          * at initialization time.
4709          */
4710         for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
4711                 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
4712                 intel_ring_emit(ring, reg_base + i);
4713                 intel_ring_emit(ring, remap_info[i/4]);
4714         }
4715
4716         intel_ring_advance(ring);
4717
4718         return ret;
4719 }
4720
4721 void i915_gem_init_swizzling(struct drm_device *dev)
4722 {
4723         struct drm_i915_private *dev_priv = dev->dev_private;
4724
4725         if (INTEL_INFO(dev)->gen < 5 ||
4726             dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
4727                 return;
4728
4729         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
4730                                  DISP_TILE_SURFACE_SWIZZLING);
4731
4732         if (IS_GEN5(dev))
4733                 return;
4734
4735         I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
4736         if (IS_GEN6(dev))
4737                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
4738         else if (IS_GEN7(dev))
4739                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
4740         else if (IS_GEN8(dev))
4741                 I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW));
4742         else
4743                 BUG();
4744 }
4745
4746 static bool
4747 intel_enable_blt(struct drm_device *dev)
4748 {
4749         if (!HAS_BLT(dev))
4750                 return false;
4751
4752         /* The blitter was dysfunctional on early prototypes */
4753         if (IS_GEN6(dev) && dev->pdev->revision < 8) {
4754                 DRM_INFO("BLT not supported on this pre-production hardware;"
4755                          " graphics performance will be degraded.\n");
4756                 return false;
4757         }
4758
4759         return true;
4760 }
4761
4762 static void init_unused_ring(struct drm_device *dev, u32 base)
4763 {
4764         struct drm_i915_private *dev_priv = dev->dev_private;
4765
4766         I915_WRITE(RING_CTL(base), 0);
4767         I915_WRITE(RING_HEAD(base), 0);
4768         I915_WRITE(RING_TAIL(base), 0);
4769         I915_WRITE(RING_START(base), 0);
4770 }
4771
4772 static void init_unused_rings(struct drm_device *dev)
4773 {
4774         if (IS_I830(dev)) {
4775                 init_unused_ring(dev, PRB1_BASE);
4776                 init_unused_ring(dev, SRB0_BASE);
4777                 init_unused_ring(dev, SRB1_BASE);
4778                 init_unused_ring(dev, SRB2_BASE);
4779                 init_unused_ring(dev, SRB3_BASE);
4780         } else if (IS_GEN2(dev)) {
4781                 init_unused_ring(dev, SRB0_BASE);
4782                 init_unused_ring(dev, SRB1_BASE);
4783         } else if (IS_GEN3(dev)) {
4784                 init_unused_ring(dev, PRB1_BASE);
4785                 init_unused_ring(dev, PRB2_BASE);
4786         }
4787 }
4788
4789 int i915_gem_init_rings(struct drm_device *dev)
4790 {
4791         struct drm_i915_private *dev_priv = dev->dev_private;
4792         int ret;
4793
4794         /*
4795          * At least 830 can leave some of the unused rings
4796          * "active" (ie. head != tail) after resume which
4797          * will prevent c3 entry. Makes sure all unused rings
4798          * are totally idle.
4799          */
4800         init_unused_rings(dev);
4801
4802         ret = intel_init_render_ring_buffer(dev);
4803         if (ret)
4804                 return ret;
4805
4806         if (HAS_BSD(dev)) {
4807                 ret = intel_init_bsd_ring_buffer(dev);
4808                 if (ret)
4809                         goto cleanup_render_ring;
4810         }
4811
4812         if (intel_enable_blt(dev)) {
4813                 ret = intel_init_blt_ring_buffer(dev);
4814                 if (ret)
4815                         goto cleanup_bsd_ring;
4816         }
4817
4818         if (HAS_VEBOX(dev)) {
4819                 ret = intel_init_vebox_ring_buffer(dev);
4820                 if (ret)
4821                         goto cleanup_blt_ring;
4822         }
4823
4824         if (HAS_BSD2(dev)) {
4825                 ret = intel_init_bsd2_ring_buffer(dev);
4826                 if (ret)
4827                         goto cleanup_vebox_ring;
4828         }
4829
4830         ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000));
4831         if (ret)
4832                 goto cleanup_bsd2_ring;
4833
4834         return 0;
4835
4836 cleanup_bsd2_ring:
4837         intel_cleanup_ring_buffer(&dev_priv->ring[VCS2]);
4838 cleanup_vebox_ring:
4839         intel_cleanup_ring_buffer(&dev_priv->ring[VECS]);
4840 cleanup_blt_ring:
4841         intel_cleanup_ring_buffer(&dev_priv->ring[BCS]);
4842 cleanup_bsd_ring:
4843         intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
4844 cleanup_render_ring:
4845         intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
4846
4847         return ret;
4848 }
4849
4850 int
4851 i915_gem_init_hw(struct drm_device *dev)
4852 {
4853         struct drm_i915_private *dev_priv = dev->dev_private;
4854         int ret, i;
4855
4856         if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
4857                 return -EIO;
4858
4859         if (dev_priv->ellc_size)
4860                 I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf));
4861
4862         if (IS_HASWELL(dev))
4863                 I915_WRITE(MI_PREDICATE_RESULT_2, IS_HSW_GT3(dev) ?
4864                            LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
4865
4866         if (HAS_PCH_NOP(dev)) {
4867                 if (IS_IVYBRIDGE(dev)) {
4868                         u32 temp = I915_READ(GEN7_MSG_CTL);
4869                         temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK);
4870                         I915_WRITE(GEN7_MSG_CTL, temp);
4871                 } else if (INTEL_INFO(dev)->gen >= 7) {
4872                         u32 temp = I915_READ(HSW_NDE_RSTWRN_OPT);
4873                         temp &= ~RESET_PCH_HANDSHAKE_ENABLE;
4874                         I915_WRITE(HSW_NDE_RSTWRN_OPT, temp);
4875                 }
4876         }
4877
4878         i915_gem_init_swizzling(dev);
4879
4880         ret = dev_priv->gt.init_rings(dev);
4881         if (ret)
4882                 return ret;
4883
4884         for (i = 0; i < NUM_L3_SLICES(dev); i++)
4885                 i915_gem_l3_remap(&dev_priv->ring[RCS], i);
4886
4887         /*
4888          * XXX: Contexts should only be initialized once. Doing a switch to the
4889          * default context switch however is something we'd like to do after
4890          * reset or thaw (the latter may not actually be necessary for HW, but
4891          * goes with our code better). Context switching requires rings (for
4892          * the do_switch), but before enabling PPGTT. So don't move this.
4893          */
4894         ret = i915_gem_context_enable(dev_priv);
4895         if (ret && ret != -EIO) {
4896                 DRM_ERROR("Context enable failed %d\n", ret);
4897                 i915_gem_cleanup_ringbuffer(dev);
4898
4899                 return ret;
4900         }
4901
4902         ret = i915_ppgtt_init_hw(dev);
4903         if (ret && ret != -EIO) {
4904                 DRM_ERROR("PPGTT enable failed %d\n", ret);
4905                 i915_gem_cleanup_ringbuffer(dev);
4906         }
4907
4908         return ret;
4909 }
4910
4911 int i915_gem_init(struct drm_device *dev)
4912 {
4913         struct drm_i915_private *dev_priv = dev->dev_private;
4914         int ret;
4915
4916         i915.enable_execlists = intel_sanitize_enable_execlists(dev,
4917                         i915.enable_execlists);
4918
4919         mutex_lock(&dev->struct_mutex);
4920
4921         if (IS_VALLEYVIEW(dev)) {
4922                 /* VLVA0 (potential hack), BIOS isn't actually waking us */
4923                 I915_WRITE(VLV_GTLC_WAKE_CTRL, VLV_GTLC_ALLOWWAKEREQ);
4924                 if (wait_for((I915_READ(VLV_GTLC_PW_STATUS) &
4925                               VLV_GTLC_ALLOWWAKEACK), 10))
4926                         DRM_DEBUG_DRIVER("allow wake ack timed out\n");
4927         }
4928
4929         if (!i915.enable_execlists) {
4930                 dev_priv->gt.do_execbuf = i915_gem_ringbuffer_submission;
4931                 dev_priv->gt.init_rings = i915_gem_init_rings;
4932                 dev_priv->gt.cleanup_ring = intel_cleanup_ring_buffer;
4933                 dev_priv->gt.stop_ring = intel_stop_ring_buffer;
4934         } else {
4935                 dev_priv->gt.do_execbuf = intel_execlists_submission;
4936                 dev_priv->gt.init_rings = intel_logical_rings_init;
4937                 dev_priv->gt.cleanup_ring = intel_logical_ring_cleanup;
4938                 dev_priv->gt.stop_ring = intel_logical_ring_stop;
4939         }
4940
4941         ret = i915_gem_init_userptr(dev);
4942         if (ret) {
4943                 mutex_unlock(&dev->struct_mutex);
4944                 return ret;
4945         }
4946
4947         i915_gem_init_global_gtt(dev);
4948
4949         ret = i915_gem_context_init(dev);
4950         if (ret) {
4951                 mutex_unlock(&dev->struct_mutex);
4952                 return ret;
4953         }
4954
4955         ret = i915_gem_init_hw(dev);
4956         if (ret == -EIO) {
4957                 /* Allow ring initialisation to fail by marking the GPU as
4958                  * wedged. But we only want to do this where the GPU is angry,
4959                  * for all other failure, such as an allocation failure, bail.
4960                  */
4961                 DRM_ERROR("Failed to initialize GPU, declaring it wedged\n");
4962                 atomic_set_mask(I915_WEDGED, &dev_priv->gpu_error.reset_counter);
4963                 ret = 0;
4964         }
4965         mutex_unlock(&dev->struct_mutex);
4966
4967         return ret;
4968 }
4969
4970 void
4971 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4972 {
4973         struct drm_i915_private *dev_priv = dev->dev_private;
4974         struct intel_engine_cs *ring;
4975         int i;
4976
4977         for_each_ring(ring, dev_priv, i)
4978                 dev_priv->gt.cleanup_ring(ring);
4979 }
4980
4981 static void
4982 init_ring_lists(struct intel_engine_cs *ring)
4983 {
4984         INIT_LIST_HEAD(&ring->active_list);
4985         INIT_LIST_HEAD(&ring->request_list);
4986 }
4987
4988 void i915_init_vm(struct drm_i915_private *dev_priv,
4989                   struct i915_address_space *vm)
4990 {
4991         if (!i915_is_ggtt(vm))
4992                 drm_mm_init(&vm->mm, vm->start, vm->total);
4993         vm->dev = dev_priv->dev;
4994         INIT_LIST_HEAD(&vm->active_list);
4995         INIT_LIST_HEAD(&vm->inactive_list);
4996         INIT_LIST_HEAD(&vm->global_link);
4997         list_add_tail(&vm->global_link, &dev_priv->vm_list);
4998 }
4999
5000 void
5001 i915_gem_load(struct drm_device *dev)
5002 {
5003         struct drm_i915_private *dev_priv = dev->dev_private;
5004         int i;
5005
5006         dev_priv->slab =
5007                 kmem_cache_create("i915_gem_object",
5008                                   sizeof(struct drm_i915_gem_object), 0,
5009                                   SLAB_HWCACHE_ALIGN,
5010                                   NULL);
5011
5012         INIT_LIST_HEAD(&dev_priv->vm_list);
5013         i915_init_vm(dev_priv, &dev_priv->gtt.base);
5014
5015         INIT_LIST_HEAD(&dev_priv->context_list);
5016         INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
5017         INIT_LIST_HEAD(&dev_priv->mm.bound_list);
5018         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
5019         for (i = 0; i < I915_NUM_RINGS; i++)
5020                 init_ring_lists(&dev_priv->ring[i]);
5021         for (i = 0; i < I915_MAX_NUM_FENCES; i++)
5022                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
5023         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
5024                           i915_gem_retire_work_handler);
5025         INIT_DELAYED_WORK(&dev_priv->mm.idle_work,
5026                           i915_gem_idle_work_handler);
5027         init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
5028
5029         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
5030         if (!drm_core_check_feature(dev, DRIVER_MODESET) && IS_GEN3(dev)) {
5031                 I915_WRITE(MI_ARB_STATE,
5032                            _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
5033         }
5034
5035         dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
5036
5037         /* Old X drivers will take 0-2 for front, back, depth buffers */
5038         if (!drm_core_check_feature(dev, DRIVER_MODESET))
5039                 dev_priv->fence_reg_start = 3;
5040
5041         if (INTEL_INFO(dev)->gen >= 7 && !IS_VALLEYVIEW(dev))
5042                 dev_priv->num_fence_regs = 32;
5043         else if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
5044                 dev_priv->num_fence_regs = 16;
5045         else
5046                 dev_priv->num_fence_regs = 8;
5047
5048         /* Initialize fence registers to zero */
5049         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
5050         i915_gem_restore_fences(dev);
5051
5052         i915_gem_detect_bit_6_swizzle(dev);
5053         init_waitqueue_head(&dev_priv->pending_flip_queue);
5054
5055         dev_priv->mm.interruptible = true;
5056
5057         dev_priv->mm.shrinker.scan_objects = i915_gem_shrinker_scan;
5058         dev_priv->mm.shrinker.count_objects = i915_gem_shrinker_count;
5059         dev_priv->mm.shrinker.seeks = DEFAULT_SEEKS;
5060         register_shrinker(&dev_priv->mm.shrinker);
5061
5062         dev_priv->mm.oom_notifier.notifier_call = i915_gem_shrinker_oom;
5063         register_oom_notifier(&dev_priv->mm.oom_notifier);
5064
5065         mutex_init(&dev_priv->fb_tracking.lock);
5066 }
5067
5068 void i915_gem_release(struct drm_device *dev, struct drm_file *file)
5069 {
5070         struct drm_i915_file_private *file_priv = file->driver_priv;
5071
5072         cancel_delayed_work_sync(&file_priv->mm.idle_work);
5073
5074         /* Clean up our request list when the client is going away, so that
5075          * later retire_requests won't dereference our soon-to-be-gone
5076          * file_priv.
5077          */
5078         spin_lock(&file_priv->mm.lock);
5079         while (!list_empty(&file_priv->mm.request_list)) {
5080                 struct drm_i915_gem_request *request;
5081
5082                 request = list_first_entry(&file_priv->mm.request_list,
5083                                            struct drm_i915_gem_request,
5084                                            client_list);
5085                 list_del(&request->client_list);
5086                 request->file_priv = NULL;
5087         }
5088         spin_unlock(&file_priv->mm.lock);
5089 }
5090
5091 static void
5092 i915_gem_file_idle_work_handler(struct work_struct *work)
5093 {
5094         struct drm_i915_file_private *file_priv =
5095                 container_of(work, typeof(*file_priv), mm.idle_work.work);
5096
5097         atomic_set(&file_priv->rps_wait_boost, false);
5098 }
5099
5100 int i915_gem_open(struct drm_device *dev, struct drm_file *file)
5101 {
5102         struct drm_i915_file_private *file_priv;
5103         int ret;
5104
5105         DRM_DEBUG_DRIVER("\n");
5106
5107         file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
5108         if (!file_priv)
5109                 return -ENOMEM;
5110
5111         file->driver_priv = file_priv;
5112         file_priv->dev_priv = dev->dev_private;
5113         file_priv->file = file;
5114
5115         spin_lock_init(&file_priv->mm.lock);
5116         INIT_LIST_HEAD(&file_priv->mm.request_list);
5117         INIT_DELAYED_WORK(&file_priv->mm.idle_work,
5118                           i915_gem_file_idle_work_handler);
5119
5120         ret = i915_gem_context_open(dev, file);
5121         if (ret)
5122                 kfree(file_priv);
5123
5124         return ret;
5125 }
5126
5127 /**
5128  * i915_gem_track_fb - update frontbuffer tracking
5129  * old: current GEM buffer for the frontbuffer slots
5130  * new: new GEM buffer for the frontbuffer slots
5131  * frontbuffer_bits: bitmask of frontbuffer slots
5132  *
5133  * This updates the frontbuffer tracking bits @frontbuffer_bits by clearing them
5134  * from @old and setting them in @new. Both @old and @new can be NULL.
5135  */
5136 void i915_gem_track_fb(struct drm_i915_gem_object *old,
5137                        struct drm_i915_gem_object *new,
5138                        unsigned frontbuffer_bits)
5139 {
5140         if (old) {
5141                 WARN_ON(!mutex_is_locked(&old->base.dev->struct_mutex));
5142                 WARN_ON(!(old->frontbuffer_bits & frontbuffer_bits));
5143                 old->frontbuffer_bits &= ~frontbuffer_bits;
5144         }
5145
5146         if (new) {
5147                 WARN_ON(!mutex_is_locked(&new->base.dev->struct_mutex));
5148                 WARN_ON(new->frontbuffer_bits & frontbuffer_bits);
5149                 new->frontbuffer_bits |= frontbuffer_bits;
5150         }
5151 }
5152
5153 static bool mutex_is_locked_by(struct mutex *mutex, struct task_struct *task)
5154 {
5155         if (!mutex_is_locked(mutex))
5156                 return false;
5157
5158 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_MUTEXES)
5159         return mutex->owner == task;
5160 #else
5161         /* Since UP may be pre-empted, we cannot assume that we own the lock */
5162         return false;
5163 #endif
5164 }
5165
5166 static bool i915_gem_shrinker_lock(struct drm_device *dev, bool *unlock)
5167 {
5168         if (!mutex_trylock(&dev->struct_mutex)) {
5169                 if (!mutex_is_locked_by(&dev->struct_mutex, current))
5170                         return false;
5171
5172                 if (to_i915(dev)->mm.shrinker_no_lock_stealing)
5173                         return false;
5174
5175                 *unlock = false;
5176         } else
5177                 *unlock = true;
5178
5179         return true;
5180 }
5181
5182 static int num_vma_bound(struct drm_i915_gem_object *obj)
5183 {
5184         struct i915_vma *vma;
5185         int count = 0;
5186
5187         list_for_each_entry(vma, &obj->vma_list, vma_link)
5188                 if (drm_mm_node_allocated(&vma->node))
5189                         count++;
5190
5191         return count;
5192 }
5193
5194 static unsigned long
5195 i915_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc)
5196 {
5197         struct drm_i915_private *dev_priv =
5198                 container_of(shrinker, struct drm_i915_private, mm.shrinker);
5199         struct drm_device *dev = dev_priv->dev;
5200         struct drm_i915_gem_object *obj;
5201         unsigned long count;
5202         bool unlock;
5203
5204         if (!i915_gem_shrinker_lock(dev, &unlock))
5205                 return 0;
5206
5207         count = 0;
5208         list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list)
5209                 if (obj->pages_pin_count == 0)
5210                         count += obj->base.size >> PAGE_SHIFT;
5211
5212         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
5213                 if (!i915_gem_obj_is_pinned(obj) &&
5214                     obj->pages_pin_count == num_vma_bound(obj))
5215                         count += obj->base.size >> PAGE_SHIFT;
5216         }
5217
5218         if (unlock)
5219                 mutex_unlock(&dev->struct_mutex);
5220
5221         return count;
5222 }
5223
5224 /* All the new VM stuff */
5225 unsigned long i915_gem_obj_offset(struct drm_i915_gem_object *o,
5226                                   struct i915_address_space *vm)
5227 {
5228         struct drm_i915_private *dev_priv = o->base.dev->dev_private;
5229         struct i915_vma *vma;
5230
5231         WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
5232
5233         list_for_each_entry(vma, &o->vma_list, vma_link) {
5234                 if (vma->vm == vm)
5235                         return vma->node.start;
5236
5237         }
5238         WARN(1, "%s vma for this object not found.\n",
5239              i915_is_ggtt(vm) ? "global" : "ppgtt");
5240         return -1;
5241 }
5242
5243 bool i915_gem_obj_bound(struct drm_i915_gem_object *o,
5244                         struct i915_address_space *vm)
5245 {
5246         struct i915_vma *vma;
5247
5248         list_for_each_entry(vma, &o->vma_list, vma_link)
5249                 if (vma->vm == vm && drm_mm_node_allocated(&vma->node))
5250                         return true;
5251
5252         return false;
5253 }
5254
5255 bool i915_gem_obj_bound_any(struct drm_i915_gem_object *o)
5256 {
5257         struct i915_vma *vma;
5258
5259         list_for_each_entry(vma, &o->vma_list, vma_link)
5260                 if (drm_mm_node_allocated(&vma->node))
5261                         return true;
5262
5263         return false;
5264 }
5265
5266 unsigned long i915_gem_obj_size(struct drm_i915_gem_object *o,
5267                                 struct i915_address_space *vm)
5268 {
5269         struct drm_i915_private *dev_priv = o->base.dev->dev_private;
5270         struct i915_vma *vma;
5271
5272         WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
5273
5274         BUG_ON(list_empty(&o->vma_list));
5275
5276         list_for_each_entry(vma, &o->vma_list, vma_link)
5277                 if (vma->vm == vm)
5278                         return vma->node.size;
5279
5280         return 0;
5281 }
5282
5283 static unsigned long
5284 i915_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
5285 {
5286         struct drm_i915_private *dev_priv =
5287                 container_of(shrinker, struct drm_i915_private, mm.shrinker);
5288         struct drm_device *dev = dev_priv->dev;
5289         unsigned long freed;
5290         bool unlock;
5291
5292         if (!i915_gem_shrinker_lock(dev, &unlock))
5293                 return SHRINK_STOP;
5294
5295         freed = i915_gem_shrink(dev_priv,
5296                                 sc->nr_to_scan,
5297                                 I915_SHRINK_BOUND |
5298                                 I915_SHRINK_UNBOUND |
5299                                 I915_SHRINK_PURGEABLE);
5300         if (freed < sc->nr_to_scan)
5301                 freed += i915_gem_shrink(dev_priv,
5302                                          sc->nr_to_scan - freed,
5303                                          I915_SHRINK_BOUND |
5304                                          I915_SHRINK_UNBOUND);
5305         if (unlock)
5306                 mutex_unlock(&dev->struct_mutex);
5307
5308         return freed;
5309 }
5310
5311 static int
5312 i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr)
5313 {
5314         struct drm_i915_private *dev_priv =
5315                 container_of(nb, struct drm_i915_private, mm.oom_notifier);
5316         struct drm_device *dev = dev_priv->dev;
5317         struct drm_i915_gem_object *obj;
5318         unsigned long timeout = msecs_to_jiffies(5000) + 1;
5319         unsigned long pinned, bound, unbound, freed_pages;
5320         bool was_interruptible;
5321         bool unlock;
5322
5323         while (!i915_gem_shrinker_lock(dev, &unlock) && --timeout) {
5324                 schedule_timeout_killable(1);
5325                 if (fatal_signal_pending(current))
5326                         return NOTIFY_DONE;
5327         }
5328         if (timeout == 0) {
5329                 pr_err("Unable to purge GPU memory due lock contention.\n");
5330                 return NOTIFY_DONE;
5331         }
5332
5333         was_interruptible = dev_priv->mm.interruptible;
5334         dev_priv->mm.interruptible = false;
5335
5336         freed_pages = i915_gem_shrink_all(dev_priv);
5337
5338         dev_priv->mm.interruptible = was_interruptible;
5339
5340         /* Because we may be allocating inside our own driver, we cannot
5341          * assert that there are no objects with pinned pages that are not
5342          * being pointed to by hardware.
5343          */
5344         unbound = bound = pinned = 0;
5345         list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) {
5346                 if (!obj->base.filp) /* not backed by a freeable object */
5347                         continue;
5348
5349                 if (obj->pages_pin_count)
5350                         pinned += obj->base.size;
5351                 else
5352                         unbound += obj->base.size;
5353         }
5354         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
5355                 if (!obj->base.filp)
5356                         continue;
5357
5358                 if (obj->pages_pin_count)
5359                         pinned += obj->base.size;
5360                 else
5361                         bound += obj->base.size;
5362         }
5363
5364         if (unlock)
5365                 mutex_unlock(&dev->struct_mutex);
5366
5367         if (freed_pages || unbound || bound)
5368                 pr_info("Purging GPU memory, %lu bytes freed, %lu bytes still pinned.\n",
5369                         freed_pages << PAGE_SHIFT, pinned);
5370         if (unbound || bound)
5371                 pr_err("%lu and %lu bytes still available in the "
5372                        "bound and unbound GPU page lists.\n",
5373                        bound, unbound);
5374
5375         *(unsigned long *)ptr += freed_pages;
5376         return NOTIFY_DONE;
5377 }
5378
5379 struct i915_vma *i915_gem_obj_to_ggtt(struct drm_i915_gem_object *obj)
5380 {
5381         struct i915_vma *vma;
5382
5383         vma = list_first_entry(&obj->vma_list, typeof(*vma), vma_link);
5384         if (vma->vm != i915_obj_to_ggtt(obj))
5385                 return NULL;
5386
5387         return vma;
5388 }