drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008-2015 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  */
  27
  28 #include <drm/drmP.h>
  29 #include <drm/drm_vma_manager.h>
  30 #include <drm/i915_drm.h>
  31 #include "i915_drv.h"
  32 #include "i915_vgpu.h"
  33 #include "i915_trace.h"
  34 #include "intel_drv.h"
  35 #include "intel_mocs.h"
  36 #include <linux/shmem_fs.h>
  37 #include <linux/slab.h>
  38 #include <linux/swap.h>
  39 #include <linux/pci.h>
  40 #include <linux/dma-buf.h>
  41
  42 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
  43 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
  44 static void
  45 i915_gem_object_retire__write(struct drm_i915_gem_object *obj);
  46 static void
  47 i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring);
  48
  49 static bool cpu_cache_is_coherent(struct drm_device *dev,
  50                                   enum i915_cache_level level)
  51 {
  52         return HAS_LLC(dev) || level != I915_CACHE_NONE;
  53 }
  54
  55 static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
  56 {
  57         if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
  58                 return true;
  59
  60         return obj->pin_display;
  61 }
  62
  63 /* some bookkeeping */
  64 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
  65                                   size_t size)
  66 {
  67         spin_lock(&dev_priv->mm.object_stat_lock);
  68         dev_priv->mm.object_count++;
  69         dev_priv->mm.object_memory += size;
  70         spin_unlock(&dev_priv->mm.object_stat_lock);
  71 }
  72
  73 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
  74                                      size_t size)
  75 {
  76         spin_lock(&dev_priv->mm.object_stat_lock);
  77         dev_priv->mm.object_count--;
  78         dev_priv->mm.object_memory -= size;
  79         spin_unlock(&dev_priv->mm.object_stat_lock);
  80 }
  81
  82 static int
  83 i915_gem_wait_for_error(struct i915_gpu_error *error)
  84 {
  85         int ret;
  86
  87         if (!i915_reset_in_progress(error))
  88                 return 0;
  89
  90         /*
  91          * Only wait 10 seconds for the gpu reset to complete to avoid hanging
  92          * userspace. If it takes that long something really bad is going on and
  93          * we should simply try to bail out and fail as gracefully as possible.
  94          */
  95         ret = wait_event_interruptible_timeout(error->reset_queue,
  96                                                !i915_reset_in_progress(error),
  97                                                10*HZ);
  98         if (ret == 0) {
  99                 DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
 100                 return -EIO;
 101         } else if (ret < 0) {
 102                 return ret;
 103         } else {
 104                 return 0;
 105         }
 106 }
 107
 108 int i915_mutex_lock_interruptible(struct drm_device *dev)
 109 {
 110         struct drm_i915_private *dev_priv = dev->dev_private;
 111         int ret;
 112
 113         ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
 114         if (ret)
 115                 return ret;
 116
 117         ret = mutex_lock_interruptible(&dev->struct_mutex);
 118         if (ret)
 119                 return ret;
 120
 121         WARN_ON(i915_verify_lists(dev));
 122         return 0;
 123 }
 124
 125 int
 126 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 127                             struct drm_file *file)
 128 {
 129         struct drm_i915_private *dev_priv = to_i915(dev);
 130         struct i915_ggtt *ggtt = &dev_priv->ggtt;
 131         struct drm_i915_gem_get_aperture *args = data;
 132         struct i915_vma *vma;
 133         size_t pinned;
 134
 135         pinned = 0;
 136         mutex_lock(&dev->struct_mutex);
 137         list_for_each_entry(vma, &ggtt->base.active_list, vm_link)
 138                 if (vma->pin_count)
 139                         pinned += vma->node.size;
 140         list_for_each_entry(vma, &ggtt->base.inactive_list, vm_link)
 141                 if (vma->pin_count)
 142                         pinned += vma->node.size;
 143         mutex_unlock(&dev->struct_mutex);
 144
 145         args->aper_size = ggtt->base.total;
 146         args->aper_available_size = args->aper_size - pinned;
 147
 148         return 0;
 149 }
 150
 151 static int
 152 i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
 153 {
 154         struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
 155         char *vaddr = obj->phys_handle->vaddr;
 156         struct sg_table *st;
 157         struct scatterlist *sg;
 158         int i;
 159
 160         if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj)))
 161                 return -EINVAL;
 162
 163         for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
 164                 struct page *page;
 165                 char *src;
 166
 167                 page = shmem_read_mapping_page(mapping, i);
 168                 if (IS_ERR(page))
 169                         return PTR_ERR(page);
 170
 171                 src = kmap_atomic(page);
 172                 memcpy(vaddr, src, PAGE_SIZE);
 173                 drm_clflush_virt_range(vaddr, PAGE_SIZE);
 174                 kunmap_atomic(src);
 175
 176                 put_page(page);
 177                 vaddr += PAGE_SIZE;
 178         }
 179
 180         i915_gem_chipset_flush(to_i915(obj->base.dev));
 181
 182         st = kmalloc(sizeof(*st), GFP_KERNEL);
 183         if (st == NULL)
 184                 return -ENOMEM;
 185
 186         if (sg_alloc_table(st, 1, GFP_KERNEL)) {
 187                 kfree(st);
 188                 return -ENOMEM;
 189         }
 190
 191         sg = st->sgl;
 192         sg->offset = 0;
 193         sg->length = obj->base.size;
 194
 195         sg_dma_address(sg) = obj->phys_handle->busaddr;
 196         sg_dma_len(sg) = obj->base.size;
 197
 198         obj->pages = st;
 199         return 0;
 200 }
 201
 202 static void
 203 i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj)
 204 {
 205         int ret;
 206
 207         BUG_ON(obj->madv == __I915_MADV_PURGED);
 208
 209         ret = i915_gem_object_set_to_cpu_domain(obj, true);
 210         if (WARN_ON(ret)) {
 211                 /* In the event of a disaster, abandon all caches and
 212                  * hope for the best.
 213                  */
 214                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
 215         }
 216
 217         if (obj->madv == I915_MADV_DONTNEED)
 218                 obj->dirty = 0;
 219
 220         if (obj->dirty) {
 221                 struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
 222                 char *vaddr = obj->phys_handle->vaddr;
 223                 int i;
 224
 225                 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
 226                         struct page *page;
 227                         char *dst;
 228
 229                         page = shmem_read_mapping_page(mapping, i);
 230                         if (IS_ERR(page))
 231                                 continue;
 232
 233                         dst = kmap_atomic(page);
 234                         drm_clflush_virt_range(vaddr, PAGE_SIZE);
 235                         memcpy(dst, vaddr, PAGE_SIZE);
 236                         kunmap_atomic(dst);
 237
 238                         set_page_dirty(page);
 239                         if (obj->madv == I915_MADV_WILLNEED)
 240                                 mark_page_accessed(page);
 241                         put_page(page);
 242                         vaddr += PAGE_SIZE;
 243                 }
 244                 obj->dirty = 0;
 245         }
 246
 247         sg_free_table(obj->pages);
 248         kfree(obj->pages);
 249 }
 250
 251 static void
 252 i915_gem_object_release_phys(struct drm_i915_gem_object *obj)
 253 {
 254         drm_pci_free(obj->base.dev, obj->phys_handle);
 255 }
 256
 257 static const struct drm_i915_gem_object_ops i915_gem_phys_ops = {
 258         .get_pages = i915_gem_object_get_pages_phys,
 259         .put_pages = i915_gem_object_put_pages_phys,
 260         .release = i915_gem_object_release_phys,
 261 };
 262
 263 static int
 264 drop_pages(struct drm_i915_gem_object *obj)
 265 {
 266         struct i915_vma *vma, *next;
 267         int ret;
 268
 269         drm_gem_object_reference(&obj->base);
 270         list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link)
 271                 if (i915_vma_unbind(vma))
 272                         break;
 273
 274         ret = i915_gem_object_put_pages(obj);
 275         drm_gem_object_unreference(&obj->base);
 276
 277         return ret;
 278 }
 279
 280 int
 281 i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
 282                             int align)
 283 {
 284         drm_dma_handle_t *phys;
 285         int ret;
 286
 287         if (obj->phys_handle) {
 288                 if ((unsigned long)obj->phys_handle->vaddr & (align -1))
 289                         return -EBUSY;
 290
 291                 return 0;
 292         }
 293
 294         if (obj->madv != I915_MADV_WILLNEED)
 295                 return -EFAULT;
 296
 297         if (obj->base.filp == NULL)
 298                 return -EINVAL;
 299
 300         ret = drop_pages(obj);
 301         if (ret)
 302                 return ret;
 303
 304         /* create a new object */
 305         phys = drm_pci_alloc(obj->base.dev, obj->base.size, align);
 306         if (!phys)
 307                 return -ENOMEM;
 308
 309         obj->phys_handle = phys;
 310         obj->ops = &i915_gem_phys_ops;
 311
 312         return i915_gem_object_get_pages(obj);
 313 }
 314
 315 static int
 316 i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
 317                      struct drm_i915_gem_pwrite *args,
 318                      struct drm_file *file_priv)
 319 {
 320         struct drm_device *dev = obj->base.dev;
 321         void *vaddr = obj->phys_handle->vaddr + args->offset;
 322         char __user *user_data = u64_to_user_ptr(args->data_ptr);
 323         int ret = 0;
 324
 325         /* We manually control the domain here and pretend that it
 326          * remains coherent i.e. in the GTT domain, like shmem_pwrite.
 327          */
 328         ret = i915_gem_object_wait_rendering(obj, false);
 329         if (ret)
 330                 return ret;
 331
 332         intel_fb_obj_invalidate(obj, ORIGIN_CPU);
 333         if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
 334                 unsigned long unwritten;
 335
 336                 /* The physical object once assigned is fixed for the lifetime
 337                  * of the obj, so we can safely drop the lock and continue
 338                  * to access vaddr.
 339                  */
 340                 mutex_unlock(&dev->struct_mutex);
 341                 unwritten = copy_from_user(vaddr, user_data, args->size);
 342                 mutex_lock(&dev->struct_mutex);
 343                 if (unwritten) {
 344                         ret = -EFAULT;
 345                         goto out;
 346                 }
 347         }
 348
 349         drm_clflush_virt_range(vaddr, args->size);
 350         i915_gem_chipset_flush(to_i915(dev));
 351
 352 out:
 353         intel_fb_obj_flush(obj, false, ORIGIN_CPU);
 354         return ret;
 355 }
 356
 357 void *i915_gem_object_alloc(struct drm_device *dev)
 358 {
 359         struct drm_i915_private *dev_priv = dev->dev_private;
 360         return kmem_cache_zalloc(dev_priv->objects, GFP_KERNEL);
 361 }
 362
 363 void i915_gem_object_free(struct drm_i915_gem_object *obj)
 364 {
 365         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
 366         kmem_cache_free(dev_priv->objects, obj);
 367 }
 368
 369 static int
 370 i915_gem_create(struct drm_file *file,
 371                 struct drm_device *dev,
 372                 uint64_t size,
 373                 uint32_t *handle_p)
 374 {
 375         struct drm_i915_gem_object *obj;
 376         int ret;
 377         u32 handle;
 378
 379         size = roundup(size, PAGE_SIZE);
 380         if (size == 0)
 381                 return -EINVAL;
 382
 383         /* Allocate the new object */
 384         obj = i915_gem_object_create(dev, size);
 385         if (IS_ERR(obj))
 386                 return PTR_ERR(obj);
 387
 388         ret = drm_gem_handle_create(file, &obj->base, &handle);
 389         /* drop reference from allocate - handle holds it now */
 390         drm_gem_object_unreference_unlocked(&obj->base);
 391         if (ret)
 392                 return ret;
 393
 394         *handle_p = handle;
 395         return 0;
 396 }
 397
 398 int
 399 i915_gem_dumb_create(struct drm_file *file,
 400                      struct drm_device *dev,
 401                      struct drm_mode_create_dumb *args)
 402 {
 403         /* have to work out size/pitch and return them */
 404         args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64);
 405         args->size = args->pitch * args->height;
 406         return i915_gem_create(file, dev,
 407                                args->size, &args->handle);
 408 }
 409
 410 /**
 411  * Creates a new mm object and returns a handle to it.
 412  */
 413 int
 414 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 415                       struct drm_file *file)
 416 {
 417         struct drm_i915_gem_create *args = data;
 418
 419         return i915_gem_create(file, dev,
 420                                args->size, &args->handle);
 421 }
 422
 423 static inline int
 424 __copy_to_user_swizzled(char __user *cpu_vaddr,
 425                         const char *gpu_vaddr, int gpu_offset,
 426                         int length)
 427 {
 428         int ret, cpu_offset = 0;
 429
 430         while (length > 0) {
 431                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 432                 int this_length = min(cacheline_end - gpu_offset, length);
 433                 int swizzled_gpu_offset = gpu_offset ^ 64;
 434
 435                 ret = __copy_to_user(cpu_vaddr + cpu_offset,
 436                                      gpu_vaddr + swizzled_gpu_offset,
 437                                      this_length);
 438                 if (ret)
 439                         return ret + length;
 440
 441                 cpu_offset += this_length;
 442                 gpu_offset += this_length;
 443                 length -= this_length;
 444         }
 445
 446         return 0;
 447 }
 448
 449 static inline int
 450 __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
 451                           const char __user *cpu_vaddr,
 452                           int length)
 453 {
 454         int ret, cpu_offset = 0;
 455
 456         while (length > 0) {
 457                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 458                 int this_length = min(cacheline_end - gpu_offset, length);
 459                 int swizzled_gpu_offset = gpu_offset ^ 64;
 460
 461                 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
 462                                        cpu_vaddr + cpu_offset,
 463                                        this_length);
 464                 if (ret)
 465                         return ret + length;
 466
 467                 cpu_offset += this_length;
 468                 gpu_offset += this_length;
 469                 length -= this_length;
 470         }
 471
 472         return 0;
 473 }
 474
 475 /*
 476  * Pins the specified object's pages and synchronizes the object with
 477  * GPU accesses. Sets needs_clflush to non-zero if the caller should
 478  * flush the object from the CPU cache.
 479  */
 480 int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
 481                                     int *needs_clflush)
 482 {
 483         int ret;
 484
 485         *needs_clflush = 0;
 486
 487         if (WARN_ON((obj->ops->flags & I915_GEM_OBJECT_HAS_STRUCT_PAGE) == 0))
 488                 return -EINVAL;
 489
 490         if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
 491                 /* If we're not in the cpu read domain, set ourself into the gtt
 492                  * read domain and manually flush cachelines (if required). This
 493                  * optimizes for the case when the gpu will dirty the data
 494                  * anyway again before the next pread happens. */
 495                 *needs_clflush = !cpu_cache_is_coherent(obj->base.dev,
 496                                                         obj->cache_level);
 497                 ret = i915_gem_object_wait_rendering(obj, true);
 498                 if (ret)
 499                         return ret;
 500         }
 501
 502         ret = i915_gem_object_get_pages(obj);
 503         if (ret)
 504                 return ret;
 505
 506         i915_gem_object_pin_pages(obj);
 507
 508         return ret;
 509 }
 510
 511 /* Per-page copy function for the shmem pread fastpath.
 512  * Flushes invalid cachelines before reading the target if
 513  * needs_clflush is set. */
 514 static int
 515 shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
 516                  char __user *user_data,
 517                  bool page_do_bit17_swizzling, bool needs_clflush)
 518 {
 519         char *vaddr;
 520         int ret;
 521
 522         if (unlikely(page_do_bit17_swizzling))
 523                 return -EINVAL;
 524
 525         vaddr = kmap_atomic(page);
 526         if (needs_clflush)
 527                 drm_clflush_virt_range(vaddr + shmem_page_offset,
 528                                        page_length);
 529         ret = __copy_to_user_inatomic(user_data,
 530                                       vaddr + shmem_page_offset,
 531                                       page_length);
 532         kunmap_atomic(vaddr);
 533
 534         return ret ? -EFAULT : 0;
 535 }
 536
 537 static void
 538 shmem_clflush_swizzled_range(char *addr, unsigned long length,
 539                              bool swizzled)
 540 {
 541         if (unlikely(swizzled)) {
 542                 unsigned long start = (unsigned long) addr;
 543                 unsigned long end = (unsigned long) addr + length;
 544
 545                 /* For swizzling simply ensure that we always flush both
 546                  * channels. Lame, but simple and it works. Swizzled
 547                  * pwrite/pread is far from a hotpath - current userspace
 548                  * doesn't use it at all. */
 549                 start = round_down(start, 128);
 550                 end = round_up(end, 128);
 551
 552                 drm_clflush_virt_range((void *)start, end - start);
 553         } else {
 554                 drm_clflush_virt_range(addr, length);
 555         }
 556
 557 }
 558
 559 /* Only difference to the fast-path function is that this can handle bit17
 560  * and uses non-atomic copy and kmap functions. */
 561 static int
 562 shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
 563                  char __user *user_data,
 564                  bool page_do_bit17_swizzling, bool needs_clflush)
 565 {
 566         char *vaddr;
 567         int ret;
 568
 569         vaddr = kmap(page);
 570         if (needs_clflush)
 571                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
 572                                              page_length,
 573                                              page_do_bit17_swizzling);
 574
 575         if (page_do_bit17_swizzling)
 576                 ret = __copy_to_user_swizzled(user_data,
 577                                               vaddr, shmem_page_offset,
 578                                               page_length);
 579         else
 580                 ret = __copy_to_user(user_data,
 581                                      vaddr + shmem_page_offset,
 582                                      page_length);
 583         kunmap(page);
 584
 585         return ret ? - EFAULT : 0;
 586 }
 587
 588 static int
 589 i915_gem_shmem_pread(struct drm_device *dev,
 590                      struct drm_i915_gem_object *obj,
 591                      struct drm_i915_gem_pread *args,
 592                      struct drm_file *file)
 593 {
 594         char __user *user_data;
 595         ssize_t remain;
 596         loff_t offset;
 597         int shmem_page_offset, page_length, ret = 0;
 598         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
 599         int prefaulted = 0;
 600         int needs_clflush = 0;
 601         struct sg_page_iter sg_iter;
 602
 603         user_data = u64_to_user_ptr(args->data_ptr);
 604         remain = args->size;
 605
 606         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 607
 608         ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
 609         if (ret)
 610                 return ret;
 611
 612         offset = args->offset;
 613
 614         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
 615                          offset >> PAGE_SHIFT) {
 616                 struct page *page = sg_page_iter_page(&sg_iter);
 617
 618                 if (remain <= 0)
 619                         break;
 620
 621                 /* Operation in this page
 622                  *
 623                  * shmem_page_offset = offset within page in shmem file
 624                  * page_length = bytes to copy for this page
 625                  */
 626                 shmem_page_offset = offset_in_page(offset);
 627                 page_length = remain;
 628                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 629                         page_length = PAGE_SIZE - shmem_page_offset;
 630
 631                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
 632                         (page_to_phys(page) & (1 << 17)) != 0;
 633
 634                 ret = shmem_pread_fast(page, shmem_page_offset, page_length,
 635                                        user_data, page_do_bit17_swizzling,
 636                                        needs_clflush);
 637                 if (ret == 0)
 638                         goto next_page;
 639
 640                 mutex_unlock(&dev->struct_mutex);
 641
 642                 if (likely(!i915.prefault_disable) && !prefaulted) {
 643                         ret = fault_in_multipages_writeable(user_data, remain);
 644                         /* Userspace is tricking us, but we've already clobbered
 645                          * its pages with the prefault and promised to write the
 646                          * data up to the first fault. Hence ignore any errors
 647                          * and just continue. */
 648                         (void)ret;
 649                         prefaulted = 1;
 650                 }
 651
 652                 ret = shmem_pread_slow(page, shmem_page_offset, page_length,
 653                                        user_data, page_do_bit17_swizzling,
 654                                        needs_clflush);
 655
 656                 mutex_lock(&dev->struct_mutex);
 657
 658                 if (ret)
 659                         goto out;
 660
 661 next_page:
 662                 remain -= page_length;
 663                 user_data += page_length;
 664                 offset += page_length;
 665         }
 666
 667 out:
 668         i915_gem_object_unpin_pages(obj);
 669
 670         return ret;
 671 }
 672
 673 /**
 674  * Reads data from the object referenced by handle.
 675  *
 676  * On error, the contents of *data are undefined.
 677  */
 678 int
 679 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 680                      struct drm_file *file)
 681 {
 682         struct drm_i915_gem_pread *args = data;
 683         struct drm_i915_gem_object *obj;
 684         int ret = 0;
 685
 686         if (args->size == 0)
 687                 return 0;
 688
 689         if (!access_ok(VERIFY_WRITE,
 690                        u64_to_user_ptr(args->data_ptr),
 691                        args->size))
 692                 return -EFAULT;
 693
 694         ret = i915_mutex_lock_interruptible(dev);
 695         if (ret)
 696                 return ret;
 697
 698         obj = to_intel_bo(drm_gem_object_lookup(file, args->handle));
 699         if (&obj->base == NULL) {
 700                 ret = -ENOENT;
 701                 goto unlock;
 702         }
 703
 704         /* Bounds check source.  */
 705         if (args->offset > obj->base.size ||
 706             args->size > obj->base.size - args->offset) {
 707                 ret = -EINVAL;
 708                 goto out;
 709         }
 710
 711         /* prime objects have no backing filp to GEM pread/pwrite
 712          * pages from.
 713          */
 714         if (!obj->base.filp) {
 715                 ret = -EINVAL;
 716                 goto out;
 717         }
 718
 719         trace_i915_gem_object_pread(obj, args->offset, args->size);
 720
 721         ret = i915_gem_shmem_pread(dev, obj, args, file);
 722
 723 out:
 724         drm_gem_object_unreference(&obj->base);
 725 unlock:
 726         mutex_unlock(&dev->struct_mutex);
 727         return ret;
 728 }
 729
 730 /* This is the fast write path which cannot handle
 731  * page faults in the source data
 732  */
 733
 734 static inline int
 735 fast_user_write(struct io_mapping *mapping,
 736                 loff_t page_base, int page_offset,
 737                 char __user *user_data,
 738                 int length)
 739 {
 740         void __iomem *vaddr_atomic;
 741         void *vaddr;
 742         unsigned long unwritten;
 743
 744         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
 745         /* We can use the cpu mem copy function because this is X86. */
 746         vaddr = (void __force*)vaddr_atomic + page_offset;
 747         unwritten = __copy_from_user_inatomic_nocache(vaddr,
 748                                                       user_data, length);
 749         io_mapping_unmap_atomic(vaddr_atomic);
 750         return unwritten;
 751 }
 752
 753 /**
 754  * This is the fast pwrite path, where we copy the data directly from the
 755  * user into the GTT, uncached.
 756  */
 757 static int
 758 i915_gem_gtt_pwrite_fast(struct drm_device *dev,
 759                          struct drm_i915_gem_object *obj,
 760                          struct drm_i915_gem_pwrite *args,
 761                          struct drm_file *file)
 762 {
 763         struct drm_i915_private *dev_priv = to_i915(dev);
 764         struct i915_ggtt *ggtt = &dev_priv->ggtt;
 765         ssize_t remain;
 766         loff_t offset, page_base;
 767         char __user *user_data;
 768         int page_offset, page_length, ret;
 769
 770         ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE | PIN_NONBLOCK);
 771         if (ret)
 772                 goto out;
 773
 774         ret = i915_gem_object_set_to_gtt_domain(obj, true);
 775         if (ret)
 776                 goto out_unpin;
 777
 778         ret = i915_gem_object_put_fence(obj);
 779         if (ret)
 780                 goto out_unpin;
 781
 782         user_data = u64_to_user_ptr(args->data_ptr);
 783         remain = args->size;
 784
 785         offset = i915_gem_obj_ggtt_offset(obj) + args->offset;
 786
 787         intel_fb_obj_invalidate(obj, ORIGIN_GTT);
 788
 789         while (remain > 0) {
 790                 /* Operation in this page
 791                  *
 792                  * page_base = page offset within aperture
 793                  * page_offset = offset within page
 794                  * page_length = bytes to copy for this page
 795                  */
 796                 page_base = offset & PAGE_MASK;
 797                 page_offset = offset_in_page(offset);
 798                 page_length = remain;
 799                 if ((page_offset + remain) > PAGE_SIZE)
 800                         page_length = PAGE_SIZE - page_offset;
 801
 802                 /* If we get a fault while copying data, then (presumably) our
 803                  * source page isn't available.  Return the error and we'll
 804                  * retry in the slow path.
 805                  */
 806                 if (fast_user_write(ggtt->mappable, page_base,
 807                                     page_offset, user_data, page_length)) {
 808                         ret = -EFAULT;
 809                         goto out_flush;
 810                 }
 811
 812                 remain -= page_length;
 813                 user_data += page_length;
 814                 offset += page_length;
 815         }
 816
 817 out_flush:
 818         intel_fb_obj_flush(obj, false, ORIGIN_GTT);
 819 out_unpin:
 820         i915_gem_object_ggtt_unpin(obj);
 821 out:
 822         return ret;
 823 }
 824
 825 /* Per-page copy function for the shmem pwrite fastpath.
 826  * Flushes invalid cachelines before writing to the target if
 827  * needs_clflush_before is set and flushes out any written cachelines after
 828  * writing if needs_clflush is set. */
 829 static int
 830 shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
 831                   char __user *user_data,
 832                   bool page_do_bit17_swizzling,
 833                   bool needs_clflush_before,
 834                   bool needs_clflush_after)
 835 {
 836         char *vaddr;
 837         int ret;
 838
 839         if (unlikely(page_do_bit17_swizzling))
 840                 return -EINVAL;
 841
 842         vaddr = kmap_atomic(page);
 843         if (needs_clflush_before)
 844                 drm_clflush_virt_range(vaddr + shmem_page_offset,
 845                                        page_length);
 846         ret = __copy_from_user_inatomic(vaddr + shmem_page_offset,
 847                                         user_data, page_length);
 848         if (needs_clflush_after)
 849                 drm_clflush_virt_range(vaddr + shmem_page_offset,
 850                                        page_length);
 851         kunmap_atomic(vaddr);
 852
 853         return ret ? -EFAULT : 0;
 854 }
 855
 856 /* Only difference to the fast-path function is that this can handle bit17
 857  * and uses non-atomic copy and kmap functions. */
 858 static int
 859 shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
 860                   char __user *user_data,
 861                   bool page_do_bit17_swizzling,
 862                   bool needs_clflush_before,
 863                   bool needs_clflush_after)
 864 {
 865         char *vaddr;
 866         int ret;
 867
 868         vaddr = kmap(page);
 869         if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
 870                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
 871                                              page_length,
 872                                              page_do_bit17_swizzling);
 873         if (page_do_bit17_swizzling)
 874                 ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
 875                                                 user_data,
 876                                                 page_length);
 877         else
 878                 ret = __copy_from_user(vaddr + shmem_page_offset,
 879                                        user_data,
 880                                        page_length);
 881         if (needs_clflush_after)
 882                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
 883                                              page_length,
 884                                              page_do_bit17_swizzling);
 885         kunmap(page);
 886
 887         return ret ? -EFAULT : 0;
 888 }
 889
 890 static int
 891 i915_gem_shmem_pwrite(struct drm_device *dev,
 892                       struct drm_i915_gem_object *obj,
 893                       struct drm_i915_gem_pwrite *args,
 894                       struct drm_file *file)
 895 {
 896         ssize_t remain;
 897         loff_t offset;
 898         char __user *user_data;
 899         int shmem_page_offset, page_length, ret = 0;
 900         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
 901         int hit_slowpath = 0;
 902         int needs_clflush_after = 0;
 903         int needs_clflush_before = 0;
 904         struct sg_page_iter sg_iter;
 905
 906         user_data = u64_to_user_ptr(args->data_ptr);
 907         remain = args->size;
 908
 909         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 910
 911         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
 912                 /* If we're not in the cpu write domain, set ourself into the gtt
 913                  * write domain and manually flush cachelines (if required). This
 914                  * optimizes for the case when the gpu will use the data
 915                  * right away and we therefore have to clflush anyway. */
 916                 needs_clflush_after = cpu_write_needs_clflush(obj);
 917                 ret = i915_gem_object_wait_rendering(obj, false);
 918                 if (ret)
 919                         return ret;
 920         }
 921         /* Same trick applies to invalidate partially written cachelines read
 922          * before writing. */
 923         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
 924                 needs_clflush_before =
 925                         !cpu_cache_is_coherent(dev, obj->cache_level);
 926
 927         ret = i915_gem_object_get_pages(obj);
 928         if (ret)
 929                 return ret;
 930
 931         intel_fb_obj_invalidate(obj, ORIGIN_CPU);
 932
 933         i915_gem_object_pin_pages(obj);
 934
 935         offset = args->offset;
 936         obj->dirty = 1;
 937
 938         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
 939                          offset >> PAGE_SHIFT) {
 940                 struct page *page = sg_page_iter_page(&sg_iter);
 941                 int partial_cacheline_write;
 942
 943                 if (remain <= 0)
 944                         break;
 945
 946                 /* Operation in this page
 947                  *
 948                  * shmem_page_offset = offset within page in shmem file
 949                  * page_length = bytes to copy for this page
 950                  */
 951                 shmem_page_offset = offset_in_page(offset);
 952
 953                 page_length = remain;
 954                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 955                         page_length = PAGE_SIZE - shmem_page_offset;
 956
 957                 /* If we don't overwrite a cacheline completely we need to be
 958                  * careful to have up-to-date data by first clflushing. Don't
 959                  * overcomplicate things and flush the entire patch. */
 960                 partial_cacheline_write = needs_clflush_before &&
 961                         ((shmem_page_offset | page_length)
 962                                 & (boot_cpu_data.x86_clflush_size - 1));
 963
 964                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
 965                         (page_to_phys(page) & (1 << 17)) != 0;
 966
 967                 ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
 968                                         user_data, page_do_bit17_swizzling,
 969                                         partial_cacheline_write,
 970                                         needs_clflush_after);
 971                 if (ret == 0)
 972                         goto next_page;
 973
 974                 hit_slowpath = 1;
 975                 mutex_unlock(&dev->struct_mutex);
 976                 ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
 977                                         user_data, page_do_bit17_swizzling,
 978                                         partial_cacheline_write,
 979                                         needs_clflush_after);
 980
 981                 mutex_lock(&dev->struct_mutex);
 982
 983                 if (ret)
 984                         goto out;
 985
 986 next_page:
 987                 remain -= page_length;
 988                 user_data += page_length;
 989                 offset += page_length;
 990         }
 991
 992 out:
 993         i915_gem_object_unpin_pages(obj);
 994
 995         if (hit_slowpath) {
 996                 /*
 997                  * Fixup: Flush cpu caches in case we didn't flush the dirty
 998                  * cachelines in-line while writing and the object moved
 999                  * out of the cpu write domain while we've dropped the lock.
1000                  */
1001                 if (!needs_clflush_after &&
1002                     obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
1003                         if (i915_gem_clflush_object(obj, obj->pin_display))
1004                                 needs_clflush_after = true;
1005                 }
1006         }
1007
1008         if (needs_clflush_after)
1009                 i915_gem_chipset_flush(to_i915(dev));
1010         else
1011                 obj->cache_dirty = true;
1012
1013         intel_fb_obj_flush(obj, false, ORIGIN_CPU);
1014         return ret;
1015 }
1016
1017 /**
1018  * Writes data to the object referenced by handle.
1019  *
1020  * On error, the contents of the buffer that were to be modified are undefined.
1021  */
1022 int
1023 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
1024                       struct drm_file *file)
1025 {
1026         struct drm_i915_private *dev_priv = dev->dev_private;
1027         struct drm_i915_gem_pwrite *args = data;
1028         struct drm_i915_gem_object *obj;
1029         int ret;
1030
1031         if (args->size == 0)
1032                 return 0;
1033
1034         if (!access_ok(VERIFY_READ,
1035                        u64_to_user_ptr(args->data_ptr),
1036                        args->size))
1037                 return -EFAULT;
1038
1039         if (likely(!i915.prefault_disable)) {
1040                 ret = fault_in_multipages_readable(u64_to_user_ptr(args->data_ptr),
1041                                                    args->size);
1042                 if (ret)
1043                         return -EFAULT;
1044         }
1045
1046         intel_runtime_pm_get(dev_priv);
1047
1048         ret = i915_mutex_lock_interruptible(dev);
1049         if (ret)
1050                 goto put_rpm;
1051
1052         obj = to_intel_bo(drm_gem_object_lookup(file, args->handle));
1053         if (&obj->base == NULL) {
1054                 ret = -ENOENT;
1055                 goto unlock;
1056         }
1057
1058         /* Bounds check destination. */
1059         if (args->offset > obj->base.size ||
1060             args->size > obj->base.size - args->offset) {
1061                 ret = -EINVAL;
1062                 goto out;
1063         }
1064
1065         /* prime objects have no backing filp to GEM pread/pwrite
1066          * pages from.
1067          */
1068         if (!obj->base.filp) {
1069                 ret = -EINVAL;
1070                 goto out;
1071         }
1072
1073         trace_i915_gem_object_pwrite(obj, args->offset, args->size);
1074
1075         ret = -EFAULT;
1076         /* We can only do the GTT pwrite on untiled buffers, as otherwise
1077          * it would end up going through the fenced access, and we'll get
1078          * different detiling behavior between reading and writing.
1079          * pread/pwrite currently are reading and writing from the CPU
1080          * perspective, requiring manual detiling by the client.
1081          */
1082         if (obj->tiling_mode == I915_TILING_NONE &&
1083             obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
1084             cpu_write_needs_clflush(obj)) {
1085                 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
1086                 /* Note that the gtt paths might fail with non-page-backed user
1087                  * pointers (e.g. gtt mappings when moving data between
1088                  * textures). Fallback to the shmem path in that case. */
1089         }
1090
1091         if (ret == -EFAULT || ret == -ENOSPC) {
1092                 if (obj->phys_handle)
1093                         ret = i915_gem_phys_pwrite(obj, args, file);
1094                 else
1095                         ret = i915_gem_shmem_pwrite(dev, obj, args, file);
1096         }
1097
1098 out:
1099         drm_gem_object_unreference(&obj->base);
1100 unlock:
1101         mutex_unlock(&dev->struct_mutex);
1102 put_rpm:
1103         intel_runtime_pm_put(dev_priv);
1104
1105         return ret;
1106 }
1107
1108 static int
1109 i915_gem_check_wedge(unsigned reset_counter, bool interruptible)
1110 {
1111         if (__i915_terminally_wedged(reset_counter))
1112                 return -EIO;
1113
1114         if (__i915_reset_in_progress(reset_counter)) {
1115                 /* Non-interruptible callers can't handle -EAGAIN, hence return
1116                  * -EIO unconditionally for these. */
1117                 if (!interruptible)
1118                         return -EIO;
1119
1120                 return -EAGAIN;
1121         }
1122
1123         return 0;
1124 }
1125
1126 static void fake_irq(unsigned long data)
1127 {
1128         wake_up_process((struct task_struct *)data);
1129 }
1130
1131 static bool missed_irq(struct drm_i915_private *dev_priv,
1132                        struct intel_engine_cs *engine)
1133 {
1134         return test_bit(engine->id, &dev_priv->gpu_error.missed_irq_rings);
1135 }
1136
1137 static unsigned long local_clock_us(unsigned *cpu)
1138 {
1139         unsigned long t;
1140
1141         /* Cheaply and approximately convert from nanoseconds to microseconds.
1142          * The result and subsequent calculations are also defined in the same
1143          * approximate microseconds units. The principal source of timing
1144          * error here is from the simple truncation.
1145          *
1146          * Note that local_clock() is only defined wrt to the current CPU;
1147          * the comparisons are no longer valid if we switch CPUs. Instead of
1148          * blocking preemption for the entire busywait, we can detect the CPU
1149          * switch and use that as indicator of system load and a reason to
1150          * stop busywaiting, see busywait_stop().
1151          */
1152         *cpu = get_cpu();
1153         t = local_clock() >> 10;
1154         put_cpu();
1155
1156         return t;
1157 }
1158
1159 static bool busywait_stop(unsigned long timeout, unsigned cpu)
1160 {
1161         unsigned this_cpu;
1162
1163         if (time_after(local_clock_us(&this_cpu), timeout))
1164                 return true;
1165
1166         return this_cpu != cpu;
1167 }
1168
1169 static int __i915_spin_request(struct drm_i915_gem_request *req, int state)
1170 {
1171         unsigned long timeout;
1172         unsigned cpu;
1173
1174         /* When waiting for high frequency requests, e.g. during synchronous
1175          * rendering split between the CPU and GPU, the finite amount of time
1176          * required to set up the irq and wait upon it limits the response
1177          * rate. By busywaiting on the request completion for a short while we
1178          * can service the high frequency waits as quick as possible. However,
1179          * if it is a slow request, we want to sleep as quickly as possible.
1180          * The tradeoff between waiting and sleeping is roughly the time it
1181          * takes to sleep on a request, on the order of a microsecond.
1182          */
1183
1184         if (req->engine->irq_refcount)
1185                 return -EBUSY;
1186
1187         /* Only spin if we know the GPU is processing this request */
1188         if (!i915_gem_request_started(req, true))
1189                 return -EAGAIN;
1190
1191         timeout = local_clock_us(&cpu) + 5;
1192         while (!need_resched()) {
1193                 if (i915_gem_request_completed(req, true))
1194                         return 0;
1195
1196                 if (signal_pending_state(state, current))
1197                         break;
1198
1199                 if (busywait_stop(timeout, cpu))
1200                         break;
1201
1202                 cpu_relax_lowlatency();
1203         }
1204
1205         if (i915_gem_request_completed(req, false))
1206                 return 0;
1207
1208         return -EAGAIN;
1209 }
1210
1211 /**
1212  * __i915_wait_request - wait until execution of request has finished
1213  * @req: duh!
1214  * @interruptible: do an interruptible wait (normally yes)
1215  * @timeout: in - how long to wait (NULL forever); out - how much time remaining
1216  *
1217  * Note: It is of utmost importance that the passed in seqno and reset_counter
1218  * values have been read by the caller in an smp safe manner. Where read-side
1219  * locks are involved, it is sufficient to read the reset_counter before
1220  * unlocking the lock that protects the seqno. For lockless tricks, the
1221  * reset_counter _must_ be read before, and an appropriate smp_rmb must be
1222  * inserted.
1223  *
1224  * Returns 0 if the request was found within the alloted time. Else returns the
1225  * errno with remaining time filled in timeout argument.
1226  */
1227 int __i915_wait_request(struct drm_i915_gem_request *req,
1228                         bool interruptible,
1229                         s64 *timeout,
1230                         struct intel_rps_client *rps)
1231 {
1232         struct intel_engine_cs *engine = i915_gem_request_get_engine(req);
1233         struct drm_i915_private *dev_priv = req->i915;
1234         const bool irq_test_in_progress =
1235                 ACCESS_ONCE(dev_priv->gpu_error.test_irq_rings) & intel_engine_flag(engine);
1236         int state = interruptible ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
1237         DEFINE_WAIT(wait);
1238         unsigned long timeout_expire;
1239         s64 before = 0; /* Only to silence a compiler warning. */
1240         int ret;
1241
1242         WARN(!intel_irqs_enabled(dev_priv), "IRQs disabled");
1243
1244         if (list_empty(&req->list))
1245                 return 0;
1246
1247         if (i915_gem_request_completed(req, true))
1248                 return 0;
1249
1250         timeout_expire = 0;
1251         if (timeout) {
1252                 if (WARN_ON(*timeout < 0))
1253                         return -EINVAL;
1254
1255                 if (*timeout == 0)
1256                         return -ETIME;
1257
1258                 timeout_expire = jiffies + nsecs_to_jiffies_timeout(*timeout);
1259
1260                 /*
1261                  * Record current time in case interrupted by signal, or wedged.
1262                  */
1263                 before = ktime_get_raw_ns();
1264         }
1265
1266         if (INTEL_INFO(dev_priv)->gen >= 6)
1267                 gen6_rps_boost(dev_priv, rps, req->emitted_jiffies);
1268
1269         trace_i915_gem_request_wait_begin(req);
1270
1271         /* Optimistic spin for the next jiffie before touching IRQs */
1272         ret = __i915_spin_request(req, state);
1273         if (ret == 0)
1274                 goto out;
1275
1276         if (!irq_test_in_progress && WARN_ON(!engine->irq_get(engine))) {
1277                 ret = -ENODEV;
1278                 goto out;
1279         }
1280
1281         for (;;) {
1282                 struct timer_list timer;
1283
1284                 prepare_to_wait(&engine->irq_queue, &wait, state);
1285
1286                 /* We need to check whether any gpu reset happened in between
1287                  * the request being submitted and now. If a reset has occurred,
1288                  * the request is effectively complete (we either are in the
1289                  * process of or have discarded the rendering and completely
1290                  * reset the GPU. The results of the request are lost and we
1291                  * are free to continue on with the original operation.
1292                  */
1293                 if (req->reset_counter != i915_reset_counter(&dev_priv->gpu_error)) {
1294                         ret = 0;
1295                         break;
1296                 }
1297
1298                 if (i915_gem_request_completed(req, false)) {
1299                         ret = 0;
1300                         break;
1301                 }
1302
1303                 if (signal_pending_state(state, current)) {
1304                         ret = -ERESTARTSYS;
1305                         break;
1306                 }
1307
1308                 if (timeout && time_after_eq(jiffies, timeout_expire)) {
1309                         ret = -ETIME;
1310                         break;
1311                 }
1312
1313                 timer.function = NULL;
1314                 if (timeout || missed_irq(dev_priv, engine)) {
1315                         unsigned long expire;
1316
1317                         setup_timer_on_stack(&timer, fake_irq, (unsigned long)current);
1318                         expire = missed_irq(dev_priv, engine) ? jiffies + 1 : timeout_expire;
1319                         mod_timer(&timer, expire);
1320                 }
1321
1322                 io_schedule();
1323
1324                 if (timer.function) {
1325                         del_singleshot_timer_sync(&timer);
1326                         destroy_timer_on_stack(&timer);
1327                 }
1328         }
1329         if (!irq_test_in_progress)
1330                 engine->irq_put(engine);
1331
1332         finish_wait(&engine->irq_queue, &wait);
1333
1334 out:
1335         trace_i915_gem_request_wait_end(req);
1336
1337         if (timeout) {
1338                 s64 tres = *timeout - (ktime_get_raw_ns() - before);
1339
1340                 *timeout = tres < 0 ? 0 : tres;
1341
1342                 /*
1343                  * Apparently ktime isn't accurate enough and occasionally has a
1344                  * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch
1345                  * things up to make the test happy. We allow up to 1 jiffy.
1346                  *
1347                  * This is a regrssion from the timespec->ktime conversion.
1348                  */
1349                 if (ret == -ETIME && *timeout < jiffies_to_usecs(1)*1000)
1350                         *timeout = 0;
1351         }
1352
1353         return ret;
1354 }
1355
1356 int i915_gem_request_add_to_client(struct drm_i915_gem_request *req,
1357                                    struct drm_file *file)
1358 {
1359         struct drm_i915_file_private *file_priv;
1360
1361         WARN_ON(!req || !file || req->file_priv);
1362
1363         if (!req || !file)
1364                 return -EINVAL;
1365
1366         if (req->file_priv)
1367                 return -EINVAL;
1368
1369         file_priv = file->driver_priv;
1370
1371         spin_lock(&file_priv->mm.lock);
1372         req->file_priv = file_priv;
1373         list_add_tail(&req->client_list, &file_priv->mm.request_list);
1374         spin_unlock(&file_priv->mm.lock);
1375
1376         req->pid = get_pid(task_pid(current));
1377
1378         return 0;
1379 }
1380
1381 static inline void
1382 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
1383 {
1384         struct drm_i915_file_private *file_priv = request->file_priv;
1385
1386         if (!file_priv)
1387                 return;
1388
1389         spin_lock(&file_priv->mm.lock);
1390         list_del(&request->client_list);
1391         request->file_priv = NULL;
1392         spin_unlock(&file_priv->mm.lock);
1393
1394         put_pid(request->pid);
1395         request->pid = NULL;
1396 }
1397
1398 static void i915_gem_request_retire(struct drm_i915_gem_request *request)
1399 {
1400         trace_i915_gem_request_retire(request);
1401
1402         /* We know the GPU must have read the request to have
1403          * sent us the seqno + interrupt, so use the position
1404          * of tail of the request to update the last known position
1405          * of the GPU head.
1406          *
1407          * Note this requires that we are always called in request
1408          * completion order.
1409          */
1410         request->ringbuf->last_retired_head = request->postfix;
1411
1412         list_del_init(&request->list);
1413         i915_gem_request_remove_from_client(request);
1414
1415         if (request->previous_context) {
1416                 if (i915.enable_execlists)
1417                         intel_lr_context_unpin(request->previous_context,
1418                                                request->engine);
1419         }
1420
1421         i915_gem_context_unreference(request->ctx);
1422         i915_gem_request_unreference(request);
1423 }
1424
1425 static void
1426 __i915_gem_request_retire__upto(struct drm_i915_gem_request *req)
1427 {
1428         struct intel_engine_cs *engine = req->engine;
1429         struct drm_i915_gem_request *tmp;
1430
1431         lockdep_assert_held(&engine->i915->dev->struct_mutex);
1432
1433         if (list_empty(&req->list))
1434                 return;
1435
1436         do {
1437                 tmp = list_first_entry(&engine->request_list,
1438                                        typeof(*tmp), list);
1439
1440                 i915_gem_request_retire(tmp);
1441         } while (tmp != req);
1442
1443         WARN_ON(i915_verify_lists(engine->dev));
1444 }
1445
1446 /**
1447  * Waits for a request to be signaled, and cleans up the
1448  * request and object lists appropriately for that event.
1449  */
1450 int
1451 i915_wait_request(struct drm_i915_gem_request *req)
1452 {
1453         struct drm_i915_private *dev_priv = req->i915;
1454         bool interruptible;
1455         int ret;
1456
1457         interruptible = dev_priv->mm.interruptible;
1458
1459         BUG_ON(!mutex_is_locked(&dev_priv->dev->struct_mutex));
1460
1461         ret = __i915_wait_request(req, interruptible, NULL, NULL);
1462         if (ret)
1463                 return ret;
1464
1465         /* If the GPU hung, we want to keep the requests to find the guilty. */
1466         if (req->reset_counter == i915_reset_counter(&dev_priv->gpu_error))
1467                 __i915_gem_request_retire__upto(req);
1468
1469         return 0;
1470 }
1471
1472 /**
1473  * Ensures that all rendering to the object has completed and the object is
1474  * safe to unbind from the GTT or access from the CPU.
1475  */
1476 int
1477 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
1478                                bool readonly)
1479 {
1480         int ret, i;
1481
1482         if (!obj->active)
1483                 return 0;
1484
1485         if (readonly) {
1486                 if (obj->last_write_req != NULL) {
1487                         ret = i915_wait_request(obj->last_write_req);
1488                         if (ret)
1489                                 return ret;
1490
1491                         i = obj->last_write_req->engine->id;
1492                         if (obj->last_read_req[i] == obj->last_write_req)
1493                                 i915_gem_object_retire__read(obj, i);
1494                         else
1495                                 i915_gem_object_retire__write(obj);
1496                 }
1497         } else {
1498                 for (i = 0; i < I915_NUM_ENGINES; i++) {
1499                         if (obj->last_read_req[i] == NULL)
1500                                 continue;
1501
1502                         ret = i915_wait_request(obj->last_read_req[i]);
1503                         if (ret)
1504                                 return ret;
1505
1506                         i915_gem_object_retire__read(obj, i);
1507                 }
1508                 GEM_BUG_ON(obj->active);
1509         }
1510
1511         return 0;
1512 }
1513
1514 static void
1515 i915_gem_object_retire_request(struct drm_i915_gem_object *obj,
1516                                struct drm_i915_gem_request *req)
1517 {
1518         int ring = req->engine->id;
1519
1520         if (obj->last_read_req[ring] == req)
1521                 i915_gem_object_retire__read(obj, ring);
1522         else if (obj->last_write_req == req)
1523                 i915_gem_object_retire__write(obj);
1524
1525         if (req->reset_counter == i915_reset_counter(&req->i915->gpu_error))
1526                 __i915_gem_request_retire__upto(req);
1527 }
1528
1529 /* A nonblocking variant of the above wait. This is a highly dangerous routine
1530  * as the object state may change during this call.
1531  */
1532 static __must_check int
1533 i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
1534                                             struct intel_rps_client *rps,
1535                                             bool readonly)
1536 {
1537         struct drm_device *dev = obj->base.dev;
1538         struct drm_i915_private *dev_priv = dev->dev_private;
1539         struct drm_i915_gem_request *requests[I915_NUM_ENGINES];
1540         int ret, i, n = 0;
1541
1542         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1543         BUG_ON(!dev_priv->mm.interruptible);
1544
1545         if (!obj->active)
1546                 return 0;
1547
1548         if (readonly) {
1549                 struct drm_i915_gem_request *req;
1550
1551                 req = obj->last_write_req;
1552                 if (req == NULL)
1553                         return 0;
1554
1555                 requests[n++] = i915_gem_request_reference(req);
1556         } else {
1557                 for (i = 0; i < I915_NUM_ENGINES; i++) {
1558                         struct drm_i915_gem_request *req;
1559
1560                         req = obj->last_read_req[i];
1561                         if (req == NULL)
1562                                 continue;
1563
1564                         requests[n++] = i915_gem_request_reference(req);
1565                 }
1566         }
1567
1568         mutex_unlock(&dev->struct_mutex);
1569         ret = 0;
1570         for (i = 0; ret == 0 && i < n; i++)
1571                 ret = __i915_wait_request(requests[i], true, NULL, rps);
1572         mutex_lock(&dev->struct_mutex);
1573
1574         for (i = 0; i < n; i++) {
1575                 if (ret == 0)
1576                         i915_gem_object_retire_request(obj, requests[i]);
1577                 i915_gem_request_unreference(requests[i]);
1578         }
1579
1580         return ret;
1581 }
1582
1583 static struct intel_rps_client *to_rps_client(struct drm_file *file)
1584 {
1585         struct drm_i915_file_private *fpriv = file->driver_priv;
1586         return &fpriv->rps;
1587 }
1588
1589 /**
1590  * Called when user space prepares to use an object with the CPU, either
1591  * through the mmap ioctl's mapping or a GTT mapping.
1592  */
1593 int
1594 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1595                           struct drm_file *file)
1596 {
1597         struct drm_i915_gem_set_domain *args = data;
1598         struct drm_i915_gem_object *obj;
1599         uint32_t read_domains = args->read_domains;
1600         uint32_t write_domain = args->write_domain;
1601         int ret;
1602
1603         /* Only handle setting domains to types used by the CPU. */
1604         if (write_domain & I915_GEM_GPU_DOMAINS)
1605                 return -EINVAL;
1606
1607         if (read_domains & I915_GEM_GPU_DOMAINS)
1608                 return -EINVAL;
1609
1610         /* Having something in the write domain implies it's in the read
1611          * domain, and only that read domain.  Enforce that in the request.
1612          */
1613         if (write_domain != 0 && read_domains != write_domain)
1614                 return -EINVAL;
1615
1616         ret = i915_mutex_lock_interruptible(dev);
1617         if (ret)
1618                 return ret;
1619
1620         obj = to_intel_bo(drm_gem_object_lookup(file, args->handle));
1621         if (&obj->base == NULL) {
1622                 ret = -ENOENT;
1623                 goto unlock;
1624         }
1625
1626         /* Try to flush the object off the GPU without holding the lock.
1627          * We will repeat the flush holding the lock in the normal manner
1628          * to catch cases where we are gazumped.
1629          */
1630         ret = i915_gem_object_wait_rendering__nonblocking(obj,
1631                                                           to_rps_client(file),
1632                                                           !write_domain);
1633         if (ret)
1634                 goto unref;
1635
1636         if (read_domains & I915_GEM_DOMAIN_GTT)
1637                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1638         else
1639                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1640
1641         if (write_domain != 0)
1642                 intel_fb_obj_invalidate(obj,
1643                                         write_domain == I915_GEM_DOMAIN_GTT ?
1644                                         ORIGIN_GTT : ORIGIN_CPU);
1645
1646 unref:
1647         drm_gem_object_unreference(&obj->base);
1648 unlock:
1649         mutex_unlock(&dev->struct_mutex);
1650         return ret;
1651 }
1652
1653 /**
1654  * Called when user space has done writes to this buffer
1655  */
1656 int
1657 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1658                          struct drm_file *file)
1659 {
1660         struct drm_i915_gem_sw_finish *args = data;
1661         struct drm_i915_gem_object *obj;
1662         int ret = 0;
1663
1664         ret = i915_mutex_lock_interruptible(dev);
1665         if (ret)
1666                 return ret;
1667
1668         obj = to_intel_bo(drm_gem_object_lookup(file, args->handle));
1669         if (&obj->base == NULL) {
1670                 ret = -ENOENT;
1671                 goto unlock;
1672         }
1673
1674         /* Pinned buffers may be scanout, so flush the cache */
1675         if (obj->pin_display)
1676                 i915_gem_object_flush_cpu_write_domain(obj);
1677
1678         drm_gem_object_unreference(&obj->base);
1679 unlock:
1680         mutex_unlock(&dev->struct_mutex);
1681         return ret;
1682 }
1683
1684 /**
1685  * Maps the contents of an object, returning the address it is mapped
1686  * into.
1687  *
1688  * While the mapping holds a reference on the contents of the object, it doesn't
1689  * imply a ref on the object itself.
1690  *
1691  * IMPORTANT:
1692  *
1693  * DRM driver writers who look a this function as an example for how to do GEM
1694  * mmap support, please don't implement mmap support like here. The modern way
1695  * to implement DRM mmap support is with an mmap offset ioctl (like
1696  * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly.
1697  * That way debug tooling like valgrind will understand what's going on, hiding
1698  * the mmap call in a driver private ioctl will break that. The i915 driver only
1699  * does cpu mmaps this way because we didn't know better.
1700  */
1701 int
1702 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1703                     struct drm_file *file)
1704 {
1705         struct drm_i915_gem_mmap *args = data;
1706         struct drm_gem_object *obj;
1707         unsigned long addr;
1708
1709         if (args->flags & ~(I915_MMAP_WC))
1710                 return -EINVAL;
1711
1712         if (args->flags & I915_MMAP_WC && !boot_cpu_has(X86_FEATURE_PAT))
1713                 return -ENODEV;
1714
1715         obj = drm_gem_object_lookup(file, args->handle);
1716         if (obj == NULL)
1717                 return -ENOENT;
1718
1719         /* prime objects have no backing filp to GEM mmap
1720          * pages from.
1721          */
1722         if (!obj->filp) {
1723                 drm_gem_object_unreference_unlocked(obj);
1724                 return -EINVAL;
1725         }
1726
1727         addr = vm_mmap(obj->filp, 0, args->size,
1728                        PROT_READ | PROT_WRITE, MAP_SHARED,
1729                        args->offset);
1730         if (args->flags & I915_MMAP_WC) {
1731                 struct mm_struct *mm = current->mm;
1732                 struct vm_area_struct *vma;
1733
1734                 if (down_write_killable(&mm->mmap_sem)) {
1735                         drm_gem_object_unreference_unlocked(obj);
1736                         return -EINTR;
1737                 }
1738                 vma = find_vma(mm, addr);
1739                 if (vma)
1740                         vma->vm_page_prot =
1741                                 pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
1742                 else
1743                         addr = -ENOMEM;
1744                 up_write(&mm->mmap_sem);
1745         }
1746         drm_gem_object_unreference_unlocked(obj);
1747         if (IS_ERR((void *)addr))
1748                 return addr;
1749
1750         args->addr_ptr = (uint64_t) addr;
1751
1752         return 0;
1753 }
1754
1755 /**
1756  * i915_gem_fault - fault a page into the GTT
1757  * @vma: VMA in question
1758  * @vmf: fault info
1759  *
1760  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1761  * from userspace.  The fault handler takes care of binding the object to
1762  * the GTT (if needed), allocating and programming a fence register (again,
1763  * only if needed based on whether the old reg is still valid or the object
1764  * is tiled) and inserting a new PTE into the faulting process.
1765  *
1766  * Note that the faulting process may involve evicting existing objects
1767  * from the GTT and/or fence registers to make room.  So performance may
1768  * suffer if the GTT working set is large or there are few fence registers
1769  * left.
1770  */
1771 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1772 {
1773         struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
1774         struct drm_device *dev = obj->base.dev;
1775         struct drm_i915_private *dev_priv = to_i915(dev);
1776         struct i915_ggtt *ggtt = &dev_priv->ggtt;
1777         struct i915_ggtt_view view = i915_ggtt_view_normal;
1778         pgoff_t page_offset;
1779         unsigned long pfn;
1780         int ret = 0;
1781         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1782
1783         intel_runtime_pm_get(dev_priv);
1784
1785         /* We don't use vmf->pgoff since that has the fake offset */
1786         page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1787                 PAGE_SHIFT;
1788
1789         ret = i915_mutex_lock_interruptible(dev);
1790         if (ret)
1791                 goto out;
1792
1793         trace_i915_gem_object_fault(obj, page_offset, true, write);
1794
1795         /* Try to flush the object off the GPU first without holding the lock.
1796          * Upon reacquiring the lock, we will perform our sanity checks and then
1797          * repeat the flush holding the lock in the normal manner to catch cases
1798          * where we are gazumped.
1799          */
1800         ret = i915_gem_object_wait_rendering__nonblocking(obj, NULL, !write);
1801         if (ret)
1802                 goto unlock;
1803
1804         /* Access to snoopable pages through the GTT is incoherent. */
1805         if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) {
1806                 ret = -EFAULT;
1807                 goto unlock;
1808         }
1809
1810         /* Use a partial view if the object is bigger than the aperture. */
1811         if (obj->base.size >= ggtt->mappable_end &&
1812             obj->tiling_mode == I915_TILING_NONE) {
1813                 static const unsigned int chunk_size = 256; // 1 MiB
1814
1815                 memset(&view, 0, sizeof(view));
1816                 view.type = I915_GGTT_VIEW_PARTIAL;
1817                 view.params.partial.offset = rounddown(page_offset, chunk_size);
1818                 view.params.partial.size =
1819                         min_t(unsigned int,
1820                               chunk_size,
1821                               (vma->vm_end - vma->vm_start)/PAGE_SIZE -
1822                               view.params.partial.offset);
1823         }
1824
1825         /* Now pin it into the GTT if needed */
1826         ret = i915_gem_object_ggtt_pin(obj, &view, 0, PIN_MAPPABLE);
1827         if (ret)
1828                 goto unlock;
1829
1830         ret = i915_gem_object_set_to_gtt_domain(obj, write);
1831         if (ret)
1832                 goto unpin;
1833
1834         ret = i915_gem_object_get_fence(obj);
1835         if (ret)
1836                 goto unpin;
1837
1838         /* Finally, remap it using the new GTT offset */
1839         pfn = ggtt->mappable_base +
1840                 i915_gem_obj_ggtt_offset_view(obj, &view);
1841         pfn >>= PAGE_SHIFT;
1842
1843         if (unlikely(view.type == I915_GGTT_VIEW_PARTIAL)) {
1844                 /* Overriding existing pages in partial view does not cause
1845                  * us any trouble as TLBs are still valid because the fault
1846                  * is due to userspace losing part of the mapping or never
1847                  * having accessed it before (at this partials' range).
1848                  */
1849                 unsigned long base = vma->vm_start +
1850                                      (view.params.partial.offset << PAGE_SHIFT);
1851                 unsigned int i;
1852
1853                 for (i = 0; i < view.params.partial.size; i++) {
1854                         ret = vm_insert_pfn(vma, base + i * PAGE_SIZE, pfn + i);
1855                         if (ret)
1856                                 break;
1857                 }
1858
1859                 obj->fault_mappable = true;
1860         } else {
1861                 if (!obj->fault_mappable) {
1862                         unsigned long size = min_t(unsigned long,
1863                                                    vma->vm_end - vma->vm_start,
1864                                                    obj->base.size);
1865                         int i;
1866
1867                         for (i = 0; i < size >> PAGE_SHIFT; i++) {
1868                                 ret = vm_insert_pfn(vma,
1869                                                     (unsigned long)vma->vm_start + i * PAGE_SIZE,
1870                                                     pfn + i);
1871                                 if (ret)
1872                                         break;
1873                         }
1874
1875                         obj->fault_mappable = true;
1876                 } else
1877                         ret = vm_insert_pfn(vma,
1878                                             (unsigned long)vmf->virtual_address,
1879                                             pfn + page_offset);
1880         }
1881 unpin:
1882         i915_gem_object_ggtt_unpin_view(obj, &view);
1883 unlock:
1884         mutex_unlock(&dev->struct_mutex);
1885 out:
1886         switch (ret) {
1887         case -EIO:
1888                 /*
1889                  * We eat errors when the gpu is terminally wedged to avoid
1890                  * userspace unduly crashing (gl has no provisions for mmaps to
1891                  * fail). But any other -EIO isn't ours (e.g. swap in failure)
1892                  * and so needs to be reported.
1893                  */
1894                 if (!i915_terminally_wedged(&dev_priv->gpu_error)) {
1895                         ret = VM_FAULT_SIGBUS;
1896                         break;
1897                 }
1898         case -EAGAIN:
1899                 /*
1900                  * EAGAIN means the gpu is hung and we'll wait for the error
1901                  * handler to reset everything when re-faulting in
1902                  * i915_mutex_lock_interruptible.
1903                  */
1904         case 0:
1905         case -ERESTARTSYS:
1906         case -EINTR:
1907         case -EBUSY:
1908                 /*
1909                  * EBUSY is ok: this just means that another thread
1910                  * already did the job.
1911                  */
1912                 ret = VM_FAULT_NOPAGE;
1913                 break;
1914         case -ENOMEM:
1915                 ret = VM_FAULT_OOM;
1916                 break;
1917         case -ENOSPC:
1918         case -EFAULT:
1919                 ret = VM_FAULT_SIGBUS;
1920                 break;
1921         default:
1922                 WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret);
1923                 ret = VM_FAULT_SIGBUS;
1924                 break;
1925         }
1926
1927         intel_runtime_pm_put(dev_priv);
1928         return ret;
1929 }
1930
1931 /**
1932  * i915_gem_release_mmap - remove physical page mappings
1933  * @obj: obj in question
1934  *
1935  * Preserve the reservation of the mmapping with the DRM core code, but
1936  * relinquish ownership of the pages back to the system.
1937  *
1938  * It is vital that we remove the page mapping if we have mapped a tiled
1939  * object through the GTT and then lose the fence register due to
1940  * resource pressure. Similarly if the object has been moved out of the
1941  * aperture, than pages mapped into userspace must be revoked. Removing the
1942  * mapping will then trigger a page fault on the next user access, allowing
1943  * fixup by i915_gem_fault().
1944  */
1945 void
1946 i915_gem_release_mmap(struct drm_i915_gem_object *obj)
1947 {
1948         /* Serialisation between user GTT access and our code depends upon
1949          * revoking the CPU's PTE whilst the mutex is held. The next user
1950          * pagefault then has to wait until we release the mutex.
1951          */
1952         lockdep_assert_held(&obj->base.dev->struct_mutex);
1953
1954         if (!obj->fault_mappable)
1955                 return;
1956
1957         drm_vma_node_unmap(&obj->base.vma_node,
1958                            obj->base.dev->anon_inode->i_mapping);
1959
1960         /* Ensure that the CPU's PTE are revoked and there are not outstanding
1961          * memory transactions from userspace before we return. The TLB
1962          * flushing implied above by changing the PTE above *should* be
1963          * sufficient, an extra barrier here just provides us with a bit
1964          * of paranoid documentation about our requirement to serialise
1965          * memory writes before touching registers / GSM.
1966          */
1967         wmb();
1968
1969         obj->fault_mappable = false;
1970 }
1971
1972 void
1973 i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv)
1974 {
1975         struct drm_i915_gem_object *obj;
1976
1977         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
1978                 i915_gem_release_mmap(obj);
1979 }
1980
1981 uint32_t
1982 i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
1983 {
1984         uint32_t gtt_size;
1985
1986         if (INTEL_INFO(dev)->gen >= 4 ||
1987             tiling_mode == I915_TILING_NONE)
1988                 return size;
1989
1990         /* Previous chips need a power-of-two fence region when tiling */
1991         if (IS_GEN3(dev))
1992                 gtt_size = 1024*1024;
1993         else
1994                 gtt_size = 512*1024;
1995
1996         while (gtt_size < size)
1997                 gtt_size <<= 1;
1998
1999         return gtt_size;
2000 }
2001
2002 /**
2003  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
2004  * @obj: object to check
2005  *
2006  * Return the required GTT alignment for an object, taking into account
2007  * potential fence register mapping.
2008  */
2009 uint32_t
2010 i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
2011                            int tiling_mode, bool fenced)
2012 {
2013         /*
2014          * Minimum alignment is 4k (GTT page size), but might be greater
2015          * if a fence register is needed for the object.
2016          */
2017         if (INTEL_INFO(dev)->gen >= 4 || (!fenced && IS_G33(dev)) ||
2018             tiling_mode == I915_TILING_NONE)
2019                 return 4096;
2020
2021         /*
2022          * Previous chips need to be aligned to the size of the smallest
2023          * fence register that can contain the object.
2024          */
2025         return i915_gem_get_gtt_size(dev, size, tiling_mode);
2026 }
2027
2028 static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
2029 {
2030         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2031         int ret;
2032
2033         dev_priv->mm.shrinker_no_lock_stealing = true;
2034
2035         ret = drm_gem_create_mmap_offset(&obj->base);
2036         if (ret != -ENOSPC)
2037                 goto out;
2038
2039         /* Badly fragmented mmap space? The only way we can recover
2040          * space is by destroying unwanted objects. We can't randomly release
2041          * mmap_offsets as userspace expects them to be persistent for the
2042          * lifetime of the objects. The closest we can is to release the
2043          * offsets on purgeable objects by truncating it and marking it purged,
2044          * which prevents userspace from ever using that object again.
2045          */
2046         i915_gem_shrink(dev_priv,
2047                         obj->base.size >> PAGE_SHIFT,
2048                         I915_SHRINK_BOUND |
2049                         I915_SHRINK_UNBOUND |
2050                         I915_SHRINK_PURGEABLE);
2051         ret = drm_gem_create_mmap_offset(&obj->base);
2052         if (ret != -ENOSPC)
2053                 goto out;
2054
2055         i915_gem_shrink_all(dev_priv);
2056         ret = drm_gem_create_mmap_offset(&obj->base);
2057 out:
2058         dev_priv->mm.shrinker_no_lock_stealing = false;
2059
2060         return ret;
2061 }
2062
2063 static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj)
2064 {
2065         drm_gem_free_mmap_offset(&obj->base);
2066 }
2067
2068 int
2069 i915_gem_mmap_gtt(struct drm_file *file,
2070                   struct drm_device *dev,
2071                   uint32_t handle,
2072                   uint64_t *offset)
2073 {
2074         struct drm_i915_gem_object *obj;
2075         int ret;
2076
2077         ret = i915_mutex_lock_interruptible(dev);
2078         if (ret)
2079                 return ret;
2080
2081         obj = to_intel_bo(drm_gem_object_lookup(file, handle));
2082         if (&obj->base == NULL) {
2083                 ret = -ENOENT;
2084                 goto unlock;
2085         }
2086
2087         if (obj->madv != I915_MADV_WILLNEED) {
2088                 DRM_DEBUG("Attempting to mmap a purgeable buffer\n");
2089                 ret = -EFAULT;
2090                 goto out;
2091         }
2092
2093         ret = i915_gem_object_create_mmap_offset(obj);
2094         if (ret)
2095                 goto out;
2096
2097         *offset = drm_vma_node_offset_addr(&obj->base.vma_node);
2098
2099 out:
2100         drm_gem_object_unreference(&obj->base);
2101 unlock:
2102         mutex_unlock(&dev->struct_mutex);
2103         return ret;
2104 }
2105
2106 /**
2107  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
2108  * @dev: DRM device
2109  * @data: GTT mapping ioctl data
2110  * @file: GEM object info
2111  *
2112  * Simply returns the fake offset to userspace so it can mmap it.
2113  * The mmap call will end up in drm_gem_mmap(), which will set things
2114  * up so we can get faults in the handler above.
2115  *
2116  * The fault handler will take care of binding the object into the GTT
2117  * (since it may have been evicted to make room for something), allocating
2118  * a fence register, and mapping the appropriate aperture address into
2119  * userspace.
2120  */
2121 int
2122 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
2123                         struct drm_file *file)
2124 {
2125         struct drm_i915_gem_mmap_gtt *args = data;
2126
2127         return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
2128 }
2129
2130 /* Immediately discard the backing storage */
2131 static void
2132 i915_gem_object_truncate(struct drm_i915_gem_object *obj)
2133 {
2134         i915_gem_object_free_mmap_offset(obj);
2135
2136         if (obj->base.filp == NULL)
2137                 return;
2138
2139         /* Our goal here is to return as much of the memory as
2140          * is possible back to the system as we are called from OOM.
2141          * To do this we must instruct the shmfs to drop all of its
2142          * backing pages, *now*.
2143          */
2144         shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);
2145         obj->madv = __I915_MADV_PURGED;
2146 }
2147
2148 /* Try to discard unwanted pages */
2149 static void
2150 i915_gem_object_invalidate(struct drm_i915_gem_object *obj)
2151 {
2152         struct address_space *mapping;
2153
2154         switch (obj->madv) {
2155         case I915_MADV_DONTNEED:
2156                 i915_gem_object_truncate(obj);
2157         case __I915_MADV_PURGED:
2158                 return;
2159         }
2160
2161         if (obj->base.filp == NULL)
2162                 return;
2163
2164         mapping = file_inode(obj->base.filp)->i_mapping,
2165         invalidate_mapping_pages(mapping, 0, (loff_t)-1);
2166 }
2167
2168 static void
2169 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
2170 {
2171         struct sgt_iter sgt_iter;
2172         struct page *page;
2173         int ret;
2174
2175         BUG_ON(obj->madv == __I915_MADV_PURGED);
2176
2177         ret = i915_gem_object_set_to_cpu_domain(obj, true);
2178         if (WARN_ON(ret)) {
2179                 /* In the event of a disaster, abandon all caches and
2180                  * hope for the best.
2181                  */
2182                 i915_gem_clflush_object(obj, true);
2183                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2184         }
2185
2186         i915_gem_gtt_finish_object(obj);
2187
2188         if (i915_gem_object_needs_bit17_swizzle(obj))
2189                 i915_gem_object_save_bit_17_swizzle(obj);
2190
2191         if (obj->madv == I915_MADV_DONTNEED)
2192                 obj->dirty = 0;
2193
2194         for_each_sgt_page(page, sgt_iter, obj->pages) {
2195                 if (obj->dirty)
2196                         set_page_dirty(page);
2197
2198                 if (obj->madv == I915_MADV_WILLNEED)
2199                         mark_page_accessed(page);
2200
2201                 put_page(page);
2202         }
2203         obj->dirty = 0;
2204
2205         sg_free_table(obj->pages);
2206         kfree(obj->pages);
2207 }
2208
2209 int
2210 i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
2211 {
2212         const struct drm_i915_gem_object_ops *ops = obj->ops;
2213
2214         if (obj->pages == NULL)
2215                 return 0;
2216
2217         if (obj->pages_pin_count)
2218                 return -EBUSY;
2219
2220         BUG_ON(i915_gem_obj_bound_any(obj));
2221
2222         /* ->put_pages might need to allocate memory for the bit17 swizzle
2223          * array, hence protect them from being reaped by removing them from gtt
2224          * lists early. */
2225         list_del(&obj->global_list);
2226
2227         if (obj->mapping) {
2228                 if (is_vmalloc_addr(obj->mapping))
2229                         vunmap(obj->mapping);
2230                 else
2231                         kunmap(kmap_to_page(obj->mapping));
2232                 obj->mapping = NULL;
2233         }
2234
2235         ops->put_pages(obj);
2236         obj->pages = NULL;
2237
2238         i915_gem_object_invalidate(obj);
2239
2240         return 0;
2241 }
2242
2243 static int
2244 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2245 {
2246         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2247         int page_count, i;
2248         struct address_space *mapping;
2249         struct sg_table *st;
2250         struct scatterlist *sg;
2251         struct sgt_iter sgt_iter;
2252         struct page *page;
2253         unsigned long last_pfn = 0;     /* suppress gcc warning */
2254         int ret;
2255         gfp_t gfp;
2256
2257         /* Assert that the object is not currently in any GPU domain. As it
2258          * wasn't in the GTT, there shouldn't be any way it could have been in
2259          * a GPU cache
2260          */
2261         BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
2262         BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
2263
2264         st = kmalloc(sizeof(*st), GFP_KERNEL);
2265         if (st == NULL)
2266                 return -ENOMEM;
2267
2268         page_count = obj->base.size / PAGE_SIZE;
2269         if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
2270                 kfree(st);
2271                 return -ENOMEM;
2272         }
2273
2274         /* Get the list of pages out of our struct file.  They'll be pinned
2275          * at this point until we release them.
2276          *
2277          * Fail silently without starting the shrinker
2278          */
2279         mapping = file_inode(obj->base.filp)->i_mapping;
2280         gfp = mapping_gfp_constraint(mapping, ~(__GFP_IO | __GFP_RECLAIM));
2281         gfp |= __GFP_NORETRY | __GFP_NOWARN;
2282         sg = st->sgl;
2283         st->nents = 0;
2284         for (i = 0; i < page_count; i++) {
2285                 page = shmem_read_mapping_page_gfp(mapping, i, gfp);
2286                 if (IS_ERR(page)) {
2287                         i915_gem_shrink(dev_priv,
2288                                         page_count,
2289                                         I915_SHRINK_BOUND |
2290                                         I915_SHRINK_UNBOUND |
2291                                         I915_SHRINK_PURGEABLE);
2292                         page = shmem_read_mapping_page_gfp(mapping, i, gfp);
2293                 }
2294                 if (IS_ERR(page)) {
2295                         /* We've tried hard to allocate the memory by reaping
2296                          * our own buffer, now let the real VM do its job and
2297                          * go down in flames if truly OOM.
2298                          */
2299                         i915_gem_shrink_all(dev_priv);
2300                         page = shmem_read_mapping_page(mapping, i);
2301                         if (IS_ERR(page)) {
2302                                 ret = PTR_ERR(page);
2303                                 goto err_pages;
2304                         }
2305                 }
2306 #ifdef CONFIG_SWIOTLB
2307                 if (swiotlb_nr_tbl()) {
2308                         st->nents++;
2309                         sg_set_page(sg, page, PAGE_SIZE, 0);
2310                         sg = sg_next(sg);
2311                         continue;
2312                 }
2313 #endif
2314                 if (!i || page_to_pfn(page) != last_pfn + 1) {
2315                         if (i)
2316                                 sg = sg_next(sg);
2317                         st->nents++;
2318                         sg_set_page(sg, page, PAGE_SIZE, 0);
2319                 } else {
2320                         sg->length += PAGE_SIZE;
2321                 }
2322                 last_pfn = page_to_pfn(page);
2323
2324                 /* Check that the i965g/gm workaround works. */
2325                 WARN_ON((gfp & __GFP_DMA32) && (last_pfn >= 0x00100000UL));
2326         }
2327 #ifdef CONFIG_SWIOTLB
2328         if (!swiotlb_nr_tbl())
2329 #endif
2330                 sg_mark_end(sg);
2331         obj->pages = st;
2332
2333         ret = i915_gem_gtt_prepare_object(obj);
2334         if (ret)
2335                 goto err_pages;
2336
2337         if (i915_gem_object_needs_bit17_swizzle(obj))
2338                 i915_gem_object_do_bit_17_swizzle(obj);
2339
2340         if (obj->tiling_mode != I915_TILING_NONE &&
2341             dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES)
2342                 i915_gem_object_pin_pages(obj);
2343
2344         return 0;
2345
2346 err_pages:
2347         sg_mark_end(sg);
2348         for_each_sgt_page(page, sgt_iter, st)
2349                 put_page(page);
2350         sg_free_table(st);
2351         kfree(st);
2352
2353         /* shmemfs first checks if there is enough memory to allocate the page
2354          * and reports ENOSPC should there be insufficient, along with the usual
2355          * ENOMEM for a genuine allocation failure.
2356          *
2357          * We use ENOSPC in our driver to mean that we have run out of aperture
2358          * space and so want to translate the error from shmemfs back to our
2359          * usual understanding of ENOMEM.
2360          */
2361         if (ret == -ENOSPC)
2362                 ret = -ENOMEM;
2363
2364         return ret;
2365 }
2366
2367 /* Ensure that the associated pages are gathered from the backing storage
2368  * and pinned into our object. i915_gem_object_get_pages() may be called
2369  * multiple times before they are released by a single call to
2370  * i915_gem_object_put_pages() - once the pages are no longer referenced
2371  * either as a result of memory pressure (reaping pages under the shrinker)
2372  * or as the object is itself released.
2373  */
2374 int
2375 i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2376 {
2377         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2378         const struct drm_i915_gem_object_ops *ops = obj->ops;
2379         int ret;
2380
2381         if (obj->pages)
2382                 return 0;
2383
2384         if (obj->madv != I915_MADV_WILLNEED) {
2385                 DRM_DEBUG("Attempting to obtain a purgeable object\n");
2386                 return -EFAULT;
2387         }
2388
2389         BUG_ON(obj->pages_pin_count);
2390
2391         ret = ops->get_pages(obj);
2392         if (ret)
2393                 return ret;
2394
2395         list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
2396
2397         obj->get_page.sg = obj->pages->sgl;
2398         obj->get_page.last = 0;
2399
2400         return 0;
2401 }
2402
2403 /* The 'mapping' part of i915_gem_object_pin_map() below */
2404 static void *i915_gem_object_map(const struct drm_i915_gem_object *obj)
2405 {
2406         unsigned long n_pages = obj->base.size >> PAGE_SHIFT;
2407         struct sg_table *sgt = obj->pages;
2408         struct sgt_iter sgt_iter;
2409         struct page *page;
2410         struct page *stack_pages[32];
2411         struct page **pages = stack_pages;
2412         unsigned long i = 0;
2413         void *addr;
2414
2415         /* A single page can always be kmapped */
2416         if (n_pages == 1)
2417                 return kmap(sg_page(sgt->sgl));
2418
2419         if (n_pages > ARRAY_SIZE(stack_pages)) {
2420                 /* Too big for stack -- allocate temporary array instead */
2421                 pages = drm_malloc_gfp(n_pages, sizeof(*pages), GFP_TEMPORARY);
2422                 if (!pages)
2423                         return NULL;
2424         }
2425
2426         for_each_sgt_page(page, sgt_iter, sgt)
2427                 pages[i++] = page;
2428
2429         /* Check that we have the expected number of pages */
2430         GEM_BUG_ON(i != n_pages);
2431
2432         addr = vmap(pages, n_pages, 0, PAGE_KERNEL);
2433
2434         if (pages != stack_pages)
2435                 drm_free_large(pages);
2436
2437         return addr;
2438 }
2439
2440 /* get, pin, and map the pages of the object into kernel space */
2441 void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj)
2442 {
2443         int ret;
2444
2445         lockdep_assert_held(&obj->base.dev->struct_mutex);
2446
2447         ret = i915_gem_object_get_pages(obj);
2448         if (ret)
2449                 return ERR_PTR(ret);
2450
2451         i915_gem_object_pin_pages(obj);
2452
2453         if (!obj->mapping) {
2454                 obj->mapping = i915_gem_object_map(obj);
2455                 if (!obj->mapping) {
2456                         i915_gem_object_unpin_pages(obj);
2457                         return ERR_PTR(-ENOMEM);
2458                 }
2459         }
2460
2461         return obj->mapping;
2462 }
2463
2464 void i915_vma_move_to_active(struct i915_vma *vma,
2465                              struct drm_i915_gem_request *req)
2466 {
2467         struct drm_i915_gem_object *obj = vma->obj;
2468         struct intel_engine_cs *engine;
2469
2470         engine = i915_gem_request_get_engine(req);
2471
2472         /* Add a reference if we're newly entering the active list. */
2473         if (obj->active == 0)
2474                 drm_gem_object_reference(&obj->base);
2475         obj->active |= intel_engine_flag(engine);
2476
2477         list_move_tail(&obj->engine_list[engine->id], &engine->active_list);
2478         i915_gem_request_assign(&obj->last_read_req[engine->id], req);
2479
2480         list_move_tail(&vma->vm_link, &vma->vm->active_list);
2481 }
2482
2483 static void
2484 i915_gem_object_retire__write(struct drm_i915_gem_object *obj)
2485 {
2486         GEM_BUG_ON(obj->last_write_req == NULL);
2487         GEM_BUG_ON(!(obj->active & intel_engine_flag(obj->last_write_req->engine)));
2488
2489         i915_gem_request_assign(&obj->last_write_req, NULL);
2490         intel_fb_obj_flush(obj, true, ORIGIN_CS);
2491 }
2492
2493 static void
2494 i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring)
2495 {
2496         struct i915_vma *vma;
2497
2498         GEM_BUG_ON(obj->last_read_req[ring] == NULL);
2499         GEM_BUG_ON(!(obj->active & (1 << ring)));
2500
2501         list_del_init(&obj->engine_list[ring]);
2502         i915_gem_request_assign(&obj->last_read_req[ring], NULL);
2503
2504         if (obj->last_write_req && obj->last_write_req->engine->id == ring)
2505                 i915_gem_object_retire__write(obj);
2506
2507         obj->active &= ~(1 << ring);
2508         if (obj->active)
2509                 return;
2510
2511         /* Bump our place on the bound list to keep it roughly in LRU order
2512          * so that we don't steal from recently used but inactive objects
2513          * (unless we are forced to ofc!)
2514          */
2515         list_move_tail(&obj->global_list,
2516                        &to_i915(obj->base.dev)->mm.bound_list);
2517
2518         list_for_each_entry(vma, &obj->vma_list, obj_link) {
2519                 if (!list_empty(&vma->vm_link))
2520                         list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
2521         }
2522
2523         i915_gem_request_assign(&obj->last_fenced_req, NULL);
2524         drm_gem_object_unreference(&obj->base);
2525 }
2526
2527 static int
2528 i915_gem_init_seqno(struct drm_i915_private *dev_priv, u32 seqno)
2529 {
2530         struct intel_engine_cs *engine;
2531         int ret;
2532
2533         /* Carefully retire all requests without writing to the rings */
2534         for_each_engine(engine, dev_priv) {
2535                 ret = intel_engine_idle(engine);
2536                 if (ret)
2537                         return ret;
2538         }
2539         i915_gem_retire_requests(dev_priv);
2540
2541         /* Finally reset hw state */
2542         for_each_engine(engine, dev_priv)
2543                 intel_ring_init_seqno(engine, seqno);
2544
2545         return 0;
2546 }
2547
2548 int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
2549 {
2550         struct drm_i915_private *dev_priv = dev->dev_private;
2551         int ret;
2552
2553         if (seqno == 0)
2554                 return -EINVAL;
2555
2556         /* HWS page needs to be set less than what we
2557          * will inject to ring
2558          */
2559         ret = i915_gem_init_seqno(dev_priv, seqno - 1);
2560         if (ret)
2561                 return ret;
2562
2563         /* Carefully set the last_seqno value so that wrap
2564          * detection still works
2565          */
2566         dev_priv->next_seqno = seqno;
2567         dev_priv->last_seqno = seqno - 1;
2568         if (dev_priv->last_seqno == 0)
2569                 dev_priv->last_seqno--;
2570
2571         return 0;
2572 }
2573
2574 int
2575 i915_gem_get_seqno(struct drm_i915_private *dev_priv, u32 *seqno)
2576 {
2577         /* reserve 0 for non-seqno */
2578         if (dev_priv->next_seqno == 0) {
2579                 int ret = i915_gem_init_seqno(dev_priv, 0);
2580                 if (ret)
2581                         return ret;
2582
2583                 dev_priv->next_seqno = 1;
2584         }
2585
2586         *seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
2587         return 0;
2588 }
2589
2590 /*
2591  * NB: This function is not allowed to fail. Doing so would mean the the
2592  * request is not being tracked for completion but the work itself is
2593  * going to happen on the hardware. This would be a Bad Thing(tm).
2594  */
2595 void __i915_add_request(struct drm_i915_gem_request *request,
2596                         struct drm_i915_gem_object *obj,
2597                         bool flush_caches)
2598 {
2599         struct intel_engine_cs *engine;
2600         struct drm_i915_private *dev_priv;
2601         struct intel_ringbuffer *ringbuf;
2602         u32 request_start;
2603         u32 reserved_tail;
2604         int ret;
2605
2606         if (WARN_ON(request == NULL))
2607                 return;
2608
2609         engine = request->engine;
2610         dev_priv = request->i915;
2611         ringbuf = request->ringbuf;
2612
2613         /*
2614          * To ensure that this call will not fail, space for its emissions
2615          * should already have been reserved in the ring buffer. Let the ring
2616          * know that it is time to use that space up.
2617          */
2618         request_start = intel_ring_get_tail(ringbuf);
2619         reserved_tail = request->reserved_space;
2620         request->reserved_space = 0;
2621
2622         /*
2623          * Emit any outstanding flushes - execbuf can fail to emit the flush
2624          * after having emitted the batchbuffer command. Hence we need to fix
2625          * things up similar to emitting the lazy request. The difference here
2626          * is that the flush _must_ happen before the next request, no matter
2627          * what.
2628          */
2629         if (flush_caches) {
2630                 if (i915.enable_execlists)
2631                         ret = logical_ring_flush_all_caches(request);
2632                 else
2633                         ret = intel_ring_flush_all_caches(request);
2634                 /* Not allowed to fail! */
2635                 WARN(ret, "*_ring_flush_all_caches failed: %d!\n", ret);
2636         }
2637
2638         trace_i915_gem_request_add(request);
2639
2640         request->head = request_start;
2641
2642         /* Whilst this request exists, batch_obj will be on the
2643          * active_list, and so will hold the active reference. Only when this
2644          * request is retired will the the batch_obj be moved onto the
2645          * inactive_list and lose its active reference. Hence we do not need
2646          * to explicitly hold another reference here.
2647          */
2648         request->batch_obj = obj;
2649
2650         /* Seal the request and mark it as pending execution. Note that
2651          * we may inspect this state, without holding any locks, during
2652          * hangcheck. Hence we apply the barrier to ensure that we do not
2653          * see a more recent value in the hws than we are tracking.
2654          */
2655         request->emitted_jiffies = jiffies;
2656         request->previous_seqno = engine->last_submitted_seqno;
2657         smp_store_mb(engine->last_submitted_seqno, request->seqno);
2658         list_add_tail(&request->list, &engine->request_list);
2659
2660         /* Record the position of the start of the request so that
2661          * should we detect the updated seqno part-way through the
2662          * GPU processing the request, we never over-estimate the
2663          * position of the head.
2664          */
2665         request->postfix = intel_ring_get_tail(ringbuf);
2666
2667         if (i915.enable_execlists)
2668                 ret = engine->emit_request(request);
2669         else {
2670                 ret = engine->add_request(request);
2671
2672                 request->tail = intel_ring_get_tail(ringbuf);
2673         }
2674         /* Not allowed to fail! */
2675         WARN(ret, "emit|add_request failed: %d!\n", ret);
2676
2677         i915_queue_hangcheck(engine->i915);
2678
2679         queue_delayed_work(dev_priv->wq,
2680                            &dev_priv->mm.retire_work,
2681                            round_jiffies_up_relative(HZ));
2682         intel_mark_busy(dev_priv);
2683
2684         /* Sanity check that the reserved size was large enough. */
2685         ret = intel_ring_get_tail(ringbuf) - request_start;
2686         if (ret < 0)
2687                 ret += ringbuf->size;
2688         WARN_ONCE(ret > reserved_tail,
2689                   "Not enough space reserved (%d bytes) "
2690                   "for adding the request (%d bytes)\n",
2691                   reserved_tail, ret);
2692 }
2693
2694 static bool i915_context_is_banned(struct drm_i915_private *dev_priv,
2695                                    const struct intel_context *ctx)
2696 {
2697         unsigned long elapsed;
2698
2699         elapsed = get_seconds() - ctx->hang_stats.guilty_ts;
2700
2701         if (ctx->hang_stats.banned)
2702                 return true;
2703
2704         if (ctx->hang_stats.ban_period_seconds &&
2705             elapsed <= ctx->hang_stats.ban_period_seconds) {
2706                 if (!i915_gem_context_is_default(ctx)) {
2707                         DRM_DEBUG("context hanging too fast, banning!\n");
2708                         return true;
2709                 } else if (i915_stop_ring_allow_ban(dev_priv)) {
2710                         if (i915_stop_ring_allow_warn(dev_priv))
2711                                 DRM_ERROR("gpu hanging too fast, banning!\n");
2712                         return true;
2713                 }
2714         }
2715
2716         return false;
2717 }
2718
2719 static void i915_set_reset_status(struct drm_i915_private *dev_priv,
2720                                   struct intel_context *ctx,
2721                                   const bool guilty)
2722 {
2723         struct i915_ctx_hang_stats *hs;
2724
2725         if (WARN_ON(!ctx))
2726                 return;
2727
2728         hs = &ctx->hang_stats;
2729
2730         if (guilty) {
2731                 hs->banned = i915_context_is_banned(dev_priv, ctx);
2732                 hs->batch_active++;
2733                 hs->guilty_ts = get_seconds();
2734         } else {
2735                 hs->batch_pending++;
2736         }
2737 }
2738
2739 void i915_gem_request_free(struct kref *req_ref)
2740 {
2741         struct drm_i915_gem_request *req = container_of(req_ref,
2742                                                  typeof(*req), ref);
2743         kmem_cache_free(req->i915->requests, req);
2744 }
2745
2746 static inline int
2747 __i915_gem_request_alloc(struct intel_engine_cs *engine,
2748                          struct intel_context *ctx,
2749                          struct drm_i915_gem_request **req_out)
2750 {
2751         struct drm_i915_private *dev_priv = engine->i915;
2752         unsigned reset_counter = i915_reset_counter(&dev_priv->gpu_error);
2753         struct drm_i915_gem_request *req;
2754         int ret;
2755
2756         if (!req_out)
2757                 return -EINVAL;
2758
2759         *req_out = NULL;
2760
2761         /* ABI: Before userspace accesses the GPU (e.g. execbuffer), report
2762          * EIO if the GPU is already wedged, or EAGAIN to drop the struct_mutex
2763          * and restart.
2764          */
2765         ret = i915_gem_check_wedge(reset_counter, dev_priv->mm.interruptible);
2766         if (ret)
2767                 return ret;
2768
2769         req = kmem_cache_zalloc(dev_priv->requests, GFP_KERNEL);
2770         if (req == NULL)
2771                 return -ENOMEM;
2772
2773         ret = i915_gem_get_seqno(engine->i915, &req->seqno);
2774         if (ret)
2775                 goto err;
2776
2777         kref_init(&req->ref);
2778         req->i915 = dev_priv;
2779         req->engine = engine;
2780         req->reset_counter = reset_counter;
2781         req->ctx  = ctx;
2782         i915_gem_context_reference(req->ctx);
2783
2784         /*
2785          * Reserve space in the ring buffer for all the commands required to
2786          * eventually emit this request. This is to guarantee that the
2787          * i915_add_request() call can't fail. Note that the reserve may need
2788          * to be redone if the request is not actually submitted straight
2789          * away, e.g. because a GPU scheduler has deferred it.
2790          */
2791         req->reserved_space = MIN_SPACE_FOR_ADD_REQUEST;
2792
2793         if (i915.enable_execlists)
2794                 ret = intel_logical_ring_alloc_request_extras(req);
2795         else
2796                 ret = intel_ring_alloc_request_extras(req);
2797         if (ret)
2798                 goto err_ctx;
2799
2800         *req_out = req;
2801         return 0;
2802
2803 err_ctx:
2804         i915_gem_context_unreference(ctx);
2805 err:
2806         kmem_cache_free(dev_priv->requests, req);
2807         return ret;
2808 }
2809
2810 /**
2811  * i915_gem_request_alloc - allocate a request structure
2812  *
2813  * @engine: engine that we wish to issue the request on.
2814  * @ctx: context that the request will be associated with.
2815  *       This can be NULL if the request is not directly related to
2816  *       any specific user context, in which case this function will
2817  *       choose an appropriate context to use.
2818  *
2819  * Returns a pointer to the allocated request if successful,
2820  * or an error code if not.
2821  */
2822 struct drm_i915_gem_request *
2823 i915_gem_request_alloc(struct intel_engine_cs *engine,
2824                        struct intel_context *ctx)
2825 {
2826         struct drm_i915_gem_request *req;
2827         int err;
2828
2829         if (ctx == NULL)
2830                 ctx = engine->i915->kernel_context;
2831         err = __i915_gem_request_alloc(engine, ctx, &req);
2832         return err ? ERR_PTR(err) : req;
2833 }
2834
2835 struct drm_i915_gem_request *
2836 i915_gem_find_active_request(struct intel_engine_cs *engine)
2837 {
2838         struct drm_i915_gem_request *request;
2839
2840         list_for_each_entry(request, &engine->request_list, list) {
2841                 if (i915_gem_request_completed(request, false))
2842                         continue;
2843
2844                 return request;
2845         }
2846
2847         return NULL;
2848 }
2849
2850 static void i915_gem_reset_engine_status(struct drm_i915_private *dev_priv,
2851                                        struct intel_engine_cs *engine)
2852 {
2853         struct drm_i915_gem_request *request;
2854         bool ring_hung;
2855
2856         request = i915_gem_find_active_request(engine);
2857
2858         if (request == NULL)
2859                 return;
2860
2861         ring_hung = engine->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG;
2862
2863         i915_set_reset_status(dev_priv, request->ctx, ring_hung);
2864
2865         list_for_each_entry_continue(request, &engine->request_list, list)
2866                 i915_set_reset_status(dev_priv, request->ctx, false);
2867 }
2868
2869 static void i915_gem_reset_engine_cleanup(struct drm_i915_private *dev_priv,
2870                                         struct intel_engine_cs *engine)
2871 {
2872         struct intel_ringbuffer *buffer;
2873
2874         while (!list_empty(&engine->active_list)) {
2875                 struct drm_i915_gem_object *obj;
2876
2877                 obj = list_first_entry(&engine->active_list,
2878                                        struct drm_i915_gem_object,
2879                                        engine_list[engine->id]);
2880
2881                 i915_gem_object_retire__read(obj, engine->id);
2882         }
2883
2884         /*
2885          * Clear the execlists queue up before freeing the requests, as those
2886          * are the ones that keep the context and ringbuffer backing objects
2887          * pinned in place.
2888          */
2889
2890         if (i915.enable_execlists) {
2891                 /* Ensure irq handler finishes or is cancelled. */
2892                 tasklet_kill(&engine->irq_tasklet);
2893
2894                 intel_execlists_cancel_requests(engine);
2895         }
2896
2897         /*
2898          * We must free the requests after all the corresponding objects have
2899          * been moved off active lists. Which is the same order as the normal
2900          * retire_requests function does. This is important if object hold
2901          * implicit references on things like e.g. ppgtt address spaces through
2902          * the request.
2903          */
2904         while (!list_empty(&engine->request_list)) {
2905                 struct drm_i915_gem_request *request;
2906
2907                 request = list_first_entry(&engine->request_list,
2908                                            struct drm_i915_gem_request,
2909                                            list);
2910
2911                 i915_gem_request_retire(request);
2912         }
2913
2914         /* Having flushed all requests from all queues, we know that all
2915          * ringbuffers must now be empty. However, since we do not reclaim
2916          * all space when retiring the request (to prevent HEADs colliding
2917          * with rapid ringbuffer wraparound) the amount of available space
2918          * upon reset is less than when we start. Do one more pass over
2919          * all the ringbuffers to reset last_retired_head.
2920          */
2921         list_for_each_entry(buffer, &engine->buffers, link) {
2922                 buffer->last_retired_head = buffer->tail;
2923                 intel_ring_update_space(buffer);
2924         }
2925
2926         intel_ring_init_seqno(engine, engine->last_submitted_seqno);
2927 }
2928
2929 void i915_gem_reset(struct drm_device *dev)
2930 {
2931         struct drm_i915_private *dev_priv = dev->dev_private;
2932         struct intel_engine_cs *engine;
2933
2934         /*
2935          * Before we free the objects from the requests, we need to inspect
2936          * them for finding the guilty party. As the requests only borrow
2937          * their reference to the objects, the inspection must be done first.
2938          */
2939         for_each_engine(engine, dev_priv)
2940                 i915_gem_reset_engine_status(dev_priv, engine);
2941
2942         for_each_engine(engine, dev_priv)
2943                 i915_gem_reset_engine_cleanup(dev_priv, engine);
2944
2945         i915_gem_context_reset(dev);
2946
2947         i915_gem_restore_fences(dev);
2948
2949         WARN_ON(i915_verify_lists(dev));
2950 }
2951
2952 /**
2953  * This function clears the request list as sequence numbers are passed.
2954  */
2955 void
2956 i915_gem_retire_requests_ring(struct intel_engine_cs *engine)
2957 {
2958         WARN_ON(i915_verify_lists(engine->dev));
2959
2960         /* Retire requests first as we use it above for the early return.
2961          * If we retire requests last, we may use a later seqno and so clear
2962          * the requests lists without clearing the active list, leading to
2963          * confusion.
2964          */
2965         while (!list_empty(&engine->request_list)) {
2966                 struct drm_i915_gem_request *request;
2967
2968                 request = list_first_entry(&engine->request_list,
2969                                            struct drm_i915_gem_request,
2970                                            list);
2971
2972                 if (!i915_gem_request_completed(request, true))
2973                         break;
2974
2975                 i915_gem_request_retire(request);
2976         }
2977
2978         /* Move any buffers on the active list that are no longer referenced
2979          * by the ringbuffer to the flushing/inactive lists as appropriate,
2980          * before we free the context associated with the requests.
2981          */
2982         while (!list_empty(&engine->active_list)) {
2983                 struct drm_i915_gem_object *obj;
2984
2985                 obj = list_first_entry(&engine->active_list,
2986                                        struct drm_i915_gem_object,
2987                                        engine_list[engine->id]);
2988
2989                 if (!list_empty(&obj->last_read_req[engine->id]->list))
2990                         break;
2991
2992                 i915_gem_object_retire__read(obj, engine->id);
2993         }
2994
2995         if (unlikely(engine->trace_irq_req &&
2996                      i915_gem_request_completed(engine->trace_irq_req, true))) {
2997                 engine->irq_put(engine);
2998                 i915_gem_request_assign(&engine->trace_irq_req, NULL);
2999         }
3000
3001         WARN_ON(i915_verify_lists(engine->dev));
3002 }
3003
3004 bool
3005 i915_gem_retire_requests(struct drm_i915_private *dev_priv)
3006 {
3007         struct intel_engine_cs *engine;
3008         bool idle = true;
3009
3010         for_each_engine(engine, dev_priv) {
3011                 i915_gem_retire_requests_ring(engine);
3012                 idle &= list_empty(&engine->request_list);
3013                 if (i915.enable_execlists) {
3014                         spin_lock_bh(&engine->execlist_lock);
3015                         idle &= list_empty(&engine->execlist_queue);
3016                         spin_unlock_bh(&engine->execlist_lock);
3017                 }
3018         }
3019
3020         if (idle)
3021                 mod_delayed_work(dev_priv->wq,
3022                                    &dev_priv->mm.idle_work,
3023                                    msecs_to_jiffies(100));
3024
3025         return idle;
3026 }
3027
3028 static void
3029 i915_gem_retire_work_handler(struct work_struct *work)
3030 {
3031         struct drm_i915_private *dev_priv =
3032                 container_of(work, typeof(*dev_priv), mm.retire_work.work);
3033         struct drm_device *dev = dev_priv->dev;
3034         bool idle;
3035
3036         /* Come back later if the device is busy... */
3037         idle = false;
3038         if (mutex_trylock(&dev->struct_mutex)) {
3039                 idle = i915_gem_retire_requests(dev_priv);
3040                 mutex_unlock(&dev->struct_mutex);
3041         }
3042         if (!idle)
3043                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
3044                                    round_jiffies_up_relative(HZ));
3045 }
3046
3047 static void
3048 i915_gem_idle_work_handler(struct work_struct *work)
3049 {
3050         struct drm_i915_private *dev_priv =
3051                 container_of(work, typeof(*dev_priv), mm.idle_work.work);
3052         struct drm_device *dev = dev_priv->dev;
3053         struct intel_engine_cs *engine;
3054
3055         for_each_engine(engine, dev_priv)
3056                 if (!list_empty(&engine->request_list))
3057                         return;
3058
3059         /* we probably should sync with hangcheck here, using cancel_work_sync.
3060          * Also locking seems to be fubar here, engine->request_list is protected
3061          * by dev->struct_mutex. */
3062
3063         intel_mark_idle(dev_priv);
3064
3065         if (mutex_trylock(&dev->struct_mutex)) {
3066                 for_each_engine(engine, dev_priv)
3067                         i915_gem_batch_pool_fini(&engine->batch_pool);
3068
3069                 mutex_unlock(&dev->struct_mutex);
3070         }
3071 }
3072
3073 /**
3074  * Ensures that an object will eventually get non-busy by flushing any required
3075  * write domains, emitting any outstanding lazy request and retiring and
3076  * completed requests.
3077  */
3078 static int
3079 i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
3080 {
3081         int i;
3082
3083         if (!obj->active)
3084                 return 0;
3085
3086         for (i = 0; i < I915_NUM_ENGINES; i++) {
3087                 struct drm_i915_gem_request *req;
3088
3089                 req = obj->last_read_req[i];
3090                 if (req == NULL)
3091                         continue;
3092
3093                 if (i915_gem_request_completed(req, true))
3094                         i915_gem_object_retire__read(obj, i);
3095         }
3096
3097         return 0;
3098 }
3099
3100 /**
3101  * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
3102  * @DRM_IOCTL_ARGS: standard ioctl arguments
3103  *
3104  * Returns 0 if successful, else an error is returned with the remaining time in
3105  * the timeout parameter.
3106  *  -ETIME: object is still busy after timeout
3107  *  -ERESTARTSYS: signal interrupted the wait
3108  *  -ENONENT: object doesn't exist
3109  * Also possible, but rare:
3110  *  -EAGAIN: GPU wedged
3111  *  -ENOMEM: damn
3112  *  -ENODEV: Internal IRQ fail
3113  *  -E?: The add request failed
3114  *
3115  * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
3116  * non-zero timeout parameter the wait ioctl will wait for the given number of
3117  * nanoseconds on an object becoming unbusy. Since the wait itself does so
3118  * without holding struct_mutex the object may become re-busied before this
3119  * function completes. A similar but shorter * race condition exists in the busy
3120  * ioctl
3121  */
3122 int
3123 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
3124 {
3125         struct drm_i915_gem_wait *args = data;
3126         struct drm_i915_gem_object *obj;
3127         struct drm_i915_gem_request *req[I915_NUM_ENGINES];
3128         int i, n = 0;
3129         int ret;
3130
3131         if (args->flags != 0)
3132                 return -EINVAL;
3133
3134         ret = i915_mutex_lock_interruptible(dev);
3135         if (ret)
3136                 return ret;
3137
3138         obj = to_intel_bo(drm_gem_object_lookup(file, args->bo_handle));
3139         if (&obj->base == NULL) {
3140                 mutex_unlock(&dev->struct_mutex);
3141                 return -ENOENT;
3142         }
3143
3144         /* Need to make sure the object gets inactive eventually. */
3145         ret = i915_gem_object_flush_active(obj);
3146         if (ret)
3147                 goto out;
3148
3149         if (!obj->active)
3150                 goto out;
3151
3152         /* Do this after OLR check to make sure we make forward progress polling
3153          * on this IOCTL with a timeout == 0 (like busy ioctl)
3154          */
3155         if (args->timeout_ns == 0) {
3156                 ret = -ETIME;
3157                 goto out;
3158         }
3159
3160         drm_gem_object_unreference(&obj->base);
3161
3162         for (i = 0; i < I915_NUM_ENGINES; i++) {
3163                 if (obj->last_read_req[i] == NULL)
3164                         continue;
3165
3166                 req[n++] = i915_gem_request_reference(obj->last_read_req[i]);
3167         }
3168
3169         mutex_unlock(&dev->struct_mutex);
3170
3171         for (i = 0; i < n; i++) {
3172                 if (ret == 0)
3173                         ret = __i915_wait_request(req[i], true,
3174                                                   args->timeout_ns > 0 ? &args->timeout_ns : NULL,
3175                                                   to_rps_client(file));
3176                 i915_gem_request_unreference(req[i]);
3177         }
3178         return ret;
3179
3180 out:
3181         drm_gem_object_unreference(&obj->base);
3182         mutex_unlock(&dev->struct_mutex);
3183         return ret;
3184 }
3185
3186 static int
3187 __i915_gem_object_sync(struct drm_i915_gem_object *obj,
3188                        struct intel_engine_cs *to,
3189                        struct drm_i915_gem_request *from_req,
3190                        struct drm_i915_gem_request **to_req)
3191 {
3192         struct intel_engine_cs *from;
3193         int ret;
3194
3195         from = i915_gem_request_get_engine(from_req);
3196         if (to == from)
3197                 return 0;
3198
3199         if (i915_gem_request_completed(from_req, true))
3200                 return 0;
3201
3202         if (!i915_semaphore_is_enabled(to_i915(obj->base.dev))) {
3203                 struct drm_i915_private *i915 = to_i915(obj->base.dev);
3204                 ret = __i915_wait_request(from_req,
3205                                           i915->mm.interruptible,
3206                                           NULL,
3207                                           &i915->rps.semaphores);
3208                 if (ret)
3209                         return ret;
3210
3211                 i915_gem_object_retire_request(obj, from_req);
3212         } else {
3213                 int idx = intel_ring_sync_index(from, to);
3214                 u32 seqno = i915_gem_request_get_seqno(from_req);
3215
3216                 WARN_ON(!to_req);
3217
3218                 if (seqno <= from->semaphore.sync_seqno[idx])
3219                         return 0;
3220
3221                 if (*to_req == NULL) {
3222                         struct drm_i915_gem_request *req;
3223
3224                         req = i915_gem_request_alloc(to, NULL);
3225                         if (IS_ERR(req))
3226                                 return PTR_ERR(req);
3227
3228                         *to_req = req;
3229                 }
3230
3231                 trace_i915_gem_ring_sync_to(*to_req, from, from_req);
3232                 ret = to->semaphore.sync_to(*to_req, from, seqno);
3233                 if (ret)
3234                         return ret;
3235
3236                 /* We use last_read_req because sync_to()
3237                  * might have just caused seqno wrap under
3238                  * the radar.
3239                  */
3240                 from->semaphore.sync_seqno[idx] =
3241                         i915_gem_request_get_seqno(obj->last_read_req[from->id]);
3242         }
3243
3244         return 0;
3245 }
3246
3247 /**
3248  * i915_gem_object_sync - sync an object to a ring.
3249  *
3250  * @obj: object which may be in use on another ring.
3251  * @to: ring we wish to use the object on. May be NULL.
3252  * @to_req: request we wish to use the object for. See below.
3253  *          This will be allocated and returned if a request is
3254  *          required but not passed in.
3255  *
3256  * This code is meant to abstract object synchronization with the GPU.
3257  * Calling with NULL implies synchronizing the object with the CPU
3258  * rather than a particular GPU ring. Conceptually we serialise writes
3259  * between engines inside the GPU. We only allow one engine to write
3260  * into a buffer at any time, but multiple readers. To ensure each has
3261  * a coherent view of memory, we must:
3262  *
3263  * - If there is an outstanding write request to the object, the new
3264  *   request must wait for it to complete (either CPU or in hw, requests
3265  *   on the same ring will be naturally ordered).
3266  *
3267  * - If we are a write request (pending_write_domain is set), the new
3268  *   request must wait for outstanding read requests to complete.
3269  *
3270  * For CPU synchronisation (NULL to) no request is required. For syncing with
3271  * rings to_req must be non-NULL. However, a request does not have to be
3272  * pre-allocated. If *to_req is NULL and sync commands will be emitted then a
3273  * request will be allocated automatically and returned through *to_req. Note
3274  * that it is not guaranteed that commands will be emitted (because the system
3275  * might already be idle). Hence there is no need to create a request that
3276  * might never have any work submitted. Note further that if a request is
3277  * returned in *to_req, it is the responsibility of the caller to submit
3278  * that request (after potentially adding more work to it).
3279  *
3280  * Returns 0 if successful, else propagates up the lower layer error.
3281  */
3282 int
3283 i915_gem_object_sync(struct drm_i915_gem_object *obj,
3284                      struct intel_engine_cs *to,
3285                      struct drm_i915_gem_request **to_req)
3286 {
3287         const bool readonly = obj->base.pending_write_domain == 0;
3288         struct drm_i915_gem_request *req[I915_NUM_ENGINES];
3289         int ret, i, n;
3290
3291         if (!obj->active)
3292                 return 0;
3293
3294         if (to == NULL)
3295                 return i915_gem_object_wait_rendering(obj, readonly);
3296
3297         n = 0;
3298         if (readonly) {
3299                 if (obj->last_write_req)
3300                         req[n++] = obj->last_write_req;
3301         } else {
3302                 for (i = 0; i < I915_NUM_ENGINES; i++)
3303                         if (obj->last_read_req[i])
3304                                 req[n++] = obj->last_read_req[i];
3305         }
3306         for (i = 0; i < n; i++) {
3307                 ret = __i915_gem_object_sync(obj, to, req[i], to_req);
3308                 if (ret)
3309                         return ret;
3310         }
3311
3312         return 0;
3313 }
3314
3315 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
3316 {
3317         u32 old_write_domain, old_read_domains;
3318
3319         /* Force a pagefault for domain tracking on next user access */
3320         i915_gem_release_mmap(obj);
3321
3322         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
3323                 return;
3324
3325         old_read_domains = obj->base.read_domains;
3326         old_write_domain = obj->base.write_domain;
3327
3328         obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
3329         obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
3330
3331         trace_i915_gem_object_change_domain(obj,
3332                                             old_read_domains,
3333                                             old_write_domain);
3334 }
3335
3336 static void __i915_vma_iounmap(struct i915_vma *vma)
3337 {
3338         GEM_BUG_ON(vma->pin_count);
3339
3340         if (vma->iomap == NULL)
3341                 return;
3342
3343         io_mapping_unmap(vma->iomap);
3344         vma->iomap = NULL;
3345 }
3346
3347 static int __i915_vma_unbind(struct i915_vma *vma, bool wait)
3348 {
3349         struct drm_i915_gem_object *obj = vma->obj;
3350         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3351         int ret;
3352
3353         if (list_empty(&vma->obj_link))
3354                 return 0;
3355
3356         if (!drm_mm_node_allocated(&vma->node)) {
3357                 i915_gem_vma_destroy(vma);
3358                 return 0;
3359         }
3360
3361         if (vma->pin_count)
3362                 return -EBUSY;
3363
3364         BUG_ON(obj->pages == NULL);
3365
3366         if (wait) {
3367                 ret = i915_gem_object_wait_rendering(obj, false);
3368                 if (ret)
3369                         return ret;
3370         }
3371
3372         if (vma->is_ggtt && vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
3373                 i915_gem_object_finish_gtt(obj);
3374
3375                 /* release the fence reg _after_ flushing */
3376                 ret = i915_gem_object_put_fence(obj);
3377                 if (ret)
3378                         return ret;
3379
3380                 __i915_vma_iounmap(vma);
3381         }
3382
3383         trace_i915_vma_unbind(vma);
3384
3385         vma->vm->unbind_vma(vma);
3386         vma->bound = 0;
3387
3388         list_del_init(&vma->vm_link);
3389         if (vma->is_ggtt) {
3390                 if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
3391                         obj->map_and_fenceable = false;
3392                 } else if (vma->ggtt_view.pages) {
3393                         sg_free_table(vma->ggtt_view.pages);
3394                         kfree(vma->ggtt_view.pages);
3395                 }
3396                 vma->ggtt_view.pages = NULL;
3397         }
3398
3399         drm_mm_remove_node(&vma->node);
3400         i915_gem_vma_destroy(vma);
3401
3402         /* Since the unbound list is global, only move to that list if
3403          * no more VMAs exist. */
3404         if (list_empty(&obj->vma_list))
3405                 list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list);
3406
3407         /* And finally now the object is completely decoupled from this vma,
3408          * we can drop its hold on the backing storage and allow it to be
3409          * reaped by the shrinker.
3410          */
3411         i915_gem_object_unpin_pages(obj);
3412
3413         return 0;
3414 }
3415
3416 int i915_vma_unbind(struct i915_vma *vma)
3417 {
3418         return __i915_vma_unbind(vma, true);
3419 }
3420
3421 int __i915_vma_unbind_no_wait(struct i915_vma *vma)
3422 {
3423         return __i915_vma_unbind(vma, false);
3424 }
3425
3426 int i915_gpu_idle(struct drm_device *dev)
3427 {
3428         struct drm_i915_private *dev_priv = dev->dev_private;
3429         struct intel_engine_cs *engine;
3430         int ret;
3431
3432         /* Flush everything onto the inactive list. */
3433         for_each_engine(engine, dev_priv) {
3434                 if (!i915.enable_execlists) {
3435                         struct drm_i915_gem_request *req;
3436
3437                         req = i915_gem_request_alloc(engine, NULL);
3438                         if (IS_ERR(req))
3439                                 return PTR_ERR(req);
3440
3441                         ret = i915_switch_context(req);
3442                         i915_add_request_no_flush(req);
3443                         if (ret)
3444                                 return ret;
3445                 }
3446
3447                 ret = intel_engine_idle(engine);
3448                 if (ret)
3449                         return ret;
3450         }
3451
3452         WARN_ON(i915_verify_lists(dev));
3453         return 0;
3454 }
3455
3456 static bool i915_gem_valid_gtt_space(struct i915_vma *vma,
3457                                      unsigned long cache_level)
3458 {
3459         struct drm_mm_node *gtt_space = &vma->node;
3460         struct drm_mm_node *other;
3461
3462         /*
3463          * On some machines we have to be careful when putting differing types
3464          * of snoopable memory together to avoid the prefetcher crossing memory
3465          * domains and dying. During vm initialisation, we decide whether or not
3466          * these constraints apply and set the drm_mm.color_adjust
3467          * appropriately.
3468          */
3469         if (vma->vm->mm.color_adjust == NULL)
3470                 return true;
3471
3472         if (!drm_mm_node_allocated(gtt_space))
3473                 return true;
3474
3475         if (list_empty(&gtt_space->node_list))
3476                 return true;
3477
3478         other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
3479         if (other->allocated && !other->hole_follows && other->color != cache_level)
3480                 return false;
3481
3482         other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
3483         if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
3484                 return false;
3485
3486         return true;
3487 }
3488
3489 /**
3490  * Finds free space in the GTT aperture and binds the object or a view of it
3491  * there.
3492  */
3493 static struct i915_vma *
3494 i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
3495                            struct i915_address_space *vm,
3496                            const struct i915_ggtt_view *ggtt_view,
3497                            unsigned alignment,
3498                            uint64_t flags)
3499 {
3500         struct drm_device *dev = obj->base.dev;
3501         struct drm_i915_private *dev_priv = to_i915(dev);
3502         struct i915_ggtt *ggtt = &dev_priv->ggtt;
3503         u32 fence_alignment, unfenced_alignment;
3504         u32 search_flag, alloc_flag;
3505         u64 start, end;
3506         u64 size, fence_size;
3507         struct i915_vma *vma;
3508         int ret;
3509
3510         if (i915_is_ggtt(vm)) {
3511                 u32 view_size;
3512
3513                 if (WARN_ON(!ggtt_view))
3514                         return ERR_PTR(-EINVAL);
3515
3516                 view_size = i915_ggtt_view_size(obj, ggtt_view);
3517
3518                 fence_size = i915_gem_get_gtt_size(dev,
3519                                                    view_size,
3520                                                    obj->tiling_mode);
3521                 fence_alignment = i915_gem_get_gtt_alignment(dev,
3522                                                              view_size,
3523                                                              obj->tiling_mode,
3524                                                              true);
3525                 unfenced_alignment = i915_gem_get_gtt_alignment(dev,
3526                                                                 view_size,
3527                                                                 obj->tiling_mode,
3528                                                                 false);
3529                 size = flags & PIN_MAPPABLE ? fence_size : view_size;
3530         } else {
3531                 fence_size = i915_gem_get_gtt_size(dev,
3532                                                    obj->base.size,
3533                                                    obj->tiling_mode);
3534                 fence_alignment = i915_gem_get_gtt_alignment(dev,
3535                                                              obj->base.size,
3536                                                              obj->tiling_mode,
3537                                                              true);
3538                 unfenced_alignment =
3539                         i915_gem_get_gtt_alignment(dev,
3540                                                    obj->base.size,
3541                                                    obj->tiling_mode,
3542                                                    false);
3543                 size = flags & PIN_MAPPABLE ? fence_size : obj->base.size;
3544         }
3545
3546         start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0;
3547         end = vm->total;
3548         if (flags & PIN_MAPPABLE)
3549                 end = min_t(u64, end, ggtt->mappable_end);
3550         if (flags & PIN_ZONE_4G)
3551                 end = min_t(u64, end, (1ULL << 32) - PAGE_SIZE);
3552
3553         if (alignment == 0)
3554                 alignment = flags & PIN_MAPPABLE ? fence_alignment :
3555                                                 unfenced_alignment;
3556         if (flags & PIN_MAPPABLE && alignment & (fence_alignment - 1)) {
3557                 DRM_DEBUG("Invalid object (view type=%u) alignment requested %u\n",
3558                           ggtt_view ? ggtt_view->type : 0,
3559                           alignment);
3560                 return ERR_PTR(-EINVAL);
3561         }
3562
3563         /* If binding the object/GGTT view requires more space than the entire
3564          * aperture has, reject it early before evicting everything in a vain
3565          * attempt to find space.
3566          */
3567         if (size > end) {
3568                 DRM_DEBUG("Attempting to bind an object (view type=%u) larger than the aperture: size=%llu > %s aperture=%llu\n",
3569                           ggtt_view ? ggtt_view->type : 0,
3570                           size,
3571                           flags & PIN_MAPPABLE ? "mappable" : "total",
3572                           end);
3573                 return ERR_PTR(-E2BIG);
3574         }
3575
3576         ret = i915_gem_object_get_pages(obj);
3577         if (ret)
3578                 return ERR_PTR(ret);
3579
3580         i915_gem_object_pin_pages(obj);
3581
3582         vma = ggtt_view ? i915_gem_obj_lookup_or_create_ggtt_vma(obj, ggtt_view) :
3583                           i915_gem_obj_lookup_or_create_vma(obj, vm);
3584
3585         if (IS_ERR(vma))
3586                 goto err_unpin;
3587
3588         if (flags & PIN_OFFSET_FIXED) {
3589                 uint64_t offset = flags & PIN_OFFSET_MASK;
3590
3591                 if (offset & (alignment - 1) || offset + size > end) {
3592                         ret = -EINVAL;
3593                         goto err_free_vma;
3594                 }
3595                 vma->node.start = offset;
3596                 vma->node.size = size;
3597                 vma->node.color = obj->cache_level;
3598                 ret = drm_mm_reserve_node(&vm->mm, &vma->node);
3599                 if (ret) {
3600                         ret = i915_gem_evict_for_vma(vma);
3601                         if (ret == 0)
3602                                 ret = drm_mm_reserve_node(&vm->mm, &vma->node);
3603                 }
3604                 if (ret)
3605                         goto err_free_vma;
3606         } else {
3607                 if (flags & PIN_HIGH) {
3608                         search_flag = DRM_MM_SEARCH_BELOW;
3609                         alloc_flag = DRM_MM_CREATE_TOP;
3610                 } else {
3611                         search_flag = DRM_MM_SEARCH_DEFAULT;
3612                         alloc_flag = DRM_MM_CREATE_DEFAULT;
3613                 }
3614
3615 search_free:
3616                 ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node,
3617                                                           size, alignment,
3618                                                           obj->cache_level,
3619                                                           start, end,
3620                                                           search_flag,
3621                                                           alloc_flag);
3622                 if (ret) {
3623                         ret = i915_gem_evict_something(dev, vm, size, alignment,
3624                                                        obj->cache_level,
3625                                                        start, end,
3626                                                        flags);
3627                         if (ret == 0)
3628                                 goto search_free;
3629
3630                         goto err_free_vma;
3631                 }
3632         }
3633         if (WARN_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level))) {
3634                 ret = -EINVAL;
3635                 goto err_remove_node;
3636         }
3637
3638         trace_i915_vma_bind(vma, flags);
3639         ret = i915_vma_bind(vma, obj->cache_level, flags);
3640         if (ret)
3641                 goto err_remove_node;
3642
3643         list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
3644         list_add_tail(&vma->vm_link, &vm->inactive_list);
3645
3646         return vma;
3647
3648 err_remove_node:
3649         drm_mm_remove_node(&vma->node);
3650 err_free_vma:
3651         i915_gem_vma_destroy(vma);
3652         vma = ERR_PTR(ret);
3653 err_unpin:
3654         i915_gem_object_unpin_pages(obj);
3655         return vma;
3656 }
3657
3658 bool
3659 i915_gem_clflush_object(struct drm_i915_gem_object *obj,
3660                         bool force)
3661 {
3662         /* If we don't have a page list set up, then we're not pinned
3663          * to GPU, and we can ignore the cache flush because it'll happen
3664          * again at bind time.
3665          */
3666         if (obj->pages == NULL)
3667                 return false;
3668
3669         /*
3670          * Stolen memory is always coherent with the GPU as it is explicitly
3671          * marked as wc by the system, or the system is cache-coherent.
3672          */
3673         if (obj->stolen || obj->phys_handle)
3674                 return false;
3675
3676         /* If the GPU is snooping the contents of the CPU cache,
3677          * we do not need to manually clear the CPU cache lines.  However,
3678          * the caches are only snooped when the render cache is
3679          * flushed/invalidated.  As we always have to emit invalidations
3680          * and flushes when moving into and out of the RENDER domain, correct
3681          * snooping behaviour occurs naturally as the result of our domain
3682          * tracking.
3683          */
3684         if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level)) {
3685                 obj->cache_dirty = true;
3686                 return false;
3687         }
3688
3689         trace_i915_gem_object_clflush(obj);
3690         drm_clflush_sg(obj->pages);
3691         obj->cache_dirty = false;
3692
3693         return true;
3694 }
3695
3696 /** Flushes the GTT write domain for the object if it's dirty. */
3697 static void
3698 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
3699 {
3700         uint32_t old_write_domain;
3701
3702         if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
3703                 return;
3704
3705         /* No actual flushing is required for the GTT write domain.  Writes
3706          * to it immediately go to main memory as far as we know, so there's
3707          * no chipset flush.  It also doesn't land in render cache.
3708          *
3709          * However, we do have to enforce the order so that all writes through
3710          * the GTT land before any writes to the device, such as updates to
3711          * the GATT itself.
3712          */
3713         wmb();
3714
3715         old_write_domain = obj->base.write_domain;
3716         obj->base.write_domain = 0;
3717
3718         intel_fb_obj_flush(obj, false, ORIGIN_GTT);
3719
3720         trace_i915_gem_object_change_domain(obj,
3721                                             obj->base.read_domains,
3722                                             old_write_domain);
3723 }
3724
3725 /** Flushes the CPU write domain for the object if it's dirty. */
3726 static void
3727 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
3728 {
3729         uint32_t old_write_domain;
3730
3731         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
3732                 return;
3733
3734         if (i915_gem_clflush_object(obj, obj->pin_display))
3735                 i915_gem_chipset_flush(to_i915(obj->base.dev));
3736
3737         old_write_domain = obj->base.write_domain;
3738         obj->base.write_domain = 0;
3739
3740         intel_fb_obj_flush(obj, false, ORIGIN_CPU);
3741
3742         trace_i915_gem_object_change_domain(obj,
3743                                             obj->base.read_domains,
3744                                             old_write_domain);
3745 }
3746
3747 /**
3748  * Moves a single object to the GTT read, and possibly write domain.
3749  *
3750  * This function returns when the move is complete, including waiting on
3751  * flushes to occur.
3752  */
3753 int
3754 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
3755 {
3756         struct drm_device *dev = obj->base.dev;
3757         struct drm_i915_private *dev_priv = to_i915(dev);
3758         struct i915_ggtt *ggtt = &dev_priv->ggtt;
3759         uint32_t old_write_domain, old_read_domains;
3760         struct i915_vma *vma;
3761         int ret;
3762
3763         if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
3764                 return 0;
3765
3766         ret = i915_gem_object_wait_rendering(obj, !write);
3767         if (ret)
3768                 return ret;
3769
3770         /* Flush and acquire obj->pages so that we are coherent through
3771          * direct access in memory with previous cached writes through
3772          * shmemfs and that our cache domain tracking remains valid.
3773          * For example, if the obj->filp was moved to swap without us
3774          * being notified and releasing the pages, we would mistakenly
3775          * continue to assume that the obj remained out of the CPU cached
3776          * domain.
3777          */
3778         ret = i915_gem_object_get_pages(obj);
3779         if (ret)
3780                 return ret;
3781
3782         i915_gem_object_flush_cpu_write_domain(obj);
3783
3784         /* Serialise direct access to this object with the barriers for
3785          * coherent writes from the GPU, by effectively invalidating the
3786          * GTT domain upon first access.
3787          */
3788         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
3789                 mb();
3790
3791         old_write_domain = obj->base.write_domain;
3792         old_read_domains = obj->base.read_domains;
3793
3794         /* It should now be out of any other write domains, and we can update
3795          * the domain values for our changes.
3796          */
3797         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
3798         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3799         if (write) {
3800                 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
3801                 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
3802                 obj->dirty = 1;
3803         }
3804
3805         trace_i915_gem_object_change_domain(obj,
3806                                             old_read_domains,
3807                                             old_write_domain);
3808
3809         /* And bump the LRU for this access */
3810         vma = i915_gem_obj_to_ggtt(obj);
3811         if (vma && drm_mm_node_allocated(&vma->node) && !obj->active)
3812                 list_move_tail(&vma->vm_link,
3813                                &ggtt->base.inactive_list);
3814
3815         return 0;
3816 }
3817
3818 /**
3819  * Changes the cache-level of an object across all VMA.
3820  *
3821  * After this function returns, the object will be in the new cache-level
3822  * across all GTT and the contents of the backing storage will be coherent,
3823  * with respect to the new cache-level. In order to keep the backing storage
3824  * coherent for all users, we only allow a single cache level to be set
3825  * globally on the object and prevent it from being changed whilst the
3826  * hardware is reading from the object. That is if the object is currently
3827  * on the scanout it will be set to uncached (or equivalent display
3828  * cache coherency) and all non-MOCS GPU access will also be uncached so
3829  * that all direct access to the scanout remains coherent.
3830  */
3831 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
3832                                     enum i915_cache_level cache_level)
3833 {
3834         struct drm_device *dev = obj->base.dev;
3835         struct i915_vma *vma, *next;
3836         bool bound = false;
3837         int ret = 0;
3838
3839         if (obj->cache_level == cache_level)
3840                 goto out;
3841
3842         /* Inspect the list of currently bound VMA and unbind any that would
3843          * be invalid given the new cache-level. This is principally to
3844          * catch the issue of the CS prefetch crossing page boundaries and
3845          * reading an invalid PTE on older architectures.
3846          */
3847         list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link) {
3848                 if (!drm_mm_node_allocated(&vma->node))
3849                         continue;
3850
3851                 if (vma->pin_count) {
3852                         DRM_DEBUG("can not change the cache level of pinned objects\n");
3853                         return -EBUSY;
3854                 }
3855
3856                 if (!i915_gem_valid_gtt_space(vma, cache_level)) {
3857                         ret = i915_vma_unbind(vma);
3858                         if (ret)
3859                                 return ret;
3860                 } else
3861                         bound = true;
3862         }
3863
3864         /* We can reuse the existing drm_mm nodes but need to change the
3865          * cache-level on the PTE. We could simply unbind them all and
3866          * rebind with the correct cache-level on next use. However since
3867          * we already have a valid slot, dma mapping, pages etc, we may as
3868          * rewrite the PTE in the belief that doing so tramples upon less
3869          * state and so involves less work.
3870          */
3871         if (bound) {
3872                 /* Before we change the PTE, the GPU must not be accessing it.
3873                  * If we wait upon the object, we know that all the bound
3874                  * VMA are no longer active.
3875                  */
3876                 ret = i915_gem_object_wait_rendering(obj, false);
3877                 if (ret)
3878                         return ret;
3879
3880                 if (!HAS_LLC(dev) && cache_level != I915_CACHE_NONE) {
3881                         /* Access to snoopable pages through the GTT is
3882                          * incoherent and on some machines causes a hard
3883                          * lockup. Relinquish the CPU mmaping to force
3884                          * userspace to refault in the pages and we can
3885                          * then double check if the GTT mapping is still
3886                          * valid for that pointer access.
3887                          */
3888                         i915_gem_release_mmap(obj);
3889
3890                         /* As we no longer need a fence for GTT access,
3891                          * we can relinquish it now (and so prevent having
3892                          * to steal a fence from someone else on the next
3893                          * fence request). Note GPU activity would have
3894                          * dropped the fence as all snoopable access is
3895                          * supposed to be linear.
3896                          */
3897                         ret = i915_gem_object_put_fence(obj);
3898                         if (ret)
3899                                 return ret;
3900                 } else {
3901                         /* We either have incoherent backing store and
3902                          * so no GTT access or the architecture is fully
3903                          * coherent. In such cases, existing GTT mmaps
3904                          * ignore the cache bit in the PTE and we can
3905                          * rewrite it without confusing the GPU or having
3906                          * to force userspace to fault back in its mmaps.
3907                          */
3908                 }
3909
3910                 list_for_each_entry(vma, &obj->vma_list, obj_link) {
3911                         if (!drm_mm_node_allocated(&vma->node))
3912                                 continue;
3913
3914                         ret = i915_vma_bind(vma, cache_level, PIN_UPDATE);
3915                         if (ret)
3916                                 return ret;
3917                 }
3918         }
3919
3920         list_for_each_entry(vma, &obj->vma_list, obj_link)
3921                 vma->node.color = cache_level;
3922         obj->cache_level = cache_level;
3923
3924 out:
3925         /* Flush the dirty CPU caches to the backing storage so that the
3926          * object is now coherent at its new cache level (with respect
3927          * to the access domain).
3928          */
3929         if (obj->cache_dirty &&
3930             obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
3931             cpu_write_needs_clflush(obj)) {
3932                 if (i915_gem_clflush_object(obj, true))
3933                         i915_gem_chipset_flush(to_i915(obj->base.dev));
3934         }
3935
3936         return 0;
3937 }
3938
3939 int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
3940                                struct drm_file *file)
3941 {
3942         struct drm_i915_gem_caching *args = data;
3943         struct drm_i915_gem_object *obj;
3944
3945         obj = to_intel_bo(drm_gem_object_lookup(file, args->handle));
3946         if (&obj->base == NULL)
3947                 return -ENOENT;
3948
3949         switch (obj->cache_level) {
3950         case I915_CACHE_LLC:
3951         case I915_CACHE_L3_LLC:
3952                 args->caching = I915_CACHING_CACHED;
3953                 break;
3954
3955         case I915_CACHE_WT:
3956                 args->caching = I915_CACHING_DISPLAY;
3957                 break;
3958
3959         default:
3960                 args->caching = I915_CACHING_NONE;
3961                 break;
3962         }
3963
3964         drm_gem_object_unreference_unlocked(&obj->base);
3965         return 0;
3966 }
3967
3968 int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
3969                                struct drm_file *file)
3970 {
3971         struct drm_i915_private *dev_priv = dev->dev_private;
3972         struct drm_i915_gem_caching *args = data;
3973         struct drm_i915_gem_object *obj;
3974         enum i915_cache_level level;
3975         int ret;
3976
3977         switch (args->caching) {
3978         case I915_CACHING_NONE:
3979                 level = I915_CACHE_NONE;
3980                 break;
3981         case I915_CACHING_CACHED:
3982                 /*
3983                  * Due to a HW issue on BXT A stepping, GPU stores via a
3984                  * snooped mapping may leave stale data in a corresponding CPU
3985                  * cacheline, whereas normally such cachelines would get
3986                  * invalidated.
3987                  */
3988                 if (!HAS_LLC(dev) && !HAS_SNOOP(dev))
3989                         return -ENODEV;
3990
3991                 level = I915_CACHE_LLC;
3992                 break;
3993         case I915_CACHING_DISPLAY:
3994                 level = HAS_WT(dev) ? I915_CACHE_WT : I915_CACHE_NONE;
3995                 break;
3996         default:
3997                 return -EINVAL;
3998         }
3999
4000         intel_runtime_pm_get(dev_priv);
4001
4002         ret = i915_mutex_lock_interruptible(dev);
4003         if (ret)
4004                 goto rpm_put;
4005
4006         obj = to_intel_bo(drm_gem_object_lookup(file, args->handle));
4007         if (&obj->base == NULL) {
4008                 ret = -ENOENT;
4009                 goto unlock;
4010         }
4011
4012         ret = i915_gem_object_set_cache_level(obj, level);
4013
4014         drm_gem_object_unreference(&obj->base);
4015 unlock:
4016         mutex_unlock(&dev->struct_mutex);
4017 rpm_put:
4018         intel_runtime_pm_put(dev_priv);
4019
4020         return ret;
4021 }
4022
4023 /*
4024  * Prepare buffer for display plane (scanout, cursors, etc).
4025  * Can be called from an uninterruptible phase (modesetting) and allows
4026  * any flushes to be pipelined (for pageflips).
4027  */
4028 int
4029 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
4030                                      u32 alignment,
4031                                      const struct i915_ggtt_view *view)
4032 {
4033         u32 old_read_domains, old_write_domain;
4034         int ret;
4035
4036         /* Mark the pin_display early so that we account for the
4037          * display coherency whilst setting up the cache domains.
4038          */
4039         obj->pin_display++;
4040
4041         /* The display engine is not coherent with the LLC cache on gen6.  As
4042          * a result, we make sure that the pinning that is about to occur is
4043          * done with uncached PTEs. This is lowest common denominator for all
4044          * chipsets.
4045          *
4046          * However for gen6+, we could do better by using the GFDT bit instead
4047          * of uncaching, which would allow us to flush all the LLC-cached data
4048          * with that bit in the PTE to main memory with just one PIPE_CONTROL.
4049          */
4050         ret = i915_gem_object_set_cache_level(obj,
4051                                               HAS_WT(obj->base.dev) ? I915_CACHE_WT : I915_CACHE_NONE);
4052         if (ret)
4053                 goto err_unpin_display;
4054
4055         /* As the user may map the buffer once pinned in the display plane
4056          * (e.g. libkms for the bootup splash), we have to ensure that we
4057          * always use map_and_fenceable for all scanout buffers.
4058          */
4059         ret = i915_gem_object_ggtt_pin(obj, view, alignment,
4060                                        view->type == I915_GGTT_VIEW_NORMAL ?
4061                                        PIN_MAPPABLE : 0);
4062         if (ret)
4063                 goto err_unpin_display;
4064
4065         i915_gem_object_flush_cpu_write_domain(obj);
4066
4067         old_write_domain = obj->base.write_domain;
4068         old_read_domains = obj->base.read_domains;
4069
4070         /* It should now be out of any other write domains, and we can update
4071          * the domain values for our changes.
4072          */
4073         obj->base.write_domain = 0;
4074         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
4075
4076         trace_i915_gem_object_change_domain(obj,
4077                                             old_read_domains,
4078                                             old_write_domain);
4079
4080         return 0;
4081
4082 err_unpin_display:
4083         obj->pin_display--;
4084         return ret;
4085 }
4086
4087 void
4088 i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj,
4089                                          const struct i915_ggtt_view *view)
4090 {
4091         if (WARN_ON(obj->pin_display == 0))
4092                 return;
4093
4094         i915_gem_object_ggtt_unpin_view(obj, view);
4095
4096         obj->pin_display--;
4097 }
4098
4099 /**
4100  * Moves a single object to the CPU read, and possibly write domain.
4101  *
4102  * This function returns when the move is complete, including waiting on
4103  * flushes to occur.
4104  */
4105 int
4106 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
4107 {
4108         uint32_t old_write_domain, old_read_domains;
4109         int ret;
4110
4111         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
4112                 return 0;
4113
4114         ret = i915_gem_object_wait_rendering(obj, !write);
4115         if (ret)
4116                 return ret;
4117
4118         i915_gem_object_flush_gtt_write_domain(obj);
4119
4120         old_write_domain = obj->base.write_domain;
4121         old_read_domains = obj->base.read_domains;
4122
4123         /* Flush the CPU cache if it's still invalid. */
4124         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
4125                 i915_gem_clflush_object(obj, false);
4126
4127                 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
4128         }
4129
4130         /* It should now be out of any other write domains, and we can update
4131          * the domain values for our changes.
4132          */
4133         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
4134
4135         /* If we're writing through the CPU, then the GPU read domains will
4136          * need to be invalidated at next use.
4137          */
4138         if (write) {
4139                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4140                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4141         }
4142
4143         trace_i915_gem_object_change_domain(obj,
4144                                             old_read_domains,
4145                                             old_write_domain);
4146
4147         return 0;
4148 }
4149
4150 /* Throttle our rendering by waiting until the ring has completed our requests
4151  * emitted over 20 msec ago.
4152  *
4153  * Note that if we were to use the current jiffies each time around the loop,
4154  * we wouldn't escape the function with any frames outstanding if the time to
4155  * render a frame was over 20ms.
4156  *
4157  * This should get us reasonable parallelism between CPU and GPU but also
4158  * relatively low latency when blocking on a particular request to finish.
4159  */
4160 static int
4161 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
4162 {
4163         struct drm_i915_private *dev_priv = dev->dev_private;
4164         struct drm_i915_file_private *file_priv = file->driver_priv;
4165         unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES;
4166         struct drm_i915_gem_request *request, *target = NULL;
4167         int ret;
4168
4169         ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
4170         if (ret)
4171                 return ret;
4172
4173         /* ABI: return -EIO if already wedged */
4174         if (i915_terminally_wedged(&dev_priv->gpu_error))
4175                 return -EIO;
4176
4177         spin_lock(&file_priv->mm.lock);
4178         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
4179                 if (time_after_eq(request->emitted_jiffies, recent_enough))
4180                         break;
4181
4182                 /*
4183                  * Note that the request might not have been submitted yet.
4184                  * In which case emitted_jiffies will be zero.
4185                  */
4186                 if (!request->emitted_jiffies)
4187                         continue;
4188
4189                 target = request;
4190         }
4191         if (target)
4192                 i915_gem_request_reference(target);
4193         spin_unlock(&file_priv->mm.lock);
4194
4195         if (target == NULL)
4196                 return 0;
4197
4198         ret = __i915_wait_request(target, true, NULL, NULL);
4199         if (ret == 0)
4200                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
4201
4202         i915_gem_request_unreference(target);
4203
4204         return ret;
4205 }
4206
4207 static bool
4208 i915_vma_misplaced(struct i915_vma *vma, uint32_t alignment, uint64_t flags)
4209 {
4210         struct drm_i915_gem_object *obj = vma->obj;
4211
4212         if (alignment &&
4213             vma->node.start & (alignment - 1))
4214                 return true;
4215
4216         if (flags & PIN_MAPPABLE && !obj->map_and_fenceable)
4217                 return true;
4218
4219         if (flags & PIN_OFFSET_BIAS &&
4220             vma->node.start < (flags & PIN_OFFSET_MASK))
4221                 return true;
4222
4223         if (flags & PIN_OFFSET_FIXED &&
4224             vma->node.start != (flags & PIN_OFFSET_MASK))
4225                 return true;
4226
4227         return false;
4228 }
4229
4230 void __i915_vma_set_map_and_fenceable(struct i915_vma *vma)
4231 {
4232         struct drm_i915_gem_object *obj = vma->obj;
4233         bool mappable, fenceable;
4234         u32 fence_size, fence_alignment;
4235
4236         fence_size = i915_gem_get_gtt_size(obj->base.dev,
4237                                            obj->base.size,
4238                                            obj->tiling_mode);
4239         fence_alignment = i915_gem_get_gtt_alignment(obj->base.dev,
4240                                                      obj->base.size,
4241                                                      obj->tiling_mode,
4242                                                      true);
4243
4244         fenceable = (vma->node.size == fence_size &&
4245                      (vma->node.start & (fence_alignment - 1)) == 0);
4246
4247         mappable = (vma->node.start + fence_size <=
4248                     to_i915(obj->base.dev)->ggtt.mappable_end);
4249
4250         obj->map_and_fenceable = mappable && fenceable;
4251 }
4252
4253 static int
4254 i915_gem_object_do_pin(struct drm_i915_gem_object *obj,
4255                        struct i915_address_space *vm,
4256                        const struct i915_ggtt_view *ggtt_view,
4257                        uint32_t alignment,
4258                        uint64_t flags)
4259 {
4260         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
4261         struct i915_vma *vma;
4262         unsigned bound;
4263         int ret;
4264
4265         if (WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base))
4266                 return -ENODEV;
4267
4268         if (WARN_ON(flags & (PIN_GLOBAL | PIN_MAPPABLE) && !i915_is_ggtt(vm)))
4269                 return -EINVAL;
4270
4271         if (WARN_ON((flags & (PIN_MAPPABLE | PIN_GLOBAL)) == PIN_MAPPABLE))
4272                 return -EINVAL;
4273
4274         if (WARN_ON(i915_is_ggtt(vm) != !!ggtt_view))
4275                 return -EINVAL;
4276
4277         vma = ggtt_view ? i915_gem_obj_to_ggtt_view(obj, ggtt_view) :
4278                           i915_gem_obj_to_vma(obj, vm);
4279
4280         if (vma) {
4281                 if (WARN_ON(vma->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
4282                         return -EBUSY;
4283
4284                 if (i915_vma_misplaced(vma, alignment, flags)) {
4285                         WARN(vma->pin_count,
4286                              "bo is already pinned in %s with incorrect alignment:"
4287                              " offset=%08x %08x, req.alignment=%x, req.map_and_fenceable=%d,"
4288                              " obj->map_and_fenceable=%d\n",
4289                              ggtt_view ? "ggtt" : "ppgtt",
4290                              upper_32_bits(vma->node.start),
4291                              lower_32_bits(vma->node.start),
4292                              alignment,
4293                              !!(flags & PIN_MAPPABLE),
4294                              obj->map_and_fenceable);
4295                         ret = i915_vma_unbind(vma);
4296                         if (ret)
4297                                 return ret;
4298
4299                         vma = NULL;
4300                 }
4301         }
4302
4303         bound = vma ? vma->bound : 0;
4304         if (vma == NULL || !drm_mm_node_allocated(&vma->node)) {
4305                 vma = i915_gem_object_bind_to_vm(obj, vm, ggtt_view, alignment,
4306                                                  flags);
4307                 if (IS_ERR(vma))
4308                         return PTR_ERR(vma);
4309         } else {
4310                 ret = i915_vma_bind(vma, obj->cache_level, flags);
4311                 if (ret)
4312                         return ret;
4313         }
4314
4315         if (ggtt_view && ggtt_view->type == I915_GGTT_VIEW_NORMAL &&
4316             (bound ^ vma->bound) & GLOBAL_BIND) {
4317                 __i915_vma_set_map_and_fenceable(vma);
4318                 WARN_ON(flags & PIN_MAPPABLE && !obj->map_and_fenceable);
4319         }
4320
4321         vma->pin_count++;
4322         return 0;
4323 }
4324
4325 int
4326 i915_gem_object_pin(struct drm_i915_gem_object *obj,
4327                     struct i915_address_space *vm,
4328                     uint32_t alignment,
4329                     uint64_t flags)
4330 {
4331         return i915_gem_object_do_pin(obj, vm,
4332                                       i915_is_ggtt(vm) ? &i915_ggtt_view_normal : NULL,
4333                                       alignment, flags);
4334 }
4335
4336 int
4337 i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
4338                          const struct i915_ggtt_view *view,
4339                          uint32_t alignment,
4340                          uint64_t flags)
4341 {
4342         struct drm_device *dev = obj->base.dev;
4343         struct drm_i915_private *dev_priv = to_i915(dev);
4344         struct i915_ggtt *ggtt = &dev_priv->ggtt;
4345
4346         BUG_ON(!view);
4347
4348         return i915_gem_object_do_pin(obj, &ggtt->base, view,
4349                                       alignment, flags | PIN_GLOBAL);
4350 }
4351
4352 void
4353 i915_gem_object_ggtt_unpin_view(struct drm_i915_gem_object *obj,
4354                                 const struct i915_ggtt_view *view)
4355 {
4356         struct i915_vma *vma = i915_gem_obj_to_ggtt_view(obj, view);
4357
4358         WARN_ON(vma->pin_count == 0);
4359         WARN_ON(!i915_gem_obj_ggtt_bound_view(obj, view));
4360
4361         --vma->pin_count;
4362 }
4363
4364 int
4365 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4366                     struct drm_file *file)
4367 {
4368         struct drm_i915_gem_busy *args = data;
4369         struct drm_i915_gem_object *obj;
4370         int ret;
4371
4372         ret = i915_mutex_lock_interruptible(dev);
4373         if (ret)
4374                 return ret;
4375
4376         obj = to_intel_bo(drm_gem_object_lookup(file, args->handle));
4377         if (&obj->base == NULL) {
4378                 ret = -ENOENT;
4379                 goto unlock;
4380         }
4381
4382         /* Count all active objects as busy, even if they are currently not used
4383          * by the gpu. Users of this interface expect objects to eventually
4384          * become non-busy without any further actions, therefore emit any
4385          * necessary flushes here.
4386          */
4387         ret = i915_gem_object_flush_active(obj);
4388         if (ret)
4389                 goto unref;
4390
4391         args->busy = 0;
4392         if (obj->active) {
4393                 int i;
4394
4395                 for (i = 0; i < I915_NUM_ENGINES; i++) {
4396                         struct drm_i915_gem_request *req;
4397
4398                         req = obj->last_read_req[i];
4399                         if (req)
4400                                 args->busy |= 1 << (16 + req->engine->exec_id);
4401                 }
4402                 if (obj->last_write_req)
4403                         args->busy |= obj->last_write_req->engine->exec_id;
4404         }
4405
4406 unref:
4407         drm_gem_object_unreference(&obj->base);
4408 unlock:
4409         mutex_unlock(&dev->struct_mutex);
4410         return ret;
4411 }
4412
4413 int
4414 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4415                         struct drm_file *file_priv)
4416 {
4417         return i915_gem_ring_throttle(dev, file_priv);
4418 }
4419
4420 int
4421 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4422                        struct drm_file *file_priv)
4423 {
4424         struct drm_i915_private *dev_priv = dev->dev_private;
4425         struct drm_i915_gem_madvise *args = data;
4426         struct drm_i915_gem_object *obj;
4427         int ret;
4428
4429         switch (args->madv) {
4430         case I915_MADV_DONTNEED:
4431         case I915_MADV_WILLNEED:
4432             break;
4433         default:
4434             return -EINVAL;
4435         }
4436
4437         ret = i915_mutex_lock_interruptible(dev);
4438         if (ret)
4439                 return ret;
4440
4441         obj = to_intel_bo(drm_gem_object_lookup(file_priv, args->handle));
4442         if (&obj->base == NULL) {
4443                 ret = -ENOENT;
4444                 goto unlock;
4445         }
4446
4447         if (i915_gem_obj_is_pinned(obj)) {
4448                 ret = -EINVAL;
4449                 goto out;
4450         }
4451
4452         if (obj->pages &&
4453             obj->tiling_mode != I915_TILING_NONE &&
4454             dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
4455                 if (obj->madv == I915_MADV_WILLNEED)
4456                         i915_gem_object_unpin_pages(obj);
4457                 if (args->madv == I915_MADV_WILLNEED)
4458                         i915_gem_object_pin_pages(obj);
4459         }
4460
4461         if (obj->madv != __I915_MADV_PURGED)
4462                 obj->madv = args->madv;
4463
4464         /* if the object is no longer attached, discard its backing storage */
4465         if (obj->madv == I915_MADV_DONTNEED && obj->pages == NULL)
4466                 i915_gem_object_truncate(obj);
4467
4468         args->retained = obj->madv != __I915_MADV_PURGED;
4469
4470 out:
4471         drm_gem_object_unreference(&obj->base);
4472 unlock:
4473         mutex_unlock(&dev->struct_mutex);
4474         return ret;
4475 }
4476
4477 void i915_gem_object_init(struct drm_i915_gem_object *obj,
4478                           const struct drm_i915_gem_object_ops *ops)
4479 {
4480         int i;
4481
4482         INIT_LIST_HEAD(&obj->global_list);
4483         for (i = 0; i < I915_NUM_ENGINES; i++)
4484                 INIT_LIST_HEAD(&obj->engine_list[i]);
4485         INIT_LIST_HEAD(&obj->obj_exec_link);
4486         INIT_LIST_HEAD(&obj->vma_list);
4487         INIT_LIST_HEAD(&obj->batch_pool_link);
4488
4489         obj->ops = ops;
4490
4491         obj->fence_reg = I915_FENCE_REG_NONE;
4492         obj->madv = I915_MADV_WILLNEED;
4493
4494         i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
4495 }
4496
4497 static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
4498         .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE,
4499         .get_pages = i915_gem_object_get_pages_gtt,
4500         .put_pages = i915_gem_object_put_pages_gtt,
4501 };
4502
4503 struct drm_i915_gem_object *i915_gem_object_create(struct drm_device *dev,
4504                                                   size_t size)
4505 {
4506         struct drm_i915_gem_object *obj;
4507         struct address_space *mapping;
4508         gfp_t mask;
4509         int ret;
4510
4511         obj = i915_gem_object_alloc(dev);
4512         if (obj == NULL)
4513                 return ERR_PTR(-ENOMEM);
4514
4515         ret = drm_gem_object_init(dev, &obj->base, size);
4516         if (ret)
4517                 goto fail;
4518
4519         mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
4520         if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) {
4521                 /* 965gm cannot relocate objects above 4GiB. */
4522                 mask &= ~__GFP_HIGHMEM;
4523                 mask |= __GFP_DMA32;
4524         }
4525
4526         mapping = file_inode(obj->base.filp)->i_mapping;
4527         mapping_set_gfp_mask(mapping, mask);
4528
4529         i915_gem_object_init(obj, &i915_gem_object_ops);
4530
4531         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4532         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4533
4534         if (HAS_LLC(dev)) {
4535                 /* On some devices, we can have the GPU use the LLC (the CPU
4536                  * cache) for about a 10% performance improvement
4537                  * compared to uncached.  Graphics requests other than
4538                  * display scanout are coherent with the CPU in
4539                  * accessing this cache.  This means in this mode we
4540                  * don't need to clflush on the CPU side, and on the
4541                  * GPU side we only need to flush internal caches to
4542                  * get data visible to the CPU.
4543                  *
4544                  * However, we maintain the display planes as UC, and so
4545                  * need to rebind when first used as such.
4546                  */
4547                 obj->cache_level = I915_CACHE_LLC;
4548         } else
4549                 obj->cache_level = I915_CACHE_NONE;
4550
4551         trace_i915_gem_object_create(obj);
4552
4553         return obj;
4554
4555 fail:
4556         i915_gem_object_free(obj);
4557
4558         return ERR_PTR(ret);
4559 }
4560
4561 static bool discard_backing_storage(struct drm_i915_gem_object *obj)
4562 {
4563         /* If we are the last user of the backing storage (be it shmemfs
4564          * pages or stolen etc), we know that the pages are going to be
4565          * immediately released. In this case, we can then skip copying
4566          * back the contents from the GPU.
4567          */
4568
4569         if (obj->madv != I915_MADV_WILLNEED)
4570                 return false;
4571
4572         if (obj->base.filp == NULL)
4573                 return true;
4574
4575         /* At first glance, this looks racy, but then again so would be
4576          * userspace racing mmap against close. However, the first external
4577          * reference to the filp can only be obtained through the
4578          * i915_gem_mmap_ioctl() which safeguards us against the user
4579          * acquiring such a reference whilst we are in the middle of
4580          * freeing the object.
4581          */
4582         return atomic_long_read(&obj->base.filp->f_count) == 1;
4583 }
4584
4585 void i915_gem_free_object(struct drm_gem_object *gem_obj)
4586 {
4587         struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
4588         struct drm_device *dev = obj->base.dev;
4589         struct drm_i915_private *dev_priv = dev->dev_private;
4590         struct i915_vma *vma, *next;
4591
4592         intel_runtime_pm_get(dev_priv);
4593
4594         trace_i915_gem_object_destroy(obj);
4595
4596         list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link) {
4597                 int ret;
4598
4599                 vma->pin_count = 0;
4600                 ret = i915_vma_unbind(vma);
4601                 if (WARN_ON(ret == -ERESTARTSYS)) {
4602                         bool was_interruptible;
4603
4604                         was_interruptible = dev_priv->mm.interruptible;
4605                         dev_priv->mm.interruptible = false;
4606
4607                         WARN_ON(i915_vma_unbind(vma));
4608
4609                         dev_priv->mm.interruptible = was_interruptible;
4610                 }
4611         }
4612
4613         /* Stolen objects don't hold a ref, but do hold pin count. Fix that up
4614          * before progressing. */
4615         if (obj->stolen)
4616                 i915_gem_object_unpin_pages(obj);
4617
4618         WARN_ON(obj->frontbuffer_bits);
4619
4620         if (obj->pages && obj->madv == I915_MADV_WILLNEED &&
4621             dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES &&
4622             obj->tiling_mode != I915_TILING_NONE)
4623                 i915_gem_object_unpin_pages(obj);
4624
4625         if (WARN_ON(obj->pages_pin_count))
4626                 obj->pages_pin_count = 0;
4627         if (discard_backing_storage(obj))
4628                 obj->madv = I915_MADV_DONTNEED;
4629         i915_gem_object_put_pages(obj);
4630         i915_gem_object_free_mmap_offset(obj);
4631
4632         BUG_ON(obj->pages);
4633
4634         if (obj->base.import_attach)
4635                 drm_prime_gem_destroy(&obj->base, NULL);
4636
4637         if (obj->ops->release)
4638                 obj->ops->release(obj);
4639
4640         drm_gem_object_release(&obj->base);
4641         i915_gem_info_remove_obj(dev_priv, obj->base.size);
4642
4643         kfree(obj->bit_17);
4644         i915_gem_object_free(obj);
4645
4646         intel_runtime_pm_put(dev_priv);
4647 }
4648
4649 struct i915_vma *i915_gem_obj_to_vma(struct drm_i915_gem_object *obj,
4650                                      struct i915_address_space *vm)
4651 {
4652         struct i915_vma *vma;
4653         list_for_each_entry(vma, &obj->vma_list, obj_link) {
4654                 if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL &&
4655                     vma->vm == vm)
4656                         return vma;
4657         }
4658         return NULL;
4659 }
4660
4661 struct i915_vma *i915_gem_obj_to_ggtt_view(struct drm_i915_gem_object *obj,
4662                                            const struct i915_ggtt_view *view)
4663 {
4664         struct i915_vma *vma;
4665
4666         GEM_BUG_ON(!view);
4667
4668         list_for_each_entry(vma, &obj->vma_list, obj_link)
4669                 if (vma->is_ggtt && i915_ggtt_view_equal(&vma->ggtt_view, view))
4670                         return vma;
4671         return NULL;
4672 }
4673
4674 void i915_gem_vma_destroy(struct i915_vma *vma)
4675 {
4676         WARN_ON(vma->node.allocated);
4677
4678         /* Keep the vma as a placeholder in the execbuffer reservation lists */
4679         if (!list_empty(&vma->exec_list))
4680                 return;
4681
4682         if (!vma->is_ggtt)
4683                 i915_ppgtt_put(i915_vm_to_ppgtt(vma->vm));
4684
4685         list_del(&vma->obj_link);
4686
4687         kmem_cache_free(to_i915(vma->obj->base.dev)->vmas, vma);
4688 }
4689
4690 static void
4691 i915_gem_stop_engines(struct drm_device *dev)
4692 {
4693         struct drm_i915_private *dev_priv = dev->dev_private;
4694         struct intel_engine_cs *engine;
4695
4696         for_each_engine(engine, dev_priv)
4697                 dev_priv->gt.stop_engine(engine);
4698 }
4699
4700 int
4701 i915_gem_suspend(struct drm_device *dev)
4702 {
4703         struct drm_i915_private *dev_priv = dev->dev_private;
4704         int ret = 0;
4705
4706         mutex_lock(&dev->struct_mutex);
4707         ret = i915_gpu_idle(dev);
4708         if (ret)
4709                 goto err;
4710
4711         i915_gem_retire_requests(dev_priv);
4712
4713         i915_gem_stop_engines(dev);
4714         i915_gem_context_lost(dev_priv);
4715         mutex_unlock(&dev->struct_mutex);
4716
4717         cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
4718         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4719         flush_delayed_work(&dev_priv->mm.idle_work);
4720
4721         /* Assert that we sucessfully flushed all the work and
4722          * reset the GPU back to its idle, low power state.
4723          */
4724         WARN_ON(dev_priv->mm.busy);
4725
4726         return 0;
4727
4728 err:
4729         mutex_unlock(&dev->struct_mutex);
4730         return ret;
4731 }
4732
4733 void i915_gem_init_swizzling(struct drm_device *dev)
4734 {
4735         struct drm_i915_private *dev_priv = dev->dev_private;
4736
4737         if (INTEL_INFO(dev)->gen < 5 ||
4738             dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
4739                 return;
4740
4741         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
4742                                  DISP_TILE_SURFACE_SWIZZLING);
4743
4744         if (IS_GEN5(dev))
4745                 return;
4746
4747         I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
4748         if (IS_GEN6(dev))
4749                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
4750         else if (IS_GEN7(dev))
4751                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
4752         else if (IS_GEN8(dev))
4753                 I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW));
4754         else
4755                 BUG();
4756 }
4757
4758 static void init_unused_ring(struct drm_device *dev, u32 base)
4759 {
4760         struct drm_i915_private *dev_priv = dev->dev_private;
4761
4762         I915_WRITE(RING_CTL(base), 0);
4763         I915_WRITE(RING_HEAD(base), 0);
4764         I915_WRITE(RING_TAIL(base), 0);
4765         I915_WRITE(RING_START(base), 0);
4766 }
4767
4768 static void init_unused_rings(struct drm_device *dev)
4769 {
4770         if (IS_I830(dev)) {
4771                 init_unused_ring(dev, PRB1_BASE);
4772                 init_unused_ring(dev, SRB0_BASE);
4773                 init_unused_ring(dev, SRB1_BASE);
4774                 init_unused_ring(dev, SRB2_BASE);
4775                 init_unused_ring(dev, SRB3_BASE);
4776         } else if (IS_GEN2(dev)) {
4777                 init_unused_ring(dev, SRB0_BASE);
4778                 init_unused_ring(dev, SRB1_BASE);
4779         } else if (IS_GEN3(dev)) {
4780                 init_unused_ring(dev, PRB1_BASE);
4781                 init_unused_ring(dev, PRB2_BASE);
4782         }
4783 }
4784
4785 int i915_gem_init_engines(struct drm_device *dev)
4786 {
4787         struct drm_i915_private *dev_priv = dev->dev_private;
4788         int ret;
4789
4790         ret = intel_init_render_ring_buffer(dev);
4791         if (ret)
4792                 return ret;
4793
4794         if (HAS_BSD(dev)) {
4795                 ret = intel_init_bsd_ring_buffer(dev);
4796                 if (ret)
4797                         goto cleanup_render_ring;
4798         }
4799
4800         if (HAS_BLT(dev)) {
4801                 ret = intel_init_blt_ring_buffer(dev);
4802                 if (ret)
4803                         goto cleanup_bsd_ring;
4804         }
4805
4806         if (HAS_VEBOX(dev)) {
4807                 ret = intel_init_vebox_ring_buffer(dev);
4808                 if (ret)
4809                         goto cleanup_blt_ring;
4810         }
4811
4812         if (HAS_BSD2(dev)) {
4813                 ret = intel_init_bsd2_ring_buffer(dev);
4814                 if (ret)
4815                         goto cleanup_vebox_ring;
4816         }
4817
4818         return 0;
4819
4820 cleanup_vebox_ring:
4821         intel_cleanup_engine(&dev_priv->engine[VECS]);
4822 cleanup_blt_ring:
4823         intel_cleanup_engine(&dev_priv->engine[BCS]);
4824 cleanup_bsd_ring:
4825         intel_cleanup_engine(&dev_priv->engine[VCS]);
4826 cleanup_render_ring:
4827         intel_cleanup_engine(&dev_priv->engine[RCS]);
4828
4829         return ret;
4830 }
4831
4832 int
4833 i915_gem_init_hw(struct drm_device *dev)
4834 {
4835         struct drm_i915_private *dev_priv = dev->dev_private;
4836         struct intel_engine_cs *engine;
4837         int ret;
4838
4839         /* Double layer security blanket, see i915_gem_init() */
4840         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4841
4842         if (HAS_EDRAM(dev) && INTEL_GEN(dev_priv) < 9)
4843                 I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf));
4844
4845         if (IS_HASWELL(dev))
4846                 I915_WRITE(MI_PREDICATE_RESULT_2, IS_HSW_GT3(dev) ?
4847                            LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
4848
4849         if (HAS_PCH_NOP(dev)) {
4850                 if (IS_IVYBRIDGE(dev)) {
4851                         u32 temp = I915_READ(GEN7_MSG_CTL);
4852                         temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK);
4853                         I915_WRITE(GEN7_MSG_CTL, temp);
4854                 } else if (INTEL_INFO(dev)->gen >= 7) {
4855                         u32 temp = I915_READ(HSW_NDE_RSTWRN_OPT);
4856                         temp &= ~RESET_PCH_HANDSHAKE_ENABLE;
4857                         I915_WRITE(HSW_NDE_RSTWRN_OPT, temp);
4858                 }
4859         }
4860
4861         i915_gem_init_swizzling(dev);
4862
4863         /*
4864          * At least 830 can leave some of the unused rings
4865          * "active" (ie. head != tail) after resume which
4866          * will prevent c3 entry. Makes sure all unused rings
4867          * are totally idle.
4868          */
4869         init_unused_rings(dev);
4870
4871         BUG_ON(!dev_priv->kernel_context);
4872
4873         ret = i915_ppgtt_init_hw(dev);
4874         if (ret) {
4875                 DRM_ERROR("PPGTT enable HW failed %d\n", ret);
4876                 goto out;
4877         }
4878
4879         /* Need to do basic initialisation of all rings first: */
4880         for_each_engine(engine, dev_priv) {
4881                 ret = engine->init_hw(engine);
4882                 if (ret)
4883                         goto out;
4884         }
4885
4886         intel_mocs_init_l3cc_table(dev);
4887
4888         /* We can't enable contexts until all firmware is loaded */
4889         if (HAS_GUC_UCODE(dev)) {
4890                 ret = intel_guc_ucode_load(dev);
4891                 if (ret) {
4892                         DRM_ERROR("Failed to initialize GuC, error %d\n", ret);
4893                         ret = -EIO;
4894                         goto out;
4895                 }
4896         }
4897
4898         /*
4899          * Increment the next seqno by 0x100 so we have a visible break
4900          * on re-initialisation
4901          */
4902         ret = i915_gem_set_seqno(dev, dev_priv->next_seqno+0x100);
4903
4904 out:
4905         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4906         return ret;
4907 }
4908
4909 int i915_gem_init(struct drm_device *dev)
4910 {
4911         struct drm_i915_private *dev_priv = dev->dev_private;
4912         int ret;
4913
4914         mutex_lock(&dev->struct_mutex);
4915
4916         if (!i915.enable_execlists) {
4917                 dev_priv->gt.execbuf_submit = i915_gem_ringbuffer_submission;
4918                 dev_priv->gt.init_engines = i915_gem_init_engines;
4919                 dev_priv->gt.cleanup_engine = intel_cleanup_engine;
4920                 dev_priv->gt.stop_engine = intel_stop_engine;
4921         } else {
4922                 dev_priv->gt.execbuf_submit = intel_execlists_submission;
4923                 dev_priv->gt.init_engines = intel_logical_rings_init;
4924                 dev_priv->gt.cleanup_engine = intel_logical_ring_cleanup;
4925                 dev_priv->gt.stop_engine = intel_logical_ring_stop;
4926         }
4927
4928         /* This is just a security blanket to placate dragons.
4929          * On some systems, we very sporadically observe that the first TLBs
4930          * used by the CS may be stale, despite us poking the TLB reset. If
4931          * we hold the forcewake during initialisation these problems
4932          * just magically go away.
4933          */
4934         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4935
4936         i915_gem_init_userptr(dev_priv);
4937         i915_gem_init_ggtt(dev);
4938
4939         ret = i915_gem_context_init(dev);
4940         if (ret)
4941                 goto out_unlock;
4942
4943         ret = dev_priv->gt.init_engines(dev);
4944         if (ret)
4945                 goto out_unlock;
4946
4947         ret = i915_gem_init_hw(dev);
4948         if (ret == -EIO) {
4949                 /* Allow ring initialisation to fail by marking the GPU as
4950                  * wedged. But we only want to do this where the GPU is angry,
4951                  * for all other failure, such as an allocation failure, bail.
4952                  */
4953                 DRM_ERROR("Failed to initialize GPU, declaring it wedged\n");
4954                 atomic_or(I915_WEDGED, &dev_priv->gpu_error.reset_counter);
4955                 ret = 0;
4956         }
4957
4958 out_unlock:
4959         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4960         mutex_unlock(&dev->struct_mutex);
4961
4962         return ret;
4963 }
4964
4965 void
4966 i915_gem_cleanup_engines(struct drm_device *dev)
4967 {
4968         struct drm_i915_private *dev_priv = dev->dev_private;
4969         struct intel_engine_cs *engine;
4970
4971         for_each_engine(engine, dev_priv)
4972                 dev_priv->gt.cleanup_engine(engine);
4973 }
4974
4975 static void
4976 init_engine_lists(struct intel_engine_cs *engine)
4977 {
4978         INIT_LIST_HEAD(&engine->active_list);
4979         INIT_LIST_HEAD(&engine->request_list);
4980 }
4981
4982 void
4983 i915_gem_load_init_fences(struct drm_i915_private *dev_priv)
4984 {
4985         struct drm_device *dev = dev_priv->dev;
4986
4987         if (INTEL_INFO(dev_priv)->gen >= 7 && !IS_VALLEYVIEW(dev_priv) &&
4988             !IS_CHERRYVIEW(dev_priv))
4989                 dev_priv->num_fence_regs = 32;
4990         else if (INTEL_INFO(dev_priv)->gen >= 4 || IS_I945G(dev_priv) ||
4991                  IS_I945GM(dev_priv) || IS_G33(dev_priv))
4992                 dev_priv->num_fence_regs = 16;
4993         else
4994                 dev_priv->num_fence_regs = 8;
4995
4996         if (intel_vgpu_active(dev_priv))
4997                 dev_priv->num_fence_regs =
4998                                 I915_READ(vgtif_reg(avail_rs.fence_num));
4999
5000         /* Initialize fence registers to zero */
5001         i915_gem_restore_fences(dev);
5002
5003         i915_gem_detect_bit_6_swizzle(dev);
5004 }
5005
5006 void
5007 i915_gem_load_init(struct drm_device *dev)
5008 {
5009         struct drm_i915_private *dev_priv = dev->dev_private;
5010         int i;
5011
5012         dev_priv->objects =
5013                 kmem_cache_create("i915_gem_object",
5014                                   sizeof(struct drm_i915_gem_object), 0,
5015                                   SLAB_HWCACHE_ALIGN,
5016                                   NULL);
5017         dev_priv->vmas =
5018                 kmem_cache_create("i915_gem_vma",
5019                                   sizeof(struct i915_vma), 0,
5020                                   SLAB_HWCACHE_ALIGN,
5021                                   NULL);
5022         dev_priv->requests =
5023                 kmem_cache_create("i915_gem_request",
5024                                   sizeof(struct drm_i915_gem_request), 0,
5025                                   SLAB_HWCACHE_ALIGN,
5026                                   NULL);
5027
5028         INIT_LIST_HEAD(&dev_priv->vm_list);
5029         INIT_LIST_HEAD(&dev_priv->context_list);
5030         INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
5031         INIT_LIST_HEAD(&dev_priv->mm.bound_list);
5032         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
5033         for (i = 0; i < I915_NUM_ENGINES; i++)
5034                 init_engine_lists(&dev_priv->engine[i]);
5035         for (i = 0; i < I915_MAX_NUM_FENCES; i++)
5036                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
5037         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
5038                           i915_gem_retire_work_handler);
5039         INIT_DELAYED_WORK(&dev_priv->mm.idle_work,
5040                           i915_gem_idle_work_handler);
5041         init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
5042
5043         dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
5044
5045         /*
5046          * Set initial sequence number for requests.
5047          * Using this number allows the wraparound to happen early,
5048          * catching any obvious problems.
5049          */
5050         dev_priv->next_seqno = ((u32)~0 - 0x1100);
5051         dev_priv->last_seqno = ((u32)~0 - 0x1101);
5052
5053         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
5054
5055         init_waitqueue_head(&dev_priv->pending_flip_queue);
5056
5057         dev_priv->mm.interruptible = true;
5058
5059         mutex_init(&dev_priv->fb_tracking.lock);
5060 }
5061
5062 void i915_gem_load_cleanup(struct drm_device *dev)
5063 {
5064         struct drm_i915_private *dev_priv = to_i915(dev);
5065
5066         kmem_cache_destroy(dev_priv->requests);
5067         kmem_cache_destroy(dev_priv->vmas);
5068         kmem_cache_destroy(dev_priv->objects);
5069 }
5070
5071 int i915_gem_freeze_late(struct drm_i915_private *dev_priv)
5072 {
5073         struct drm_i915_gem_object *obj;
5074
5075         /* Called just before we write the hibernation image.
5076          *
5077          * We need to update the domain tracking to reflect that the CPU
5078          * will be accessing all the pages to create and restore from the
5079          * hibernation, and so upon restoration those pages will be in the
5080          * CPU domain.
5081          *
5082          * To make sure the hibernation image contains the latest state,
5083          * we update that state just before writing out the image.
5084          */
5085
5086         list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) {
5087                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
5088                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
5089         }
5090
5091         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
5092                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
5093                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
5094         }
5095
5096         return 0;
5097 }
5098
5099 void i915_gem_release(struct drm_device *dev, struct drm_file *file)
5100 {
5101         struct drm_i915_file_private *file_priv = file->driver_priv;
5102
5103         /* Clean up our request list when the client is going away, so that
5104          * later retire_requests won't dereference our soon-to-be-gone
5105          * file_priv.
5106          */
5107         spin_lock(&file_priv->mm.lock);
5108         while (!list_empty(&file_priv->mm.request_list)) {
5109                 struct drm_i915_gem_request *request;
5110
5111                 request = list_first_entry(&file_priv->mm.request_list,
5112                                            struct drm_i915_gem_request,
5113                                            client_list);
5114                 list_del(&request->client_list);
5115                 request->file_priv = NULL;
5116         }
5117         spin_unlock(&file_priv->mm.lock);
5118
5119         if (!list_empty(&file_priv->rps.link)) {
5120                 spin_lock(&to_i915(dev)->rps.client_lock);
5121                 list_del(&file_priv->rps.link);
5122                 spin_unlock(&to_i915(dev)->rps.client_lock);
5123         }
5124 }
5125
5126 int i915_gem_open(struct drm_device *dev, struct drm_file *file)
5127 {
5128         struct drm_i915_file_private *file_priv;
5129         int ret;
5130
5131         DRM_DEBUG_DRIVER("\n");
5132
5133         file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
5134         if (!file_priv)
5135                 return -ENOMEM;
5136
5137         file->driver_priv = file_priv;
5138         file_priv->dev_priv = dev->dev_private;
5139         file_priv->file = file;
5140         INIT_LIST_HEAD(&file_priv->rps.link);
5141
5142         spin_lock_init(&file_priv->mm.lock);
5143         INIT_LIST_HEAD(&file_priv->mm.request_list);
5144
5145         file_priv->bsd_ring = -1;
5146
5147         ret = i915_gem_context_open(dev, file);
5148         if (ret)
5149                 kfree(file_priv);
5150
5151         return ret;
5152 }
5153
5154 /**
5155  * i915_gem_track_fb - update frontbuffer tracking
5156  * @old: current GEM buffer for the frontbuffer slots
5157  * @new: new GEM buffer for the frontbuffer slots
5158  * @frontbuffer_bits: bitmask of frontbuffer slots
5159  *
5160  * This updates the frontbuffer tracking bits @frontbuffer_bits by clearing them
5161  * from @old and setting them in @new. Both @old and @new can be NULL.
5162  */
5163 void i915_gem_track_fb(struct drm_i915_gem_object *old,
5164                        struct drm_i915_gem_object *new,
5165                        unsigned frontbuffer_bits)
5166 {
5167         if (old) {
5168                 WARN_ON(!mutex_is_locked(&old->base.dev->struct_mutex));
5169                 WARN_ON(!(old->frontbuffer_bits & frontbuffer_bits));
5170                 old->frontbuffer_bits &= ~frontbuffer_bits;
5171         }
5172
5173         if (new) {
5174                 WARN_ON(!mutex_is_locked(&new->base.dev->struct_mutex));
5175                 WARN_ON(new->frontbuffer_bits & frontbuffer_bits);
5176                 new->frontbuffer_bits |= frontbuffer_bits;
5177         }
5178 }
5179
5180 /* All the new VM stuff */
5181 u64 i915_gem_obj_offset(struct drm_i915_gem_object *o,
5182                         struct i915_address_space *vm)
5183 {
5184         struct drm_i915_private *dev_priv = o->base.dev->dev_private;
5185         struct i915_vma *vma;
5186
5187         WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
5188
5189         list_for_each_entry(vma, &o->vma_list, obj_link) {
5190                 if (vma->is_ggtt &&
5191                     vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
5192                         continue;
5193                 if (vma->vm == vm)
5194                         return vma->node.start;
5195         }
5196
5197         WARN(1, "%s vma for this object not found.\n",
5198              i915_is_ggtt(vm) ? "global" : "ppgtt");
5199         return -1;
5200 }
5201
5202 u64 i915_gem_obj_ggtt_offset_view(struct drm_i915_gem_object *o,
5203                                   const struct i915_ggtt_view *view)
5204 {
5205         struct i915_vma *vma;
5206
5207         list_for_each_entry(vma, &o->vma_list, obj_link)
5208                 if (vma->is_ggtt && i915_ggtt_view_equal(&vma->ggtt_view, view))
5209                         return vma->node.start;
5210
5211         WARN(1, "global vma for this object not found. (view=%u)\n", view->type);
5212         return -1;
5213 }
5214
5215 bool i915_gem_obj_bound(struct drm_i915_gem_object *o,
5216                         struct i915_address_space *vm)
5217 {
5218         struct i915_vma *vma;
5219
5220         list_for_each_entry(vma, &o->vma_list, obj_link) {
5221                 if (vma->is_ggtt &&
5222                     vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
5223                         continue;
5224                 if (vma->vm == vm && drm_mm_node_allocated(&vma->node))
5225                         return true;
5226         }
5227
5228         return false;
5229 }
5230
5231 bool i915_gem_obj_ggtt_bound_view(struct drm_i915_gem_object *o,
5232                                   const struct i915_ggtt_view *view)
5233 {
5234         struct i915_vma *vma;
5235
5236         list_for_each_entry(vma, &o->vma_list, obj_link)
5237                 if (vma->is_ggtt &&
5238                     i915_ggtt_view_equal(&vma->ggtt_view, view) &&
5239                     drm_mm_node_allocated(&vma->node))
5240                         return true;
5241
5242         return false;
5243 }
5244
5245 bool i915_gem_obj_bound_any(struct drm_i915_gem_object *o)
5246 {
5247         struct i915_vma *vma;
5248
5249         list_for_each_entry(vma, &o->vma_list, obj_link)
5250                 if (drm_mm_node_allocated(&vma->node))
5251                         return true;
5252
5253         return false;
5254 }
5255
5256 unsigned long i915_gem_obj_ggtt_size(struct drm_i915_gem_object *o)
5257 {
5258         struct i915_vma *vma;
5259
5260         GEM_BUG_ON(list_empty(&o->vma_list));
5261
5262         list_for_each_entry(vma, &o->vma_list, obj_link) {
5263                 if (vma->is_ggtt &&
5264                     vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL)
5265                         return vma->node.size;
5266         }
5267
5268         return 0;
5269 }
5270
5271 bool i915_gem_obj_is_pinned(struct drm_i915_gem_object *obj)
5272 {
5273         struct i915_vma *vma;
5274         list_for_each_entry(vma, &obj->vma_list, obj_link)
5275                 if (vma->pin_count > 0)
5276                         return true;
5277
5278         return false;
5279 }
5280
5281 /* Like i915_gem_object_get_page(), but mark the returned page dirty */
5282 struct page *
5283 i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, int n)
5284 {
5285         struct page *page;
5286
5287         /* Only default objects have per-page dirty tracking */
5288         if (WARN_ON((obj->ops->flags & I915_GEM_OBJECT_HAS_STRUCT_PAGE) == 0))
5289                 return NULL;
5290
5291         page = i915_gem_object_get_page(obj, n);
5292         set_page_dirty(page);
5293         return page;
5294 }
5295
5296 /* Allocate a new GEM object and fill it with the supplied data */
5297 struct drm_i915_gem_object *
5298 i915_gem_object_create_from_data(struct drm_device *dev,
5299                                  const void *data, size_t size)
5300 {
5301         struct drm_i915_gem_object *obj;
5302         struct sg_table *sg;
5303         size_t bytes;
5304         int ret;
5305
5306         obj = i915_gem_object_create(dev, round_up(size, PAGE_SIZE));
5307         if (IS_ERR(obj))
5308                 return obj;
5309
5310         ret = i915_gem_object_set_to_cpu_domain(obj, true);
5311         if (ret)
5312                 goto fail;
5313
5314         ret = i915_gem_object_get_pages(obj);
5315         if (ret)
5316                 goto fail;
5317
5318         i915_gem_object_pin_pages(obj);
5319         sg = obj->pages;
5320         bytes = sg_copy_from_buffer(sg->sgl, sg->nents, (void *)data, size);
5321         obj->dirty = 1;         /* Backing store is now out of date */
5322         i915_gem_object_unpin_pages(obj);
5323
5324         if (WARN_ON(bytes != size)) {
5325                 DRM_ERROR("Incomplete copy, wrote %zu of %zu", bytes, size);
5326                 ret = -EFAULT;
5327                 goto fail;
5328         }
5329
5330         return obj;
5331
5332 fail:
5333         drm_gem_object_unreference(&obj->base);
5334         return ERR_PTR(ret);
5335 }