drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  */
  27
  28 #include "drmP.h"
  29 #include "drm.h"
  30 #include "i915_drm.h"
  31 #include "i915_drv.h"
  32 #include "i915_trace.h"
  33 #include "intel_drv.h"
  34 #include <linux/slab.h>
  35 #include <linux/swap.h>
  36 #include <linux/pci.h>
  37 #include <linux/intel-gtt.h>
  38
  39 static uint32_t i915_gem_get_gtt_alignment(struct drm_gem_object *obj);
  40
  41 static int i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj,
  42                                                   bool pipelined);
  43 static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
  44 static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
  45 static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
  46                                              int write);
  47 static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
  48                                                      uint64_t offset,
  49                                                      uint64_t size);
  50 static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
  51 static int i915_gem_object_wait_rendering(struct drm_gem_object *obj,
  52                                           bool interruptible);
  53 static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
  54                                            unsigned alignment);
  55 static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
  56 static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
  57                                 struct drm_i915_gem_pwrite *args,
  58                                 struct drm_file *file_priv);
  59 static void i915_gem_free_object_tail(struct drm_gem_object *obj);
  60
  61 static LIST_HEAD(shrink_list);
  62 static DEFINE_SPINLOCK(shrink_list_lock);
  63
  64 static inline bool
  65 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj_priv)
  66 {
  67         return obj_priv->gtt_space &&
  68                 !obj_priv->active &&
  69                 obj_priv->pin_count == 0;
  70 }
  71
  72 int i915_gem_do_init(struct drm_device *dev, unsigned long start,
  73                      unsigned long end)
  74 {
  75         drm_i915_private_t *dev_priv = dev->dev_private;
  76
  77         if (start >= end ||
  78             (start & (PAGE_SIZE - 1)) != 0 ||
  79             (end & (PAGE_SIZE - 1)) != 0) {
  80                 return -EINVAL;
  81         }
  82
  83         drm_mm_init(&dev_priv->mm.gtt_space, start,
  84                     end - start);
  85
  86         dev->gtt_total = (uint32_t) (end - start);
  87
  88         return 0;
  89 }
  90
  91 int
  92 i915_gem_init_ioctl(struct drm_device *dev, void *data,
  93                     struct drm_file *file_priv)
  94 {
  95         struct drm_i915_gem_init *args = data;
  96         int ret;
  97
  98         mutex_lock(&dev->struct_mutex);
  99         ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end);
 100         mutex_unlock(&dev->struct_mutex);
 101
 102         return ret;
 103 }
 104
 105 int
 106 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 107                             struct drm_file *file_priv)
 108 {
 109         struct drm_i915_gem_get_aperture *args = data;
 110
 111         if (!(dev->driver->driver_features & DRIVER_GEM))
 112                 return -ENODEV;
 113
 114         args->aper_size = dev->gtt_total;
 115         args->aper_available_size = (args->aper_size -
 116                                      atomic_read(&dev->pin_memory));
 117
 118         return 0;
 119 }
 120
 121
 122 /**
 123  * Creates a new mm object and returns a handle to it.
 124  */
 125 int
 126 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 127                       struct drm_file *file_priv)
 128 {
 129         struct drm_i915_gem_create *args = data;
 130         struct drm_gem_object *obj;
 131         int ret;
 132         u32 handle;
 133
 134         args->size = roundup(args->size, PAGE_SIZE);
 135
 136         /* Allocate the new object */
 137         obj = i915_gem_alloc_object(dev, args->size);
 138         if (obj == NULL)
 139                 return -ENOMEM;
 140
 141         ret = drm_gem_handle_create(file_priv, obj, &handle);
 142         if (ret) {
 143                 drm_gem_object_unreference_unlocked(obj);
 144                 return ret;
 145         }
 146
 147         /* Sink the floating reference from kref_init(handlecount) */
 148         drm_gem_object_handle_unreference_unlocked(obj);
 149
 150         args->handle = handle;
 151         return 0;
 152 }
 153
 154 static inline int
 155 fast_shmem_read(struct page **pages,
 156                 loff_t page_base, int page_offset,
 157                 char __user *data,
 158                 int length)
 159 {
 160         char __iomem *vaddr;
 161         int unwritten;
 162
 163         vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
 164         if (vaddr == NULL)
 165                 return -ENOMEM;
 166         unwritten = __copy_to_user_inatomic(data, vaddr + page_offset, length);
 167         kunmap_atomic(vaddr, KM_USER0);
 168
 169         if (unwritten)
 170                 return -EFAULT;
 171
 172         return 0;
 173 }
 174
 175 static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
 176 {
 177         drm_i915_private_t *dev_priv = obj->dev->dev_private;
 178         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 179
 180         return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
 181                 obj_priv->tiling_mode != I915_TILING_NONE;
 182 }
 183
 184 static inline void
 185 slow_shmem_copy(struct page *dst_page,
 186                 int dst_offset,
 187                 struct page *src_page,
 188                 int src_offset,
 189                 int length)
 190 {
 191         char *dst_vaddr, *src_vaddr;
 192
 193         dst_vaddr = kmap(dst_page);
 194         src_vaddr = kmap(src_page);
 195
 196         memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
 197
 198         kunmap(src_page);
 199         kunmap(dst_page);
 200 }
 201
 202 static inline void
 203 slow_shmem_bit17_copy(struct page *gpu_page,
 204                       int gpu_offset,
 205                       struct page *cpu_page,
 206                       int cpu_offset,
 207                       int length,
 208                       int is_read)
 209 {
 210         char *gpu_vaddr, *cpu_vaddr;
 211
 212         /* Use the unswizzled path if this page isn't affected. */
 213         if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
 214                 if (is_read)
 215                         return slow_shmem_copy(cpu_page, cpu_offset,
 216                                                gpu_page, gpu_offset, length);
 217                 else
 218                         return slow_shmem_copy(gpu_page, gpu_offset,
 219                                                cpu_page, cpu_offset, length);
 220         }
 221
 222         gpu_vaddr = kmap(gpu_page);
 223         cpu_vaddr = kmap(cpu_page);
 224
 225         /* Copy the data, XORing A6 with A17 (1). The user already knows he's
 226          * XORing with the other bits (A9 for Y, A9 and A10 for X)
 227          */
 228         while (length > 0) {
 229                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 230                 int this_length = min(cacheline_end - gpu_offset, length);
 231                 int swizzled_gpu_offset = gpu_offset ^ 64;
 232
 233                 if (is_read) {
 234                         memcpy(cpu_vaddr + cpu_offset,
 235                                gpu_vaddr + swizzled_gpu_offset,
 236                                this_length);
 237                 } else {
 238                         memcpy(gpu_vaddr + swizzled_gpu_offset,
 239                                cpu_vaddr + cpu_offset,
 240                                this_length);
 241                 }
 242                 cpu_offset += this_length;
 243                 gpu_offset += this_length;
 244                 length -= this_length;
 245         }
 246
 247         kunmap(cpu_page);
 248         kunmap(gpu_page);
 249 }
 250
 251 /**
 252  * This is the fast shmem pread path, which attempts to copy_from_user directly
 253  * from the backing pages of the object to the user's address space.  On a
 254  * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
 255  */
 256 static int
 257 i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
 258                           struct drm_i915_gem_pread *args,
 259                           struct drm_file *file_priv)
 260 {
 261         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 262         ssize_t remain;
 263         loff_t offset, page_base;
 264         char __user *user_data;
 265         int page_offset, page_length;
 266         int ret;
 267
 268         user_data = (char __user *) (uintptr_t) args->data_ptr;
 269         remain = args->size;
 270
 271         mutex_lock(&dev->struct_mutex);
 272
 273         ret = i915_gem_object_get_pages(obj, 0);
 274         if (ret != 0)
 275                 goto fail_unlock;
 276
 277         ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
 278                                                         args->size);
 279         if (ret != 0)
 280                 goto fail_put_pages;
 281
 282         obj_priv = to_intel_bo(obj);
 283         offset = args->offset;
 284
 285         while (remain > 0) {
 286                 /* Operation in this page
 287                  *
 288                  * page_base = page offset within aperture
 289                  * page_offset = offset within page
 290                  * page_length = bytes to copy for this page
 291                  */
 292                 page_base = (offset & ~(PAGE_SIZE-1));
 293                 page_offset = offset & (PAGE_SIZE-1);
 294                 page_length = remain;
 295                 if ((page_offset + remain) > PAGE_SIZE)
 296                         page_length = PAGE_SIZE - page_offset;
 297
 298                 ret = fast_shmem_read(obj_priv->pages,
 299                                       page_base, page_offset,
 300                                       user_data, page_length);
 301                 if (ret)
 302                         goto fail_put_pages;
 303
 304                 remain -= page_length;
 305                 user_data += page_length;
 306                 offset += page_length;
 307         }
 308
 309 fail_put_pages:
 310         i915_gem_object_put_pages(obj);
 311 fail_unlock:
 312         mutex_unlock(&dev->struct_mutex);
 313
 314         return ret;
 315 }
 316
 317 static int
 318 i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
 319 {
 320         int ret;
 321
 322         ret = i915_gem_object_get_pages(obj, __GFP_NORETRY | __GFP_NOWARN);
 323
 324         /* If we've insufficient memory to map in the pages, attempt
 325          * to make some space by throwing out some old buffers.
 326          */
 327         if (ret == -ENOMEM) {
 328                 struct drm_device *dev = obj->dev;
 329
 330                 ret = i915_gem_evict_something(dev, obj->size,
 331                                                i915_gem_get_gtt_alignment(obj));
 332                 if (ret)
 333                         return ret;
 334
 335                 ret = i915_gem_object_get_pages(obj, 0);
 336         }
 337
 338         return ret;
 339 }
 340
 341 /**
 342  * This is the fallback shmem pread path, which allocates temporary storage
 343  * in kernel space to copy_to_user into outside of the struct_mutex, so we
 344  * can copy out of the object's backing pages while holding the struct mutex
 345  * and not take page faults.
 346  */
 347 static int
 348 i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
 349                           struct drm_i915_gem_pread *args,
 350                           struct drm_file *file_priv)
 351 {
 352         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 353         struct mm_struct *mm = current->mm;
 354         struct page **user_pages;
 355         ssize_t remain;
 356         loff_t offset, pinned_pages, i;
 357         loff_t first_data_page, last_data_page, num_pages;
 358         int shmem_page_index, shmem_page_offset;
 359         int data_page_index,  data_page_offset;
 360         int page_length;
 361         int ret;
 362         uint64_t data_ptr = args->data_ptr;
 363         int do_bit17_swizzling;
 364
 365         remain = args->size;
 366
 367         /* Pin the user pages containing the data.  We can't fault while
 368          * holding the struct mutex, yet we want to hold it while
 369          * dereferencing the user data.
 370          */
 371         first_data_page = data_ptr / PAGE_SIZE;
 372         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 373         num_pages = last_data_page - first_data_page + 1;
 374
 375         user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
 376         if (user_pages == NULL)
 377                 return -ENOMEM;
 378
 379         down_read(&mm->mmap_sem);
 380         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 381                                       num_pages, 1, 0, user_pages, NULL);
 382         up_read(&mm->mmap_sem);
 383         if (pinned_pages < num_pages) {
 384                 ret = -EFAULT;
 385                 goto fail_put_user_pages;
 386         }
 387
 388         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 389
 390         mutex_lock(&dev->struct_mutex);
 391
 392         ret = i915_gem_object_get_pages_or_evict(obj);
 393         if (ret)
 394                 goto fail_unlock;
 395
 396         ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
 397                                                         args->size);
 398         if (ret != 0)
 399                 goto fail_put_pages;
 400
 401         obj_priv = to_intel_bo(obj);
 402         offset = args->offset;
 403
 404         while (remain > 0) {
 405                 /* Operation in this page
 406                  *
 407                  * shmem_page_index = page number within shmem file
 408                  * shmem_page_offset = offset within page in shmem file
 409                  * data_page_index = page number in get_user_pages return
 410                  * data_page_offset = offset with data_page_index page.
 411                  * page_length = bytes to copy for this page
 412                  */
 413                 shmem_page_index = offset / PAGE_SIZE;
 414                 shmem_page_offset = offset & ~PAGE_MASK;
 415                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 416                 data_page_offset = data_ptr & ~PAGE_MASK;
 417
 418                 page_length = remain;
 419                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 420                         page_length = PAGE_SIZE - shmem_page_offset;
 421                 if ((data_page_offset + page_length) > PAGE_SIZE)
 422                         page_length = PAGE_SIZE - data_page_offset;
 423
 424                 if (do_bit17_swizzling) {
 425                         slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
 426                                               shmem_page_offset,
 427                                               user_pages[data_page_index],
 428                                               data_page_offset,
 429                                               page_length,
 430                                               1);
 431                 } else {
 432                         slow_shmem_copy(user_pages[data_page_index],
 433                                         data_page_offset,
 434                                         obj_priv->pages[shmem_page_index],
 435                                         shmem_page_offset,
 436                                         page_length);
 437                 }
 438
 439                 remain -= page_length;
 440                 data_ptr += page_length;
 441                 offset += page_length;
 442         }
 443
 444 fail_put_pages:
 445         i915_gem_object_put_pages(obj);
 446 fail_unlock:
 447         mutex_unlock(&dev->struct_mutex);
 448 fail_put_user_pages:
 449         for (i = 0; i < pinned_pages; i++) {
 450                 SetPageDirty(user_pages[i]);
 451                 page_cache_release(user_pages[i]);
 452         }
 453         drm_free_large(user_pages);
 454
 455         return ret;
 456 }
 457
 458 /**
 459  * Reads data from the object referenced by handle.
 460  *
 461  * On error, the contents of *data are undefined.
 462  */
 463 int
 464 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 465                      struct drm_file *file_priv)
 466 {
 467         struct drm_i915_gem_pread *args = data;
 468         struct drm_gem_object *obj;
 469         struct drm_i915_gem_object *obj_priv;
 470         int ret;
 471
 472         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 473         if (obj == NULL)
 474                 return -ENOENT;
 475         obj_priv = to_intel_bo(obj);
 476
 477         /* Bounds check source.
 478          *
 479          * XXX: This could use review for overflow issues...
 480          */
 481         if (args->offset > obj->size || args->size > obj->size ||
 482             args->offset + args->size > obj->size) {
 483                 drm_gem_object_unreference_unlocked(obj);
 484                 return -EINVAL;
 485         }
 486
 487         if (i915_gem_object_needs_bit17_swizzle(obj)) {
 488                 ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv);
 489         } else {
 490                 ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
 491                 if (ret != 0)
 492                         ret = i915_gem_shmem_pread_slow(dev, obj, args,
 493                                                         file_priv);
 494         }
 495
 496         drm_gem_object_unreference_unlocked(obj);
 497
 498         return ret;
 499 }
 500
 501 /* This is the fast write path which cannot handle
 502  * page faults in the source data
 503  */
 504
 505 static inline int
 506 fast_user_write(struct io_mapping *mapping,
 507                 loff_t page_base, int page_offset,
 508                 char __user *user_data,
 509                 int length)
 510 {
 511         char *vaddr_atomic;
 512         unsigned long unwritten;
 513
 514         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base, KM_USER0);
 515         unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
 516                                                       user_data, length);
 517         io_mapping_unmap_atomic(vaddr_atomic, KM_USER0);
 518         if (unwritten)
 519                 return -EFAULT;
 520         return 0;
 521 }
 522
 523 /* Here's the write path which can sleep for
 524  * page faults
 525  */
 526
 527 static inline void
 528 slow_kernel_write(struct io_mapping *mapping,
 529                   loff_t gtt_base, int gtt_offset,
 530                   struct page *user_page, int user_offset,
 531                   int length)
 532 {
 533         char __iomem *dst_vaddr;
 534         char *src_vaddr;
 535
 536         dst_vaddr = io_mapping_map_wc(mapping, gtt_base);
 537         src_vaddr = kmap(user_page);
 538
 539         memcpy_toio(dst_vaddr + gtt_offset,
 540                     src_vaddr + user_offset,
 541                     length);
 542
 543         kunmap(user_page);
 544         io_mapping_unmap(dst_vaddr);
 545 }
 546
 547 static inline int
 548 fast_shmem_write(struct page **pages,
 549                  loff_t page_base, int page_offset,
 550                  char __user *data,
 551                  int length)
 552 {
 553         char __iomem *vaddr;
 554         unsigned long unwritten;
 555
 556         vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
 557         if (vaddr == NULL)
 558                 return -ENOMEM;
 559         unwritten = __copy_from_user_inatomic(vaddr + page_offset, data, length);
 560         kunmap_atomic(vaddr, KM_USER0);
 561
 562         if (unwritten)
 563                 return -EFAULT;
 564         return 0;
 565 }
 566
 567 /**
 568  * This is the fast pwrite path, where we copy the data directly from the
 569  * user into the GTT, uncached.
 570  */
 571 static int
 572 i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
 573                          struct drm_i915_gem_pwrite *args,
 574                          struct drm_file *file_priv)
 575 {
 576         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 577         drm_i915_private_t *dev_priv = dev->dev_private;
 578         ssize_t remain;
 579         loff_t offset, page_base;
 580         char __user *user_data;
 581         int page_offset, page_length;
 582         int ret;
 583
 584         user_data = (char __user *) (uintptr_t) args->data_ptr;
 585         remain = args->size;
 586         if (!access_ok(VERIFY_READ, user_data, remain))
 587                 return -EFAULT;
 588
 589
 590         mutex_lock(&dev->struct_mutex);
 591         ret = i915_gem_object_pin(obj, 0);
 592         if (ret) {
 593                 mutex_unlock(&dev->struct_mutex);
 594                 return ret;
 595         }
 596         ret = i915_gem_object_set_to_gtt_domain(obj, 1);
 597         if (ret)
 598                 goto fail;
 599
 600         obj_priv = to_intel_bo(obj);
 601         offset = obj_priv->gtt_offset + args->offset;
 602
 603         while (remain > 0) {
 604                 /* Operation in this page
 605                  *
 606                  * page_base = page offset within aperture
 607                  * page_offset = offset within page
 608                  * page_length = bytes to copy for this page
 609                  */
 610                 page_base = (offset & ~(PAGE_SIZE-1));
 611                 page_offset = offset & (PAGE_SIZE-1);
 612                 page_length = remain;
 613                 if ((page_offset + remain) > PAGE_SIZE)
 614                         page_length = PAGE_SIZE - page_offset;
 615
 616                 ret = fast_user_write (dev_priv->mm.gtt_mapping, page_base,
 617                                        page_offset, user_data, page_length);
 618
 619                 /* If we get a fault while copying data, then (presumably) our
 620                  * source page isn't available.  Return the error and we'll
 621                  * retry in the slow path.
 622                  */
 623                 if (ret)
 624                         goto fail;
 625
 626                 remain -= page_length;
 627                 user_data += page_length;
 628                 offset += page_length;
 629         }
 630
 631 fail:
 632         i915_gem_object_unpin(obj);
 633         mutex_unlock(&dev->struct_mutex);
 634
 635         return ret;
 636 }
 637
 638 /**
 639  * This is the fallback GTT pwrite path, which uses get_user_pages to pin
 640  * the memory and maps it using kmap_atomic for copying.
 641  *
 642  * This code resulted in x11perf -rgb10text consuming about 10% more CPU
 643  * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
 644  */
 645 static int
 646 i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
 647                          struct drm_i915_gem_pwrite *args,
 648                          struct drm_file *file_priv)
 649 {
 650         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 651         drm_i915_private_t *dev_priv = dev->dev_private;
 652         ssize_t remain;
 653         loff_t gtt_page_base, offset;
 654         loff_t first_data_page, last_data_page, num_pages;
 655         loff_t pinned_pages, i;
 656         struct page **user_pages;
 657         struct mm_struct *mm = current->mm;
 658         int gtt_page_offset, data_page_offset, data_page_index, page_length;
 659         int ret;
 660         uint64_t data_ptr = args->data_ptr;
 661
 662         remain = args->size;
 663
 664         /* Pin the user pages containing the data.  We can't fault while
 665          * holding the struct mutex, and all of the pwrite implementations
 666          * want to hold it while dereferencing the user data.
 667          */
 668         first_data_page = data_ptr / PAGE_SIZE;
 669         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 670         num_pages = last_data_page - first_data_page + 1;
 671
 672         user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
 673         if (user_pages == NULL)
 674                 return -ENOMEM;
 675
 676         down_read(&mm->mmap_sem);
 677         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 678                                       num_pages, 0, 0, user_pages, NULL);
 679         up_read(&mm->mmap_sem);
 680         if (pinned_pages < num_pages) {
 681                 ret = -EFAULT;
 682                 goto out_unpin_pages;
 683         }
 684
 685         mutex_lock(&dev->struct_mutex);
 686         ret = i915_gem_object_pin(obj, 0);
 687         if (ret)
 688                 goto out_unlock;
 689
 690         ret = i915_gem_object_set_to_gtt_domain(obj, 1);
 691         if (ret)
 692                 goto out_unpin_object;
 693
 694         obj_priv = to_intel_bo(obj);
 695         offset = obj_priv->gtt_offset + args->offset;
 696
 697         while (remain > 0) {
 698                 /* Operation in this page
 699                  *
 700                  * gtt_page_base = page offset within aperture
 701                  * gtt_page_offset = offset within page in aperture
 702                  * data_page_index = page number in get_user_pages return
 703                  * data_page_offset = offset with data_page_index page.
 704                  * page_length = bytes to copy for this page
 705                  */
 706                 gtt_page_base = offset & PAGE_MASK;
 707                 gtt_page_offset = offset & ~PAGE_MASK;
 708                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 709                 data_page_offset = data_ptr & ~PAGE_MASK;
 710
 711                 page_length = remain;
 712                 if ((gtt_page_offset + page_length) > PAGE_SIZE)
 713                         page_length = PAGE_SIZE - gtt_page_offset;
 714                 if ((data_page_offset + page_length) > PAGE_SIZE)
 715                         page_length = PAGE_SIZE - data_page_offset;
 716
 717                 slow_kernel_write(dev_priv->mm.gtt_mapping,
 718                                   gtt_page_base, gtt_page_offset,
 719                                   user_pages[data_page_index],
 720                                   data_page_offset,
 721                                   page_length);
 722
 723                 remain -= page_length;
 724                 offset += page_length;
 725                 data_ptr += page_length;
 726         }
 727
 728 out_unpin_object:
 729         i915_gem_object_unpin(obj);
 730 out_unlock:
 731         mutex_unlock(&dev->struct_mutex);
 732 out_unpin_pages:
 733         for (i = 0; i < pinned_pages; i++)
 734                 page_cache_release(user_pages[i]);
 735         drm_free_large(user_pages);
 736
 737         return ret;
 738 }
 739
 740 /**
 741  * This is the fast shmem pwrite path, which attempts to directly
 742  * copy_from_user into the kmapped pages backing the object.
 743  */
 744 static int
 745 i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
 746                            struct drm_i915_gem_pwrite *args,
 747                            struct drm_file *file_priv)
 748 {
 749         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 750         ssize_t remain;
 751         loff_t offset, page_base;
 752         char __user *user_data;
 753         int page_offset, page_length;
 754         int ret;
 755
 756         user_data = (char __user *) (uintptr_t) args->data_ptr;
 757         remain = args->size;
 758
 759         mutex_lock(&dev->struct_mutex);
 760
 761         ret = i915_gem_object_get_pages(obj, 0);
 762         if (ret != 0)
 763                 goto fail_unlock;
 764
 765         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
 766         if (ret != 0)
 767                 goto fail_put_pages;
 768
 769         obj_priv = to_intel_bo(obj);
 770         offset = args->offset;
 771         obj_priv->dirty = 1;
 772
 773         while (remain > 0) {
 774                 /* Operation in this page
 775                  *
 776                  * page_base = page offset within aperture
 777                  * page_offset = offset within page
 778                  * page_length = bytes to copy for this page
 779                  */
 780                 page_base = (offset & ~(PAGE_SIZE-1));
 781                 page_offset = offset & (PAGE_SIZE-1);
 782                 page_length = remain;
 783                 if ((page_offset + remain) > PAGE_SIZE)
 784                         page_length = PAGE_SIZE - page_offset;
 785
 786                 ret = fast_shmem_write(obj_priv->pages,
 787                                        page_base, page_offset,
 788                                        user_data, page_length);
 789                 if (ret)
 790                         goto fail_put_pages;
 791
 792                 remain -= page_length;
 793                 user_data += page_length;
 794                 offset += page_length;
 795         }
 796
 797 fail_put_pages:
 798         i915_gem_object_put_pages(obj);
 799 fail_unlock:
 800         mutex_unlock(&dev->struct_mutex);
 801
 802         return ret;
 803 }
 804
 805 /**
 806  * This is the fallback shmem pwrite path, which uses get_user_pages to pin
 807  * the memory and maps it using kmap_atomic for copying.
 808  *
 809  * This avoids taking mmap_sem for faulting on the user's address while the
 810  * struct_mutex is held.
 811  */
 812 static int
 813 i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
 814                            struct drm_i915_gem_pwrite *args,
 815                            struct drm_file *file_priv)
 816 {
 817         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 818         struct mm_struct *mm = current->mm;
 819         struct page **user_pages;
 820         ssize_t remain;
 821         loff_t offset, pinned_pages, i;
 822         loff_t first_data_page, last_data_page, num_pages;
 823         int shmem_page_index, shmem_page_offset;
 824         int data_page_index,  data_page_offset;
 825         int page_length;
 826         int ret;
 827         uint64_t data_ptr = args->data_ptr;
 828         int do_bit17_swizzling;
 829
 830         remain = args->size;
 831
 832         /* Pin the user pages containing the data.  We can't fault while
 833          * holding the struct mutex, and all of the pwrite implementations
 834          * want to hold it while dereferencing the user data.
 835          */
 836         first_data_page = data_ptr / PAGE_SIZE;
 837         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 838         num_pages = last_data_page - first_data_page + 1;
 839
 840         user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
 841         if (user_pages == NULL)
 842                 return -ENOMEM;
 843
 844         down_read(&mm->mmap_sem);
 845         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 846                                       num_pages, 0, 0, user_pages, NULL);
 847         up_read(&mm->mmap_sem);
 848         if (pinned_pages < num_pages) {
 849                 ret = -EFAULT;
 850                 goto fail_put_user_pages;
 851         }
 852
 853         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 854
 855         mutex_lock(&dev->struct_mutex);
 856
 857         ret = i915_gem_object_get_pages_or_evict(obj);
 858         if (ret)
 859                 goto fail_unlock;
 860
 861         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
 862         if (ret != 0)
 863                 goto fail_put_pages;
 864
 865         obj_priv = to_intel_bo(obj);
 866         offset = args->offset;
 867         obj_priv->dirty = 1;
 868
 869         while (remain > 0) {
 870                 /* Operation in this page
 871                  *
 872                  * shmem_page_index = page number within shmem file
 873                  * shmem_page_offset = offset within page in shmem file
 874                  * data_page_index = page number in get_user_pages return
 875                  * data_page_offset = offset with data_page_index page.
 876                  * page_length = bytes to copy for this page
 877                  */
 878                 shmem_page_index = offset / PAGE_SIZE;
 879                 shmem_page_offset = offset & ~PAGE_MASK;
 880                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 881                 data_page_offset = data_ptr & ~PAGE_MASK;
 882
 883                 page_length = remain;
 884                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 885                         page_length = PAGE_SIZE - shmem_page_offset;
 886                 if ((data_page_offset + page_length) > PAGE_SIZE)
 887                         page_length = PAGE_SIZE - data_page_offset;
 888
 889                 if (do_bit17_swizzling) {
 890                         slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
 891                                               shmem_page_offset,
 892                                               user_pages[data_page_index],
 893                                               data_page_offset,
 894                                               page_length,
 895                                               0);
 896                 } else {
 897                         slow_shmem_copy(obj_priv->pages[shmem_page_index],
 898                                         shmem_page_offset,
 899                                         user_pages[data_page_index],
 900                                         data_page_offset,
 901                                         page_length);
 902                 }
 903
 904                 remain -= page_length;
 905                 data_ptr += page_length;
 906                 offset += page_length;
 907         }
 908
 909 fail_put_pages:
 910         i915_gem_object_put_pages(obj);
 911 fail_unlock:
 912         mutex_unlock(&dev->struct_mutex);
 913 fail_put_user_pages:
 914         for (i = 0; i < pinned_pages; i++)
 915                 page_cache_release(user_pages[i]);
 916         drm_free_large(user_pages);
 917
 918         return ret;
 919 }
 920
 921 /**
 922  * Writes data to the object referenced by handle.
 923  *
 924  * On error, the contents of the buffer that were to be modified are undefined.
 925  */
 926 int
 927 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 928                       struct drm_file *file_priv)
 929 {
 930         struct drm_i915_gem_pwrite *args = data;
 931         struct drm_gem_object *obj;
 932         struct drm_i915_gem_object *obj_priv;
 933         int ret = 0;
 934
 935         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 936         if (obj == NULL)
 937                 return -ENOENT;
 938         obj_priv = to_intel_bo(obj);
 939
 940         /* Bounds check destination.
 941          *
 942          * XXX: This could use review for overflow issues...
 943          */
 944         if (args->offset > obj->size || args->size > obj->size ||
 945             args->offset + args->size > obj->size) {
 946                 drm_gem_object_unreference_unlocked(obj);
 947                 return -EINVAL;
 948         }
 949
 950         /* We can only do the GTT pwrite on untiled buffers, as otherwise
 951          * it would end up going through the fenced access, and we'll get
 952          * different detiling behavior between reading and writing.
 953          * pread/pwrite currently are reading and writing from the CPU
 954          * perspective, requiring manual detiling by the client.
 955          */
 956         if (obj_priv->phys_obj)
 957                 ret = i915_gem_phys_pwrite(dev, obj, args, file_priv);
 958         else if (obj_priv->tiling_mode == I915_TILING_NONE &&
 959                  dev->gtt_total != 0 &&
 960                  obj->write_domain != I915_GEM_DOMAIN_CPU) {
 961                 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file_priv);
 962                 if (ret == -EFAULT) {
 963                         ret = i915_gem_gtt_pwrite_slow(dev, obj, args,
 964                                                        file_priv);
 965                 }
 966         } else if (i915_gem_object_needs_bit17_swizzle(obj)) {
 967                 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file_priv);
 968         } else {
 969                 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file_priv);
 970                 if (ret == -EFAULT) {
 971                         ret = i915_gem_shmem_pwrite_slow(dev, obj, args,
 972                                                          file_priv);
 973                 }
 974         }
 975
 976 #if WATCH_PWRITE
 977         if (ret)
 978                 DRM_INFO("pwrite failed %d\n", ret);
 979 #endif
 980
 981         drm_gem_object_unreference_unlocked(obj);
 982
 983         return ret;
 984 }
 985
 986 /**
 987  * Called when user space prepares to use an object with the CPU, either
 988  * through the mmap ioctl's mapping or a GTT mapping.
 989  */
 990 int
 991 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 992                           struct drm_file *file_priv)
 993 {
 994         struct drm_i915_private *dev_priv = dev->dev_private;
 995         struct drm_i915_gem_set_domain *args = data;
 996         struct drm_gem_object *obj;
 997         struct drm_i915_gem_object *obj_priv;
 998         uint32_t read_domains = args->read_domains;
 999         uint32_t write_domain = args->write_domain;
1000         int ret;
1001
1002         if (!(dev->driver->driver_features & DRIVER_GEM))
1003                 return -ENODEV;
1004
1005         /* Only handle setting domains to types used by the CPU. */
1006         if (write_domain & I915_GEM_GPU_DOMAINS)
1007                 return -EINVAL;
1008
1009         if (read_domains & I915_GEM_GPU_DOMAINS)
1010                 return -EINVAL;
1011
1012         /* Having something in the write domain implies it's in the read
1013          * domain, and only that read domain.  Enforce that in the request.
1014          */
1015         if (write_domain != 0 && read_domains != write_domain)
1016                 return -EINVAL;
1017
1018         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1019         if (obj == NULL)
1020                 return -ENOENT;
1021         obj_priv = to_intel_bo(obj);
1022
1023         mutex_lock(&dev->struct_mutex);
1024
1025         intel_mark_busy(dev, obj);
1026
1027 #if WATCH_BUF
1028         DRM_INFO("set_domain_ioctl %p(%zd), %08x %08x\n",
1029                  obj, obj->size, read_domains, write_domain);
1030 #endif
1031         if (read_domains & I915_GEM_DOMAIN_GTT) {
1032                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1033
1034                 /* Update the LRU on the fence for the CPU access that's
1035                  * about to occur.
1036                  */
1037                 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1038                         struct drm_i915_fence_reg *reg =
1039                                 &dev_priv->fence_regs[obj_priv->fence_reg];
1040                         list_move_tail(&reg->lru_list,
1041                                        &dev_priv->mm.fence_list);
1042                 }
1043
1044                 /* Silently promote "you're not bound, there was nothing to do"
1045                  * to success, since the client was just asking us to
1046                  * make sure everything was done.
1047                  */
1048                 if (ret == -EINVAL)
1049                         ret = 0;
1050         } else {
1051                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1052         }
1053
1054         /* Maintain LRU order of "inactive" objects */
1055         if (ret == 0 && i915_gem_object_is_inactive(obj_priv))
1056                 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1057
1058         drm_gem_object_unreference(obj);
1059         mutex_unlock(&dev->struct_mutex);
1060         return ret;
1061 }
1062
1063 /**
1064  * Called when user space has done writes to this buffer
1065  */
1066 int
1067 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1068                       struct drm_file *file_priv)
1069 {
1070         struct drm_i915_gem_sw_finish *args = data;
1071         struct drm_gem_object *obj;
1072         struct drm_i915_gem_object *obj_priv;
1073         int ret = 0;
1074
1075         if (!(dev->driver->driver_features & DRIVER_GEM))
1076                 return -ENODEV;
1077
1078         mutex_lock(&dev->struct_mutex);
1079         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1080         if (obj == NULL) {
1081                 mutex_unlock(&dev->struct_mutex);
1082                 return -ENOENT;
1083         }
1084
1085 #if WATCH_BUF
1086         DRM_INFO("%s: sw_finish %d (%p %zd)\n",
1087                  __func__, args->handle, obj, obj->size);
1088 #endif
1089         obj_priv = to_intel_bo(obj);
1090
1091         /* Pinned buffers may be scanout, so flush the cache */
1092         if (obj_priv->pin_count)
1093                 i915_gem_object_flush_cpu_write_domain(obj);
1094
1095         drm_gem_object_unreference(obj);
1096         mutex_unlock(&dev->struct_mutex);
1097         return ret;
1098 }
1099
1100 /**
1101  * Maps the contents of an object, returning the address it is mapped
1102  * into.
1103  *
1104  * While the mapping holds a reference on the contents of the object, it doesn't
1105  * imply a ref on the object itself.
1106  */
1107 int
1108 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1109                    struct drm_file *file_priv)
1110 {
1111         struct drm_i915_gem_mmap *args = data;
1112         struct drm_gem_object *obj;
1113         loff_t offset;
1114         unsigned long addr;
1115
1116         if (!(dev->driver->driver_features & DRIVER_GEM))
1117                 return -ENODEV;
1118
1119         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1120         if (obj == NULL)
1121                 return -ENOENT;
1122
1123         offset = args->offset;
1124
1125         down_write(&current->mm->mmap_sem);
1126         addr = do_mmap(obj->filp, 0, args->size,
1127                        PROT_READ | PROT_WRITE, MAP_SHARED,
1128                        args->offset);
1129         up_write(&current->mm->mmap_sem);
1130         drm_gem_object_unreference_unlocked(obj);
1131         if (IS_ERR((void *)addr))
1132                 return addr;
1133
1134         args->addr_ptr = (uint64_t) addr;
1135
1136         return 0;
1137 }
1138
1139 /**
1140  * i915_gem_fault - fault a page into the GTT
1141  * vma: VMA in question
1142  * vmf: fault info
1143  *
1144  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1145  * from userspace.  The fault handler takes care of binding the object to
1146  * the GTT (if needed), allocating and programming a fence register (again,
1147  * only if needed based on whether the old reg is still valid or the object
1148  * is tiled) and inserting a new PTE into the faulting process.
1149  *
1150  * Note that the faulting process may involve evicting existing objects
1151  * from the GTT and/or fence registers to make room.  So performance may
1152  * suffer if the GTT working set is large or there are few fence registers
1153  * left.
1154  */
1155 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1156 {
1157         struct drm_gem_object *obj = vma->vm_private_data;
1158         struct drm_device *dev = obj->dev;
1159         drm_i915_private_t *dev_priv = dev->dev_private;
1160         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1161         pgoff_t page_offset;
1162         unsigned long pfn;
1163         int ret = 0;
1164         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1165
1166         /* We don't use vmf->pgoff since that has the fake offset */
1167         page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1168                 PAGE_SHIFT;
1169
1170         /* Now bind it into the GTT if needed */
1171         mutex_lock(&dev->struct_mutex);
1172         if (!obj_priv->gtt_space) {
1173                 ret = i915_gem_object_bind_to_gtt(obj, 0);
1174                 if (ret)
1175                         goto unlock;
1176
1177                 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1178                 if (ret)
1179                         goto unlock;
1180         }
1181
1182         /* Need a new fence register? */
1183         if (obj_priv->tiling_mode != I915_TILING_NONE) {
1184                 ret = i915_gem_object_get_fence_reg(obj, true);
1185                 if (ret)
1186                         goto unlock;
1187         }
1188
1189         if (i915_gem_object_is_inactive(obj_priv))
1190                 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1191
1192         pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
1193                 page_offset;
1194
1195         /* Finally, remap it using the new GTT offset */
1196         ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1197 unlock:
1198         mutex_unlock(&dev->struct_mutex);
1199
1200         switch (ret) {
1201         case 0:
1202         case -ERESTARTSYS:
1203                 return VM_FAULT_NOPAGE;
1204         case -ENOMEM:
1205         case -EAGAIN:
1206                 return VM_FAULT_OOM;
1207         default:
1208                 return VM_FAULT_SIGBUS;
1209         }
1210 }
1211
1212 /**
1213  * i915_gem_create_mmap_offset - create a fake mmap offset for an object
1214  * @obj: obj in question
1215  *
1216  * GEM memory mapping works by handing back to userspace a fake mmap offset
1217  * it can use in a subsequent mmap(2) call.  The DRM core code then looks
1218  * up the object based on the offset and sets up the various memory mapping
1219  * structures.
1220  *
1221  * This routine allocates and attaches a fake offset for @obj.
1222  */
1223 static int
1224 i915_gem_create_mmap_offset(struct drm_gem_object *obj)
1225 {
1226         struct drm_device *dev = obj->dev;
1227         struct drm_gem_mm *mm = dev->mm_private;
1228         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1229         struct drm_map_list *list;
1230         struct drm_local_map *map;
1231         int ret = 0;
1232
1233         /* Set the object up for mmap'ing */
1234         list = &obj->map_list;
1235         list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
1236         if (!list->map)
1237                 return -ENOMEM;
1238
1239         map = list->map;
1240         map->type = _DRM_GEM;
1241         map->size = obj->size;
1242         map->handle = obj;
1243
1244         /* Get a DRM GEM mmap offset allocated... */
1245         list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
1246                                                     obj->size / PAGE_SIZE, 0, 0);
1247         if (!list->file_offset_node) {
1248                 DRM_ERROR("failed to allocate offset for bo %d\n", obj->name);
1249                 ret = -ENOSPC;
1250                 goto out_free_list;
1251         }
1252
1253         list->file_offset_node = drm_mm_get_block(list->file_offset_node,
1254                                                   obj->size / PAGE_SIZE, 0);
1255         if (!list->file_offset_node) {
1256                 ret = -ENOMEM;
1257                 goto out_free_list;
1258         }
1259
1260         list->hash.key = list->file_offset_node->start;
1261         ret = drm_ht_insert_item(&mm->offset_hash, &list->hash);
1262         if (ret) {
1263                 DRM_ERROR("failed to add to map hash\n");
1264                 goto out_free_mm;
1265         }
1266
1267         /* By now we should be all set, any drm_mmap request on the offset
1268          * below will get to our mmap & fault handler */
1269         obj_priv->mmap_offset = ((uint64_t) list->hash.key) << PAGE_SHIFT;
1270
1271         return 0;
1272
1273 out_free_mm:
1274         drm_mm_put_block(list->file_offset_node);
1275 out_free_list:
1276         kfree(list->map);
1277
1278         return ret;
1279 }
1280
1281 /**
1282  * i915_gem_release_mmap - remove physical page mappings
1283  * @obj: obj in question
1284  *
1285  * Preserve the reservation of the mmapping with the DRM core code, but
1286  * relinquish ownership of the pages back to the system.
1287  *
1288  * It is vital that we remove the page mapping if we have mapped a tiled
1289  * object through the GTT and then lose the fence register due to
1290  * resource pressure. Similarly if the object has been moved out of the
1291  * aperture, than pages mapped into userspace must be revoked. Removing the
1292  * mapping will then trigger a page fault on the next user access, allowing
1293  * fixup by i915_gem_fault().
1294  */
1295 void
1296 i915_gem_release_mmap(struct drm_gem_object *obj)
1297 {
1298         struct drm_device *dev = obj->dev;
1299         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1300
1301         if (dev->dev_mapping)
1302                 unmap_mapping_range(dev->dev_mapping,
1303                                     obj_priv->mmap_offset, obj->size, 1);
1304 }
1305
1306 static void
1307 i915_gem_free_mmap_offset(struct drm_gem_object *obj)
1308 {
1309         struct drm_device *dev = obj->dev;
1310         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1311         struct drm_gem_mm *mm = dev->mm_private;
1312         struct drm_map_list *list;
1313
1314         list = &obj->map_list;
1315         drm_ht_remove_item(&mm->offset_hash, &list->hash);
1316
1317         if (list->file_offset_node) {
1318                 drm_mm_put_block(list->file_offset_node);
1319                 list->file_offset_node = NULL;
1320         }
1321
1322         if (list->map) {
1323                 kfree(list->map);
1324                 list->map = NULL;
1325         }
1326
1327         obj_priv->mmap_offset = 0;
1328 }
1329
1330 /**
1331  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1332  * @obj: object to check
1333  *
1334  * Return the required GTT alignment for an object, taking into account
1335  * potential fence register mapping if needed.
1336  */
1337 static uint32_t
1338 i915_gem_get_gtt_alignment(struct drm_gem_object *obj)
1339 {
1340         struct drm_device *dev = obj->dev;
1341         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1342         int start, i;
1343
1344         /*
1345          * Minimum alignment is 4k (GTT page size), but might be greater
1346          * if a fence register is needed for the object.
1347          */
1348         if (INTEL_INFO(dev)->gen >= 4 || obj_priv->tiling_mode == I915_TILING_NONE)
1349                 return 4096;
1350
1351         /*
1352          * Previous chips need to be aligned to the size of the smallest
1353          * fence register that can contain the object.
1354          */
1355         if (INTEL_INFO(dev)->gen == 3)
1356                 start = 1024*1024;
1357         else
1358                 start = 512*1024;
1359
1360         for (i = start; i < obj->size; i <<= 1)
1361                 ;
1362
1363         return i;
1364 }
1365
1366 /**
1367  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1368  * @dev: DRM device
1369  * @data: GTT mapping ioctl data
1370  * @file_priv: GEM object info
1371  *
1372  * Simply returns the fake offset to userspace so it can mmap it.
1373  * The mmap call will end up in drm_gem_mmap(), which will set things
1374  * up so we can get faults in the handler above.
1375  *
1376  * The fault handler will take care of binding the object into the GTT
1377  * (since it may have been evicted to make room for something), allocating
1378  * a fence register, and mapping the appropriate aperture address into
1379  * userspace.
1380  */
1381 int
1382 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1383                         struct drm_file *file_priv)
1384 {
1385         struct drm_i915_gem_mmap_gtt *args = data;
1386         struct drm_gem_object *obj;
1387         struct drm_i915_gem_object *obj_priv;
1388         int ret;
1389
1390         if (!(dev->driver->driver_features & DRIVER_GEM))
1391                 return -ENODEV;
1392
1393         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1394         if (obj == NULL)
1395                 return -ENOENT;
1396
1397         mutex_lock(&dev->struct_mutex);
1398
1399         obj_priv = to_intel_bo(obj);
1400
1401         if (obj_priv->madv != I915_MADV_WILLNEED) {
1402                 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1403                 drm_gem_object_unreference(obj);
1404                 mutex_unlock(&dev->struct_mutex);
1405                 return -EINVAL;
1406         }
1407
1408
1409         if (!obj_priv->mmap_offset) {
1410                 ret = i915_gem_create_mmap_offset(obj);
1411                 if (ret) {
1412                         drm_gem_object_unreference(obj);
1413                         mutex_unlock(&dev->struct_mutex);
1414                         return ret;
1415                 }
1416         }
1417
1418         args->offset = obj_priv->mmap_offset;
1419
1420         /*
1421          * Pull it into the GTT so that we have a page list (makes the
1422          * initial fault faster and any subsequent flushing possible).
1423          */
1424         if (!obj_priv->agp_mem) {
1425                 ret = i915_gem_object_bind_to_gtt(obj, 0);
1426                 if (ret) {
1427                         drm_gem_object_unreference(obj);
1428                         mutex_unlock(&dev->struct_mutex);
1429                         return ret;
1430                 }
1431         }
1432
1433         drm_gem_object_unreference(obj);
1434         mutex_unlock(&dev->struct_mutex);
1435
1436         return 0;
1437 }
1438
1439 void
1440 i915_gem_object_put_pages(struct drm_gem_object *obj)
1441 {
1442         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1443         int page_count = obj->size / PAGE_SIZE;
1444         int i;
1445
1446         BUG_ON(obj_priv->pages_refcount == 0);
1447         BUG_ON(obj_priv->madv == __I915_MADV_PURGED);
1448
1449         if (--obj_priv->pages_refcount != 0)
1450                 return;
1451
1452         if (obj_priv->tiling_mode != I915_TILING_NONE)
1453                 i915_gem_object_save_bit_17_swizzle(obj);
1454
1455         if (obj_priv->madv == I915_MADV_DONTNEED)
1456                 obj_priv->dirty = 0;
1457
1458         for (i = 0; i < page_count; i++) {
1459                 if (obj_priv->dirty)
1460                         set_page_dirty(obj_priv->pages[i]);
1461
1462                 if (obj_priv->madv == I915_MADV_WILLNEED)
1463                         mark_page_accessed(obj_priv->pages[i]);
1464
1465                 page_cache_release(obj_priv->pages[i]);
1466         }
1467         obj_priv->dirty = 0;
1468
1469         drm_free_large(obj_priv->pages);
1470         obj_priv->pages = NULL;
1471 }
1472
1473 static void
1474 i915_gem_object_move_to_active(struct drm_gem_object *obj,
1475                                struct intel_ring_buffer *ring)
1476 {
1477         struct drm_i915_private *dev_priv = obj->dev->dev_private;
1478         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1479
1480         BUG_ON(ring == NULL);
1481         obj_priv->ring = ring;
1482
1483         /* Add a reference if we're newly entering the active list. */
1484         if (!obj_priv->active) {
1485                 drm_gem_object_reference(obj);
1486                 obj_priv->active = 1;
1487         }
1488
1489         /* Move from whatever list we were on to the tail of execution. */
1490         list_move_tail(&obj_priv->list, &ring->active_list);
1491         obj_priv->last_rendering_seqno = dev_priv->next_seqno;
1492 }
1493
1494 static void
1495 i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
1496 {
1497         struct drm_device *dev = obj->dev;
1498         drm_i915_private_t *dev_priv = dev->dev_private;
1499         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1500
1501         BUG_ON(!obj_priv->active);
1502         list_move_tail(&obj_priv->list, &dev_priv->mm.flushing_list);
1503         obj_priv->last_rendering_seqno = 0;
1504 }
1505
1506 /* Immediately discard the backing storage */
1507 static void
1508 i915_gem_object_truncate(struct drm_gem_object *obj)
1509 {
1510         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1511         struct inode *inode;
1512
1513         /* Our goal here is to return as much of the memory as
1514          * is possible back to the system as we are called from OOM.
1515          * To do this we must instruct the shmfs to drop all of its
1516          * backing pages, *now*. Here we mirror the actions taken
1517          * when by shmem_delete_inode() to release the backing store.
1518          */
1519         inode = obj->filp->f_path.dentry->d_inode;
1520         truncate_inode_pages(inode->i_mapping, 0);
1521         if (inode->i_op->truncate_range)
1522                 inode->i_op->truncate_range(inode, 0, (loff_t)-1);
1523
1524         obj_priv->madv = __I915_MADV_PURGED;
1525 }
1526
1527 static inline int
1528 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj_priv)
1529 {
1530         return obj_priv->madv == I915_MADV_DONTNEED;
1531 }
1532
1533 static void
1534 i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
1535 {
1536         struct drm_device *dev = obj->dev;
1537         drm_i915_private_t *dev_priv = dev->dev_private;
1538         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1539
1540         i915_verify_inactive(dev, __FILE__, __LINE__);
1541         if (obj_priv->pin_count != 0)
1542                 list_move_tail(&obj_priv->list, &dev_priv->mm.pinned_list);
1543         else
1544                 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1545
1546         BUG_ON(!list_empty(&obj_priv->gpu_write_list));
1547
1548         obj_priv->last_rendering_seqno = 0;
1549         obj_priv->ring = NULL;
1550         if (obj_priv->active) {
1551                 obj_priv->active = 0;
1552                 drm_gem_object_unreference(obj);
1553         }
1554         i915_verify_inactive(dev, __FILE__, __LINE__);
1555 }
1556
1557 static void
1558 i915_gem_process_flushing_list(struct drm_device *dev,
1559                                uint32_t flush_domains,
1560                                struct intel_ring_buffer *ring)
1561 {
1562         drm_i915_private_t *dev_priv = dev->dev_private;
1563         struct drm_i915_gem_object *obj_priv, *next;
1564
1565         list_for_each_entry_safe(obj_priv, next,
1566                                  &dev_priv->mm.gpu_write_list,
1567                                  gpu_write_list) {
1568                 struct drm_gem_object *obj = &obj_priv->base;
1569
1570                 if (obj->write_domain & flush_domains &&
1571                     obj_priv->ring == ring) {
1572                         uint32_t old_write_domain = obj->write_domain;
1573
1574                         obj->write_domain = 0;
1575                         list_del_init(&obj_priv->gpu_write_list);
1576                         i915_gem_object_move_to_active(obj, ring);
1577
1578                         /* update the fence lru list */
1579                         if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1580                                 struct drm_i915_fence_reg *reg =
1581                                         &dev_priv->fence_regs[obj_priv->fence_reg];
1582                                 list_move_tail(&reg->lru_list,
1583                                                 &dev_priv->mm.fence_list);
1584                         }
1585
1586                         trace_i915_gem_object_change_domain(obj,
1587                                                             obj->read_domains,
1588                                                             old_write_domain);
1589                 }
1590         }
1591 }
1592
1593 uint32_t
1594 i915_add_request(struct drm_device *dev,
1595                  struct drm_file *file_priv,
1596                  struct drm_i915_gem_request *request,
1597                  struct intel_ring_buffer *ring)
1598 {
1599         drm_i915_private_t *dev_priv = dev->dev_private;
1600         struct drm_i915_file_private *i915_file_priv = NULL;
1601         uint32_t seqno;
1602         int was_empty;
1603
1604         if (file_priv != NULL)
1605                 i915_file_priv = file_priv->driver_priv;
1606
1607         if (request == NULL) {
1608                 request = kzalloc(sizeof(*request), GFP_KERNEL);
1609                 if (request == NULL)
1610                         return 0;
1611         }
1612
1613         seqno = ring->add_request(dev, ring, file_priv, 0);
1614
1615         request->seqno = seqno;
1616         request->ring = ring;
1617         request->emitted_jiffies = jiffies;
1618         was_empty = list_empty(&ring->request_list);
1619         list_add_tail(&request->list, &ring->request_list);
1620
1621         if (i915_file_priv) {
1622                 list_add_tail(&request->client_list,
1623                               &i915_file_priv->mm.request_list);
1624         } else {
1625                 INIT_LIST_HEAD(&request->client_list);
1626         }
1627
1628         if (!dev_priv->mm.suspended) {
1629                 mod_timer(&dev_priv->hangcheck_timer,
1630                           jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
1631                 if (was_empty)
1632                         queue_delayed_work(dev_priv->wq,
1633                                            &dev_priv->mm.retire_work, HZ);
1634         }
1635         return seqno;
1636 }
1637
1638 /**
1639  * Command execution barrier
1640  *
1641  * Ensures that all commands in the ring are finished
1642  * before signalling the CPU
1643  */
1644 static void
1645 i915_retire_commands(struct drm_device *dev, struct intel_ring_buffer *ring)
1646 {
1647         uint32_t flush_domains = 0;
1648
1649         /* The sampler always gets flushed on i965 (sigh) */
1650         if (INTEL_INFO(dev)->gen >= 4)
1651                 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
1652
1653         ring->flush(dev, ring,
1654                         I915_GEM_DOMAIN_COMMAND, flush_domains);
1655 }
1656
1657 /**
1658  * Returns true if seq1 is later than seq2.
1659  */
1660 bool
1661 i915_seqno_passed(uint32_t seq1, uint32_t seq2)
1662 {
1663         return (int32_t)(seq1 - seq2) >= 0;
1664 }
1665
1666 uint32_t
1667 i915_get_gem_seqno(struct drm_device *dev,
1668                    struct intel_ring_buffer *ring)
1669 {
1670         return ring->get_gem_seqno(dev, ring);
1671 }
1672
1673 static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
1674                                       struct intel_ring_buffer *ring)
1675 {
1676         while (!list_empty(&ring->request_list)) {
1677                 struct drm_i915_gem_request *request;
1678
1679                 request = list_first_entry(&ring->request_list,
1680                                            struct drm_i915_gem_request,
1681                                            list);
1682
1683                 list_del(&request->list);
1684                 list_del(&request->client_list);
1685                 kfree(request);
1686         }
1687
1688         while (!list_empty(&ring->active_list)) {
1689                 struct drm_i915_gem_object *obj_priv;
1690
1691                 obj_priv = list_first_entry(&ring->active_list,
1692                                             struct drm_i915_gem_object,
1693                                             list);
1694
1695                 obj_priv->base.write_domain = 0;
1696                 list_del_init(&obj_priv->gpu_write_list);
1697                 i915_gem_object_move_to_inactive(&obj_priv->base);
1698         }
1699 }
1700
1701 void i915_gem_reset_lists(struct drm_device *dev)
1702 {
1703         struct drm_i915_private *dev_priv = dev->dev_private;
1704         struct drm_i915_gem_object *obj_priv;
1705
1706         i915_gem_reset_ring_lists(dev_priv, &dev_priv->render_ring);
1707         if (HAS_BSD(dev))
1708                 i915_gem_reset_ring_lists(dev_priv, &dev_priv->bsd_ring);
1709
1710         /* Remove anything from the flushing lists. The GPU cache is likely
1711          * to be lost on reset along with the data, so simply move the
1712          * lost bo to the inactive list.
1713          */
1714         while (!list_empty(&dev_priv->mm.flushing_list)) {
1715                 obj_priv = list_first_entry(&dev_priv->mm.flushing_list,
1716                                             struct drm_i915_gem_object,
1717                                             list);
1718
1719                 obj_priv->base.write_domain = 0;
1720                 list_del_init(&obj_priv->gpu_write_list);
1721                 i915_gem_object_move_to_inactive(&obj_priv->base);
1722         }
1723
1724         /* Move everything out of the GPU domains to ensure we do any
1725          * necessary invalidation upon reuse.
1726          */
1727         list_for_each_entry(obj_priv,
1728                             &dev_priv->mm.inactive_list,
1729                             list)
1730         {
1731                 obj_priv->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
1732         }
1733 }
1734
1735 /**
1736  * This function clears the request list as sequence numbers are passed.
1737  */
1738 static void
1739 i915_gem_retire_requests_ring(struct drm_device *dev,
1740                               struct intel_ring_buffer *ring)
1741 {
1742         drm_i915_private_t *dev_priv = dev->dev_private;
1743         uint32_t seqno;
1744
1745         if (!ring->status_page.page_addr ||
1746             list_empty(&ring->request_list))
1747                 return;
1748
1749         seqno = i915_get_gem_seqno(dev, ring);
1750         while (!list_empty(&ring->request_list)) {
1751                 struct drm_i915_gem_request *request;
1752
1753                 request = list_first_entry(&ring->request_list,
1754                                            struct drm_i915_gem_request,
1755                                            list);
1756
1757                 if (!i915_seqno_passed(seqno, request->seqno))
1758                         break;
1759
1760                 trace_i915_gem_request_retire(dev, request->seqno);
1761
1762                 list_del(&request->list);
1763                 list_del(&request->client_list);
1764                 kfree(request);
1765         }
1766
1767         /* Move any buffers on the active list that are no longer referenced
1768          * by the ringbuffer to the flushing/inactive lists as appropriate.
1769          */
1770         while (!list_empty(&ring->active_list)) {
1771                 struct drm_gem_object *obj;
1772                 struct drm_i915_gem_object *obj_priv;
1773
1774                 obj_priv = list_first_entry(&ring->active_list,
1775                                             struct drm_i915_gem_object,
1776                                             list);
1777
1778                 if (!i915_seqno_passed(seqno, obj_priv->last_rendering_seqno))
1779                         break;
1780
1781                 obj = &obj_priv->base;
1782
1783 #if WATCH_LRU
1784                 DRM_INFO("%s: retire %d moves to inactive list %p\n",
1785                          __func__, request->seqno, obj);
1786 #endif
1787
1788                 if (obj->write_domain != 0)
1789                         i915_gem_object_move_to_flushing(obj);
1790                 else
1791                         i915_gem_object_move_to_inactive(obj);
1792         }
1793
1794         if (unlikely (dev_priv->trace_irq_seqno &&
1795                       i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) {
1796                 ring->user_irq_put(dev, ring);
1797                 dev_priv->trace_irq_seqno = 0;
1798         }
1799 }
1800
1801 void
1802 i915_gem_retire_requests(struct drm_device *dev)
1803 {
1804         drm_i915_private_t *dev_priv = dev->dev_private;
1805
1806         if (!list_empty(&dev_priv->mm.deferred_free_list)) {
1807             struct drm_i915_gem_object *obj_priv, *tmp;
1808
1809             /* We must be careful that during unbind() we do not
1810              * accidentally infinitely recurse into retire requests.
1811              * Currently:
1812              *   retire -> free -> unbind -> wait -> retire_ring
1813              */
1814             list_for_each_entry_safe(obj_priv, tmp,
1815                                      &dev_priv->mm.deferred_free_list,
1816                                      list)
1817                     i915_gem_free_object_tail(&obj_priv->base);
1818         }
1819
1820         i915_gem_retire_requests_ring(dev, &dev_priv->render_ring);
1821         if (HAS_BSD(dev))
1822                 i915_gem_retire_requests_ring(dev, &dev_priv->bsd_ring);
1823 }
1824
1825 static void
1826 i915_gem_retire_work_handler(struct work_struct *work)
1827 {
1828         drm_i915_private_t *dev_priv;
1829         struct drm_device *dev;
1830
1831         dev_priv = container_of(work, drm_i915_private_t,
1832                                 mm.retire_work.work);
1833         dev = dev_priv->dev;
1834
1835         mutex_lock(&dev->struct_mutex);
1836         i915_gem_retire_requests(dev);
1837
1838         if (!dev_priv->mm.suspended &&
1839                 (!list_empty(&dev_priv->render_ring.request_list) ||
1840                         (HAS_BSD(dev) &&
1841                          !list_empty(&dev_priv->bsd_ring.request_list))))
1842                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1843         mutex_unlock(&dev->struct_mutex);
1844 }
1845
1846 int
1847 i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
1848                      bool interruptible, struct intel_ring_buffer *ring)
1849 {
1850         drm_i915_private_t *dev_priv = dev->dev_private;
1851         u32 ier;
1852         int ret = 0;
1853
1854         BUG_ON(seqno == 0);
1855
1856         if (seqno == dev_priv->next_seqno) {
1857                 seqno = i915_add_request(dev, NULL, NULL, ring);
1858                 if (seqno == 0)
1859                         return -ENOMEM;
1860         }
1861
1862         if (atomic_read(&dev_priv->mm.wedged))
1863                 return -EIO;
1864
1865         if (!i915_seqno_passed(ring->get_gem_seqno(dev, ring), seqno)) {
1866                 if (HAS_PCH_SPLIT(dev))
1867                         ier = I915_READ(DEIER) | I915_READ(GTIER);
1868                 else
1869                         ier = I915_READ(IER);
1870                 if (!ier) {
1871                         DRM_ERROR("something (likely vbetool) disabled "
1872                                   "interrupts, re-enabling\n");
1873                         i915_driver_irq_preinstall(dev);
1874                         i915_driver_irq_postinstall(dev);
1875                 }
1876
1877                 trace_i915_gem_request_wait_begin(dev, seqno);
1878
1879                 ring->waiting_gem_seqno = seqno;
1880                 ring->user_irq_get(dev, ring);
1881                 if (interruptible)
1882                         ret = wait_event_interruptible(ring->irq_queue,
1883                                 i915_seqno_passed(
1884                                         ring->get_gem_seqno(dev, ring), seqno)
1885                                 || atomic_read(&dev_priv->mm.wedged));
1886                 else
1887                         wait_event(ring->irq_queue,
1888                                 i915_seqno_passed(
1889                                         ring->get_gem_seqno(dev, ring), seqno)
1890                                 || atomic_read(&dev_priv->mm.wedged));
1891
1892                 ring->user_irq_put(dev, ring);
1893                 ring->waiting_gem_seqno = 0;
1894
1895                 trace_i915_gem_request_wait_end(dev, seqno);
1896         }
1897         if (atomic_read(&dev_priv->mm.wedged))
1898                 ret = -EIO;
1899
1900         if (ret && ret != -ERESTARTSYS)
1901                 DRM_ERROR("%s returns %d (awaiting %d at %d, next %d)\n",
1902                           __func__, ret, seqno, ring->get_gem_seqno(dev, ring),
1903                           dev_priv->next_seqno);
1904
1905         /* Directly dispatch request retiring.  While we have the work queue
1906          * to handle this, the waiter on a request often wants an associated
1907          * buffer to have made it to the inactive list, and we would need
1908          * a separate wait queue to handle that.
1909          */
1910         if (ret == 0)
1911                 i915_gem_retire_requests_ring(dev, ring);
1912
1913         return ret;
1914 }
1915
1916 /**
1917  * Waits for a sequence number to be signaled, and cleans up the
1918  * request and object lists appropriately for that event.
1919  */
1920 static int
1921 i915_wait_request(struct drm_device *dev, uint32_t seqno,
1922                 struct intel_ring_buffer *ring)
1923 {
1924         return i915_do_wait_request(dev, seqno, 1, ring);
1925 }
1926
1927 static void
1928 i915_gem_flush_ring(struct drm_device *dev,
1929                     struct drm_file *file_priv,
1930                     struct intel_ring_buffer *ring,
1931                     uint32_t invalidate_domains,
1932                     uint32_t flush_domains)
1933 {
1934         ring->flush(dev, ring, invalidate_domains, flush_domains);
1935         i915_gem_process_flushing_list(dev, flush_domains, ring);
1936 }
1937
1938 static void
1939 i915_gem_flush(struct drm_device *dev,
1940                struct drm_file *file_priv,
1941                uint32_t invalidate_domains,
1942                uint32_t flush_domains,
1943                uint32_t flush_rings)
1944 {
1945         drm_i915_private_t *dev_priv = dev->dev_private;
1946
1947         if (flush_domains & I915_GEM_DOMAIN_CPU)
1948                 drm_agp_chipset_flush(dev);
1949
1950         if ((flush_domains | invalidate_domains) & I915_GEM_GPU_DOMAINS) {
1951                 if (flush_rings & RING_RENDER)
1952                         i915_gem_flush_ring(dev, file_priv,
1953                                             &dev_priv->render_ring,
1954                                             invalidate_domains, flush_domains);
1955                 if (flush_rings & RING_BSD)
1956                         i915_gem_flush_ring(dev, file_priv,
1957                                             &dev_priv->bsd_ring,
1958                                             invalidate_domains, flush_domains);
1959         }
1960 }
1961
1962 /**
1963  * Ensures that all rendering to the object has completed and the object is
1964  * safe to unbind from the GTT or access from the CPU.
1965  */
1966 static int
1967 i915_gem_object_wait_rendering(struct drm_gem_object *obj,
1968                                bool interruptible)
1969 {
1970         struct drm_device *dev = obj->dev;
1971         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1972         int ret;
1973
1974         /* This function only exists to support waiting for existing rendering,
1975          * not for emitting required flushes.
1976          */
1977         BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0);
1978
1979         /* If there is rendering queued on the buffer being evicted, wait for
1980          * it.
1981          */
1982         if (obj_priv->active) {
1983 #if WATCH_BUF
1984                 DRM_INFO("%s: object %p wait for seqno %08x\n",
1985                           __func__, obj, obj_priv->last_rendering_seqno);
1986 #endif
1987                 ret = i915_do_wait_request(dev,
1988                                            obj_priv->last_rendering_seqno,
1989                                            interruptible,
1990                                            obj_priv->ring);
1991                 if (ret)
1992                         return ret;
1993         }
1994
1995         return 0;
1996 }
1997
1998 /**
1999  * Unbinds an object from the GTT aperture.
2000  */
2001 int
2002 i915_gem_object_unbind(struct drm_gem_object *obj)
2003 {
2004         struct drm_device *dev = obj->dev;
2005         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2006         int ret = 0;
2007
2008 #if WATCH_BUF
2009         DRM_INFO("%s:%d %p\n", __func__, __LINE__, obj);
2010         DRM_INFO("gtt_space %p\n", obj_priv->gtt_space);
2011 #endif
2012         if (obj_priv->gtt_space == NULL)
2013                 return 0;
2014
2015         if (obj_priv->pin_count != 0) {
2016                 DRM_ERROR("Attempting to unbind pinned buffer\n");
2017                 return -EINVAL;
2018         }
2019
2020         /* blow away mappings if mapped through GTT */
2021         i915_gem_release_mmap(obj);
2022
2023         /* Move the object to the CPU domain to ensure that
2024          * any possible CPU writes while it's not in the GTT
2025          * are flushed when we go to remap it. This will
2026          * also ensure that all pending GPU writes are finished
2027          * before we unbind.
2028          */
2029         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
2030         if (ret == -ERESTARTSYS)
2031                 return ret;
2032         /* Continue on if we fail due to EIO, the GPU is hung so we
2033          * should be safe and we need to cleanup or else we might
2034          * cause memory corruption through use-after-free.
2035          */
2036
2037         /* release the fence reg _after_ flushing */
2038         if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
2039                 i915_gem_clear_fence_reg(obj);
2040
2041         if (obj_priv->agp_mem != NULL) {
2042                 drm_unbind_agp(obj_priv->agp_mem);
2043                 drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
2044                 obj_priv->agp_mem = NULL;
2045         }
2046
2047         i915_gem_object_put_pages(obj);
2048         BUG_ON(obj_priv->pages_refcount);
2049
2050         if (obj_priv->gtt_space) {
2051                 atomic_dec(&dev->gtt_count);
2052                 atomic_sub(obj->size, &dev->gtt_memory);
2053
2054                 drm_mm_put_block(obj_priv->gtt_space);
2055                 obj_priv->gtt_space = NULL;
2056         }
2057
2058         list_del_init(&obj_priv->list);
2059
2060         if (i915_gem_object_is_purgeable(obj_priv))
2061                 i915_gem_object_truncate(obj);
2062
2063         trace_i915_gem_object_unbind(obj);
2064
2065         return ret;
2066 }
2067
2068 int
2069 i915_gpu_idle(struct drm_device *dev)
2070 {
2071         drm_i915_private_t *dev_priv = dev->dev_private;
2072         bool lists_empty;
2073         u32 seqno;
2074         int ret;
2075
2076         lists_empty = (list_empty(&dev_priv->mm.flushing_list) &&
2077                        list_empty(&dev_priv->render_ring.active_list) &&
2078                        (!HAS_BSD(dev) ||
2079                         list_empty(&dev_priv->bsd_ring.active_list)));
2080         if (lists_empty)
2081                 return 0;
2082
2083         /* Flush everything onto the inactive list. */
2084         seqno = dev_priv->next_seqno;
2085         i915_gem_flush_ring(dev, NULL, &dev_priv->render_ring,
2086                             I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2087         ret = i915_wait_request(dev, seqno, &dev_priv->render_ring);
2088         if (ret)
2089                 return ret;
2090
2091         if (HAS_BSD(dev)) {
2092                 seqno = dev_priv->next_seqno;
2093                 i915_gem_flush_ring(dev, NULL, &dev_priv->bsd_ring,
2094                                     I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2095                 ret = i915_wait_request(dev, seqno, &dev_priv->bsd_ring);
2096                 if (ret)
2097                         return ret;
2098         }
2099
2100         return 0;
2101 }
2102
2103 int
2104 i915_gem_object_get_pages(struct drm_gem_object *obj,
2105                           gfp_t gfpmask)
2106 {
2107         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2108         int page_count, i;
2109         struct address_space *mapping;
2110         struct inode *inode;
2111         struct page *page;
2112
2113         BUG_ON(obj_priv->pages_refcount
2114                         == DRM_I915_GEM_OBJECT_MAX_PAGES_REFCOUNT);
2115
2116         if (obj_priv->pages_refcount++ != 0)
2117                 return 0;
2118
2119         /* Get the list of pages out of our struct file.  They'll be pinned
2120          * at this point until we release them.
2121          */
2122         page_count = obj->size / PAGE_SIZE;
2123         BUG_ON(obj_priv->pages != NULL);
2124         obj_priv->pages = drm_calloc_large(page_count, sizeof(struct page *));
2125         if (obj_priv->pages == NULL) {
2126                 obj_priv->pages_refcount--;
2127                 return -ENOMEM;
2128         }
2129
2130         inode = obj->filp->f_path.dentry->d_inode;
2131         mapping = inode->i_mapping;
2132         for (i = 0; i < page_count; i++) {
2133                 page = read_cache_page_gfp(mapping, i,
2134                                            GFP_HIGHUSER |
2135                                            __GFP_COLD |
2136                                            __GFP_RECLAIMABLE |
2137                                            gfpmask);
2138                 if (IS_ERR(page))
2139                         goto err_pages;
2140
2141                 obj_priv->pages[i] = page;
2142         }
2143
2144         if (obj_priv->tiling_mode != I915_TILING_NONE)
2145                 i915_gem_object_do_bit_17_swizzle(obj);
2146
2147         return 0;
2148
2149 err_pages:
2150         while (i--)
2151                 page_cache_release(obj_priv->pages[i]);
2152
2153         drm_free_large(obj_priv->pages);
2154         obj_priv->pages = NULL;
2155         obj_priv->pages_refcount--;
2156         return PTR_ERR(page);
2157 }
2158
2159 static void sandybridge_write_fence_reg(struct drm_i915_fence_reg *reg)
2160 {
2161         struct drm_gem_object *obj = reg->obj;
2162         struct drm_device *dev = obj->dev;
2163         drm_i915_private_t *dev_priv = dev->dev_private;
2164         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2165         int regnum = obj_priv->fence_reg;
2166         uint64_t val;
2167
2168         val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2169                     0xfffff000) << 32;
2170         val |= obj_priv->gtt_offset & 0xfffff000;
2171         val |= (uint64_t)((obj_priv->stride / 128) - 1) <<
2172                 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2173
2174         if (obj_priv->tiling_mode == I915_TILING_Y)
2175                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2176         val |= I965_FENCE_REG_VALID;
2177
2178         I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (regnum * 8), val);
2179 }
2180
2181 static void i965_write_fence_reg(struct drm_i915_fence_reg *reg)
2182 {
2183         struct drm_gem_object *obj = reg->obj;
2184         struct drm_device *dev = obj->dev;
2185         drm_i915_private_t *dev_priv = dev->dev_private;
2186         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2187         int regnum = obj_priv->fence_reg;
2188         uint64_t val;
2189
2190         val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2191                     0xfffff000) << 32;
2192         val |= obj_priv->gtt_offset & 0xfffff000;
2193         val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2194         if (obj_priv->tiling_mode == I915_TILING_Y)
2195                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2196         val |= I965_FENCE_REG_VALID;
2197
2198         I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val);
2199 }
2200
2201 static void i915_write_fence_reg(struct drm_i915_fence_reg *reg)
2202 {
2203         struct drm_gem_object *obj = reg->obj;
2204         struct drm_device *dev = obj->dev;
2205         drm_i915_private_t *dev_priv = dev->dev_private;
2206         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2207         int regnum = obj_priv->fence_reg;
2208         int tile_width;
2209         uint32_t fence_reg, val;
2210         uint32_t pitch_val;
2211
2212         if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) ||
2213             (obj_priv->gtt_offset & (obj->size - 1))) {
2214                 WARN(1, "%s: object 0x%08x not 1M or size (0x%zx) aligned\n",
2215                      __func__, obj_priv->gtt_offset, obj->size);
2216                 return;
2217         }
2218
2219         if (obj_priv->tiling_mode == I915_TILING_Y &&
2220             HAS_128_BYTE_Y_TILING(dev))
2221                 tile_width = 128;
2222         else
2223                 tile_width = 512;
2224
2225         /* Note: pitch better be a power of two tile widths */
2226         pitch_val = obj_priv->stride / tile_width;
2227         pitch_val = ffs(pitch_val) - 1;
2228
2229         if (obj_priv->tiling_mode == I915_TILING_Y &&
2230             HAS_128_BYTE_Y_TILING(dev))
2231                 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2232         else
2233                 WARN_ON(pitch_val > I915_FENCE_MAX_PITCH_VAL);
2234
2235         val = obj_priv->gtt_offset;
2236         if (obj_priv->tiling_mode == I915_TILING_Y)
2237                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2238         val |= I915_FENCE_SIZE_BITS(obj->size);
2239         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2240         val |= I830_FENCE_REG_VALID;
2241
2242         if (regnum < 8)
2243                 fence_reg = FENCE_REG_830_0 + (regnum * 4);
2244         else
2245                 fence_reg = FENCE_REG_945_8 + ((regnum - 8) * 4);
2246         I915_WRITE(fence_reg, val);
2247 }
2248
2249 static void i830_write_fence_reg(struct drm_i915_fence_reg *reg)
2250 {
2251         struct drm_gem_object *obj = reg->obj;
2252         struct drm_device *dev = obj->dev;
2253         drm_i915_private_t *dev_priv = dev->dev_private;
2254         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2255         int regnum = obj_priv->fence_reg;
2256         uint32_t val;
2257         uint32_t pitch_val;
2258         uint32_t fence_size_bits;
2259
2260         if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) ||
2261             (obj_priv->gtt_offset & (obj->size - 1))) {
2262                 WARN(1, "%s: object 0x%08x not 512K or size aligned\n",
2263                      __func__, obj_priv->gtt_offset);
2264                 return;
2265         }
2266
2267         pitch_val = obj_priv->stride / 128;
2268         pitch_val = ffs(pitch_val) - 1;
2269         WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2270
2271         val = obj_priv->gtt_offset;
2272         if (obj_priv->tiling_mode == I915_TILING_Y)
2273                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2274         fence_size_bits = I830_FENCE_SIZE_BITS(obj->size);
2275         WARN_ON(fence_size_bits & ~0x00000f00);
2276         val |= fence_size_bits;
2277         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2278         val |= I830_FENCE_REG_VALID;
2279
2280         I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val);
2281 }
2282
2283 static int i915_find_fence_reg(struct drm_device *dev,
2284                                bool interruptible)
2285 {
2286         struct drm_i915_fence_reg *reg = NULL;
2287         struct drm_i915_gem_object *obj_priv = NULL;
2288         struct drm_i915_private *dev_priv = dev->dev_private;
2289         struct drm_gem_object *obj = NULL;
2290         int i, avail, ret;
2291
2292         /* First try to find a free reg */
2293         avail = 0;
2294         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2295                 reg = &dev_priv->fence_regs[i];
2296                 if (!reg->obj)
2297                         return i;
2298
2299                 obj_priv = to_intel_bo(reg->obj);
2300                 if (!obj_priv->pin_count)
2301                     avail++;
2302         }
2303
2304         if (avail == 0)
2305                 return -ENOSPC;
2306
2307         /* None available, try to steal one or wait for a user to finish */
2308         i = I915_FENCE_REG_NONE;
2309         list_for_each_entry(reg, &dev_priv->mm.fence_list,
2310                             lru_list) {
2311                 obj = reg->obj;
2312                 obj_priv = to_intel_bo(obj);
2313
2314                 if (obj_priv->pin_count)
2315                         continue;
2316
2317                 /* found one! */
2318                 i = obj_priv->fence_reg;
2319                 break;
2320         }
2321
2322         BUG_ON(i == I915_FENCE_REG_NONE);
2323
2324         /* We only have a reference on obj from the active list. put_fence_reg
2325          * might drop that one, causing a use-after-free in it. So hold a
2326          * private reference to obj like the other callers of put_fence_reg
2327          * (set_tiling ioctl) do. */
2328         drm_gem_object_reference(obj);
2329         ret = i915_gem_object_put_fence_reg(obj, interruptible);
2330         drm_gem_object_unreference(obj);
2331         if (ret != 0)
2332                 return ret;
2333
2334         return i;
2335 }
2336
2337 /**
2338  * i915_gem_object_get_fence_reg - set up a fence reg for an object
2339  * @obj: object to map through a fence reg
2340  *
2341  * When mapping objects through the GTT, userspace wants to be able to write
2342  * to them without having to worry about swizzling if the object is tiled.
2343  *
2344  * This function walks the fence regs looking for a free one for @obj,
2345  * stealing one if it can't find any.
2346  *
2347  * It then sets up the reg based on the object's properties: address, pitch
2348  * and tiling format.
2349  */
2350 int
2351 i915_gem_object_get_fence_reg(struct drm_gem_object *obj,
2352                               bool interruptible)
2353 {
2354         struct drm_device *dev = obj->dev;
2355         struct drm_i915_private *dev_priv = dev->dev_private;
2356         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2357         struct drm_i915_fence_reg *reg = NULL;
2358         int ret;
2359
2360         /* Just update our place in the LRU if our fence is getting used. */
2361         if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
2362                 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2363                 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2364                 return 0;
2365         }
2366
2367         switch (obj_priv->tiling_mode) {
2368         case I915_TILING_NONE:
2369                 WARN(1, "allocating a fence for non-tiled object?\n");
2370                 break;
2371         case I915_TILING_X:
2372                 if (!obj_priv->stride)
2373                         return -EINVAL;
2374                 WARN((obj_priv->stride & (512 - 1)),
2375                      "object 0x%08x is X tiled but has non-512B pitch\n",
2376                      obj_priv->gtt_offset);
2377                 break;
2378         case I915_TILING_Y:
2379                 if (!obj_priv->stride)
2380                         return -EINVAL;
2381                 WARN((obj_priv->stride & (128 - 1)),
2382                      "object 0x%08x is Y tiled but has non-128B pitch\n",
2383                      obj_priv->gtt_offset);
2384                 break;
2385         }
2386
2387         ret = i915_find_fence_reg(dev, interruptible);
2388         if (ret < 0)
2389                 return ret;
2390
2391         obj_priv->fence_reg = ret;
2392         reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2393         list_add_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2394
2395         reg->obj = obj;
2396
2397         switch (INTEL_INFO(dev)->gen) {
2398         case 6:
2399                 sandybridge_write_fence_reg(reg);
2400                 break;
2401         case 5:
2402         case 4:
2403                 i965_write_fence_reg(reg);
2404                 break;
2405         case 3:
2406                 i915_write_fence_reg(reg);
2407                 break;
2408         case 2:
2409                 i830_write_fence_reg(reg);
2410                 break;
2411         }
2412
2413         trace_i915_gem_object_get_fence(obj, obj_priv->fence_reg,
2414                         obj_priv->tiling_mode);
2415
2416         return 0;
2417 }
2418
2419 /**
2420  * i915_gem_clear_fence_reg - clear out fence register info
2421  * @obj: object to clear
2422  *
2423  * Zeroes out the fence register itself and clears out the associated
2424  * data structures in dev_priv and obj_priv.
2425  */
2426 static void
2427 i915_gem_clear_fence_reg(struct drm_gem_object *obj)
2428 {
2429         struct drm_device *dev = obj->dev;
2430         drm_i915_private_t *dev_priv = dev->dev_private;
2431         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2432         struct drm_i915_fence_reg *reg =
2433                 &dev_priv->fence_regs[obj_priv->fence_reg];
2434         uint32_t fence_reg;
2435
2436         switch (INTEL_INFO(dev)->gen) {
2437         case 6:
2438                 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 +
2439                              (obj_priv->fence_reg * 8), 0);
2440                 break;
2441         case 5:
2442         case 4:
2443                 I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0);
2444                 break;
2445         case 3:
2446                 if (obj_priv->fence_reg > 8)
2447                         fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg - 8) * 4;
2448                 else
2449         case 2:
2450                         fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4;
2451
2452                 I915_WRITE(fence_reg, 0);
2453                 break;
2454         }
2455
2456         reg->obj = NULL;
2457         obj_priv->fence_reg = I915_FENCE_REG_NONE;
2458         list_del_init(&reg->lru_list);
2459 }
2460
2461 /**
2462  * i915_gem_object_put_fence_reg - waits on outstanding fenced access
2463  * to the buffer to finish, and then resets the fence register.
2464  * @obj: tiled object holding a fence register.
2465  * @bool: whether the wait upon the fence is interruptible
2466  *
2467  * Zeroes out the fence register itself and clears out the associated
2468  * data structures in dev_priv and obj_priv.
2469  */
2470 int
2471 i915_gem_object_put_fence_reg(struct drm_gem_object *obj,
2472                               bool interruptible)
2473 {
2474         struct drm_device *dev = obj->dev;
2475         struct drm_i915_private *dev_priv = dev->dev_private;
2476         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2477         struct drm_i915_fence_reg *reg;
2478
2479         if (obj_priv->fence_reg == I915_FENCE_REG_NONE)
2480                 return 0;
2481
2482         /* If we've changed tiling, GTT-mappings of the object
2483          * need to re-fault to ensure that the correct fence register
2484          * setup is in place.
2485          */
2486         i915_gem_release_mmap(obj);
2487
2488         /* On the i915, GPU access to tiled buffers is via a fence,
2489          * therefore we must wait for any outstanding access to complete
2490          * before clearing the fence.
2491          */
2492         reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2493         if (reg->gpu) {
2494                 int ret;
2495
2496                 ret = i915_gem_object_flush_gpu_write_domain(obj, true);
2497                 if (ret)
2498                         return ret;
2499
2500                 ret = i915_gem_object_wait_rendering(obj, interruptible);
2501                 if (ret)
2502                         return ret;
2503
2504                 reg->gpu = false;
2505         }
2506
2507         i915_gem_object_flush_gtt_write_domain(obj);
2508         i915_gem_clear_fence_reg(obj);
2509
2510         return 0;
2511 }
2512
2513 /**
2514  * Finds free space in the GTT aperture and binds the object there.
2515  */
2516 static int
2517 i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
2518 {
2519         struct drm_device *dev = obj->dev;
2520         drm_i915_private_t *dev_priv = dev->dev_private;
2521         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2522         struct drm_mm_node *free_space;
2523         gfp_t gfpmask =  __GFP_NORETRY | __GFP_NOWARN;
2524         int ret;
2525
2526         if (obj_priv->madv != I915_MADV_WILLNEED) {
2527                 DRM_ERROR("Attempting to bind a purgeable object\n");
2528                 return -EINVAL;
2529         }
2530
2531         if (alignment == 0)
2532                 alignment = i915_gem_get_gtt_alignment(obj);
2533         if (alignment & (i915_gem_get_gtt_alignment(obj) - 1)) {
2534                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2535                 return -EINVAL;
2536         }
2537
2538         /* If the object is bigger than the entire aperture, reject it early
2539          * before evicting everything in a vain attempt to find space.
2540          */
2541         if (obj->size > dev->gtt_total) {
2542                 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
2543                 return -E2BIG;
2544         }
2545
2546  search_free:
2547         free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2548                                         obj->size, alignment, 0);
2549         if (free_space != NULL) {
2550                 obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size,
2551                                                        alignment);
2552                 if (obj_priv->gtt_space != NULL)
2553                         obj_priv->gtt_offset = obj_priv->gtt_space->start;
2554         }
2555         if (obj_priv->gtt_space == NULL) {
2556                 /* If the gtt is empty and we're still having trouble
2557                  * fitting our object in, we're out of memory.
2558                  */
2559 #if WATCH_LRU
2560                 DRM_INFO("%s: GTT full, evicting something\n", __func__);
2561 #endif
2562                 ret = i915_gem_evict_something(dev, obj->size, alignment);
2563                 if (ret)
2564                         return ret;
2565
2566                 goto search_free;
2567         }
2568
2569 #if WATCH_BUF
2570         DRM_INFO("Binding object of size %zd at 0x%08x\n",
2571                  obj->size, obj_priv->gtt_offset);
2572 #endif
2573         ret = i915_gem_object_get_pages(obj, gfpmask);
2574         if (ret) {
2575                 drm_mm_put_block(obj_priv->gtt_space);
2576                 obj_priv->gtt_space = NULL;
2577
2578                 if (ret == -ENOMEM) {
2579                         /* first try to clear up some space from the GTT */
2580                         ret = i915_gem_evict_something(dev, obj->size,
2581                                                        alignment);
2582                         if (ret) {
2583                                 /* now try to shrink everyone else */
2584                                 if (gfpmask) {
2585                                         gfpmask = 0;
2586                                         goto search_free;
2587                                 }
2588
2589                                 return ret;
2590                         }
2591
2592                         goto search_free;
2593                 }
2594
2595                 return ret;
2596         }
2597
2598         /* Create an AGP memory structure pointing at our pages, and bind it
2599          * into the GTT.
2600          */
2601         obj_priv->agp_mem = drm_agp_bind_pages(dev,
2602                                                obj_priv->pages,
2603                                                obj->size >> PAGE_SHIFT,
2604                                                obj_priv->gtt_offset,
2605                                                obj_priv->agp_type);
2606         if (obj_priv->agp_mem == NULL) {
2607                 i915_gem_object_put_pages(obj);
2608                 drm_mm_put_block(obj_priv->gtt_space);
2609                 obj_priv->gtt_space = NULL;
2610
2611                 ret = i915_gem_evict_something(dev, obj->size, alignment);
2612                 if (ret)
2613                         return ret;
2614
2615                 goto search_free;
2616         }
2617         atomic_inc(&dev->gtt_count);
2618         atomic_add(obj->size, &dev->gtt_memory);
2619
2620         /* keep track of bounds object by adding it to the inactive list */
2621         list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
2622
2623         /* Assert that the object is not currently in any GPU domain. As it
2624          * wasn't in the GTT, there shouldn't be any way it could have been in
2625          * a GPU cache
2626          */
2627         BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
2628         BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
2629
2630         trace_i915_gem_object_bind(obj, obj_priv->gtt_offset);
2631
2632         return 0;
2633 }
2634
2635 void
2636 i915_gem_clflush_object(struct drm_gem_object *obj)
2637 {
2638         struct drm_i915_gem_object      *obj_priv = to_intel_bo(obj);
2639
2640         /* If we don't have a page list set up, then we're not pinned
2641          * to GPU, and we can ignore the cache flush because it'll happen
2642          * again at bind time.
2643          */
2644         if (obj_priv->pages == NULL)
2645                 return;
2646
2647         trace_i915_gem_object_clflush(obj);
2648
2649         drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE);
2650 }
2651
2652 /** Flushes any GPU write domain for the object if it's dirty. */
2653 static int
2654 i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj,
2655                                        bool pipelined)
2656 {
2657         struct drm_device *dev = obj->dev;
2658         uint32_t old_write_domain;
2659
2660         if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
2661                 return 0;
2662
2663         /* Queue the GPU write cache flushing we need. */
2664         old_write_domain = obj->write_domain;
2665         i915_gem_flush_ring(dev, NULL,
2666                             to_intel_bo(obj)->ring,
2667                             0, obj->write_domain);
2668         BUG_ON(obj->write_domain);
2669
2670         trace_i915_gem_object_change_domain(obj,
2671                                             obj->read_domains,
2672                                             old_write_domain);
2673
2674         if (pipelined)
2675                 return 0;
2676
2677         return i915_gem_object_wait_rendering(obj, true);
2678 }
2679
2680 /** Flushes the GTT write domain for the object if it's dirty. */
2681 static void
2682 i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
2683 {
2684         uint32_t old_write_domain;
2685
2686         if (obj->write_domain != I915_GEM_DOMAIN_GTT)
2687                 return;
2688
2689         /* No actual flushing is required for the GTT write domain.   Writes
2690          * to it immediately go to main memory as far as we know, so there's
2691          * no chipset flush.  It also doesn't land in render cache.
2692          */
2693         old_write_domain = obj->write_domain;
2694         obj->write_domain = 0;
2695
2696         trace_i915_gem_object_change_domain(obj,
2697                                             obj->read_domains,
2698                                             old_write_domain);
2699 }
2700
2701 /** Flushes the CPU write domain for the object if it's dirty. */
2702 static void
2703 i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
2704 {
2705         struct drm_device *dev = obj->dev;
2706         uint32_t old_write_domain;
2707
2708         if (obj->write_domain != I915_GEM_DOMAIN_CPU)
2709                 return;
2710
2711         i915_gem_clflush_object(obj);
2712         drm_agp_chipset_flush(dev);
2713         old_write_domain = obj->write_domain;
2714         obj->write_domain = 0;
2715
2716         trace_i915_gem_object_change_domain(obj,
2717                                             obj->read_domains,
2718                                             old_write_domain);
2719 }
2720
2721 /**
2722  * Moves a single object to the GTT read, and possibly write domain.
2723  *
2724  * This function returns when the move is complete, including waiting on
2725  * flushes to occur.
2726  */
2727 int
2728 i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
2729 {
2730         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2731         uint32_t old_write_domain, old_read_domains;
2732         int ret;
2733
2734         /* Not valid to be called on unbound objects. */
2735         if (obj_priv->gtt_space == NULL)
2736                 return -EINVAL;
2737
2738         ret = i915_gem_object_flush_gpu_write_domain(obj, false);
2739         if (ret != 0)
2740                 return ret;
2741
2742         i915_gem_object_flush_cpu_write_domain(obj);
2743
2744         if (write) {
2745                 ret = i915_gem_object_wait_rendering(obj, true);
2746                 if (ret)
2747                         return ret;
2748         }
2749
2750         old_write_domain = obj->write_domain;
2751         old_read_domains = obj->read_domains;
2752
2753         /* It should now be out of any other write domains, and we can update
2754          * the domain values for our changes.
2755          */
2756         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2757         obj->read_domains |= I915_GEM_DOMAIN_GTT;
2758         if (write) {
2759                 obj->read_domains = I915_GEM_DOMAIN_GTT;
2760                 obj->write_domain = I915_GEM_DOMAIN_GTT;
2761                 obj_priv->dirty = 1;
2762         }
2763
2764         trace_i915_gem_object_change_domain(obj,
2765                                             old_read_domains,
2766                                             old_write_domain);
2767
2768         return 0;
2769 }
2770
2771 /*
2772  * Prepare buffer for display plane. Use uninterruptible for possible flush
2773  * wait, as in modesetting process we're not supposed to be interrupted.
2774  */
2775 int
2776 i915_gem_object_set_to_display_plane(struct drm_gem_object *obj,
2777                                      bool pipelined)
2778 {
2779         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2780         uint32_t old_read_domains;
2781         int ret;
2782
2783         /* Not valid to be called on unbound objects. */
2784         if (obj_priv->gtt_space == NULL)
2785                 return -EINVAL;
2786
2787         ret = i915_gem_object_flush_gpu_write_domain(obj, pipelined);
2788         if (ret)
2789                 return ret;
2790
2791         i915_gem_object_flush_cpu_write_domain(obj);
2792
2793         old_read_domains = obj->read_domains;
2794         obj->read_domains |= I915_GEM_DOMAIN_GTT;
2795
2796         trace_i915_gem_object_change_domain(obj,
2797                                             old_read_domains,
2798                                             obj->write_domain);
2799
2800         return 0;
2801 }
2802
2803 /**
2804  * Moves a single object to the CPU read, and possibly write domain.
2805  *
2806  * This function returns when the move is complete, including waiting on
2807  * flushes to occur.
2808  */
2809 static int
2810 i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
2811 {
2812         uint32_t old_write_domain, old_read_domains;
2813         int ret;
2814
2815         ret = i915_gem_object_flush_gpu_write_domain(obj, false);
2816         if (ret != 0)
2817                 return ret;
2818
2819         i915_gem_object_flush_gtt_write_domain(obj);
2820
2821         /* If we have a partially-valid cache of the object in the CPU,
2822          * finish invalidating it and free the per-page flags.
2823          */
2824         i915_gem_object_set_to_full_cpu_read_domain(obj);
2825
2826         if (write) {
2827                 ret = i915_gem_object_wait_rendering(obj, true);
2828                 if (ret)
2829                         return ret;
2830         }
2831
2832         old_write_domain = obj->write_domain;
2833         old_read_domains = obj->read_domains;
2834
2835         /* Flush the CPU cache if it's still invalid. */
2836         if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2837                 i915_gem_clflush_object(obj);
2838
2839                 obj->read_domains |= I915_GEM_DOMAIN_CPU;
2840         }
2841
2842         /* It should now be out of any other write domains, and we can update
2843          * the domain values for our changes.
2844          */
2845         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2846
2847         /* If we're writing through the CPU, then the GPU read domains will
2848          * need to be invalidated at next use.
2849          */
2850         if (write) {
2851                 obj->read_domains = I915_GEM_DOMAIN_CPU;
2852                 obj->write_domain = I915_GEM_DOMAIN_CPU;
2853         }
2854
2855         trace_i915_gem_object_change_domain(obj,
2856                                             old_read_domains,
2857                                             old_write_domain);
2858
2859         return 0;
2860 }
2861
2862 /*
2863  * Set the next domain for the specified object. This
2864  * may not actually perform the necessary flushing/invaliding though,
2865  * as that may want to be batched with other set_domain operations
2866  *
2867  * This is (we hope) the only really tricky part of gem. The goal
2868  * is fairly simple -- track which caches hold bits of the object
2869  * and make sure they remain coherent. A few concrete examples may
2870  * help to explain how it works. For shorthand, we use the notation
2871  * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
2872  * a pair of read and write domain masks.
2873  *
2874  * Case 1: the batch buffer
2875  *
2876  *      1. Allocated
2877  *      2. Written by CPU
2878  *      3. Mapped to GTT
2879  *      4. Read by GPU
2880  *      5. Unmapped from GTT
2881  *      6. Freed
2882  *
2883  *      Let's take these a step at a time
2884  *
2885  *      1. Allocated
2886  *              Pages allocated from the kernel may still have
2887  *              cache contents, so we set them to (CPU, CPU) always.
2888  *      2. Written by CPU (using pwrite)
2889  *              The pwrite function calls set_domain (CPU, CPU) and
2890  *              this function does nothing (as nothing changes)
2891  *      3. Mapped by GTT
2892  *              This function asserts that the object is not
2893  *              currently in any GPU-based read or write domains
2894  *      4. Read by GPU
2895  *              i915_gem_execbuffer calls set_domain (COMMAND, 0).
2896  *              As write_domain is zero, this function adds in the
2897  *              current read domains (CPU+COMMAND, 0).
2898  *              flush_domains is set to CPU.
2899  *              invalidate_domains is set to COMMAND
2900  *              clflush is run to get data out of the CPU caches
2901  *              then i915_dev_set_domain calls i915_gem_flush to
2902  *              emit an MI_FLUSH and drm_agp_chipset_flush
2903  *      5. Unmapped from GTT
2904  *              i915_gem_object_unbind calls set_domain (CPU, CPU)
2905  *              flush_domains and invalidate_domains end up both zero
2906  *              so no flushing/invalidating happens
2907  *      6. Freed
2908  *              yay, done
2909  *
2910  * Case 2: The shared render buffer
2911  *
2912  *      1. Allocated
2913  *      2. Mapped to GTT
2914  *      3. Read/written by GPU
2915  *      4. set_domain to (CPU,CPU)
2916  *      5. Read/written by CPU
2917  *      6. Read/written by GPU
2918  *
2919  *      1. Allocated
2920  *              Same as last example, (CPU, CPU)
2921  *      2. Mapped to GTT
2922  *              Nothing changes (assertions find that it is not in the GPU)
2923  *      3. Read/written by GPU
2924  *              execbuffer calls set_domain (RENDER, RENDER)
2925  *              flush_domains gets CPU
2926  *              invalidate_domains gets GPU
2927  *              clflush (obj)
2928  *              MI_FLUSH and drm_agp_chipset_flush
2929  *      4. set_domain (CPU, CPU)
2930  *              flush_domains gets GPU
2931  *              invalidate_domains gets CPU
2932  *              wait_rendering (obj) to make sure all drawing is complete.
2933  *              This will include an MI_FLUSH to get the data from GPU
2934  *              to memory
2935  *              clflush (obj) to invalidate the CPU cache
2936  *              Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
2937  *      5. Read/written by CPU
2938  *              cache lines are loaded and dirtied
2939  *      6. Read written by GPU
2940  *              Same as last GPU access
2941  *
2942  * Case 3: The constant buffer
2943  *
2944  *      1. Allocated
2945  *      2. Written by CPU
2946  *      3. Read by GPU
2947  *      4. Updated (written) by CPU again
2948  *      5. Read by GPU
2949  *
2950  *      1. Allocated
2951  *              (CPU, CPU)
2952  *      2. Written by CPU
2953  *              (CPU, CPU)
2954  *      3. Read by GPU
2955  *              (CPU+RENDER, 0)
2956  *              flush_domains = CPU
2957  *              invalidate_domains = RENDER
2958  *              clflush (obj)
2959  *              MI_FLUSH
2960  *              drm_agp_chipset_flush
2961  *      4. Updated (written) by CPU again
2962  *              (CPU, CPU)
2963  *              flush_domains = 0 (no previous write domain)
2964  *              invalidate_domains = 0 (no new read domains)
2965  *      5. Read by GPU
2966  *              (CPU+RENDER, 0)
2967  *              flush_domains = CPU
2968  *              invalidate_domains = RENDER
2969  *              clflush (obj)
2970  *              MI_FLUSH
2971  *              drm_agp_chipset_flush
2972  */
2973 static void
2974 i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
2975 {
2976         struct drm_device               *dev = obj->dev;
2977         struct drm_i915_private         *dev_priv = dev->dev_private;
2978         struct drm_i915_gem_object      *obj_priv = to_intel_bo(obj);
2979         uint32_t                        invalidate_domains = 0;
2980         uint32_t                        flush_domains = 0;
2981         uint32_t                        old_read_domains;
2982
2983         BUG_ON(obj->pending_read_domains & I915_GEM_DOMAIN_CPU);
2984         BUG_ON(obj->pending_write_domain == I915_GEM_DOMAIN_CPU);
2985
2986         intel_mark_busy(dev, obj);
2987
2988 #if WATCH_BUF
2989         DRM_INFO("%s: object %p read %08x -> %08x write %08x -> %08x\n",
2990                  __func__, obj,
2991                  obj->read_domains, obj->pending_read_domains,
2992                  obj->write_domain, obj->pending_write_domain);
2993 #endif
2994         /*
2995          * If the object isn't moving to a new write domain,
2996          * let the object stay in multiple read domains
2997          */
2998         if (obj->pending_write_domain == 0)
2999                 obj->pending_read_domains |= obj->read_domains;
3000         else
3001                 obj_priv->dirty = 1;
3002
3003         /*
3004          * Flush the current write domain if
3005          * the new read domains don't match. Invalidate
3006          * any read domains which differ from the old
3007          * write domain
3008          */
3009         if (obj->write_domain &&
3010             obj->write_domain != obj->pending_read_domains) {
3011                 flush_domains |= obj->write_domain;
3012                 invalidate_domains |=
3013                         obj->pending_read_domains & ~obj->write_domain;
3014         }
3015         /*
3016          * Invalidate any read caches which may have
3017          * stale data. That is, any new read domains.
3018          */
3019         invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
3020         if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) {
3021 #if WATCH_BUF
3022                 DRM_INFO("%s: CPU domain flush %08x invalidate %08x\n",
3023                          __func__, flush_domains, invalidate_domains);
3024 #endif
3025                 i915_gem_clflush_object(obj);
3026         }
3027
3028         old_read_domains = obj->read_domains;
3029
3030         /* The actual obj->write_domain will be updated with
3031          * pending_write_domain after we emit the accumulated flush for all
3032          * of our domain changes in execbuffers (which clears objects'
3033          * write_domains).  So if we have a current write domain that we
3034          * aren't changing, set pending_write_domain to that.
3035          */
3036         if (flush_domains == 0 && obj->pending_write_domain == 0)
3037                 obj->pending_write_domain = obj->write_domain;
3038         obj->read_domains = obj->pending_read_domains;
3039
3040         dev->invalidate_domains |= invalidate_domains;
3041         dev->flush_domains |= flush_domains;
3042         if (obj_priv->ring)
3043                 dev_priv->mm.flush_rings |= obj_priv->ring->id;
3044 #if WATCH_BUF
3045         DRM_INFO("%s: read %08x write %08x invalidate %08x flush %08x\n",
3046                  __func__,
3047                  obj->read_domains, obj->write_domain,
3048                  dev->invalidate_domains, dev->flush_domains);
3049 #endif
3050
3051         trace_i915_gem_object_change_domain(obj,
3052                                             old_read_domains,
3053                                             obj->write_domain);
3054 }
3055
3056 /**
3057  * Moves the object from a partially CPU read to a full one.
3058  *
3059  * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
3060  * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
3061  */
3062 static void
3063 i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
3064 {
3065         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3066
3067         if (!obj_priv->page_cpu_valid)
3068                 return;
3069
3070         /* If we're partially in the CPU read domain, finish moving it in.
3071          */
3072         if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
3073                 int i;
3074
3075                 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
3076                         if (obj_priv->page_cpu_valid[i])
3077                                 continue;
3078                         drm_clflush_pages(obj_priv->pages + i, 1);
3079                 }
3080         }
3081
3082         /* Free the page_cpu_valid mappings which are now stale, whether
3083          * or not we've got I915_GEM_DOMAIN_CPU.
3084          */
3085         kfree(obj_priv->page_cpu_valid);
3086         obj_priv->page_cpu_valid = NULL;
3087 }
3088
3089 /**
3090  * Set the CPU read domain on a range of the object.
3091  *
3092  * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
3093  * not entirely valid.  The page_cpu_valid member of the object flags which
3094  * pages have been flushed, and will be respected by
3095  * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
3096  * of the whole object.
3097  *
3098  * This function returns when the move is complete, including waiting on
3099  * flushes to occur.
3100  */
3101 static int
3102 i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
3103                                           uint64_t offset, uint64_t size)
3104 {
3105         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3106         uint32_t old_read_domains;
3107         int i, ret;
3108
3109         if (offset == 0 && size == obj->size)
3110                 return i915_gem_object_set_to_cpu_domain(obj, 0);
3111
3112         ret = i915_gem_object_flush_gpu_write_domain(obj, false);
3113         if (ret != 0)
3114                 return ret;
3115         i915_gem_object_flush_gtt_write_domain(obj);
3116
3117         /* If we're already fully in the CPU read domain, we're done. */
3118         if (obj_priv->page_cpu_valid == NULL &&
3119             (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
3120                 return 0;
3121
3122         /* Otherwise, create/clear the per-page CPU read domain flag if we're
3123          * newly adding I915_GEM_DOMAIN_CPU
3124          */
3125         if (obj_priv->page_cpu_valid == NULL) {
3126                 obj_priv->page_cpu_valid = kzalloc(obj->size / PAGE_SIZE,
3127                                                    GFP_KERNEL);
3128                 if (obj_priv->page_cpu_valid == NULL)
3129                         return -ENOMEM;
3130         } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
3131                 memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
3132
3133         /* Flush the cache on any pages that are still invalid from the CPU's
3134          * perspective.
3135          */
3136         for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3137              i++) {
3138                 if (obj_priv->page_cpu_valid[i])
3139                         continue;
3140
3141                 drm_clflush_pages(obj_priv->pages + i, 1);
3142
3143                 obj_priv->page_cpu_valid[i] = 1;
3144         }
3145
3146         /* It should now be out of any other write domains, and we can update
3147          * the domain values for our changes.
3148          */
3149         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3150
3151         old_read_domains = obj->read_domains;
3152         obj->read_domains |= I915_GEM_DOMAIN_CPU;
3153
3154         trace_i915_gem_object_change_domain(obj,
3155                                             old_read_domains,
3156                                             obj->write_domain);
3157
3158         return 0;
3159 }
3160
3161 /**
3162  * Pin an object to the GTT and evaluate the relocations landing in it.
3163  */
3164 static int
3165 i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
3166                                  struct drm_file *file_priv,
3167                                  struct drm_i915_gem_exec_object2 *entry,
3168                                  struct drm_i915_gem_relocation_entry *relocs)
3169 {
3170         struct drm_device *dev = obj->dev;
3171         drm_i915_private_t *dev_priv = dev->dev_private;
3172         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3173         int i, ret;
3174         void __iomem *reloc_page;
3175         bool need_fence;
3176
3177         need_fence = entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
3178                      obj_priv->tiling_mode != I915_TILING_NONE;
3179
3180         /* Check fence reg constraints and rebind if necessary */
3181         if (need_fence &&
3182             !i915_gem_object_fence_offset_ok(obj,
3183                                              obj_priv->tiling_mode)) {
3184                 ret = i915_gem_object_unbind(obj);
3185                 if (ret)
3186                         return ret;
3187         }
3188
3189         /* Choose the GTT offset for our buffer and put it there. */
3190         ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment);
3191         if (ret)
3192                 return ret;
3193
3194         /*
3195          * Pre-965 chips need a fence register set up in order to
3196          * properly handle blits to/from tiled surfaces.
3197          */
3198         if (need_fence) {
3199                 ret = i915_gem_object_get_fence_reg(obj, true);
3200                 if (ret != 0) {
3201                         i915_gem_object_unpin(obj);
3202                         return ret;
3203                 }
3204
3205                 dev_priv->fence_regs[obj_priv->fence_reg].gpu = true;
3206         }
3207
3208         entry->offset = obj_priv->gtt_offset;
3209
3210         /* Apply the relocations, using the GTT aperture to avoid cache
3211          * flushing requirements.
3212          */
3213         for (i = 0; i < entry->relocation_count; i++) {
3214                 struct drm_i915_gem_relocation_entry *reloc= &relocs[i];
3215                 struct drm_gem_object *target_obj;
3216                 struct drm_i915_gem_object *target_obj_priv;
3217                 uint32_t reloc_val, reloc_offset;
3218                 uint32_t __iomem *reloc_entry;
3219
3220                 target_obj = drm_gem_object_lookup(obj->dev, file_priv,
3221                                                    reloc->target_handle);
3222                 if (target_obj == NULL) {
3223                         i915_gem_object_unpin(obj);
3224                         return -ENOENT;
3225                 }
3226                 target_obj_priv = to_intel_bo(target_obj);
3227
3228 #if WATCH_RELOC
3229                 DRM_INFO("%s: obj %p offset %08x target %d "
3230                          "read %08x write %08x gtt %08x "
3231                          "presumed %08x delta %08x\n",
3232                          __func__,
3233                          obj,
3234                          (int) reloc->offset,
3235                          (int) reloc->target_handle,
3236                          (int) reloc->read_domains,
3237                          (int) reloc->write_domain,
3238                          (int) target_obj_priv->gtt_offset,
3239                          (int) reloc->presumed_offset,
3240                          reloc->delta);
3241 #endif
3242
3243                 /* The target buffer should have appeared before us in the
3244                  * exec_object list, so it should have a GTT space bound by now.
3245                  */
3246                 if (target_obj_priv->gtt_space == NULL) {
3247                         DRM_ERROR("No GTT space found for object %d\n",
3248                                   reloc->target_handle);
3249                         drm_gem_object_unreference(target_obj);
3250                         i915_gem_object_unpin(obj);
3251                         return -EINVAL;
3252                 }
3253
3254                 /* Validate that the target is in a valid r/w GPU domain */
3255                 if (reloc->write_domain & (reloc->write_domain - 1)) {
3256                         DRM_ERROR("reloc with multiple write domains: "
3257                                   "obj %p target %d offset %d "
3258                                   "read %08x write %08x",
3259                                   obj, reloc->target_handle,
3260                                   (int) reloc->offset,
3261                                   reloc->read_domains,
3262                                   reloc->write_domain);
3263                         return -EINVAL;
3264                 }
3265                 if (reloc->write_domain & I915_GEM_DOMAIN_CPU ||
3266                     reloc->read_domains & I915_GEM_DOMAIN_CPU) {
3267                         DRM_ERROR("reloc with read/write CPU domains: "
3268                                   "obj %p target %d offset %d "
3269                                   "read %08x write %08x",
3270                                   obj, reloc->target_handle,
3271                                   (int) reloc->offset,
3272                                   reloc->read_domains,
3273                                   reloc->write_domain);
3274                         drm_gem_object_unreference(target_obj);
3275                         i915_gem_object_unpin(obj);
3276                         return -EINVAL;
3277                 }
3278                 if (reloc->write_domain && target_obj->pending_write_domain &&
3279                     reloc->write_domain != target_obj->pending_write_domain) {
3280                         DRM_ERROR("Write domain conflict: "
3281                                   "obj %p target %d offset %d "
3282                                   "new %08x old %08x\n",
3283                                   obj, reloc->target_handle,
3284                                   (int) reloc->offset,
3285                                   reloc->write_domain,
3286                                   target_obj->pending_write_domain);
3287                         drm_gem_object_unreference(target_obj);
3288                         i915_gem_object_unpin(obj);
3289                         return -EINVAL;
3290                 }
3291
3292                 target_obj->pending_read_domains |= reloc->read_domains;
3293                 target_obj->pending_write_domain |= reloc->write_domain;
3294
3295                 /* If the relocation already has the right value in it, no
3296                  * more work needs to be done.
3297                  */
3298                 if (target_obj_priv->gtt_offset == reloc->presumed_offset) {
3299                         drm_gem_object_unreference(target_obj);
3300                         continue;
3301                 }
3302
3303                 /* Check that the relocation address is valid... */
3304                 if (reloc->offset > obj->size - 4) {
3305                         DRM_ERROR("Relocation beyond object bounds: "
3306                                   "obj %p target %d offset %d size %d.\n",
3307                                   obj, reloc->target_handle,
3308                                   (int) reloc->offset, (int) obj->size);
3309                         drm_gem_object_unreference(target_obj);
3310                         i915_gem_object_unpin(obj);
3311                         return -EINVAL;
3312                 }
3313                 if (reloc->offset & 3) {
3314                         DRM_ERROR("Relocation not 4-byte aligned: "
3315                                   "obj %p target %d offset %d.\n",
3316                                   obj, reloc->target_handle,
3317                                   (int) reloc->offset);
3318                         drm_gem_object_unreference(target_obj);
3319                         i915_gem_object_unpin(obj);
3320                         return -EINVAL;
3321                 }
3322
3323                 /* and points to somewhere within the target object. */
3324                 if (reloc->delta >= target_obj->size) {
3325                         DRM_ERROR("Relocation beyond target object bounds: "
3326                                   "obj %p target %d delta %d size %d.\n",
3327                                   obj, reloc->target_handle,
3328                                   (int) reloc->delta, (int) target_obj->size);
3329                         drm_gem_object_unreference(target_obj);
3330                         i915_gem_object_unpin(obj);
3331                         return -EINVAL;
3332                 }
3333
3334                 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
3335                 if (ret != 0) {
3336                         drm_gem_object_unreference(target_obj);
3337                         i915_gem_object_unpin(obj);
3338                         return -EINVAL;
3339                 }
3340
3341                 /* Map the page containing the relocation we're going to
3342                  * perform.
3343                  */
3344                 reloc_offset = obj_priv->gtt_offset + reloc->offset;
3345                 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
3346                                                       (reloc_offset &
3347                                                        ~(PAGE_SIZE - 1)),
3348                                                       KM_USER0);
3349                 reloc_entry = (uint32_t __iomem *)(reloc_page +
3350                                                    (reloc_offset & (PAGE_SIZE - 1)));
3351                 reloc_val = target_obj_priv->gtt_offset + reloc->delta;
3352
3353 #if WATCH_BUF
3354                 DRM_INFO("Applied relocation: %p@0x%08x %08x -> %08x\n",
3355                           obj, (unsigned int) reloc->offset,
3356                           readl(reloc_entry), reloc_val);
3357 #endif
3358                 writel(reloc_val, reloc_entry);
3359                 io_mapping_unmap_atomic(reloc_page, KM_USER0);
3360
3361                 /* The updated presumed offset for this entry will be
3362                  * copied back out to the user.
3363                  */
3364                 reloc->presumed_offset = target_obj_priv->gtt_offset;
3365
3366                 drm_gem_object_unreference(target_obj);
3367         }
3368
3369 #if WATCH_BUF
3370         if (0)
3371                 i915_gem_dump_object(obj, 128, __func__, ~0);
3372 #endif
3373         return 0;
3374 }
3375
3376 /* Throttle our rendering by waiting until the ring has completed our requests
3377  * emitted over 20 msec ago.
3378  *
3379  * Note that if we were to use the current jiffies each time around the loop,
3380  * we wouldn't escape the function with any frames outstanding if the time to
3381  * render a frame was over 20ms.
3382  *
3383  * This should get us reasonable parallelism between CPU and GPU but also
3384  * relatively low latency when blocking on a particular request to finish.
3385  */
3386 static int
3387 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file_priv)
3388 {
3389         struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
3390         int ret = 0;
3391         unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
3392
3393         mutex_lock(&dev->struct_mutex);
3394         while (!list_empty(&i915_file_priv->mm.request_list)) {
3395                 struct drm_i915_gem_request *request;
3396
3397                 request = list_first_entry(&i915_file_priv->mm.request_list,
3398                                            struct drm_i915_gem_request,
3399                                            client_list);
3400
3401                 if (time_after_eq(request->emitted_jiffies, recent_enough))
3402                         break;
3403
3404                 ret = i915_wait_request(dev, request->seqno, request->ring);
3405                 if (ret != 0)
3406                         break;
3407         }
3408         mutex_unlock(&dev->struct_mutex);
3409
3410         return ret;
3411 }
3412
3413 static int
3414 i915_gem_get_relocs_from_user(struct drm_i915_gem_exec_object2 *exec_list,
3415                               uint32_t buffer_count,
3416                               struct drm_i915_gem_relocation_entry **relocs)
3417 {
3418         uint32_t reloc_count = 0, reloc_index = 0, i;
3419         int ret;
3420
3421         *relocs = NULL;
3422         for (i = 0; i < buffer_count; i++) {
3423                 if (reloc_count + exec_list[i].relocation_count < reloc_count)
3424                         return -EINVAL;
3425                 reloc_count += exec_list[i].relocation_count;
3426         }
3427
3428         *relocs = drm_calloc_large(reloc_count, sizeof(**relocs));
3429         if (*relocs == NULL) {
3430                 DRM_ERROR("failed to alloc relocs, count %d\n", reloc_count);
3431                 return -ENOMEM;
3432         }
3433
3434         for (i = 0; i < buffer_count; i++) {
3435                 struct drm_i915_gem_relocation_entry __user *user_relocs;
3436
3437                 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3438
3439                 ret = copy_from_user(&(*relocs)[reloc_index],
3440                                      user_relocs,
3441                                      exec_list[i].relocation_count *
3442                                      sizeof(**relocs));
3443                 if (ret != 0) {
3444                         drm_free_large(*relocs);
3445                         *relocs = NULL;
3446                         return -EFAULT;
3447                 }
3448
3449                 reloc_index += exec_list[i].relocation_count;
3450         }
3451
3452         return 0;
3453 }
3454
3455 static int
3456 i915_gem_put_relocs_to_user(struct drm_i915_gem_exec_object2 *exec_list,
3457                             uint32_t buffer_count,
3458                             struct drm_i915_gem_relocation_entry *relocs)
3459 {
3460         uint32_t reloc_count = 0, i;
3461         int ret = 0;
3462
3463         if (relocs == NULL)
3464             return 0;
3465
3466         for (i = 0; i < buffer_count; i++) {
3467                 struct drm_i915_gem_relocation_entry __user *user_relocs;
3468                 int unwritten;
3469
3470                 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3471
3472                 unwritten = copy_to_user(user_relocs,
3473                                          &relocs[reloc_count],
3474                                          exec_list[i].relocation_count *
3475                                          sizeof(*relocs));
3476
3477                 if (unwritten) {
3478                         ret = -EFAULT;
3479                         goto err;
3480                 }
3481
3482                 reloc_count += exec_list[i].relocation_count;
3483         }
3484
3485 err:
3486         drm_free_large(relocs);
3487
3488         return ret;
3489 }
3490
3491 static int
3492 i915_gem_check_execbuffer (struct drm_i915_gem_execbuffer2 *exec,
3493                            uint64_t exec_offset)
3494 {
3495         uint32_t exec_start, exec_len;
3496
3497         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3498         exec_len = (uint32_t) exec->batch_len;
3499
3500         if ((exec_start | exec_len) & 0x7)
3501                 return -EINVAL;
3502
3503         if (!exec_start)
3504                 return -EINVAL;
3505
3506         return 0;
3507 }
3508
3509 static int
3510 i915_gem_wait_for_pending_flip(struct drm_device *dev,
3511                                struct drm_gem_object **object_list,
3512                                int count)
3513 {
3514         drm_i915_private_t *dev_priv = dev->dev_private;
3515         struct drm_i915_gem_object *obj_priv;
3516         DEFINE_WAIT(wait);
3517         int i, ret = 0;
3518
3519         for (;;) {
3520                 prepare_to_wait(&dev_priv->pending_flip_queue,
3521                                 &wait, TASK_INTERRUPTIBLE);
3522                 for (i = 0; i < count; i++) {
3523                         obj_priv = to_intel_bo(object_list[i]);
3524                         if (atomic_read(&obj_priv->pending_flip) > 0)
3525                                 break;
3526                 }
3527                 if (i == count)
3528                         break;
3529
3530                 if (!signal_pending(current)) {
3531                         mutex_unlock(&dev->struct_mutex);
3532                         schedule();
3533                         mutex_lock(&dev->struct_mutex);
3534                         continue;
3535                 }
3536                 ret = -ERESTARTSYS;
3537                 break;
3538         }
3539         finish_wait(&dev_priv->pending_flip_queue, &wait);
3540
3541         return ret;
3542 }
3543
3544 static int
3545 i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3546                        struct drm_file *file_priv,
3547                        struct drm_i915_gem_execbuffer2 *args,
3548                        struct drm_i915_gem_exec_object2 *exec_list)
3549 {
3550         drm_i915_private_t *dev_priv = dev->dev_private;
3551         struct drm_gem_object **object_list = NULL;
3552         struct drm_gem_object *batch_obj;
3553         struct drm_i915_gem_object *obj_priv;
3554         struct drm_clip_rect *cliprects = NULL;
3555         struct drm_i915_gem_relocation_entry *relocs = NULL;
3556         struct drm_i915_gem_request *request = NULL;
3557         int ret = 0, ret2, i, pinned = 0;
3558         uint64_t exec_offset;
3559         uint32_t reloc_index;
3560         int pin_tries, flips;
3561
3562         struct intel_ring_buffer *ring = NULL;
3563
3564 #if WATCH_EXEC
3565         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3566                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3567 #endif
3568         if (args->flags & I915_EXEC_BSD) {
3569                 if (!HAS_BSD(dev)) {
3570                         DRM_ERROR("execbuf with wrong flag\n");
3571                         return -EINVAL;
3572                 }
3573                 ring = &dev_priv->bsd_ring;
3574         } else {
3575                 ring = &dev_priv->render_ring;
3576         }
3577
3578         if (args->buffer_count < 1) {
3579                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3580                 return -EINVAL;
3581         }
3582         object_list = drm_malloc_ab(sizeof(*object_list), args->buffer_count);
3583         if (object_list == NULL) {
3584                 DRM_ERROR("Failed to allocate object list for %d buffers\n",
3585                           args->buffer_count);
3586                 ret = -ENOMEM;
3587                 goto pre_mutex_err;
3588         }
3589
3590         if (args->num_cliprects != 0) {
3591                 cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects),
3592                                     GFP_KERNEL);
3593                 if (cliprects == NULL) {
3594                         ret = -ENOMEM;
3595                         goto pre_mutex_err;
3596                 }
3597
3598                 ret = copy_from_user(cliprects,
3599                                      (struct drm_clip_rect __user *)
3600                                      (uintptr_t) args->cliprects_ptr,
3601                                      sizeof(*cliprects) * args->num_cliprects);
3602                 if (ret != 0) {
3603                         DRM_ERROR("copy %d cliprects failed: %d\n",
3604                                   args->num_cliprects, ret);
3605                         ret = -EFAULT;
3606                         goto pre_mutex_err;
3607                 }
3608         }
3609
3610         request = kzalloc(sizeof(*request), GFP_KERNEL);
3611         if (request == NULL) {
3612                 ret = -ENOMEM;
3613                 goto pre_mutex_err;
3614         }
3615
3616         ret = i915_gem_get_relocs_from_user(exec_list, args->buffer_count,
3617                                             &relocs);
3618         if (ret != 0)
3619                 goto pre_mutex_err;
3620
3621         mutex_lock(&dev->struct_mutex);
3622
3623         i915_verify_inactive(dev, __FILE__, __LINE__);
3624
3625         if (atomic_read(&dev_priv->mm.wedged)) {
3626                 mutex_unlock(&dev->struct_mutex);
3627                 ret = -EIO;
3628                 goto pre_mutex_err;
3629         }
3630
3631         if (dev_priv->mm.suspended) {
3632                 mutex_unlock(&dev->struct_mutex);
3633                 ret = -EBUSY;
3634                 goto pre_mutex_err;
3635         }
3636
3637         /* Look up object handles */
3638         flips = 0;
3639         for (i = 0; i < args->buffer_count; i++) {
3640                 object_list[i] = drm_gem_object_lookup(dev, file_priv,
3641                                                        exec_list[i].handle);
3642                 if (object_list[i] == NULL) {
3643                         DRM_ERROR("Invalid object handle %d at index %d\n",
3644                                    exec_list[i].handle, i);
3645                         /* prevent error path from reading uninitialized data */
3646                         args->buffer_count = i + 1;
3647                         ret = -ENOENT;
3648                         goto err;
3649                 }
3650
3651                 obj_priv = to_intel_bo(object_list[i]);
3652                 if (obj_priv->in_execbuffer) {
3653                         DRM_ERROR("Object %p appears more than once in object list\n",
3654                                    object_list[i]);
3655                         /* prevent error path from reading uninitialized data */
3656                         args->buffer_count = i + 1;
3657                         ret = -EINVAL;
3658                         goto err;
3659                 }
3660                 obj_priv->in_execbuffer = true;
3661                 flips += atomic_read(&obj_priv->pending_flip);
3662         }
3663
3664         if (flips > 0) {
3665                 ret = i915_gem_wait_for_pending_flip(dev, object_list,
3666                                                      args->buffer_count);
3667                 if (ret)
3668                         goto err;
3669         }
3670
3671         /* Pin and relocate */
3672         for (pin_tries = 0; ; pin_tries++) {
3673                 ret = 0;
3674                 reloc_index = 0;
3675
3676                 for (i = 0; i < args->buffer_count; i++) {
3677                         object_list[i]->pending_read_domains = 0;
3678                         object_list[i]->pending_write_domain = 0;
3679                         ret = i915_gem_object_pin_and_relocate(object_list[i],
3680                                                                file_priv,
3681                                                                &exec_list[i],
3682                                                                &relocs[reloc_index]);
3683                         if (ret)
3684                                 break;
3685                         pinned = i + 1;
3686                         reloc_index += exec_list[i].relocation_count;
3687                 }
3688                 /* success */
3689                 if (ret == 0)
3690                         break;
3691
3692                 /* error other than GTT full, or we've already tried again */
3693                 if (ret != -ENOSPC || pin_tries >= 1) {
3694                         if (ret != -ERESTARTSYS) {
3695                                 unsigned long long total_size = 0;
3696                                 int num_fences = 0;
3697                                 for (i = 0; i < args->buffer_count; i++) {
3698                                         obj_priv = to_intel_bo(object_list[i]);
3699
3700                                         total_size += object_list[i]->size;
3701                                         num_fences +=
3702                                                 exec_list[i].flags & EXEC_OBJECT_NEEDS_FENCE &&
3703                                                 obj_priv->tiling_mode != I915_TILING_NONE;
3704                                 }
3705                                 DRM_ERROR("Failed to pin buffer %d of %d, total %llu bytes, %d fences: %d\n",
3706                                           pinned+1, args->buffer_count,
3707                                           total_size, num_fences,
3708                                           ret);
3709                                 DRM_ERROR("%d objects [%d pinned], "
3710                                           "%d object bytes [%d pinned], "
3711                                           "%d/%d gtt bytes\n",
3712                                           atomic_read(&dev->object_count),
3713                                           atomic_read(&dev->pin_count),
3714                                           atomic_read(&dev->object_memory),
3715                                           atomic_read(&dev->pin_memory),
3716                                           atomic_read(&dev->gtt_memory),
3717                                           dev->gtt_total);
3718                         }
3719                         goto err;
3720                 }
3721
3722                 /* unpin all of our buffers */
3723                 for (i = 0; i < pinned; i++)
3724                         i915_gem_object_unpin(object_list[i]);
3725                 pinned = 0;
3726
3727                 /* evict everyone we can from the aperture */
3728                 ret = i915_gem_evict_everything(dev);
3729                 if (ret && ret != -ENOSPC)
3730                         goto err;
3731         }
3732
3733         /* Set the pending read domains for the batch buffer to COMMAND */
3734         batch_obj = object_list[args->buffer_count-1];
3735         if (batch_obj->pending_write_domain) {
3736                 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
3737                 ret = -EINVAL;
3738                 goto err;
3739         }
3740         batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
3741
3742         /* Sanity check the batch buffer, prior to moving objects */
3743         exec_offset = exec_list[args->buffer_count - 1].offset;
3744         ret = i915_gem_check_execbuffer (args, exec_offset);
3745         if (ret != 0) {
3746                 DRM_ERROR("execbuf with invalid offset/length\n");
3747                 goto err;
3748         }
3749
3750         i915_verify_inactive(dev, __FILE__, __LINE__);
3751
3752         /* Zero the global flush/invalidate flags. These
3753          * will be modified as new domains are computed
3754          * for each object
3755          */
3756         dev->invalidate_domains = 0;
3757         dev->flush_domains = 0;
3758         dev_priv->mm.flush_rings = 0;
3759
3760         for (i = 0; i < args->buffer_count; i++) {
3761                 struct drm_gem_object *obj = object_list[i];
3762
3763                 /* Compute new gpu domains and update invalidate/flush */
3764                 i915_gem_object_set_to_gpu_domain(obj);
3765         }
3766
3767         i915_verify_inactive(dev, __FILE__, __LINE__);
3768
3769         if (dev->invalidate_domains | dev->flush_domains) {
3770 #if WATCH_EXEC
3771                 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
3772                           __func__,
3773                          dev->invalidate_domains,
3774                          dev->flush_domains);
3775 #endif
3776                 i915_gem_flush(dev, file_priv,
3777                                dev->invalidate_domains,
3778                                dev->flush_domains,
3779                                dev_priv->mm.flush_rings);
3780         }
3781
3782         for (i = 0; i < args->buffer_count; i++) {
3783                 struct drm_gem_object *obj = object_list[i];
3784                 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3785                 uint32_t old_write_domain = obj->write_domain;
3786
3787                 obj->write_domain = obj->pending_write_domain;
3788                 if (obj->write_domain)
3789                         list_move_tail(&obj_priv->gpu_write_list,
3790                                        &dev_priv->mm.gpu_write_list);
3791                 else
3792                         list_del_init(&obj_priv->gpu_write_list);
3793
3794                 trace_i915_gem_object_change_domain(obj,
3795                                                     obj->read_domains,
3796                                                     old_write_domain);
3797         }
3798
3799         i915_verify_inactive(dev, __FILE__, __LINE__);
3800
3801 #if WATCH_COHERENCY
3802         for (i = 0; i < args->buffer_count; i++) {
3803                 i915_gem_object_check_coherency(object_list[i],
3804                                                 exec_list[i].handle);
3805         }
3806 #endif
3807
3808 #if WATCH_EXEC
3809         i915_gem_dump_object(batch_obj,
3810                               args->batch_len,
3811                               __func__,
3812                               ~0);
3813 #endif
3814
3815         /* Exec the batchbuffer */
3816         ret = ring->dispatch_gem_execbuffer(dev, ring, args,
3817                         cliprects, exec_offset);
3818         if (ret) {
3819                 DRM_ERROR("dispatch failed %d\n", ret);
3820                 goto err;
3821         }
3822
3823         /*
3824          * Ensure that the commands in the batch buffer are
3825          * finished before the interrupt fires
3826          */
3827         i915_retire_commands(dev, ring);
3828
3829         i915_verify_inactive(dev, __FILE__, __LINE__);
3830
3831         for (i = 0; i < args->buffer_count; i++) {
3832                 struct drm_gem_object *obj = object_list[i];
3833                 obj_priv = to_intel_bo(obj);
3834
3835                 i915_gem_object_move_to_active(obj, ring);
3836 #if WATCH_LRU
3837                 DRM_INFO("%s: move to exec list %p\n", __func__, obj);
3838 #endif
3839         }
3840         i915_add_request(dev, file_priv, request, ring);
3841         request = NULL;
3842
3843 #if WATCH_LRU
3844         i915_dump_lru(dev, __func__);
3845 #endif
3846
3847         i915_verify_inactive(dev, __FILE__, __LINE__);
3848
3849 err:
3850         for (i = 0; i < pinned; i++)
3851                 i915_gem_object_unpin(object_list[i]);
3852
3853         for (i = 0; i < args->buffer_count; i++) {
3854                 if (object_list[i]) {
3855                         obj_priv = to_intel_bo(object_list[i]);
3856                         obj_priv->in_execbuffer = false;
3857                 }
3858                 drm_gem_object_unreference(object_list[i]);
3859         }
3860
3861         mutex_unlock(&dev->struct_mutex);
3862
3863 pre_mutex_err:
3864         /* Copy the updated relocations out regardless of current error
3865          * state.  Failure to update the relocs would mean that the next
3866          * time userland calls execbuf, it would do so with presumed offset
3867          * state that didn't match the actual object state.
3868          */
3869         ret2 = i915_gem_put_relocs_to_user(exec_list, args->buffer_count,
3870                                            relocs);
3871         if (ret2 != 0) {
3872                 DRM_ERROR("Failed to copy relocations back out: %d\n", ret2);
3873
3874                 if (ret == 0)
3875                         ret = ret2;
3876         }
3877
3878         drm_free_large(object_list);
3879         kfree(cliprects);
3880         kfree(request);
3881
3882         return ret;
3883 }
3884
3885 /*
3886  * Legacy execbuffer just creates an exec2 list from the original exec object
3887  * list array and passes it to the real function.
3888  */
3889 int
3890 i915_gem_execbuffer(struct drm_device *dev, void *data,
3891                     struct drm_file *file_priv)
3892 {
3893         struct drm_i915_gem_execbuffer *args = data;
3894         struct drm_i915_gem_execbuffer2 exec2;
3895         struct drm_i915_gem_exec_object *exec_list = NULL;
3896         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
3897         int ret, i;
3898
3899 #if WATCH_EXEC
3900         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3901                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3902 #endif
3903
3904         if (args->buffer_count < 1) {
3905                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3906                 return -EINVAL;
3907         }
3908
3909         /* Copy in the exec list from userland */
3910         exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
3911         exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
3912         if (exec_list == NULL || exec2_list == NULL) {
3913                 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
3914                           args->buffer_count);
3915                 drm_free_large(exec_list);
3916                 drm_free_large(exec2_list);
3917                 return -ENOMEM;
3918         }
3919         ret = copy_from_user(exec_list,
3920                              (struct drm_i915_relocation_entry __user *)
3921                              (uintptr_t) args->buffers_ptr,
3922                              sizeof(*exec_list) * args->buffer_count);
3923         if (ret != 0) {
3924                 DRM_ERROR("copy %d exec entries failed %d\n",
3925                           args->buffer_count, ret);
3926                 drm_free_large(exec_list);
3927                 drm_free_large(exec2_list);
3928                 return -EFAULT;
3929         }
3930
3931         for (i = 0; i < args->buffer_count; i++) {
3932                 exec2_list[i].handle = exec_list[i].handle;
3933                 exec2_list[i].relocation_count = exec_list[i].relocation_count;
3934                 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
3935                 exec2_list[i].alignment = exec_list[i].alignment;
3936                 exec2_list[i].offset = exec_list[i].offset;
3937                 if (INTEL_INFO(dev)->gen < 4)
3938                         exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
3939                 else
3940                         exec2_list[i].flags = 0;
3941         }
3942
3943         exec2.buffers_ptr = args->buffers_ptr;
3944         exec2.buffer_count = args->buffer_count;
3945         exec2.batch_start_offset = args->batch_start_offset;
3946         exec2.batch_len = args->batch_len;
3947         exec2.DR1 = args->DR1;
3948         exec2.DR4 = args->DR4;
3949         exec2.num_cliprects = args->num_cliprects;
3950         exec2.cliprects_ptr = args->cliprects_ptr;
3951         exec2.flags = I915_EXEC_RENDER;
3952
3953         ret = i915_gem_do_execbuffer(dev, data, file_priv, &exec2, exec2_list);
3954         if (!ret) {
3955                 /* Copy the new buffer offsets back to the user's exec list. */
3956                 for (i = 0; i < args->buffer_count; i++)
3957                         exec_list[i].offset = exec2_list[i].offset;
3958                 /* ... and back out to userspace */
3959                 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
3960                                    (uintptr_t) args->buffers_ptr,
3961                                    exec_list,
3962                                    sizeof(*exec_list) * args->buffer_count);
3963                 if (ret) {
3964                         ret = -EFAULT;
3965                         DRM_ERROR("failed to copy %d exec entries "
3966                                   "back to user (%d)\n",
3967                                   args->buffer_count, ret);
3968                 }
3969         }
3970
3971         drm_free_large(exec_list);
3972         drm_free_large(exec2_list);
3973         return ret;
3974 }
3975
3976 int
3977 i915_gem_execbuffer2(struct drm_device *dev, void *data,
3978                      struct drm_file *file_priv)
3979 {
3980         struct drm_i915_gem_execbuffer2 *args = data;
3981         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
3982         int ret;
3983
3984 #if WATCH_EXEC
3985         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3986                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3987 #endif
3988
3989         if (args->buffer_count < 1) {
3990                 DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
3991                 return -EINVAL;
3992         }
3993
3994         exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
3995         if (exec2_list == NULL) {
3996                 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
3997                           args->buffer_count);
3998                 return -ENOMEM;
3999         }
4000         ret = copy_from_user(exec2_list,
4001                              (struct drm_i915_relocation_entry __user *)
4002                              (uintptr_t) args->buffers_ptr,
4003                              sizeof(*exec2_list) * args->buffer_count);
4004         if (ret != 0) {
4005                 DRM_ERROR("copy %d exec entries failed %d\n",
4006                           args->buffer_count, ret);
4007                 drm_free_large(exec2_list);
4008                 return -EFAULT;
4009         }
4010
4011         ret = i915_gem_do_execbuffer(dev, data, file_priv, args, exec2_list);
4012         if (!ret) {
4013                 /* Copy the new buffer offsets back to the user's exec list. */
4014                 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4015                                    (uintptr_t) args->buffers_ptr,
4016                                    exec2_list,
4017                                    sizeof(*exec2_list) * args->buffer_count);
4018                 if (ret) {
4019                         ret = -EFAULT;
4020                         DRM_ERROR("failed to copy %d exec entries "
4021                                   "back to user (%d)\n",
4022                                   args->buffer_count, ret);
4023                 }
4024         }
4025
4026         drm_free_large(exec2_list);
4027         return ret;
4028 }
4029
4030 int
4031 i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
4032 {
4033         struct drm_device *dev = obj->dev;
4034         struct drm_i915_private *dev_priv = dev->dev_private;
4035         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4036         int ret;
4037
4038         BUG_ON(obj_priv->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
4039
4040         i915_verify_inactive(dev, __FILE__, __LINE__);
4041
4042         if (obj_priv->gtt_space != NULL) {
4043                 if (alignment == 0)
4044                         alignment = i915_gem_get_gtt_alignment(obj);
4045                 if (obj_priv->gtt_offset & (alignment - 1)) {
4046                         WARN(obj_priv->pin_count,
4047                              "bo is already pinned with incorrect alignment:"
4048                              " offset=%x, req.alignment=%x\n",
4049                              obj_priv->gtt_offset, alignment);
4050                         ret = i915_gem_object_unbind(obj);
4051                         if (ret)
4052                                 return ret;
4053                 }
4054         }
4055
4056         if (obj_priv->gtt_space == NULL) {
4057                 ret = i915_gem_object_bind_to_gtt(obj, alignment);
4058                 if (ret)
4059                         return ret;
4060         }
4061
4062         obj_priv->pin_count++;
4063
4064         /* If the object is not active and not pending a flush,
4065          * remove it from the inactive list
4066          */
4067         if (obj_priv->pin_count == 1) {
4068                 atomic_inc(&dev->pin_count);
4069                 atomic_add(obj->size, &dev->pin_memory);
4070                 if (!obj_priv->active)
4071                         list_move_tail(&obj_priv->list,
4072                                        &dev_priv->mm.pinned_list);
4073         }
4074         i915_verify_inactive(dev, __FILE__, __LINE__);
4075
4076         return 0;
4077 }
4078
4079 void
4080 i915_gem_object_unpin(struct drm_gem_object *obj)
4081 {
4082         struct drm_device *dev = obj->dev;
4083         drm_i915_private_t *dev_priv = dev->dev_private;
4084         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4085
4086         i915_verify_inactive(dev, __FILE__, __LINE__);
4087         obj_priv->pin_count--;
4088         BUG_ON(obj_priv->pin_count < 0);
4089         BUG_ON(obj_priv->gtt_space == NULL);
4090
4091         /* If the object is no longer pinned, and is
4092          * neither active nor being flushed, then stick it on
4093          * the inactive list
4094          */
4095         if (obj_priv->pin_count == 0) {
4096                 if (!obj_priv->active)
4097                         list_move_tail(&obj_priv->list,
4098                                        &dev_priv->mm.inactive_list);
4099                 atomic_dec(&dev->pin_count);
4100                 atomic_sub(obj->size, &dev->pin_memory);
4101         }
4102         i915_verify_inactive(dev, __FILE__, __LINE__);
4103 }
4104
4105 int
4106 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
4107                    struct drm_file *file_priv)
4108 {
4109         struct drm_i915_gem_pin *args = data;
4110         struct drm_gem_object *obj;
4111         struct drm_i915_gem_object *obj_priv;
4112         int ret;
4113
4114         mutex_lock(&dev->struct_mutex);
4115
4116         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4117         if (obj == NULL) {
4118                 DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n",
4119                           args->handle);
4120                 mutex_unlock(&dev->struct_mutex);
4121                 return -ENOENT;
4122         }
4123         obj_priv = to_intel_bo(obj);
4124
4125         if (obj_priv->madv != I915_MADV_WILLNEED) {
4126                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
4127                 drm_gem_object_unreference(obj);
4128                 mutex_unlock(&dev->struct_mutex);
4129                 return -EINVAL;
4130         }
4131
4132         if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) {
4133                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
4134                           args->handle);
4135                 drm_gem_object_unreference(obj);
4136                 mutex_unlock(&dev->struct_mutex);
4137                 return -EINVAL;
4138         }
4139
4140         obj_priv->user_pin_count++;
4141         obj_priv->pin_filp = file_priv;
4142         if (obj_priv->user_pin_count == 1) {
4143                 ret = i915_gem_object_pin(obj, args->alignment);
4144                 if (ret != 0) {
4145                         drm_gem_object_unreference(obj);
4146                         mutex_unlock(&dev->struct_mutex);
4147                         return ret;
4148                 }
4149         }
4150
4151         /* XXX - flush the CPU caches for pinned objects
4152          * as the X server doesn't manage domains yet
4153          */
4154         i915_gem_object_flush_cpu_write_domain(obj);
4155         args->offset = obj_priv->gtt_offset;
4156         drm_gem_object_unreference(obj);
4157         mutex_unlock(&dev->struct_mutex);
4158
4159         return 0;
4160 }
4161
4162 int
4163 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
4164                      struct drm_file *file_priv)
4165 {
4166         struct drm_i915_gem_pin *args = data;
4167         struct drm_gem_object *obj;
4168         struct drm_i915_gem_object *obj_priv;
4169
4170         mutex_lock(&dev->struct_mutex);
4171
4172         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4173         if (obj == NULL) {
4174                 DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n",
4175                           args->handle);
4176                 mutex_unlock(&dev->struct_mutex);
4177                 return -ENOENT;
4178         }
4179
4180         obj_priv = to_intel_bo(obj);
4181         if (obj_priv->pin_filp != file_priv) {
4182                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
4183                           args->handle);
4184                 drm_gem_object_unreference(obj);
4185                 mutex_unlock(&dev->struct_mutex);
4186                 return -EINVAL;
4187         }
4188         obj_priv->user_pin_count--;
4189         if (obj_priv->user_pin_count == 0) {
4190                 obj_priv->pin_filp = NULL;
4191                 i915_gem_object_unpin(obj);
4192         }
4193
4194         drm_gem_object_unreference(obj);
4195         mutex_unlock(&dev->struct_mutex);
4196         return 0;
4197 }
4198
4199 int
4200 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4201                     struct drm_file *file_priv)
4202 {
4203         struct drm_i915_gem_busy *args = data;
4204         struct drm_gem_object *obj;
4205         struct drm_i915_gem_object *obj_priv;
4206
4207         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4208         if (obj == NULL) {
4209                 DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n",
4210                           args->handle);
4211                 return -ENOENT;
4212         }
4213
4214         mutex_lock(&dev->struct_mutex);
4215
4216         /* Count all active objects as busy, even if they are currently not used
4217          * by the gpu. Users of this interface expect objects to eventually
4218          * become non-busy without any further actions, therefore emit any
4219          * necessary flushes here.
4220          */
4221         obj_priv = to_intel_bo(obj);
4222         args->busy = obj_priv->active;
4223         if (args->busy) {
4224                 /* Unconditionally flush objects, even when the gpu still uses this
4225                  * object. Userspace calling this function indicates that it wants to
4226                  * use this buffer rather sooner than later, so issuing the required
4227                  * flush earlier is beneficial.
4228                  */
4229                 if (obj->write_domain & I915_GEM_GPU_DOMAINS)
4230                         i915_gem_flush_ring(dev, file_priv,
4231                                             obj_priv->ring,
4232                                             0, obj->write_domain);
4233
4234                 /* Update the active list for the hardware's current position.
4235                  * Otherwise this only updates on a delayed timer or when irqs
4236                  * are actually unmasked, and our working set ends up being
4237                  * larger than required.
4238                  */
4239                 i915_gem_retire_requests_ring(dev, obj_priv->ring);
4240
4241                 args->busy = obj_priv->active;
4242         }
4243
4244         drm_gem_object_unreference(obj);
4245         mutex_unlock(&dev->struct_mutex);
4246         return 0;
4247 }
4248
4249 int
4250 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4251                         struct drm_file *file_priv)
4252 {
4253     return i915_gem_ring_throttle(dev, file_priv);
4254 }
4255
4256 int
4257 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4258                        struct drm_file *file_priv)
4259 {
4260         struct drm_i915_gem_madvise *args = data;
4261         struct drm_gem_object *obj;
4262         struct drm_i915_gem_object *obj_priv;
4263
4264         switch (args->madv) {
4265         case I915_MADV_DONTNEED:
4266         case I915_MADV_WILLNEED:
4267             break;
4268         default:
4269             return -EINVAL;
4270         }
4271
4272         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4273         if (obj == NULL) {
4274                 DRM_ERROR("Bad handle in i915_gem_madvise_ioctl(): %d\n",
4275                           args->handle);
4276                 return -ENOENT;
4277         }
4278
4279         mutex_lock(&dev->struct_mutex);
4280         obj_priv = to_intel_bo(obj);
4281
4282         if (obj_priv->pin_count) {
4283                 drm_gem_object_unreference(obj);
4284                 mutex_unlock(&dev->struct_mutex);
4285
4286                 DRM_ERROR("Attempted i915_gem_madvise_ioctl() on a pinned object\n");
4287                 return -EINVAL;
4288         }
4289
4290         if (obj_priv->madv != __I915_MADV_PURGED)
4291                 obj_priv->madv = args->madv;
4292
4293         /* if the object is no longer bound, discard its backing storage */
4294         if (i915_gem_object_is_purgeable(obj_priv) &&
4295             obj_priv->gtt_space == NULL)
4296                 i915_gem_object_truncate(obj);
4297
4298         args->retained = obj_priv->madv != __I915_MADV_PURGED;
4299
4300         drm_gem_object_unreference(obj);
4301         mutex_unlock(&dev->struct_mutex);
4302
4303         return 0;
4304 }
4305
4306 struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev,
4307                                               size_t size)
4308 {
4309         struct drm_i915_gem_object *obj;
4310
4311         obj = kzalloc(sizeof(*obj), GFP_KERNEL);
4312         if (obj == NULL)
4313                 return NULL;
4314
4315         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4316                 kfree(obj);
4317                 return NULL;
4318         }
4319
4320         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4321         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4322
4323         obj->agp_type = AGP_USER_MEMORY;
4324         obj->base.driver_private = NULL;
4325         obj->fence_reg = I915_FENCE_REG_NONE;
4326         INIT_LIST_HEAD(&obj->list);
4327         INIT_LIST_HEAD(&obj->gpu_write_list);
4328         obj->madv = I915_MADV_WILLNEED;
4329
4330         trace_i915_gem_object_create(&obj->base);
4331
4332         return &obj->base;
4333 }
4334
4335 int i915_gem_init_object(struct drm_gem_object *obj)
4336 {
4337         BUG();
4338
4339         return 0;
4340 }
4341
4342 static void i915_gem_free_object_tail(struct drm_gem_object *obj)
4343 {
4344         struct drm_device *dev = obj->dev;
4345         drm_i915_private_t *dev_priv = dev->dev_private;
4346         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4347         int ret;
4348
4349         ret = i915_gem_object_unbind(obj);
4350         if (ret == -ERESTARTSYS) {
4351                 list_move(&obj_priv->list,
4352                           &dev_priv->mm.deferred_free_list);
4353                 return;
4354         }
4355
4356         if (obj_priv->mmap_offset)
4357                 i915_gem_free_mmap_offset(obj);
4358
4359         drm_gem_object_release(obj);
4360
4361         kfree(obj_priv->page_cpu_valid);
4362         kfree(obj_priv->bit_17);
4363         kfree(obj_priv);
4364 }
4365
4366 void i915_gem_free_object(struct drm_gem_object *obj)
4367 {
4368         struct drm_device *dev = obj->dev;
4369         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4370
4371         trace_i915_gem_object_destroy(obj);
4372
4373         while (obj_priv->pin_count > 0)
4374                 i915_gem_object_unpin(obj);
4375
4376         if (obj_priv->phys_obj)
4377                 i915_gem_detach_phys_object(dev, obj);
4378
4379         i915_gem_free_object_tail(obj);
4380 }
4381
4382 int
4383 i915_gem_idle(struct drm_device *dev)
4384 {
4385         drm_i915_private_t *dev_priv = dev->dev_private;
4386         int ret;
4387
4388         mutex_lock(&dev->struct_mutex);
4389
4390         if (dev_priv->mm.suspended ||
4391                         (dev_priv->render_ring.gem_object == NULL) ||
4392                         (HAS_BSD(dev) &&
4393                          dev_priv->bsd_ring.gem_object == NULL)) {
4394                 mutex_unlock(&dev->struct_mutex);
4395                 return 0;
4396         }
4397
4398         ret = i915_gpu_idle(dev);
4399         if (ret) {
4400                 mutex_unlock(&dev->struct_mutex);
4401                 return ret;
4402         }
4403
4404         /* Under UMS, be paranoid and evict. */
4405         if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
4406                 ret = i915_gem_evict_inactive(dev);
4407                 if (ret) {
4408                         mutex_unlock(&dev->struct_mutex);
4409                         return ret;
4410                 }
4411         }
4412
4413         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
4414          * We need to replace this with a semaphore, or something.
4415          * And not confound mm.suspended!
4416          */
4417         dev_priv->mm.suspended = 1;
4418         del_timer_sync(&dev_priv->hangcheck_timer);
4419
4420         i915_kernel_lost_context(dev);
4421         i915_gem_cleanup_ringbuffer(dev);
4422
4423         mutex_unlock(&dev->struct_mutex);
4424
4425         /* Cancel the retire work handler, which should be idle now. */
4426         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4427
4428         return 0;
4429 }
4430
4431 /*
4432  * 965+ support PIPE_CONTROL commands, which provide finer grained control
4433  * over cache flushing.
4434  */
4435 static int
4436 i915_gem_init_pipe_control(struct drm_device *dev)
4437 {
4438         drm_i915_private_t *dev_priv = dev->dev_private;
4439         struct drm_gem_object *obj;
4440         struct drm_i915_gem_object *obj_priv;
4441         int ret;
4442
4443         obj = i915_gem_alloc_object(dev, 4096);
4444         if (obj == NULL) {
4445                 DRM_ERROR("Failed to allocate seqno page\n");
4446                 ret = -ENOMEM;
4447                 goto err;
4448         }
4449         obj_priv = to_intel_bo(obj);
4450         obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
4451
4452         ret = i915_gem_object_pin(obj, 4096);
4453         if (ret)
4454                 goto err_unref;
4455
4456         dev_priv->seqno_gfx_addr = obj_priv->gtt_offset;
4457         dev_priv->seqno_page =  kmap(obj_priv->pages[0]);
4458         if (dev_priv->seqno_page == NULL)
4459                 goto err_unpin;
4460
4461         dev_priv->seqno_obj = obj;
4462         memset(dev_priv->seqno_page, 0, PAGE_SIZE);
4463
4464         return 0;
4465
4466 err_unpin:
4467         i915_gem_object_unpin(obj);
4468 err_unref:
4469         drm_gem_object_unreference(obj);
4470 err:
4471         return ret;
4472 }
4473
4474
4475 static void
4476 i915_gem_cleanup_pipe_control(struct drm_device *dev)
4477 {
4478         drm_i915_private_t *dev_priv = dev->dev_private;
4479         struct drm_gem_object *obj;
4480         struct drm_i915_gem_object *obj_priv;
4481
4482         obj = dev_priv->seqno_obj;
4483         obj_priv = to_intel_bo(obj);
4484         kunmap(obj_priv->pages[0]);
4485         i915_gem_object_unpin(obj);
4486         drm_gem_object_unreference(obj);
4487         dev_priv->seqno_obj = NULL;
4488
4489         dev_priv->seqno_page = NULL;
4490 }
4491
4492 int
4493 i915_gem_init_ringbuffer(struct drm_device *dev)
4494 {
4495         drm_i915_private_t *dev_priv = dev->dev_private;
4496         int ret;
4497
4498         if (HAS_PIPE_CONTROL(dev)) {
4499                 ret = i915_gem_init_pipe_control(dev);
4500                 if (ret)
4501                         return ret;
4502         }
4503
4504         ret = intel_init_render_ring_buffer(dev);
4505         if (ret)
4506                 goto cleanup_pipe_control;
4507
4508         if (HAS_BSD(dev)) {
4509                 ret = intel_init_bsd_ring_buffer(dev);
4510                 if (ret)
4511                         goto cleanup_render_ring;
4512         }
4513
4514         dev_priv->next_seqno = 1;
4515
4516         return 0;
4517
4518 cleanup_render_ring:
4519         intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
4520 cleanup_pipe_control:
4521         if (HAS_PIPE_CONTROL(dev))
4522                 i915_gem_cleanup_pipe_control(dev);
4523         return ret;
4524 }
4525
4526 void
4527 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4528 {
4529         drm_i915_private_t *dev_priv = dev->dev_private;
4530
4531         intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
4532         if (HAS_BSD(dev))
4533                 intel_cleanup_ring_buffer(dev, &dev_priv->bsd_ring);
4534         if (HAS_PIPE_CONTROL(dev))
4535                 i915_gem_cleanup_pipe_control(dev);
4536 }
4537
4538 int
4539 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4540                        struct drm_file *file_priv)
4541 {
4542         drm_i915_private_t *dev_priv = dev->dev_private;
4543         int ret;
4544
4545         if (drm_core_check_feature(dev, DRIVER_MODESET))
4546                 return 0;
4547
4548         if (atomic_read(&dev_priv->mm.wedged)) {
4549                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
4550                 atomic_set(&dev_priv->mm.wedged, 0);
4551         }
4552
4553         mutex_lock(&dev->struct_mutex);
4554         dev_priv->mm.suspended = 0;
4555
4556         ret = i915_gem_init_ringbuffer(dev);
4557         if (ret != 0) {
4558                 mutex_unlock(&dev->struct_mutex);
4559                 return ret;
4560         }
4561
4562         BUG_ON(!list_empty(&dev_priv->render_ring.active_list));
4563         BUG_ON(HAS_BSD(dev) && !list_empty(&dev_priv->bsd_ring.active_list));
4564         BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
4565         BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
4566         BUG_ON(!list_empty(&dev_priv->render_ring.request_list));
4567         BUG_ON(HAS_BSD(dev) && !list_empty(&dev_priv->bsd_ring.request_list));
4568         mutex_unlock(&dev->struct_mutex);
4569
4570         ret = drm_irq_install(dev);
4571         if (ret)
4572                 goto cleanup_ringbuffer;
4573
4574         return 0;
4575
4576 cleanup_ringbuffer:
4577         mutex_lock(&dev->struct_mutex);
4578         i915_gem_cleanup_ringbuffer(dev);
4579         dev_priv->mm.suspended = 1;
4580         mutex_unlock(&dev->struct_mutex);
4581
4582         return ret;
4583 }
4584
4585 int
4586 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
4587                        struct drm_file *file_priv)
4588 {
4589         if (drm_core_check_feature(dev, DRIVER_MODESET))
4590                 return 0;
4591
4592         drm_irq_uninstall(dev);
4593         return i915_gem_idle(dev);
4594 }
4595
4596 void
4597 i915_gem_lastclose(struct drm_device *dev)
4598 {
4599         int ret;
4600
4601         if (drm_core_check_feature(dev, DRIVER_MODESET))
4602                 return;
4603
4604         ret = i915_gem_idle(dev);
4605         if (ret)
4606                 DRM_ERROR("failed to idle hardware: %d\n", ret);
4607 }
4608
4609 void
4610 i915_gem_load(struct drm_device *dev)
4611 {
4612         int i;
4613         drm_i915_private_t *dev_priv = dev->dev_private;
4614
4615         INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
4616         INIT_LIST_HEAD(&dev_priv->mm.gpu_write_list);
4617         INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
4618         INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
4619         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4620         INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
4621         INIT_LIST_HEAD(&dev_priv->render_ring.active_list);
4622         INIT_LIST_HEAD(&dev_priv->render_ring.request_list);
4623         if (HAS_BSD(dev)) {
4624                 INIT_LIST_HEAD(&dev_priv->bsd_ring.active_list);
4625                 INIT_LIST_HEAD(&dev_priv->bsd_ring.request_list);
4626         }
4627         for (i = 0; i < 16; i++)
4628                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
4629         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4630                           i915_gem_retire_work_handler);
4631         spin_lock(&shrink_list_lock);
4632         list_add(&dev_priv->mm.shrink_list, &shrink_list);
4633         spin_unlock(&shrink_list_lock);
4634
4635         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
4636         if (IS_GEN3(dev)) {
4637                 u32 tmp = I915_READ(MI_ARB_STATE);
4638                 if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) {
4639                         /* arb state is a masked write, so set bit + bit in mask */
4640                         tmp = MI_ARB_C3_LP_WRITE_ENABLE | (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT);
4641                         I915_WRITE(MI_ARB_STATE, tmp);
4642                 }
4643         }
4644
4645         /* Old X drivers will take 0-2 for front, back, depth buffers */
4646         if (!drm_core_check_feature(dev, DRIVER_MODESET))
4647                 dev_priv->fence_reg_start = 3;
4648
4649         if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4650                 dev_priv->num_fence_regs = 16;
4651         else
4652                 dev_priv->num_fence_regs = 8;
4653
4654         /* Initialize fence registers to zero */
4655         switch (INTEL_INFO(dev)->gen) {
4656         case 6:
4657                 for (i = 0; i < 16; i++)
4658                         I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (i * 8), 0);
4659                 break;
4660         case 5:
4661         case 4:
4662                 for (i = 0; i < 16; i++)
4663                         I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0);
4664                 break;
4665         case 3:
4666                 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4667                         for (i = 0; i < 8; i++)
4668                                 I915_WRITE(FENCE_REG_945_8 + (i * 4), 0);
4669         case 2:
4670                 for (i = 0; i < 8; i++)
4671                         I915_WRITE(FENCE_REG_830_0 + (i * 4), 0);
4672                 break;
4673         }
4674         i915_gem_detect_bit_6_swizzle(dev);
4675         init_waitqueue_head(&dev_priv->pending_flip_queue);
4676 }
4677
4678 /*
4679  * Create a physically contiguous memory object for this object
4680  * e.g. for cursor + overlay regs
4681  */
4682 static int i915_gem_init_phys_object(struct drm_device *dev,
4683                                      int id, int size, int align)
4684 {
4685         drm_i915_private_t *dev_priv = dev->dev_private;
4686         struct drm_i915_gem_phys_object *phys_obj;
4687         int ret;
4688
4689         if (dev_priv->mm.phys_objs[id - 1] || !size)
4690                 return 0;
4691
4692         phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
4693         if (!phys_obj)
4694                 return -ENOMEM;
4695
4696         phys_obj->id = id;
4697
4698         phys_obj->handle = drm_pci_alloc(dev, size, align);
4699         if (!phys_obj->handle) {
4700                 ret = -ENOMEM;
4701                 goto kfree_obj;
4702         }
4703 #ifdef CONFIG_X86
4704         set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4705 #endif
4706
4707         dev_priv->mm.phys_objs[id - 1] = phys_obj;
4708
4709         return 0;
4710 kfree_obj:
4711         kfree(phys_obj);
4712         return ret;
4713 }
4714
4715 static void i915_gem_free_phys_object(struct drm_device *dev, int id)
4716 {
4717         drm_i915_private_t *dev_priv = dev->dev_private;
4718         struct drm_i915_gem_phys_object *phys_obj;
4719
4720         if (!dev_priv->mm.phys_objs[id - 1])
4721                 return;
4722
4723         phys_obj = dev_priv->mm.phys_objs[id - 1];
4724         if (phys_obj->cur_obj) {
4725                 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4726         }
4727
4728 #ifdef CONFIG_X86
4729         set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4730 #endif
4731         drm_pci_free(dev, phys_obj->handle);
4732         kfree(phys_obj);
4733         dev_priv->mm.phys_objs[id - 1] = NULL;
4734 }
4735
4736 void i915_gem_free_all_phys_object(struct drm_device *dev)
4737 {
4738         int i;
4739
4740         for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
4741                 i915_gem_free_phys_object(dev, i);
4742 }
4743
4744 void i915_gem_detach_phys_object(struct drm_device *dev,
4745                                  struct drm_gem_object *obj)
4746 {
4747         struct drm_i915_gem_object *obj_priv;
4748         int i;
4749         int ret;
4750         int page_count;
4751
4752         obj_priv = to_intel_bo(obj);
4753         if (!obj_priv->phys_obj)
4754                 return;
4755
4756         ret = i915_gem_object_get_pages(obj, 0);
4757         if (ret)
4758                 goto out;
4759
4760         page_count = obj->size / PAGE_SIZE;
4761
4762         for (i = 0; i < page_count; i++) {
4763                 char *dst = kmap_atomic(obj_priv->pages[i], KM_USER0);
4764                 char *src = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4765
4766                 memcpy(dst, src, PAGE_SIZE);
4767                 kunmap_atomic(dst, KM_USER0);
4768         }
4769         drm_clflush_pages(obj_priv->pages, page_count);
4770         drm_agp_chipset_flush(dev);
4771
4772         i915_gem_object_put_pages(obj);
4773 out:
4774         obj_priv->phys_obj->cur_obj = NULL;
4775         obj_priv->phys_obj = NULL;
4776 }
4777
4778 int
4779 i915_gem_attach_phys_object(struct drm_device *dev,
4780                             struct drm_gem_object *obj,
4781                             int id,
4782                             int align)
4783 {
4784         drm_i915_private_t *dev_priv = dev->dev_private;
4785         struct drm_i915_gem_object *obj_priv;
4786         int ret = 0;
4787         int page_count;
4788         int i;
4789
4790         if (id > I915_MAX_PHYS_OBJECT)
4791                 return -EINVAL;
4792
4793         obj_priv = to_intel_bo(obj);
4794
4795         if (obj_priv->phys_obj) {
4796                 if (obj_priv->phys_obj->id == id)
4797                         return 0;
4798                 i915_gem_detach_phys_object(dev, obj);
4799         }
4800
4801         /* create a new object */
4802         if (!dev_priv->mm.phys_objs[id - 1]) {
4803                 ret = i915_gem_init_phys_object(dev, id,
4804                                                 obj->size, align);
4805                 if (ret) {
4806                         DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size);
4807                         goto out;
4808                 }
4809         }
4810
4811         /* bind to the object */
4812         obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1];
4813         obj_priv->phys_obj->cur_obj = obj;
4814
4815         ret = i915_gem_object_get_pages(obj, 0);
4816         if (ret) {
4817                 DRM_ERROR("failed to get page list\n");
4818                 goto out;
4819         }
4820
4821         page_count = obj->size / PAGE_SIZE;
4822
4823         for (i = 0; i < page_count; i++) {
4824                 char *src = kmap_atomic(obj_priv->pages[i], KM_USER0);
4825                 char *dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4826
4827                 memcpy(dst, src, PAGE_SIZE);
4828                 kunmap_atomic(src, KM_USER0);
4829         }
4830
4831         i915_gem_object_put_pages(obj);
4832
4833         return 0;
4834 out:
4835         return ret;
4836 }
4837
4838 static int
4839 i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
4840                      struct drm_i915_gem_pwrite *args,
4841                      struct drm_file *file_priv)
4842 {
4843         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4844         void *obj_addr;
4845         int ret;
4846         char __user *user_data;
4847
4848         user_data = (char __user *) (uintptr_t) args->data_ptr;
4849         obj_addr = obj_priv->phys_obj->handle->vaddr + args->offset;
4850
4851         DRM_DEBUG_DRIVER("obj_addr %p, %lld\n", obj_addr, args->size);
4852         ret = copy_from_user(obj_addr, user_data, args->size);
4853         if (ret)
4854                 return -EFAULT;
4855
4856         drm_agp_chipset_flush(dev);
4857         return 0;
4858 }
4859
4860 void i915_gem_release(struct drm_device * dev, struct drm_file *file_priv)
4861 {
4862         struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
4863
4864         /* Clean up our request list when the client is going away, so that
4865          * later retire_requests won't dereference our soon-to-be-gone
4866          * file_priv.
4867          */
4868         mutex_lock(&dev->struct_mutex);
4869         while (!list_empty(&i915_file_priv->mm.request_list))
4870                 list_del_init(i915_file_priv->mm.request_list.next);
4871         mutex_unlock(&dev->struct_mutex);
4872 }
4873
4874 static int
4875 i915_gpu_is_active(struct drm_device *dev)
4876 {
4877         drm_i915_private_t *dev_priv = dev->dev_private;
4878         int lists_empty;
4879
4880         lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
4881                       list_empty(&dev_priv->render_ring.active_list);
4882         if (HAS_BSD(dev))
4883                 lists_empty &= list_empty(&dev_priv->bsd_ring.active_list);
4884
4885         return !lists_empty;
4886 }
4887
4888 static int
4889 i915_gem_shrink(struct shrinker *shrink, int nr_to_scan, gfp_t gfp_mask)
4890 {
4891         drm_i915_private_t *dev_priv, *next_dev;
4892         struct drm_i915_gem_object *obj_priv, *next_obj;
4893         int cnt = 0;
4894         int would_deadlock = 1;
4895
4896         /* "fast-path" to count number of available objects */
4897         if (nr_to_scan == 0) {
4898                 spin_lock(&shrink_list_lock);
4899                 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
4900                         struct drm_device *dev = dev_priv->dev;
4901
4902                         if (mutex_trylock(&dev->struct_mutex)) {
4903                                 list_for_each_entry(obj_priv,
4904                                                     &dev_priv->mm.inactive_list,
4905                                                     list)
4906                                         cnt++;
4907                                 mutex_unlock(&dev->struct_mutex);
4908                         }
4909                 }
4910                 spin_unlock(&shrink_list_lock);
4911
4912                 return (cnt / 100) * sysctl_vfs_cache_pressure;
4913         }
4914
4915         spin_lock(&shrink_list_lock);
4916
4917 rescan:
4918         /* first scan for clean buffers */
4919         list_for_each_entry_safe(dev_priv, next_dev,
4920                                  &shrink_list, mm.shrink_list) {
4921                 struct drm_device *dev = dev_priv->dev;
4922
4923                 if (! mutex_trylock(&dev->struct_mutex))
4924                         continue;
4925
4926                 spin_unlock(&shrink_list_lock);
4927                 i915_gem_retire_requests(dev);
4928
4929                 list_for_each_entry_safe(obj_priv, next_obj,
4930                                          &dev_priv->mm.inactive_list,
4931                                          list) {
4932                         if (i915_gem_object_is_purgeable(obj_priv)) {
4933                                 i915_gem_object_unbind(&obj_priv->base);
4934                                 if (--nr_to_scan <= 0)
4935                                         break;
4936                         }
4937                 }
4938
4939                 spin_lock(&shrink_list_lock);
4940                 mutex_unlock(&dev->struct_mutex);
4941
4942                 would_deadlock = 0;
4943
4944                 if (nr_to_scan <= 0)
4945                         break;
4946         }
4947
4948         /* second pass, evict/count anything still on the inactive list */
4949         list_for_each_entry_safe(dev_priv, next_dev,
4950                                  &shrink_list, mm.shrink_list) {
4951                 struct drm_device *dev = dev_priv->dev;
4952
4953                 if (! mutex_trylock(&dev->struct_mutex))
4954                         continue;
4955
4956                 spin_unlock(&shrink_list_lock);
4957
4958                 list_for_each_entry_safe(obj_priv, next_obj,
4959                                          &dev_priv->mm.inactive_list,
4960                                          list) {
4961                         if (nr_to_scan > 0) {
4962                                 i915_gem_object_unbind(&obj_priv->base);
4963                                 nr_to_scan--;
4964                         } else
4965                                 cnt++;
4966                 }
4967
4968                 spin_lock(&shrink_list_lock);
4969                 mutex_unlock(&dev->struct_mutex);
4970
4971                 would_deadlock = 0;
4972         }
4973
4974         if (nr_to_scan) {
4975                 int active = 0;
4976
4977                 /*
4978                  * We are desperate for pages, so as a last resort, wait
4979                  * for the GPU to finish and discard whatever we can.
4980                  * This has a dramatic impact to reduce the number of
4981                  * OOM-killer events whilst running the GPU aggressively.
4982                  */
4983                 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
4984                         struct drm_device *dev = dev_priv->dev;
4985
4986                         if (!mutex_trylock(&dev->struct_mutex))
4987                                 continue;
4988
4989                         spin_unlock(&shrink_list_lock);
4990
4991                         if (i915_gpu_is_active(dev)) {
4992                                 i915_gpu_idle(dev);
4993                                 active++;
4994                         }
4995
4996                         spin_lock(&shrink_list_lock);
4997                         mutex_unlock(&dev->struct_mutex);
4998                 }
4999
5000                 if (active)
5001                         goto rescan;
5002         }
5003
5004         spin_unlock(&shrink_list_lock);
5005
5006         if (would_deadlock)
5007                 return -1;
5008         else if (cnt > 0)
5009                 return (cnt / 100) * sysctl_vfs_cache_pressure;
5010         else
5011                 return 0;
5012 }
5013
5014 static struct shrinker shrinker = {
5015         .shrink = i915_gem_shrink,
5016         .seeks = DEFAULT_SEEKS,
5017 };
5018
5019 __init void
5020 i915_gem_shrinker_init(void)
5021 {
5022     register_shrinker(&shrinker);
5023 }
5024
5025 __exit void
5026 i915_gem_shrinker_exit(void)
5027 {
5028     unregister_shrinker(&shrinker);
5029 }