drivers/gpu/drm/i915/i915_gem_execbuffer.c

   1 /*
   2  * Copyright © 2008,2010 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *    Chris Wilson <chris@chris-wilson.co.uk>
  26  *
  27  */
  28
  29 #include "drmP.h"
  30 #include "drm.h"
  31 #include "i915_drm.h"
  32 #include "i915_drv.h"
  33 #include "i915_trace.h"
  34 #include "intel_drv.h"
  35
  36 struct change_domains {
  37         uint32_t invalidate_domains;
  38         uint32_t flush_domains;
  39         uint32_t flush_rings;
  40         uint32_t flips;
  41 };
  42
  43 /*
  44  * Set the next domain for the specified object. This
  45  * may not actually perform the necessary flushing/invaliding though,
  46  * as that may want to be batched with other set_domain operations
  47  *
  48  * This is (we hope) the only really tricky part of gem. The goal
  49  * is fairly simple -- track which caches hold bits of the object
  50  * and make sure they remain coherent. A few concrete examples may
  51  * help to explain how it works. For shorthand, we use the notation
  52  * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
  53  * a pair of read and write domain masks.
  54  *
  55  * Case 1: the batch buffer
  56  *
  57  *      1. Allocated
  58  *      2. Written by CPU
  59  *      3. Mapped to GTT
  60  *      4. Read by GPU
  61  *      5. Unmapped from GTT
  62  *      6. Freed
  63  *
  64  *      Let's take these a step at a time
  65  *
  66  *      1. Allocated
  67  *              Pages allocated from the kernel may still have
  68  *              cache contents, so we set them to (CPU, CPU) always.
  69  *      2. Written by CPU (using pwrite)
  70  *              The pwrite function calls set_domain (CPU, CPU) and
  71  *              this function does nothing (as nothing changes)
  72  *      3. Mapped by GTT
  73  *              This function asserts that the object is not
  74  *              currently in any GPU-based read or write domains
  75  *      4. Read by GPU
  76  *              i915_gem_execbuffer calls set_domain (COMMAND, 0).
  77  *              As write_domain is zero, this function adds in the
  78  *              current read domains (CPU+COMMAND, 0).
  79  *              flush_domains is set to CPU.
  80  *              invalidate_domains is set to COMMAND
  81  *              clflush is run to get data out of the CPU caches
  82  *              then i915_dev_set_domain calls i915_gem_flush to
  83  *              emit an MI_FLUSH and drm_agp_chipset_flush
  84  *      5. Unmapped from GTT
  85  *              i915_gem_object_unbind calls set_domain (CPU, CPU)
  86  *              flush_domains and invalidate_domains end up both zero
  87  *              so no flushing/invalidating happens
  88  *      6. Freed
  89  *              yay, done
  90  *
  91  * Case 2: The shared render buffer
  92  *
  93  *      1. Allocated
  94  *      2. Mapped to GTT
  95  *      3. Read/written by GPU
  96  *      4. set_domain to (CPU,CPU)
  97  *      5. Read/written by CPU
  98  *      6. Read/written by GPU
  99  *
 100  *      1. Allocated
 101  *              Same as last example, (CPU, CPU)
 102  *      2. Mapped to GTT
 103  *              Nothing changes (assertions find that it is not in the GPU)
 104  *      3. Read/written by GPU
 105  *              execbuffer calls set_domain (RENDER, RENDER)
 106  *              flush_domains gets CPU
 107  *              invalidate_domains gets GPU
 108  *              clflush (obj)
 109  *              MI_FLUSH and drm_agp_chipset_flush
 110  *      4. set_domain (CPU, CPU)
 111  *              flush_domains gets GPU
 112  *              invalidate_domains gets CPU
 113  *              wait_rendering (obj) to make sure all drawing is complete.
 114  *              This will include an MI_FLUSH to get the data from GPU
 115  *              to memory
 116  *              clflush (obj) to invalidate the CPU cache
 117  *              Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
 118  *      5. Read/written by CPU
 119  *              cache lines are loaded and dirtied
 120  *      6. Read written by GPU
 121  *              Same as last GPU access
 122  *
 123  * Case 3: The constant buffer
 124  *
 125  *      1. Allocated
 126  *      2. Written by CPU
 127  *      3. Read by GPU
 128  *      4. Updated (written) by CPU again
 129  *      5. Read by GPU
 130  *
 131  *      1. Allocated
 132  *              (CPU, CPU)
 133  *      2. Written by CPU
 134  *              (CPU, CPU)
 135  *      3. Read by GPU
 136  *              (CPU+RENDER, 0)
 137  *              flush_domains = CPU
 138  *              invalidate_domains = RENDER
 139  *              clflush (obj)
 140  *              MI_FLUSH
 141  *              drm_agp_chipset_flush
 142  *      4. Updated (written) by CPU again
 143  *              (CPU, CPU)
 144  *              flush_domains = 0 (no previous write domain)
 145  *              invalidate_domains = 0 (no new read domains)
 146  *      5. Read by GPU
 147  *              (CPU+RENDER, 0)
 148  *              flush_domains = CPU
 149  *              invalidate_domains = RENDER
 150  *              clflush (obj)
 151  *              MI_FLUSH
 152  *              drm_agp_chipset_flush
 153  */
 154 static void
 155 i915_gem_object_set_to_gpu_domain(struct drm_i915_gem_object *obj,
 156                                   struct intel_ring_buffer *ring,
 157                                   struct change_domains *cd)
 158 {
 159         uint32_t invalidate_domains = 0, flush_domains = 0;
 160
 161         /*
 162          * If the object isn't moving to a new write domain,
 163          * let the object stay in multiple read domains
 164          */
 165         if (obj->base.pending_write_domain == 0)
 166                 obj->base.pending_read_domains |= obj->base.read_domains;
 167
 168         /*
 169          * Flush the current write domain if
 170          * the new read domains don't match. Invalidate
 171          * any read domains which differ from the old
 172          * write domain
 173          */
 174         if (obj->base.write_domain &&
 175             (((obj->base.write_domain != obj->base.pending_read_domains ||
 176                obj->ring != ring)) ||
 177              (obj->fenced_gpu_access && !obj->pending_fenced_gpu_access))) {
 178                 flush_domains |= obj->base.write_domain;
 179                 invalidate_domains |=
 180                         obj->base.pending_read_domains & ~obj->base.write_domain;
 181         }
 182         /*
 183          * Invalidate any read caches which may have
 184          * stale data. That is, any new read domains.
 185          */
 186         invalidate_domains |= obj->base.pending_read_domains & ~obj->base.read_domains;
 187         if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU)
 188                 i915_gem_clflush_object(obj);
 189
 190         /* blow away mappings if mapped through GTT */
 191         if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_GTT)
 192                 i915_gem_release_mmap(obj);
 193
 194         if (obj->base.pending_write_domain)
 195                 cd->flips |= atomic_read(&obj->pending_flip);
 196
 197         /* The actual obj->write_domain will be updated with
 198          * pending_write_domain after we emit the accumulated flush for all
 199          * of our domain changes in execbuffers (which clears objects'
 200          * write_domains).  So if we have a current write domain that we
 201          * aren't changing, set pending_write_domain to that.
 202          */
 203         if (flush_domains == 0 && obj->base.pending_write_domain == 0)
 204                 obj->base.pending_write_domain = obj->base.write_domain;
 205
 206         cd->invalidate_domains |= invalidate_domains;
 207         cd->flush_domains |= flush_domains;
 208         if (flush_domains & I915_GEM_GPU_DOMAINS)
 209                 cd->flush_rings |= obj->ring->id;
 210         if (invalidate_domains & I915_GEM_GPU_DOMAINS)
 211                 cd->flush_rings |= ring->id;
 212 }
 213
 214 struct eb_objects {
 215         int and;
 216         struct hlist_head buckets[0];
 217 };
 218
 219 static struct eb_objects *
 220 eb_create(int size)
 221 {
 222         struct eb_objects *eb;
 223         int count = PAGE_SIZE / sizeof(struct hlist_head) / 2;
 224         while (count > size)
 225                 count >>= 1;
 226         eb = kzalloc(count*sizeof(struct hlist_head) +
 227                      sizeof(struct eb_objects),
 228                      GFP_KERNEL);
 229         if (eb == NULL)
 230                 return eb;
 231
 232         eb->and = count - 1;
 233         return eb;
 234 }
 235
 236 static void
 237 eb_reset(struct eb_objects *eb)
 238 {
 239         memset(eb->buckets, 0, (eb->and+1)*sizeof(struct hlist_head));
 240 }
 241
 242 static void
 243 eb_add_object(struct eb_objects *eb, struct drm_i915_gem_object *obj)
 244 {
 245         hlist_add_head(&obj->exec_node,
 246                        &eb->buckets[obj->exec_handle & eb->and]);
 247 }
 248
 249 static struct drm_i915_gem_object *
 250 eb_get_object(struct eb_objects *eb, unsigned long handle)
 251 {
 252         struct hlist_head *head;
 253         struct hlist_node *node;
 254         struct drm_i915_gem_object *obj;
 255
 256         head = &eb->buckets[handle & eb->and];
 257         hlist_for_each(node, head) {
 258                 obj = hlist_entry(node, struct drm_i915_gem_object, exec_node);
 259                 if (obj->exec_handle == handle)
 260                         return obj;
 261         }
 262
 263         return NULL;
 264 }
 265
 266 static void
 267 eb_destroy(struct eb_objects *eb)
 268 {
 269         kfree(eb);
 270 }
 271
 272 static int
 273 i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
 274                                    struct eb_objects *eb,
 275                                    struct drm_i915_gem_relocation_entry *reloc)
 276 {
 277         struct drm_device *dev = obj->base.dev;
 278         struct drm_gem_object *target_obj;
 279         uint32_t target_offset;
 280         int ret = -EINVAL;
 281
 282         /* we've already hold a reference to all valid objects */
 283         target_obj = &eb_get_object(eb, reloc->target_handle)->base;
 284         if (unlikely(target_obj == NULL))
 285                 return -ENOENT;
 286
 287         target_offset = to_intel_bo(target_obj)->gtt_offset;
 288
 289         /* The target buffer should have appeared before us in the
 290          * exec_object list, so it should have a GTT space bound by now.
 291          */
 292         if (unlikely(target_offset == 0)) {
 293                 DRM_ERROR("No GTT space found for object %d\n",
 294                           reloc->target_handle);
 295                 return ret;
 296         }
 297
 298         /* Validate that the target is in a valid r/w GPU domain */
 299         if (unlikely(reloc->write_domain & (reloc->write_domain - 1))) {
 300                 DRM_ERROR("reloc with multiple write domains: "
 301                           "obj %p target %d offset %d "
 302                           "read %08x write %08x",
 303                           obj, reloc->target_handle,
 304                           (int) reloc->offset,
 305                           reloc->read_domains,
 306                           reloc->write_domain);
 307                 return ret;
 308         }
 309         if (unlikely((reloc->write_domain | reloc->read_domains) & I915_GEM_DOMAIN_CPU)) {
 310                 DRM_ERROR("reloc with read/write CPU domains: "
 311                           "obj %p target %d offset %d "
 312                           "read %08x write %08x",
 313                           obj, reloc->target_handle,
 314                           (int) reloc->offset,
 315                           reloc->read_domains,
 316                           reloc->write_domain);
 317                 return ret;
 318         }
 319         if (unlikely(reloc->write_domain && target_obj->pending_write_domain &&
 320                      reloc->write_domain != target_obj->pending_write_domain)) {
 321                 DRM_ERROR("Write domain conflict: "
 322                           "obj %p target %d offset %d "
 323                           "new %08x old %08x\n",
 324                           obj, reloc->target_handle,
 325                           (int) reloc->offset,
 326                           reloc->write_domain,
 327                           target_obj->pending_write_domain);
 328                 return ret;
 329         }
 330
 331         target_obj->pending_read_domains |= reloc->read_domains;
 332         target_obj->pending_write_domain |= reloc->write_domain;
 333
 334         /* If the relocation already has the right value in it, no
 335          * more work needs to be done.
 336          */
 337         if (target_offset == reloc->presumed_offset)
 338                 return 0;
 339
 340         /* Check that the relocation address is valid... */
 341         if (unlikely(reloc->offset > obj->base.size - 4)) {
 342                 DRM_ERROR("Relocation beyond object bounds: "
 343                           "obj %p target %d offset %d size %d.\n",
 344                           obj, reloc->target_handle,
 345                           (int) reloc->offset,
 346                           (int) obj->base.size);
 347                 return ret;
 348         }
 349         if (unlikely(reloc->offset & 3)) {
 350                 DRM_ERROR("Relocation not 4-byte aligned: "
 351                           "obj %p target %d offset %d.\n",
 352                           obj, reloc->target_handle,
 353                           (int) reloc->offset);
 354                 return ret;
 355         }
 356
 357         reloc->delta += target_offset;
 358         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) {
 359                 uint32_t page_offset = reloc->offset & ~PAGE_MASK;
 360                 char *vaddr;
 361
 362                 vaddr = kmap_atomic(obj->pages[reloc->offset >> PAGE_SHIFT]);
 363                 *(uint32_t *)(vaddr + page_offset) = reloc->delta;
 364                 kunmap_atomic(vaddr);
 365         } else {
 366                 struct drm_i915_private *dev_priv = dev->dev_private;
 367                 uint32_t __iomem *reloc_entry;
 368                 void __iomem *reloc_page;
 369
 370                 /* We can't wait for rendering with pagefaults disabled */
 371                 if (obj->active && in_atomic())
 372                         return -EFAULT;
 373
 374                 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
 375                 if (ret)
 376                         return ret;
 377
 378                 /* Map the page containing the relocation we're going to perform.  */
 379                 reloc->offset += obj->gtt_offset;
 380                 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
 381                                                       reloc->offset & PAGE_MASK);
 382                 reloc_entry = (uint32_t __iomem *)
 383                         (reloc_page + (reloc->offset & ~PAGE_MASK));
 384                 iowrite32(reloc->delta, reloc_entry);
 385                 io_mapping_unmap_atomic(reloc_page);
 386         }
 387
 388         /* and update the user's relocation entry */
 389         reloc->presumed_offset = target_offset;
 390
 391         return 0;
 392 }
 393
 394 static int
 395 i915_gem_execbuffer_relocate_object(struct drm_i915_gem_object *obj,
 396                                     struct eb_objects *eb)
 397 {
 398         struct drm_i915_gem_relocation_entry __user *user_relocs;
 399         struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
 400         int i, ret;
 401
 402         user_relocs = (void __user *)(uintptr_t)entry->relocs_ptr;
 403         for (i = 0; i < entry->relocation_count; i++) {
 404                 struct drm_i915_gem_relocation_entry reloc;
 405
 406                 if (__copy_from_user_inatomic(&reloc,
 407                                               user_relocs+i,
 408                                               sizeof(reloc)))
 409                         return -EFAULT;
 410
 411                 ret = i915_gem_execbuffer_relocate_entry(obj, eb, &reloc);
 412                 if (ret)
 413                         return ret;
 414
 415                 if (__copy_to_user_inatomic(&user_relocs[i].presumed_offset,
 416                                             &reloc.presumed_offset,
 417                                             sizeof(reloc.presumed_offset)))
 418                         return -EFAULT;
 419         }
 420
 421         return 0;
 422 }
 423
 424 static int
 425 i915_gem_execbuffer_relocate_object_slow(struct drm_i915_gem_object *obj,
 426                                          struct eb_objects *eb,
 427                                          struct drm_i915_gem_relocation_entry *relocs)
 428 {
 429         const struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
 430         int i, ret;
 431
 432         for (i = 0; i < entry->relocation_count; i++) {
 433                 ret = i915_gem_execbuffer_relocate_entry(obj, eb, &relocs[i]);
 434                 if (ret)
 435                         return ret;
 436         }
 437
 438         return 0;
 439 }
 440
 441 static int
 442 i915_gem_execbuffer_relocate(struct drm_device *dev,
 443                              struct eb_objects *eb,
 444                              struct list_head *objects)
 445 {
 446         struct drm_i915_gem_object *obj;
 447         int ret = 0;
 448
 449         /* This is the fast path and we cannot handle a pagefault whilst
 450          * holding the struct mutex lest the user pass in the relocations
 451          * contained within a mmaped bo. For in such a case we, the page
 452          * fault handler would call i915_gem_fault() and we would try to
 453          * acquire the struct mutex again. Obviously this is bad and so
 454          * lockdep complains vehemently.
 455          */
 456         pagefault_disable();
 457         list_for_each_entry(obj, objects, exec_list) {
 458                 ret = i915_gem_execbuffer_relocate_object(obj, eb);
 459                 if (ret)
 460                         break;
 461         }
 462         pagefault_enable();
 463
 464         return ret;
 465 }
 466
 467 static int
 468 i915_gem_execbuffer_reserve(struct intel_ring_buffer *ring,
 469                             struct drm_file *file,
 470                             struct list_head *objects)
 471 {
 472         struct drm_i915_gem_object *obj;
 473         int ret, retry;
 474         bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4;
 475         struct list_head ordered_objects;
 476
 477         INIT_LIST_HEAD(&ordered_objects);
 478         while (!list_empty(objects)) {
 479                 struct drm_i915_gem_exec_object2 *entry;
 480                 bool need_fence, need_mappable;
 481
 482                 obj = list_first_entry(objects,
 483                                        struct drm_i915_gem_object,
 484                                        exec_list);
 485                 entry = obj->exec_entry;
 486
 487                 need_fence =
 488                         has_fenced_gpu_access &&
 489                         entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
 490                         obj->tiling_mode != I915_TILING_NONE;
 491                 need_mappable =
 492                         entry->relocation_count ? true : need_fence;
 493
 494                 if (need_mappable)
 495                         list_move(&obj->exec_list, &ordered_objects);
 496                 else
 497                         list_move_tail(&obj->exec_list, &ordered_objects);
 498
 499                 obj->base.pending_read_domains = 0;
 500                 obj->base.pending_write_domain = 0;
 501         }
 502         list_splice(&ordered_objects, objects);
 503
 504         /* Attempt to pin all of the buffers into the GTT.
 505          * This is done in 3 phases:
 506          *
 507          * 1a. Unbind all objects that do not match the GTT constraints for
 508          *     the execbuffer (fenceable, mappable, alignment etc).
 509          * 1b. Increment pin count for already bound objects.
 510          * 2.  Bind new objects.
 511          * 3.  Decrement pin count.
 512          *
 513          * This avoid unnecessary unbinding of later objects in order to makr
 514          * room for the earlier objects *unless* we need to defragment.
 515          */
 516         retry = 0;
 517         do {
 518                 ret = 0;
 519
 520                 /* Unbind any ill-fitting objects or pin. */
 521                 list_for_each_entry(obj, objects, exec_list) {
 522                         struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
 523                         bool need_fence, need_mappable;
 524                         if (!obj->gtt_space)
 525                                 continue;
 526
 527                         need_fence =
 528                                 has_fenced_gpu_access &&
 529                                 entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
 530                                 obj->tiling_mode != I915_TILING_NONE;
 531                         need_mappable =
 532                                 entry->relocation_count ? true : need_fence;
 533
 534                         if ((entry->alignment && obj->gtt_offset & (entry->alignment - 1)) ||
 535                             (need_mappable && !obj->map_and_fenceable))
 536                                 ret = i915_gem_object_unbind(obj);
 537                         else
 538                                 ret = i915_gem_object_pin(obj,
 539                                                           entry->alignment,
 540                                                           need_mappable);
 541                         if (ret)
 542                                 goto err;
 543
 544                         entry++;
 545                 }
 546
 547                 /* Bind fresh objects */
 548                 list_for_each_entry(obj, objects, exec_list) {
 549                         struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
 550                         bool need_fence;
 551
 552                         need_fence =
 553                                 has_fenced_gpu_access &&
 554                                 entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
 555                                 obj->tiling_mode != I915_TILING_NONE;
 556
 557                         if (!obj->gtt_space) {
 558                                 bool need_mappable =
 559                                         entry->relocation_count ? true : need_fence;
 560
 561                                 ret = i915_gem_object_pin(obj,
 562                                                           entry->alignment,
 563                                                           need_mappable);
 564                                 if (ret)
 565                                         break;
 566                         }
 567
 568                         if (has_fenced_gpu_access) {
 569                                 if (need_fence) {
 570                                         ret = i915_gem_object_get_fence(obj, ring);
 571                                         if (ret)
 572                                                 break;
 573                                 } else if (entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
 574                                            obj->tiling_mode == I915_TILING_NONE) {
 575                                         /* XXX pipelined! */
 576                                         ret = i915_gem_object_put_fence(obj);
 577                                         if (ret)
 578                                                 break;
 579                                 }
 580                                 obj->pending_fenced_gpu_access = need_fence;
 581                         }
 582
 583                         entry->offset = obj->gtt_offset;
 584                 }
 585
 586                 /* Decrement pin count for bound objects */
 587                 list_for_each_entry(obj, objects, exec_list) {
 588                         if (obj->gtt_space)
 589                                 i915_gem_object_unpin(obj);
 590                 }
 591
 592                 if (ret != -ENOSPC || retry > 1)
 593                         return ret;
 594
 595                 /* First attempt, just clear anything that is purgeable.
 596                  * Second attempt, clear the entire GTT.
 597                  */
 598                 ret = i915_gem_evict_everything(ring->dev, retry == 0);
 599                 if (ret)
 600                         return ret;
 601
 602                 retry++;
 603         } while (1);
 604
 605 err:
 606         obj = list_entry(obj->exec_list.prev,
 607                          struct drm_i915_gem_object,
 608                          exec_list);
 609         while (objects != &obj->exec_list) {
 610                 if (obj->gtt_space)
 611                         i915_gem_object_unpin(obj);
 612
 613                 obj = list_entry(obj->exec_list.prev,
 614                                  struct drm_i915_gem_object,
 615                                  exec_list);
 616         }
 617
 618         return ret;
 619 }
 620
 621 static int
 622 i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
 623                                   struct drm_file *file,
 624                                   struct intel_ring_buffer *ring,
 625                                   struct list_head *objects,
 626                                   struct eb_objects *eb,
 627                                   struct drm_i915_gem_exec_object2 *exec,
 628                                   int count)
 629 {
 630         struct drm_i915_gem_relocation_entry *reloc;
 631         struct drm_i915_gem_object *obj;
 632         int *reloc_offset;
 633         int i, total, ret;
 634
 635         /* We may process another execbuffer during the unlock... */
 636         while (!list_empty(objects)) {
 637                 obj = list_first_entry(objects,
 638                                        struct drm_i915_gem_object,
 639                                        exec_list);
 640                 list_del_init(&obj->exec_list);
 641                 drm_gem_object_unreference(&obj->base);
 642         }
 643
 644         mutex_unlock(&dev->struct_mutex);
 645
 646         total = 0;
 647         for (i = 0; i < count; i++)
 648                 total += exec[i].relocation_count;
 649
 650         reloc_offset = drm_malloc_ab(count, sizeof(*reloc_offset));
 651         reloc = drm_malloc_ab(total, sizeof(*reloc));
 652         if (reloc == NULL || reloc_offset == NULL) {
 653                 drm_free_large(reloc);
 654                 drm_free_large(reloc_offset);
 655                 mutex_lock(&dev->struct_mutex);
 656                 return -ENOMEM;
 657         }
 658
 659         total = 0;
 660         for (i = 0; i < count; i++) {
 661                 struct drm_i915_gem_relocation_entry __user *user_relocs;
 662
 663                 user_relocs = (void __user *)(uintptr_t)exec[i].relocs_ptr;
 664
 665                 if (copy_from_user(reloc+total, user_relocs,
 666                                    exec[i].relocation_count * sizeof(*reloc))) {
 667                         ret = -EFAULT;
 668                         mutex_lock(&dev->struct_mutex);
 669                         goto err;
 670                 }
 671
 672                 reloc_offset[i] = total;
 673                 total += exec[i].relocation_count;
 674         }
 675
 676         ret = i915_mutex_lock_interruptible(dev);
 677         if (ret) {
 678                 mutex_lock(&dev->struct_mutex);
 679                 goto err;
 680         }
 681
 682         /* reacquire the objects */
 683         eb_reset(eb);
 684         for (i = 0; i < count; i++) {
 685                 obj = to_intel_bo(drm_gem_object_lookup(dev, file,
 686                                                         exec[i].handle));
 687                 if (&obj->base == NULL) {
 688                         DRM_ERROR("Invalid object handle %d at index %d\n",
 689                                    exec[i].handle, i);
 690                         ret = -ENOENT;
 691                         goto err;
 692                 }
 693
 694                 list_add_tail(&obj->exec_list, objects);
 695                 obj->exec_handle = exec[i].handle;
 696                 obj->exec_entry = &exec[i];
 697                 eb_add_object(eb, obj);
 698         }
 699
 700         ret = i915_gem_execbuffer_reserve(ring, file, objects);
 701         if (ret)
 702                 goto err;
 703
 704         list_for_each_entry(obj, objects, exec_list) {
 705                 int offset = obj->exec_entry - exec;
 706                 ret = i915_gem_execbuffer_relocate_object_slow(obj, eb,
 707                                                                reloc + reloc_offset[offset]);
 708                 if (ret)
 709                         goto err;
 710         }
 711
 712         /* Leave the user relocations as are, this is the painfully slow path,
 713          * and we want to avoid the complication of dropping the lock whilst
 714          * having buffers reserved in the aperture and so causing spurious
 715          * ENOSPC for random operations.
 716          */
 717
 718 err:
 719         drm_free_large(reloc);
 720         drm_free_large(reloc_offset);
 721         return ret;
 722 }
 723
 724 static int
 725 i915_gem_execbuffer_flush(struct drm_device *dev,
 726                           uint32_t invalidate_domains,
 727                           uint32_t flush_domains,
 728                           uint32_t flush_rings)
 729 {
 730         drm_i915_private_t *dev_priv = dev->dev_private;
 731         int i, ret;
 732
 733         if (flush_domains & I915_GEM_DOMAIN_CPU)
 734                 intel_gtt_chipset_flush();
 735
 736         if (flush_domains & I915_GEM_DOMAIN_GTT)
 737                 wmb();
 738
 739         if ((flush_domains | invalidate_domains) & I915_GEM_GPU_DOMAINS) {
 740                 for (i = 0; i < I915_NUM_RINGS; i++)
 741                         if (flush_rings & (1 << i)) {
 742                                 ret = i915_gem_flush_ring(&dev_priv->ring[i],
 743                                                           invalidate_domains,
 744                                                           flush_domains);
 745                                 if (ret)
 746                                         return ret;
 747                         }
 748         }
 749
 750         return 0;
 751 }
 752
 753 static int
 754 i915_gem_execbuffer_sync_rings(struct drm_i915_gem_object *obj,
 755                                struct intel_ring_buffer *to)
 756 {
 757         struct intel_ring_buffer *from = obj->ring;
 758         u32 seqno;
 759         int ret, idx;
 760
 761         if (from == NULL || to == from)
 762                 return 0;
 763
 764         /* XXX gpu semaphores are implicated in various hard hangs on SNB */
 765         if (INTEL_INFO(obj->base.dev)->gen < 6 || !i915_semaphores)
 766                 return i915_gem_object_wait_rendering(obj);
 767
 768         idx = intel_ring_sync_index(from, to);
 769
 770         seqno = obj->last_rendering_seqno;
 771         if (seqno <= from->sync_seqno[idx])
 772                 return 0;
 773
 774         if (seqno == from->outstanding_lazy_request) {
 775                 struct drm_i915_gem_request *request;
 776
 777                 request = kzalloc(sizeof(*request), GFP_KERNEL);
 778                 if (request == NULL)
 779                         return -ENOMEM;
 780
 781                 ret = i915_add_request(from, NULL, request);
 782                 if (ret) {
 783                         kfree(request);
 784                         return ret;
 785                 }
 786
 787                 seqno = request->seqno;
 788         }
 789
 790         from->sync_seqno[idx] = seqno;
 791         return intel_ring_sync(to, from, seqno - 1);
 792 }
 793
 794 static int
 795 i915_gem_execbuffer_wait_for_flips(struct intel_ring_buffer *ring, u32 flips)
 796 {
 797         u32 plane, flip_mask;
 798         int ret;
 799
 800         /* Check for any pending flips. As we only maintain a flip queue depth
 801          * of 1, we can simply insert a WAIT for the next display flip prior
 802          * to executing the batch and avoid stalling the CPU.
 803          */
 804
 805         for (plane = 0; flips >> plane; plane++) {
 806                 if (((flips >> plane) & 1) == 0)
 807                         continue;
 808
 809                 if (plane)
 810                         flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
 811                 else
 812                         flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
 813
 814                 ret = intel_ring_begin(ring, 2);
 815                 if (ret)
 816                         return ret;
 817
 818                 intel_ring_emit(ring, MI_WAIT_FOR_EVENT | flip_mask);
 819                 intel_ring_emit(ring, MI_NOOP);
 820                 intel_ring_advance(ring);
 821         }
 822
 823         return 0;
 824 }
 825
 826
 827 static int
 828 i915_gem_execbuffer_move_to_gpu(struct intel_ring_buffer *ring,
 829                                 struct list_head *objects)
 830 {
 831         struct drm_i915_gem_object *obj;
 832         struct change_domains cd;
 833         int ret;
 834
 835         memset(&cd, 0, sizeof(cd));
 836         list_for_each_entry(obj, objects, exec_list)
 837                 i915_gem_object_set_to_gpu_domain(obj, ring, &cd);
 838
 839         if (cd.invalidate_domains | cd.flush_domains) {
 840                 ret = i915_gem_execbuffer_flush(ring->dev,
 841                                                 cd.invalidate_domains,
 842                                                 cd.flush_domains,
 843                                                 cd.flush_rings);
 844                 if (ret)
 845                         return ret;
 846         }
 847
 848         if (cd.flips) {
 849                 ret = i915_gem_execbuffer_wait_for_flips(ring, cd.flips);
 850                 if (ret)
 851                         return ret;
 852         }
 853
 854         list_for_each_entry(obj, objects, exec_list) {
 855                 ret = i915_gem_execbuffer_sync_rings(obj, ring);
 856                 if (ret)
 857                         return ret;
 858         }
 859
 860         return 0;
 861 }
 862
 863 static bool
 864 i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
 865 {
 866         return ((exec->batch_start_offset | exec->batch_len) & 0x7) == 0;
 867 }
 868
 869 static int
 870 validate_exec_list(struct drm_i915_gem_exec_object2 *exec,
 871                    int count)
 872 {
 873         int i;
 874
 875         for (i = 0; i < count; i++) {
 876                 char __user *ptr = (char __user *)(uintptr_t)exec[i].relocs_ptr;
 877                 int length; /* limited by fault_in_pages_readable() */
 878
 879                 /* First check for malicious input causing overflow */
 880                 if (exec[i].relocation_count >
 881                     INT_MAX / sizeof(struct drm_i915_gem_relocation_entry))
 882                         return -EINVAL;
 883
 884                 length = exec[i].relocation_count *
 885                         sizeof(struct drm_i915_gem_relocation_entry);
 886                 if (!access_ok(VERIFY_READ, ptr, length))
 887                         return -EFAULT;
 888
 889                 /* we may also need to update the presumed offsets */
 890                 if (!access_ok(VERIFY_WRITE, ptr, length))
 891                         return -EFAULT;
 892
 893                 if (fault_in_pages_readable(ptr, length))
 894                         return -EFAULT;
 895         }
 896
 897         return 0;
 898 }
 899
 900 static void
 901 i915_gem_execbuffer_move_to_active(struct list_head *objects,
 902                                    struct intel_ring_buffer *ring,
 903                                    u32 seqno)
 904 {
 905         struct drm_i915_gem_object *obj;
 906
 907         list_for_each_entry(obj, objects, exec_list) {
 908                   u32 old_read = obj->base.read_domains;
 909                   u32 old_write = obj->base.write_domain;
 910
 911
 912                 obj->base.read_domains = obj->base.pending_read_domains;
 913                 obj->base.write_domain = obj->base.pending_write_domain;
 914                 obj->fenced_gpu_access = obj->pending_fenced_gpu_access;
 915
 916                 i915_gem_object_move_to_active(obj, ring, seqno);
 917                 if (obj->base.write_domain) {
 918                         obj->dirty = 1;
 919                         obj->pending_gpu_write = true;
 920                         list_move_tail(&obj->gpu_write_list,
 921                                        &ring->gpu_write_list);
 922                         intel_mark_busy(ring->dev, obj);
 923                 }
 924
 925                 trace_i915_gem_object_change_domain(obj, old_read, old_write);
 926         }
 927 }
 928
 929 static void
 930 i915_gem_execbuffer_retire_commands(struct drm_device *dev,
 931                                     struct drm_file *file,
 932                                     struct intel_ring_buffer *ring)
 933 {
 934         struct drm_i915_gem_request *request;
 935         u32 invalidate;
 936
 937         /*
 938          * Ensure that the commands in the batch buffer are
 939          * finished before the interrupt fires.
 940          *
 941          * The sampler always gets flushed on i965 (sigh).
 942          */
 943         invalidate = I915_GEM_DOMAIN_COMMAND;
 944         if (INTEL_INFO(dev)->gen >= 4)
 945                 invalidate |= I915_GEM_DOMAIN_SAMPLER;
 946         if (ring->flush(ring, invalidate, 0)) {
 947                 i915_gem_next_request_seqno(ring);
 948                 return;
 949         }
 950
 951         /* Add a breadcrumb for the completion of the batch buffer */
 952         request = kzalloc(sizeof(*request), GFP_KERNEL);
 953         if (request == NULL || i915_add_request(ring, file, request)) {
 954                 i915_gem_next_request_seqno(ring);
 955                 kfree(request);
 956         }
 957 }
 958
 959 static int
 960 i915_gem_do_execbuffer(struct drm_device *dev, void *data,
 961                        struct drm_file *file,
 962                        struct drm_i915_gem_execbuffer2 *args,
 963                        struct drm_i915_gem_exec_object2 *exec)
 964 {
 965         drm_i915_private_t *dev_priv = dev->dev_private;
 966         struct list_head objects;
 967         struct eb_objects *eb;
 968         struct drm_i915_gem_object *batch_obj;
 969         struct drm_clip_rect *cliprects = NULL;
 970         struct intel_ring_buffer *ring;
 971         u32 exec_start, exec_len;
 972         u32 seqno;
 973         int ret, mode, i;
 974
 975         if (!i915_gem_check_execbuffer(args)) {
 976                 DRM_ERROR("execbuf with invalid offset/length\n");
 977                 return -EINVAL;
 978         }
 979
 980         ret = validate_exec_list(exec, args->buffer_count);
 981         if (ret)
 982                 return ret;
 983
 984         switch (args->flags & I915_EXEC_RING_MASK) {
 985         case I915_EXEC_DEFAULT:
 986         case I915_EXEC_RENDER:
 987                 ring = &dev_priv->ring[RCS];
 988                 break;
 989         case I915_EXEC_BSD:
 990                 if (!HAS_BSD(dev)) {
 991                         DRM_ERROR("execbuf with invalid ring (BSD)\n");
 992                         return -EINVAL;
 993                 }
 994                 ring = &dev_priv->ring[VCS];
 995                 break;
 996         case I915_EXEC_BLT:
 997                 if (!HAS_BLT(dev)) {
 998                         DRM_ERROR("execbuf with invalid ring (BLT)\n");
 999                         return -EINVAL;
1000                 }
1001                 ring = &dev_priv->ring[BCS];
1002                 break;
1003         default:
1004                 DRM_ERROR("execbuf with unknown ring: %d\n",
1005                           (int)(args->flags & I915_EXEC_RING_MASK));
1006                 return -EINVAL;
1007         }
1008
1009         mode = args->flags & I915_EXEC_CONSTANTS_MASK;
1010         switch (mode) {
1011         case I915_EXEC_CONSTANTS_REL_GENERAL:
1012         case I915_EXEC_CONSTANTS_ABSOLUTE:
1013         case I915_EXEC_CONSTANTS_REL_SURFACE:
1014                 if (ring == &dev_priv->ring[RCS] &&
1015                     mode != dev_priv->relative_constants_mode) {
1016                         if (INTEL_INFO(dev)->gen < 4)
1017                                 return -EINVAL;
1018
1019                         if (INTEL_INFO(dev)->gen > 5 &&
1020                             mode == I915_EXEC_CONSTANTS_REL_SURFACE)
1021                                 return -EINVAL;
1022
1023                         ret = intel_ring_begin(ring, 4);
1024                         if (ret)
1025                                 return ret;
1026
1027                         intel_ring_emit(ring, MI_NOOP);
1028                         intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
1029                         intel_ring_emit(ring, INSTPM);
1030                         intel_ring_emit(ring,
1031                                         I915_EXEC_CONSTANTS_MASK << 16 | mode);
1032                         intel_ring_advance(ring);
1033
1034                         dev_priv->relative_constants_mode = mode;
1035                 }
1036                 break;
1037         default:
1038                 DRM_ERROR("execbuf with unknown constants: %d\n", mode);
1039                 return -EINVAL;
1040         }
1041
1042         if (args->buffer_count < 1) {
1043                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
1044                 return -EINVAL;
1045         }
1046
1047         if (args->num_cliprects != 0) {
1048                 if (ring != &dev_priv->ring[RCS]) {
1049                         DRM_ERROR("clip rectangles are only valid with the render ring\n");
1050                         return -EINVAL;
1051                 }
1052
1053                 cliprects = kmalloc(args->num_cliprects * sizeof(*cliprects),
1054                                     GFP_KERNEL);
1055                 if (cliprects == NULL) {
1056                         ret = -ENOMEM;
1057                         goto pre_mutex_err;
1058                 }
1059
1060                 if (copy_from_user(cliprects,
1061                                      (struct drm_clip_rect __user *)(uintptr_t)
1062                                      args->cliprects_ptr,
1063                                      sizeof(*cliprects)*args->num_cliprects)) {
1064                         ret = -EFAULT;
1065                         goto pre_mutex_err;
1066                 }
1067         }
1068
1069         ret = i915_mutex_lock_interruptible(dev);
1070         if (ret)
1071                 goto pre_mutex_err;
1072
1073         if (dev_priv->mm.suspended) {
1074                 mutex_unlock(&dev->struct_mutex);
1075                 ret = -EBUSY;
1076                 goto pre_mutex_err;
1077         }
1078
1079         eb = eb_create(args->buffer_count);
1080         if (eb == NULL) {
1081                 mutex_unlock(&dev->struct_mutex);
1082                 ret = -ENOMEM;
1083                 goto pre_mutex_err;
1084         }
1085
1086         /* Look up object handles */
1087         INIT_LIST_HEAD(&objects);
1088         for (i = 0; i < args->buffer_count; i++) {
1089                 struct drm_i915_gem_object *obj;
1090
1091                 obj = to_intel_bo(drm_gem_object_lookup(dev, file,
1092                                                         exec[i].handle));
1093                 if (&obj->base == NULL) {
1094                         DRM_ERROR("Invalid object handle %d at index %d\n",
1095                                    exec[i].handle, i);
1096                         /* prevent error path from reading uninitialized data */
1097                         ret = -ENOENT;
1098                         goto err;
1099                 }
1100
1101                 if (!list_empty(&obj->exec_list)) {
1102                         DRM_ERROR("Object %p [handle %d, index %d] appears more than once in object list\n",
1103                                    obj, exec[i].handle, i);
1104                         ret = -EINVAL;
1105                         goto err;
1106                 }
1107
1108                 list_add_tail(&obj->exec_list, &objects);
1109                 obj->exec_handle = exec[i].handle;
1110                 obj->exec_entry = &exec[i];
1111                 eb_add_object(eb, obj);
1112         }
1113
1114         /* take note of the batch buffer before we might reorder the lists */
1115         batch_obj = list_entry(objects.prev,
1116                                struct drm_i915_gem_object,
1117                                exec_list);
1118
1119         /* Move the objects en-masse into the GTT, evicting if necessary. */
1120         ret = i915_gem_execbuffer_reserve(ring, file, &objects);
1121         if (ret)
1122                 goto err;
1123
1124         /* The objects are in their final locations, apply the relocations. */
1125         ret = i915_gem_execbuffer_relocate(dev, eb, &objects);
1126         if (ret) {
1127                 if (ret == -EFAULT) {
1128                         ret = i915_gem_execbuffer_relocate_slow(dev, file, ring,
1129                                                                 &objects, eb,
1130                                                                 exec,
1131                                                                 args->buffer_count);
1132                         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1133                 }
1134                 if (ret)
1135                         goto err;
1136         }
1137
1138         /* Set the pending read domains for the batch buffer to COMMAND */
1139         if (batch_obj->base.pending_write_domain) {
1140                 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
1141                 ret = -EINVAL;
1142                 goto err;
1143         }
1144         batch_obj->base.pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
1145
1146         ret = i915_gem_execbuffer_move_to_gpu(ring, &objects);
1147         if (ret)
1148                 goto err;
1149
1150         seqno = i915_gem_next_request_seqno(ring);
1151         for (i = 0; i < ARRAY_SIZE(ring->sync_seqno); i++) {
1152                 if (seqno < ring->sync_seqno[i]) {
1153                         /* The GPU can not handle its semaphore value wrapping,
1154                          * so every billion or so execbuffers, we need to stall
1155                          * the GPU in order to reset the counters.
1156                          */
1157                         ret = i915_gpu_idle(dev);
1158                         if (ret)
1159                                 goto err;
1160
1161                         BUG_ON(ring->sync_seqno[i]);
1162                 }
1163         }
1164
1165         trace_i915_gem_ring_dispatch(ring, seqno);
1166
1167         exec_start = batch_obj->gtt_offset + args->batch_start_offset;
1168         exec_len = args->batch_len;
1169         if (cliprects) {
1170                 for (i = 0; i < args->num_cliprects; i++) {
1171                         ret = i915_emit_box(dev, &cliprects[i],
1172                                             args->DR1, args->DR4);
1173                         if (ret)
1174                                 goto err;
1175
1176                         ret = ring->dispatch_execbuffer(ring,
1177                                                         exec_start, exec_len);
1178                         if (ret)
1179                                 goto err;
1180                 }
1181         } else {
1182                 ret = ring->dispatch_execbuffer(ring, exec_start, exec_len);
1183                 if (ret)
1184                         goto err;
1185         }
1186
1187         i915_gem_execbuffer_move_to_active(&objects, ring, seqno);
1188         i915_gem_execbuffer_retire_commands(dev, file, ring);
1189
1190 err:
1191         eb_destroy(eb);
1192         while (!list_empty(&objects)) {
1193                 struct drm_i915_gem_object *obj;
1194
1195                 obj = list_first_entry(&objects,
1196                                        struct drm_i915_gem_object,
1197                                        exec_list);
1198                 list_del_init(&obj->exec_list);
1199                 drm_gem_object_unreference(&obj->base);
1200         }
1201
1202         mutex_unlock(&dev->struct_mutex);
1203
1204 pre_mutex_err:
1205         kfree(cliprects);
1206         return ret;
1207 }
1208
1209 /*
1210  * Legacy execbuffer just creates an exec2 list from the original exec object
1211  * list array and passes it to the real function.
1212  */
1213 int
1214 i915_gem_execbuffer(struct drm_device *dev, void *data,
1215                     struct drm_file *file)
1216 {
1217         struct drm_i915_gem_execbuffer *args = data;
1218         struct drm_i915_gem_execbuffer2 exec2;
1219         struct drm_i915_gem_exec_object *exec_list = NULL;
1220         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
1221         int ret, i;
1222
1223         if (args->buffer_count < 1) {
1224                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
1225                 return -EINVAL;
1226         }
1227
1228         /* Copy in the exec list from userland */
1229         exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
1230         exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
1231         if (exec_list == NULL || exec2_list == NULL) {
1232                 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
1233                           args->buffer_count);
1234                 drm_free_large(exec_list);
1235                 drm_free_large(exec2_list);
1236                 return -ENOMEM;
1237         }
1238         ret = copy_from_user(exec_list,
1239                              (struct drm_i915_relocation_entry __user *)
1240                              (uintptr_t) args->buffers_ptr,
1241                              sizeof(*exec_list) * args->buffer_count);
1242         if (ret != 0) {
1243                 DRM_ERROR("copy %d exec entries failed %d\n",
1244                           args->buffer_count, ret);
1245                 drm_free_large(exec_list);
1246                 drm_free_large(exec2_list);
1247                 return -EFAULT;
1248         }
1249
1250         for (i = 0; i < args->buffer_count; i++) {
1251                 exec2_list[i].handle = exec_list[i].handle;
1252                 exec2_list[i].relocation_count = exec_list[i].relocation_count;
1253                 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
1254                 exec2_list[i].alignment = exec_list[i].alignment;
1255                 exec2_list[i].offset = exec_list[i].offset;
1256                 if (INTEL_INFO(dev)->gen < 4)
1257                         exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
1258                 else
1259                         exec2_list[i].flags = 0;
1260         }
1261
1262         exec2.buffers_ptr = args->buffers_ptr;
1263         exec2.buffer_count = args->buffer_count;
1264         exec2.batch_start_offset = args->batch_start_offset;
1265         exec2.batch_len = args->batch_len;
1266         exec2.DR1 = args->DR1;
1267         exec2.DR4 = args->DR4;
1268         exec2.num_cliprects = args->num_cliprects;
1269         exec2.cliprects_ptr = args->cliprects_ptr;
1270         exec2.flags = I915_EXEC_RENDER;
1271
1272         ret = i915_gem_do_execbuffer(dev, data, file, &exec2, exec2_list);
1273         if (!ret) {
1274                 /* Copy the new buffer offsets back to the user's exec list. */
1275                 for (i = 0; i < args->buffer_count; i++)
1276                         exec_list[i].offset = exec2_list[i].offset;
1277                 /* ... and back out to userspace */
1278                 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
1279                                    (uintptr_t) args->buffers_ptr,
1280                                    exec_list,
1281                                    sizeof(*exec_list) * args->buffer_count);
1282                 if (ret) {
1283                         ret = -EFAULT;
1284                         DRM_ERROR("failed to copy %d exec entries "
1285                                   "back to user (%d)\n",
1286                                   args->buffer_count, ret);
1287                 }
1288         }
1289
1290         drm_free_large(exec_list);
1291         drm_free_large(exec2_list);
1292         return ret;
1293 }
1294
1295 int
1296 i915_gem_execbuffer2(struct drm_device *dev, void *data,
1297                      struct drm_file *file)
1298 {
1299         struct drm_i915_gem_execbuffer2 *args = data;
1300         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
1301         int ret;
1302
1303         if (args->buffer_count < 1) {
1304                 DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
1305                 return -EINVAL;
1306         }
1307
1308         exec2_list = kmalloc(sizeof(*exec2_list)*args->buffer_count,
1309                              GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY);
1310         if (exec2_list == NULL)
1311                 exec2_list = drm_malloc_ab(sizeof(*exec2_list),
1312                                            args->buffer_count);
1313         if (exec2_list == NULL) {
1314                 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
1315                           args->buffer_count);
1316                 return -ENOMEM;
1317         }
1318         ret = copy_from_user(exec2_list,
1319                              (struct drm_i915_relocation_entry __user *)
1320                              (uintptr_t) args->buffers_ptr,
1321                              sizeof(*exec2_list) * args->buffer_count);
1322         if (ret != 0) {
1323                 DRM_ERROR("copy %d exec entries failed %d\n",
1324                           args->buffer_count, ret);
1325                 drm_free_large(exec2_list);
1326                 return -EFAULT;
1327         }
1328
1329         ret = i915_gem_do_execbuffer(dev, data, file, args, exec2_list);
1330         if (!ret) {
1331                 /* Copy the new buffer offsets back to the user's exec list. */
1332                 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
1333                                    (uintptr_t) args->buffers_ptr,
1334                                    exec2_list,
1335                                    sizeof(*exec2_list) * args->buffer_count);
1336                 if (ret) {
1337                         ret = -EFAULT;
1338                         DRM_ERROR("failed to copy %d exec entries "
1339                                   "back to user (%d)\n",
1340                                   args->buffer_count, ret);
1341                 }
1342         }
1343
1344         drm_free_large(exec2_list);
1345         return ret;
1346 }