drivers/gpu/drm/i915/i915_gem_execbuffer.c

   1 /*
   2  * Copyright © 2008,2010 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *    Chris Wilson <chris@chris-wilson.co.uk>
  26  *
  27  */
  28
  29 #include <drm/drmP.h>
  30 #include <drm/i915_drm.h>
  31 #include "i915_drv.h"
  32 #include "i915_trace.h"
  33 #include "intel_drv.h"
  34 #include <linux/dma_remapping.h>
  35 #include <linux/uaccess.h>
  36
  37 #define  __EXEC_OBJECT_HAS_PIN          (1<<31)
  38 #define  __EXEC_OBJECT_HAS_FENCE        (1<<30)
  39 #define  __EXEC_OBJECT_NEEDS_MAP        (1<<29)
  40 #define  __EXEC_OBJECT_NEEDS_BIAS       (1<<28)
  41 #define  __EXEC_OBJECT_INTERNAL_FLAGS (0xf<<28) /* all of the above */
  42
  43 #define BATCH_OFFSET_BIAS (256*1024)
  44
  45 struct i915_execbuffer_params {
  46         struct drm_device               *dev;
  47         struct drm_file                 *file;
  48         u32                              dispatch_flags;
  49         u32                              args_batch_start_offset;
  50         u32                              batch_obj_vm_offset;
  51         struct intel_engine_cs          *engine;
  52         struct drm_i915_gem_object      *batch_obj;
  53         struct i915_gem_context         *ctx;
  54         struct drm_i915_gem_request     *request;
  55 };
  56
  57 struct eb_vmas {
  58         struct list_head vmas;
  59         int and;
  60         union {
  61                 struct i915_vma *lut[0];
  62                 struct hlist_head buckets[0];
  63         };
  64 };
  65
  66 static struct eb_vmas *
  67 eb_create(struct drm_i915_gem_execbuffer2 *args)
  68 {
  69         struct eb_vmas *eb = NULL;
  70
  71         if (args->flags & I915_EXEC_HANDLE_LUT) {
  72                 unsigned size = args->buffer_count;
  73                 size *= sizeof(struct i915_vma *);
  74                 size += sizeof(struct eb_vmas);
  75                 eb = kmalloc(size, GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY);
  76         }
  77
  78         if (eb == NULL) {
  79                 unsigned size = args->buffer_count;
  80                 unsigned count = PAGE_SIZE / sizeof(struct hlist_head) / 2;
  81                 BUILD_BUG_ON_NOT_POWER_OF_2(PAGE_SIZE / sizeof(struct hlist_head));
  82                 while (count > 2*size)
  83                         count >>= 1;
  84                 eb = kzalloc(count*sizeof(struct hlist_head) +
  85                              sizeof(struct eb_vmas),
  86                              GFP_TEMPORARY);
  87                 if (eb == NULL)
  88                         return eb;
  89
  90                 eb->and = count - 1;
  91         } else
  92                 eb->and = -args->buffer_count;
  93
  94         INIT_LIST_HEAD(&eb->vmas);
  95         return eb;
  96 }
  97
  98 static void
  99 eb_reset(struct eb_vmas *eb)
 100 {
 101         if (eb->and >= 0)
 102                 memset(eb->buckets, 0, (eb->and+1)*sizeof(struct hlist_head));
 103 }
 104
 105 static int
 106 eb_lookup_vmas(struct eb_vmas *eb,
 107                struct drm_i915_gem_exec_object2 *exec,
 108                const struct drm_i915_gem_execbuffer2 *args,
 109                struct i915_address_space *vm,
 110                struct drm_file *file)
 111 {
 112         struct drm_i915_gem_object *obj;
 113         struct list_head objects;
 114         int i, ret;
 115
 116         INIT_LIST_HEAD(&objects);
 117         spin_lock(&file->table_lock);
 118         /* Grab a reference to the object and release the lock so we can lookup
 119          * or create the VMA without using GFP_ATOMIC */
 120         for (i = 0; i < args->buffer_count; i++) {
 121                 obj = to_intel_bo(idr_find(&file->object_idr, exec[i].handle));
 122                 if (obj == NULL) {
 123                         spin_unlock(&file->table_lock);
 124                         DRM_DEBUG("Invalid object handle %d at index %d\n",
 125                                    exec[i].handle, i);
 126                         ret = -ENOENT;
 127                         goto err;
 128                 }
 129
 130                 if (!list_empty(&obj->obj_exec_link)) {
 131                         spin_unlock(&file->table_lock);
 132                         DRM_DEBUG("Object %p [handle %d, index %d] appears more than once in object list\n",
 133                                    obj, exec[i].handle, i);
 134                         ret = -EINVAL;
 135                         goto err;
 136                 }
 137
 138                 i915_gem_object_get(obj);
 139                 list_add_tail(&obj->obj_exec_link, &objects);
 140         }
 141         spin_unlock(&file->table_lock);
 142
 143         i = 0;
 144         while (!list_empty(&objects)) {
 145                 struct i915_vma *vma;
 146
 147                 obj = list_first_entry(&objects,
 148                                        struct drm_i915_gem_object,
 149                                        obj_exec_link);
 150
 151                 /*
 152                  * NOTE: We can leak any vmas created here when something fails
 153                  * later on. But that's no issue since vma_unbind can deal with
 154                  * vmas which are not actually bound. And since only
 155                  * lookup_or_create exists as an interface to get at the vma
 156                  * from the (obj, vm) we don't run the risk of creating
 157                  * duplicated vmas for the same vm.
 158                  */
 159                 vma = i915_gem_obj_lookup_or_create_vma(obj, vm);
 160                 if (IS_ERR(vma)) {
 161                         DRM_DEBUG("Failed to lookup VMA\n");
 162                         ret = PTR_ERR(vma);
 163                         goto err;
 164                 }
 165
 166                 /* Transfer ownership from the objects list to the vmas list. */
 167                 list_add_tail(&vma->exec_list, &eb->vmas);
 168                 list_del_init(&obj->obj_exec_link);
 169
 170                 vma->exec_entry = &exec[i];
 171                 if (eb->and < 0) {
 172                         eb->lut[i] = vma;
 173                 } else {
 174                         uint32_t handle = args->flags & I915_EXEC_HANDLE_LUT ? i : exec[i].handle;
 175                         vma->exec_handle = handle;
 176                         hlist_add_head(&vma->exec_node,
 177                                        &eb->buckets[handle & eb->and]);
 178                 }
 179                 ++i;
 180         }
 181
 182         return 0;
 183
 184
 185 err:
 186         while (!list_empty(&objects)) {
 187                 obj = list_first_entry(&objects,
 188                                        struct drm_i915_gem_object,
 189                                        obj_exec_link);
 190                 list_del_init(&obj->obj_exec_link);
 191                 i915_gem_object_put(obj);
 192         }
 193         /*
 194          * Objects already transfered to the vmas list will be unreferenced by
 195          * eb_destroy.
 196          */
 197
 198         return ret;
 199 }
 200
 201 static inline struct i915_vma *
 202 eb_get_batch_vma(struct eb_vmas *eb)
 203 {
 204         /* The batch is always the LAST item in the VMA list */
 205         struct i915_vma *vma = list_last_entry(&eb->vmas, typeof(*vma), exec_list);
 206
 207         return vma;
 208 }
 209
 210 static struct drm_i915_gem_object *
 211 eb_get_batch(struct eb_vmas *eb)
 212 {
 213         struct i915_vma *vma = eb_get_batch_vma(eb);
 214
 215         /*
 216          * SNA is doing fancy tricks with compressing batch buffers, which leads
 217          * to negative relocation deltas. Usually that works out ok since the
 218          * relocate address is still positive, except when the batch is placed
 219          * very low in the GTT. Ensure this doesn't happen.
 220          *
 221          * Note that actual hangs have only been observed on gen7, but for
 222          * paranoia do it everywhere.
 223          */
 224         if ((vma->exec_entry->flags & EXEC_OBJECT_PINNED) == 0)
 225                 vma->exec_entry->flags |= __EXEC_OBJECT_NEEDS_BIAS;
 226
 227         return vma->obj;
 228 }
 229
 230 static struct i915_vma *eb_get_vma(struct eb_vmas *eb, unsigned long handle)
 231 {
 232         if (eb->and < 0) {
 233                 if (handle >= -eb->and)
 234                         return NULL;
 235                 return eb->lut[handle];
 236         } else {
 237                 struct hlist_head *head;
 238                 struct i915_vma *vma;
 239
 240                 head = &eb->buckets[handle & eb->and];
 241                 hlist_for_each_entry(vma, head, exec_node) {
 242                         if (vma->exec_handle == handle)
 243                                 return vma;
 244                 }
 245                 return NULL;
 246         }
 247 }
 248
 249 static void
 250 i915_gem_execbuffer_unreserve_vma(struct i915_vma *vma)
 251 {
 252         struct drm_i915_gem_exec_object2 *entry;
 253         struct drm_i915_gem_object *obj = vma->obj;
 254
 255         if (!drm_mm_node_allocated(&vma->node))
 256                 return;
 257
 258         entry = vma->exec_entry;
 259
 260         if (entry->flags & __EXEC_OBJECT_HAS_FENCE)
 261                 i915_gem_object_unpin_fence(obj);
 262
 263         if (entry->flags & __EXEC_OBJECT_HAS_PIN)
 264                 vma->pin_count--;
 265
 266         entry->flags &= ~(__EXEC_OBJECT_HAS_FENCE | __EXEC_OBJECT_HAS_PIN);
 267 }
 268
 269 static void eb_destroy(struct eb_vmas *eb)
 270 {
 271         while (!list_empty(&eb->vmas)) {
 272                 struct i915_vma *vma;
 273
 274                 vma = list_first_entry(&eb->vmas,
 275                                        struct i915_vma,
 276                                        exec_list);
 277                 list_del_init(&vma->exec_list);
 278                 i915_gem_execbuffer_unreserve_vma(vma);
 279                 i915_gem_object_put(vma->obj);
 280         }
 281         kfree(eb);
 282 }
 283
 284 static inline int use_cpu_reloc(struct drm_i915_gem_object *obj)
 285 {
 286         return (HAS_LLC(obj->base.dev) ||
 287                 obj->base.write_domain == I915_GEM_DOMAIN_CPU ||
 288                 obj->cache_level != I915_CACHE_NONE);
 289 }
 290
 291 /* Used to convert any address to canonical form.
 292  * Starting from gen8, some commands (e.g. STATE_BASE_ADDRESS,
 293  * MI_LOAD_REGISTER_MEM and others, see Broadwell PRM Vol2a) require the
 294  * addresses to be in a canonical form:
 295  * "GraphicsAddress[63:48] are ignored by the HW and assumed to be in correct
 296  * canonical form [63:48] == [47]."
 297  */
 298 #define GEN8_HIGH_ADDRESS_BIT 47
 299 static inline uint64_t gen8_canonical_addr(uint64_t address)
 300 {
 301         return sign_extend64(address, GEN8_HIGH_ADDRESS_BIT);
 302 }
 303
 304 static inline uint64_t gen8_noncanonical_addr(uint64_t address)
 305 {
 306         return address & ((1ULL << (GEN8_HIGH_ADDRESS_BIT + 1)) - 1);
 307 }
 308
 309 static inline uint64_t
 310 relocation_target(struct drm_i915_gem_relocation_entry *reloc,
 311                   uint64_t target_offset)
 312 {
 313         return gen8_canonical_addr((int)reloc->delta + target_offset);
 314 }
 315
 316 static int
 317 relocate_entry_cpu(struct drm_i915_gem_object *obj,
 318                    struct drm_i915_gem_relocation_entry *reloc,
 319                    uint64_t target_offset)
 320 {
 321         struct drm_device *dev = obj->base.dev;
 322         uint32_t page_offset = offset_in_page(reloc->offset);
 323         uint64_t delta = relocation_target(reloc, target_offset);
 324         char *vaddr;
 325         int ret;
 326
 327         ret = i915_gem_object_set_to_cpu_domain(obj, true);
 328         if (ret)
 329                 return ret;
 330
 331         vaddr = kmap_atomic(i915_gem_object_get_dirty_page(obj,
 332                                 reloc->offset >> PAGE_SHIFT));
 333         *(uint32_t *)(vaddr + page_offset) = lower_32_bits(delta);
 334
 335         if (INTEL_INFO(dev)->gen >= 8) {
 336                 page_offset = offset_in_page(page_offset + sizeof(uint32_t));
 337
 338                 if (page_offset == 0) {
 339                         kunmap_atomic(vaddr);
 340                         vaddr = kmap_atomic(i915_gem_object_get_dirty_page(obj,
 341                             (reloc->offset + sizeof(uint32_t)) >> PAGE_SHIFT));
 342                 }
 343
 344                 *(uint32_t *)(vaddr + page_offset) = upper_32_bits(delta);
 345         }
 346
 347         kunmap_atomic(vaddr);
 348
 349         return 0;
 350 }
 351
 352 static int
 353 relocate_entry_gtt(struct drm_i915_gem_object *obj,
 354                    struct drm_i915_gem_relocation_entry *reloc,
 355                    uint64_t target_offset)
 356 {
 357         struct drm_device *dev = obj->base.dev;
 358         struct drm_i915_private *dev_priv = to_i915(dev);
 359         struct i915_ggtt *ggtt = &dev_priv->ggtt;
 360         uint64_t delta = relocation_target(reloc, target_offset);
 361         uint64_t offset;
 362         void __iomem *reloc_page;
 363         int ret;
 364
 365         ret = i915_gem_object_set_to_gtt_domain(obj, true);
 366         if (ret)
 367                 return ret;
 368
 369         ret = i915_gem_object_put_fence(obj);
 370         if (ret)
 371                 return ret;
 372
 373         /* Map the page containing the relocation we're going to perform.  */
 374         offset = i915_gem_obj_ggtt_offset(obj);
 375         offset += reloc->offset;
 376         reloc_page = io_mapping_map_atomic_wc(ggtt->mappable,
 377                                               offset & PAGE_MASK);
 378         iowrite32(lower_32_bits(delta), reloc_page + offset_in_page(offset));
 379
 380         if (INTEL_INFO(dev)->gen >= 8) {
 381                 offset += sizeof(uint32_t);
 382
 383                 if (offset_in_page(offset) == 0) {
 384                         io_mapping_unmap_atomic(reloc_page);
 385                         reloc_page =
 386                                 io_mapping_map_atomic_wc(ggtt->mappable,
 387                                                          offset);
 388                 }
 389
 390                 iowrite32(upper_32_bits(delta),
 391                           reloc_page + offset_in_page(offset));
 392         }
 393
 394         io_mapping_unmap_atomic(reloc_page);
 395
 396         return 0;
 397 }
 398
 399 static void
 400 clflush_write32(void *addr, uint32_t value)
 401 {
 402         /* This is not a fast path, so KISS. */
 403         drm_clflush_virt_range(addr, sizeof(uint32_t));
 404         *(uint32_t *)addr = value;
 405         drm_clflush_virt_range(addr, sizeof(uint32_t));
 406 }
 407
 408 static int
 409 relocate_entry_clflush(struct drm_i915_gem_object *obj,
 410                        struct drm_i915_gem_relocation_entry *reloc,
 411                        uint64_t target_offset)
 412 {
 413         struct drm_device *dev = obj->base.dev;
 414         uint32_t page_offset = offset_in_page(reloc->offset);
 415         uint64_t delta = relocation_target(reloc, target_offset);
 416         char *vaddr;
 417         int ret;
 418
 419         ret = i915_gem_object_set_to_gtt_domain(obj, true);
 420         if (ret)
 421                 return ret;
 422
 423         vaddr = kmap_atomic(i915_gem_object_get_dirty_page(obj,
 424                                 reloc->offset >> PAGE_SHIFT));
 425         clflush_write32(vaddr + page_offset, lower_32_bits(delta));
 426
 427         if (INTEL_INFO(dev)->gen >= 8) {
 428                 page_offset = offset_in_page(page_offset + sizeof(uint32_t));
 429
 430                 if (page_offset == 0) {
 431                         kunmap_atomic(vaddr);
 432                         vaddr = kmap_atomic(i915_gem_object_get_dirty_page(obj,
 433                             (reloc->offset + sizeof(uint32_t)) >> PAGE_SHIFT));
 434                 }
 435
 436                 clflush_write32(vaddr + page_offset, upper_32_bits(delta));
 437         }
 438
 439         kunmap_atomic(vaddr);
 440
 441         return 0;
 442 }
 443
 444 static bool object_is_idle(struct drm_i915_gem_object *obj)
 445 {
 446         unsigned long active = obj->active;
 447         int idx;
 448
 449         for_each_active(active, idx) {
 450                 if (!i915_gem_active_is_idle(&obj->last_read[idx],
 451                                              &obj->base.dev->struct_mutex))
 452                         return false;
 453         }
 454
 455         return true;
 456 }
 457
 458 static int
 459 i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
 460                                    struct eb_vmas *eb,
 461                                    struct drm_i915_gem_relocation_entry *reloc)
 462 {
 463         struct drm_device *dev = obj->base.dev;
 464         struct drm_gem_object *target_obj;
 465         struct drm_i915_gem_object *target_i915_obj;
 466         struct i915_vma *target_vma;
 467         uint64_t target_offset;
 468         int ret;
 469
 470         /* we've already hold a reference to all valid objects */
 471         target_vma = eb_get_vma(eb, reloc->target_handle);
 472         if (unlikely(target_vma == NULL))
 473                 return -ENOENT;
 474         target_i915_obj = target_vma->obj;
 475         target_obj = &target_vma->obj->base;
 476
 477         target_offset = gen8_canonical_addr(target_vma->node.start);
 478
 479         /* Sandybridge PPGTT errata: We need a global gtt mapping for MI and
 480          * pipe_control writes because the gpu doesn't properly redirect them
 481          * through the ppgtt for non_secure batchbuffers. */
 482         if (unlikely(IS_GEN6(dev) &&
 483             reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION)) {
 484                 ret = i915_vma_bind(target_vma, target_i915_obj->cache_level,
 485                                     PIN_GLOBAL);
 486                 if (WARN_ONCE(ret, "Unexpected failure to bind target VMA!"))
 487                         return ret;
 488         }
 489
 490         /* Validate that the target is in a valid r/w GPU domain */
 491         if (unlikely(reloc->write_domain & (reloc->write_domain - 1))) {
 492                 DRM_DEBUG("reloc with multiple write domains: "
 493                           "obj %p target %d offset %d "
 494                           "read %08x write %08x",
 495                           obj, reloc->target_handle,
 496                           (int) reloc->offset,
 497                           reloc->read_domains,
 498                           reloc->write_domain);
 499                 return -EINVAL;
 500         }
 501         if (unlikely((reloc->write_domain | reloc->read_domains)
 502                      & ~I915_GEM_GPU_DOMAINS)) {
 503                 DRM_DEBUG("reloc with read/write non-GPU domains: "
 504                           "obj %p target %d offset %d "
 505                           "read %08x write %08x",
 506                           obj, reloc->target_handle,
 507                           (int) reloc->offset,
 508                           reloc->read_domains,
 509                           reloc->write_domain);
 510                 return -EINVAL;
 511         }
 512
 513         target_obj->pending_read_domains |= reloc->read_domains;
 514         target_obj->pending_write_domain |= reloc->write_domain;
 515
 516         /* If the relocation already has the right value in it, no
 517          * more work needs to be done.
 518          */
 519         if (target_offset == reloc->presumed_offset)
 520                 return 0;
 521
 522         /* Check that the relocation address is valid... */
 523         if (unlikely(reloc->offset >
 524                 obj->base.size - (INTEL_INFO(dev)->gen >= 8 ? 8 : 4))) {
 525                 DRM_DEBUG("Relocation beyond object bounds: "
 526                           "obj %p target %d offset %d size %d.\n",
 527                           obj, reloc->target_handle,
 528                           (int) reloc->offset,
 529                           (int) obj->base.size);
 530                 return -EINVAL;
 531         }
 532         if (unlikely(reloc->offset & 3)) {
 533                 DRM_DEBUG("Relocation not 4-byte aligned: "
 534                           "obj %p target %d offset %d.\n",
 535                           obj, reloc->target_handle,
 536                           (int) reloc->offset);
 537                 return -EINVAL;
 538         }
 539
 540         /* We can't wait for rendering with pagefaults disabled */
 541         if (pagefault_disabled() && !object_is_idle(obj))
 542                 return -EFAULT;
 543
 544         if (use_cpu_reloc(obj))
 545                 ret = relocate_entry_cpu(obj, reloc, target_offset);
 546         else if (obj->map_and_fenceable)
 547                 ret = relocate_entry_gtt(obj, reloc, target_offset);
 548         else if (static_cpu_has(X86_FEATURE_CLFLUSH))
 549                 ret = relocate_entry_clflush(obj, reloc, target_offset);
 550         else {
 551                 WARN_ONCE(1, "Impossible case in relocation handling\n");
 552                 ret = -ENODEV;
 553         }
 554
 555         if (ret)
 556                 return ret;
 557
 558         /* and update the user's relocation entry */
 559         reloc->presumed_offset = target_offset;
 560
 561         return 0;
 562 }
 563
 564 static int
 565 i915_gem_execbuffer_relocate_vma(struct i915_vma *vma,
 566                                  struct eb_vmas *eb)
 567 {
 568 #define N_RELOC(x) ((x) / sizeof(struct drm_i915_gem_relocation_entry))
 569         struct drm_i915_gem_relocation_entry stack_reloc[N_RELOC(512)];
 570         struct drm_i915_gem_relocation_entry __user *user_relocs;
 571         struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
 572         int remain, ret;
 573
 574         user_relocs = u64_to_user_ptr(entry->relocs_ptr);
 575
 576         remain = entry->relocation_count;
 577         while (remain) {
 578                 struct drm_i915_gem_relocation_entry *r = stack_reloc;
 579                 int count = remain;
 580                 if (count > ARRAY_SIZE(stack_reloc))
 581                         count = ARRAY_SIZE(stack_reloc);
 582                 remain -= count;
 583
 584                 if (__copy_from_user_inatomic(r, user_relocs, count*sizeof(r[0])))
 585                         return -EFAULT;
 586
 587                 do {
 588                         u64 offset = r->presumed_offset;
 589
 590                         ret = i915_gem_execbuffer_relocate_entry(vma->obj, eb, r);
 591                         if (ret)
 592                                 return ret;
 593
 594                         if (r->presumed_offset != offset &&
 595                             __put_user(r->presumed_offset, &user_relocs->presumed_offset)) {
 596                                 return -EFAULT;
 597                         }
 598
 599                         user_relocs++;
 600                         r++;
 601                 } while (--count);
 602         }
 603
 604         return 0;
 605 #undef N_RELOC
 606 }
 607
 608 static int
 609 i915_gem_execbuffer_relocate_vma_slow(struct i915_vma *vma,
 610                                       struct eb_vmas *eb,
 611                                       struct drm_i915_gem_relocation_entry *relocs)
 612 {
 613         const struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
 614         int i, ret;
 615
 616         for (i = 0; i < entry->relocation_count; i++) {
 617                 ret = i915_gem_execbuffer_relocate_entry(vma->obj, eb, &relocs[i]);
 618                 if (ret)
 619                         return ret;
 620         }
 621
 622         return 0;
 623 }
 624
 625 static int
 626 i915_gem_execbuffer_relocate(struct eb_vmas *eb)
 627 {
 628         struct i915_vma *vma;
 629         int ret = 0;
 630
 631         /* This is the fast path and we cannot handle a pagefault whilst
 632          * holding the struct mutex lest the user pass in the relocations
 633          * contained within a mmaped bo. For in such a case we, the page
 634          * fault handler would call i915_gem_fault() and we would try to
 635          * acquire the struct mutex again. Obviously this is bad and so
 636          * lockdep complains vehemently.
 637          */
 638         pagefault_disable();
 639         list_for_each_entry(vma, &eb->vmas, exec_list) {
 640                 ret = i915_gem_execbuffer_relocate_vma(vma, eb);
 641                 if (ret)
 642                         break;
 643         }
 644         pagefault_enable();
 645
 646         return ret;
 647 }
 648
 649 static bool only_mappable_for_reloc(unsigned int flags)
 650 {
 651         return (flags & (EXEC_OBJECT_NEEDS_FENCE | __EXEC_OBJECT_NEEDS_MAP)) ==
 652                 __EXEC_OBJECT_NEEDS_MAP;
 653 }
 654
 655 static int
 656 i915_gem_execbuffer_reserve_vma(struct i915_vma *vma,
 657                                 struct intel_engine_cs *engine,
 658                                 bool *need_reloc)
 659 {
 660         struct drm_i915_gem_object *obj = vma->obj;
 661         struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
 662         uint64_t flags;
 663         int ret;
 664
 665         flags = PIN_USER;
 666         if (entry->flags & EXEC_OBJECT_NEEDS_GTT)
 667                 flags |= PIN_GLOBAL;
 668
 669         if (!drm_mm_node_allocated(&vma->node)) {
 670                 /* Wa32bitGeneralStateOffset & Wa32bitInstructionBaseOffset,
 671                  * limit address to the first 4GBs for unflagged objects.
 672                  */
 673                 if ((entry->flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) == 0)
 674                         flags |= PIN_ZONE_4G;
 675                 if (entry->flags & __EXEC_OBJECT_NEEDS_MAP)
 676                         flags |= PIN_GLOBAL | PIN_MAPPABLE;
 677                 if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS)
 678                         flags |= BATCH_OFFSET_BIAS | PIN_OFFSET_BIAS;
 679                 if (entry->flags & EXEC_OBJECT_PINNED)
 680                         flags |= entry->offset | PIN_OFFSET_FIXED;
 681                 if ((flags & PIN_MAPPABLE) == 0)
 682                         flags |= PIN_HIGH;
 683         }
 684
 685         ret = i915_gem_object_pin(obj, vma->vm, entry->alignment, flags);
 686         if ((ret == -ENOSPC  || ret == -E2BIG) &&
 687             only_mappable_for_reloc(entry->flags))
 688                 ret = i915_gem_object_pin(obj, vma->vm,
 689                                           entry->alignment,
 690                                           flags & ~PIN_MAPPABLE);
 691         if (ret)
 692                 return ret;
 693
 694         entry->flags |= __EXEC_OBJECT_HAS_PIN;
 695
 696         if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) {
 697                 ret = i915_gem_object_get_fence(obj);
 698                 if (ret)
 699                         return ret;
 700
 701                 if (i915_gem_object_pin_fence(obj))
 702                         entry->flags |= __EXEC_OBJECT_HAS_FENCE;
 703         }
 704
 705         if (entry->offset != vma->node.start) {
 706                 entry->offset = vma->node.start;
 707                 *need_reloc = true;
 708         }
 709
 710         if (entry->flags & EXEC_OBJECT_WRITE) {
 711                 obj->base.pending_read_domains = I915_GEM_DOMAIN_RENDER;
 712                 obj->base.pending_write_domain = I915_GEM_DOMAIN_RENDER;
 713         }
 714
 715         return 0;
 716 }
 717
 718 static bool
 719 need_reloc_mappable(struct i915_vma *vma)
 720 {
 721         struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
 722
 723         if (entry->relocation_count == 0)
 724                 return false;
 725
 726         if (!vma->is_ggtt)
 727                 return false;
 728
 729         /* See also use_cpu_reloc() */
 730         if (HAS_LLC(vma->obj->base.dev))
 731                 return false;
 732
 733         if (vma->obj->base.write_domain == I915_GEM_DOMAIN_CPU)
 734                 return false;
 735
 736         return true;
 737 }
 738
 739 static bool
 740 eb_vma_misplaced(struct i915_vma *vma)
 741 {
 742         struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
 743         struct drm_i915_gem_object *obj = vma->obj;
 744
 745         WARN_ON(entry->flags & __EXEC_OBJECT_NEEDS_MAP && !vma->is_ggtt);
 746
 747         if (entry->alignment &&
 748             vma->node.start & (entry->alignment - 1))
 749                 return true;
 750
 751         if (entry->flags & EXEC_OBJECT_PINNED &&
 752             vma->node.start != entry->offset)
 753                 return true;
 754
 755         if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS &&
 756             vma->node.start < BATCH_OFFSET_BIAS)
 757                 return true;
 758
 759         /* avoid costly ping-pong once a batch bo ended up non-mappable */
 760         if (entry->flags & __EXEC_OBJECT_NEEDS_MAP && !obj->map_and_fenceable)
 761                 return !only_mappable_for_reloc(entry->flags);
 762
 763         if ((entry->flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) == 0 &&
 764             (vma->node.start + vma->node.size - 1) >> 32)
 765                 return true;
 766
 767         return false;
 768 }
 769
 770 static int
 771 i915_gem_execbuffer_reserve(struct intel_engine_cs *engine,
 772                             struct list_head *vmas,
 773                             struct i915_gem_context *ctx,
 774                             bool *need_relocs)
 775 {
 776         struct drm_i915_gem_object *obj;
 777         struct i915_vma *vma;
 778         struct i915_address_space *vm;
 779         struct list_head ordered_vmas;
 780         struct list_head pinned_vmas;
 781         bool has_fenced_gpu_access = INTEL_GEN(engine->i915) < 4;
 782         int retry;
 783
 784         vm = list_first_entry(vmas, struct i915_vma, exec_list)->vm;
 785
 786         INIT_LIST_HEAD(&ordered_vmas);
 787         INIT_LIST_HEAD(&pinned_vmas);
 788         while (!list_empty(vmas)) {
 789                 struct drm_i915_gem_exec_object2 *entry;
 790                 bool need_fence, need_mappable;
 791
 792                 vma = list_first_entry(vmas, struct i915_vma, exec_list);
 793                 obj = vma->obj;
 794                 entry = vma->exec_entry;
 795
 796                 if (ctx->flags & CONTEXT_NO_ZEROMAP)
 797                         entry->flags |= __EXEC_OBJECT_NEEDS_BIAS;
 798
 799                 if (!has_fenced_gpu_access)
 800                         entry->flags &= ~EXEC_OBJECT_NEEDS_FENCE;
 801                 need_fence =
 802                         entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
 803                         obj->tiling_mode != I915_TILING_NONE;
 804                 need_mappable = need_fence || need_reloc_mappable(vma);
 805
 806                 if (entry->flags & EXEC_OBJECT_PINNED)
 807                         list_move_tail(&vma->exec_list, &pinned_vmas);
 808                 else if (need_mappable) {
 809                         entry->flags |= __EXEC_OBJECT_NEEDS_MAP;
 810                         list_move(&vma->exec_list, &ordered_vmas);
 811                 } else
 812                         list_move_tail(&vma->exec_list, &ordered_vmas);
 813
 814                 obj->base.pending_read_domains = I915_GEM_GPU_DOMAINS & ~I915_GEM_DOMAIN_COMMAND;
 815                 obj->base.pending_write_domain = 0;
 816         }
 817         list_splice(&ordered_vmas, vmas);
 818         list_splice(&pinned_vmas, vmas);
 819
 820         /* Attempt to pin all of the buffers into the GTT.
 821          * This is done in 3 phases:
 822          *
 823          * 1a. Unbind all objects that do not match the GTT constraints for
 824          *     the execbuffer (fenceable, mappable, alignment etc).
 825          * 1b. Increment pin count for already bound objects.
 826          * 2.  Bind new objects.
 827          * 3.  Decrement pin count.
 828          *
 829          * This avoid unnecessary unbinding of later objects in order to make
 830          * room for the earlier objects *unless* we need to defragment.
 831          */
 832         retry = 0;
 833         do {
 834                 int ret = 0;
 835
 836                 /* Unbind any ill-fitting objects or pin. */
 837                 list_for_each_entry(vma, vmas, exec_list) {
 838                         if (!drm_mm_node_allocated(&vma->node))
 839                                 continue;
 840
 841                         if (eb_vma_misplaced(vma))
 842                                 ret = i915_vma_unbind(vma);
 843                         else
 844                                 ret = i915_gem_execbuffer_reserve_vma(vma,
 845                                                                       engine,
 846                                                                       need_relocs);
 847                         if (ret)
 848                                 goto err;
 849                 }
 850
 851                 /* Bind fresh objects */
 852                 list_for_each_entry(vma, vmas, exec_list) {
 853                         if (drm_mm_node_allocated(&vma->node))
 854                                 continue;
 855
 856                         ret = i915_gem_execbuffer_reserve_vma(vma, engine,
 857                                                               need_relocs);
 858                         if (ret)
 859                                 goto err;
 860                 }
 861
 862 err:
 863                 if (ret != -ENOSPC || retry++)
 864                         return ret;
 865
 866                 /* Decrement pin count for bound objects */
 867                 list_for_each_entry(vma, vmas, exec_list)
 868                         i915_gem_execbuffer_unreserve_vma(vma);
 869
 870                 ret = i915_gem_evict_vm(vm, true);
 871                 if (ret)
 872                         return ret;
 873         } while (1);
 874 }
 875
 876 static int
 877 i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
 878                                   struct drm_i915_gem_execbuffer2 *args,
 879                                   struct drm_file *file,
 880                                   struct intel_engine_cs *engine,
 881                                   struct eb_vmas *eb,
 882                                   struct drm_i915_gem_exec_object2 *exec,
 883                                   struct i915_gem_context *ctx)
 884 {
 885         struct drm_i915_gem_relocation_entry *reloc;
 886         struct i915_address_space *vm;
 887         struct i915_vma *vma;
 888         bool need_relocs;
 889         int *reloc_offset;
 890         int i, total, ret;
 891         unsigned count = args->buffer_count;
 892
 893         vm = list_first_entry(&eb->vmas, struct i915_vma, exec_list)->vm;
 894
 895         /* We may process another execbuffer during the unlock... */
 896         while (!list_empty(&eb->vmas)) {
 897                 vma = list_first_entry(&eb->vmas, struct i915_vma, exec_list);
 898                 list_del_init(&vma->exec_list);
 899                 i915_gem_execbuffer_unreserve_vma(vma);
 900                 i915_gem_object_put(vma->obj);
 901         }
 902
 903         mutex_unlock(&dev->struct_mutex);
 904
 905         total = 0;
 906         for (i = 0; i < count; i++)
 907                 total += exec[i].relocation_count;
 908
 909         reloc_offset = drm_malloc_ab(count, sizeof(*reloc_offset));
 910         reloc = drm_malloc_ab(total, sizeof(*reloc));
 911         if (reloc == NULL || reloc_offset == NULL) {
 912                 drm_free_large(reloc);
 913                 drm_free_large(reloc_offset);
 914                 mutex_lock(&dev->struct_mutex);
 915                 return -ENOMEM;
 916         }
 917
 918         total = 0;
 919         for (i = 0; i < count; i++) {
 920                 struct drm_i915_gem_relocation_entry __user *user_relocs;
 921                 u64 invalid_offset = (u64)-1;
 922                 int j;
 923
 924                 user_relocs = u64_to_user_ptr(exec[i].relocs_ptr);
 925
 926                 if (copy_from_user(reloc+total, user_relocs,
 927                                    exec[i].relocation_count * sizeof(*reloc))) {
 928                         ret = -EFAULT;
 929                         mutex_lock(&dev->struct_mutex);
 930                         goto err;
 931                 }
 932
 933                 /* As we do not update the known relocation offsets after
 934                  * relocating (due to the complexities in lock handling),
 935                  * we need to mark them as invalid now so that we force the
 936                  * relocation processing next time. Just in case the target
 937                  * object is evicted and then rebound into its old
 938                  * presumed_offset before the next execbuffer - if that
 939                  * happened we would make the mistake of assuming that the
 940                  * relocations were valid.
 941                  */
 942                 for (j = 0; j < exec[i].relocation_count; j++) {
 943                         if (__copy_to_user(&user_relocs[j].presumed_offset,
 944                                            &invalid_offset,
 945                                            sizeof(invalid_offset))) {
 946                                 ret = -EFAULT;
 947                                 mutex_lock(&dev->struct_mutex);
 948                                 goto err;
 949                         }
 950                 }
 951
 952                 reloc_offset[i] = total;
 953                 total += exec[i].relocation_count;
 954         }
 955
 956         ret = i915_mutex_lock_interruptible(dev);
 957         if (ret) {
 958                 mutex_lock(&dev->struct_mutex);
 959                 goto err;
 960         }
 961
 962         /* reacquire the objects */
 963         eb_reset(eb);
 964         ret = eb_lookup_vmas(eb, exec, args, vm, file);
 965         if (ret)
 966                 goto err;
 967
 968         need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0;
 969         ret = i915_gem_execbuffer_reserve(engine, &eb->vmas, ctx,
 970                                           &need_relocs);
 971         if (ret)
 972                 goto err;
 973
 974         list_for_each_entry(vma, &eb->vmas, exec_list) {
 975                 int offset = vma->exec_entry - exec;
 976                 ret = i915_gem_execbuffer_relocate_vma_slow(vma, eb,
 977                                                             reloc + reloc_offset[offset]);
 978                 if (ret)
 979                         goto err;
 980         }
 981
 982         /* Leave the user relocations as are, this is the painfully slow path,
 983          * and we want to avoid the complication of dropping the lock whilst
 984          * having buffers reserved in the aperture and so causing spurious
 985          * ENOSPC for random operations.
 986          */
 987
 988 err:
 989         drm_free_large(reloc);
 990         drm_free_large(reloc_offset);
 991         return ret;
 992 }
 993
 994 static int
 995 i915_gem_execbuffer_move_to_gpu(struct drm_i915_gem_request *req,
 996                                 struct list_head *vmas)
 997 {
 998         const unsigned other_rings = ~intel_engine_flag(req->engine);
 999         struct i915_vma *vma;
1000         uint32_t flush_domains = 0;
1001         bool flush_chipset = false;
1002         int ret;
1003
1004         list_for_each_entry(vma, vmas, exec_list) {
1005                 struct drm_i915_gem_object *obj = vma->obj;
1006
1007                 if (obj->active & other_rings) {
1008                         ret = i915_gem_object_sync(obj, req);
1009                         if (ret)
1010                                 return ret;
1011                 }
1012
1013                 if (obj->base.write_domain & I915_GEM_DOMAIN_CPU)
1014                         flush_chipset |= i915_gem_clflush_object(obj, false);
1015
1016                 flush_domains |= obj->base.write_domain;
1017         }
1018
1019         if (flush_chipset)
1020                 i915_gem_chipset_flush(req->engine->i915);
1021
1022         if (flush_domains & I915_GEM_DOMAIN_GTT)
1023                 wmb();
1024
1025         /* Unconditionally invalidate GPU caches and TLBs. */
1026         return req->engine->emit_flush(req, EMIT_INVALIDATE);
1027 }
1028
1029 static bool
1030 i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
1031 {
1032         if (exec->flags & __I915_EXEC_UNKNOWN_FLAGS)
1033                 return false;
1034
1035         /* Kernel clipping was a DRI1 misfeature */
1036         if (exec->num_cliprects || exec->cliprects_ptr)
1037                 return false;
1038
1039         if (exec->DR4 == 0xffffffff) {
1040                 DRM_DEBUG("UXA submitting garbage DR4, fixing up\n");
1041                 exec->DR4 = 0;
1042         }
1043         if (exec->DR1 || exec->DR4)
1044                 return false;
1045
1046         if ((exec->batch_start_offset | exec->batch_len) & 0x7)
1047                 return false;
1048
1049         return true;
1050 }
1051
1052 static int
1053 validate_exec_list(struct drm_device *dev,
1054                    struct drm_i915_gem_exec_object2 *exec,
1055                    int count)
1056 {
1057         unsigned relocs_total = 0;
1058         unsigned relocs_max = UINT_MAX / sizeof(struct drm_i915_gem_relocation_entry);
1059         unsigned invalid_flags;
1060         int i;
1061
1062         /* INTERNAL flags must not overlap with external ones */
1063         BUILD_BUG_ON(__EXEC_OBJECT_INTERNAL_FLAGS & ~__EXEC_OBJECT_UNKNOWN_FLAGS);
1064
1065         invalid_flags = __EXEC_OBJECT_UNKNOWN_FLAGS;
1066         if (USES_FULL_PPGTT(dev))
1067                 invalid_flags |= EXEC_OBJECT_NEEDS_GTT;
1068
1069         for (i = 0; i < count; i++) {
1070                 char __user *ptr = u64_to_user_ptr(exec[i].relocs_ptr);
1071                 int length; /* limited by fault_in_pages_readable() */
1072
1073                 if (exec[i].flags & invalid_flags)
1074                         return -EINVAL;
1075
1076                 /* Offset can be used as input (EXEC_OBJECT_PINNED), reject
1077                  * any non-page-aligned or non-canonical addresses.
1078                  */
1079                 if (exec[i].flags & EXEC_OBJECT_PINNED) {
1080                         if (exec[i].offset !=
1081                             gen8_canonical_addr(exec[i].offset & PAGE_MASK))
1082                                 return -EINVAL;
1083
1084                         /* From drm_mm perspective address space is continuous,
1085                          * so from this point we're always using non-canonical
1086                          * form internally.
1087                          */
1088                         exec[i].offset = gen8_noncanonical_addr(exec[i].offset);
1089                 }
1090
1091                 if (exec[i].alignment && !is_power_of_2(exec[i].alignment))
1092                         return -EINVAL;
1093
1094                 /* First check for malicious input causing overflow in
1095                  * the worst case where we need to allocate the entire
1096                  * relocation tree as a single array.
1097                  */
1098                 if (exec[i].relocation_count > relocs_max - relocs_total)
1099                         return -EINVAL;
1100                 relocs_total += exec[i].relocation_count;
1101
1102                 length = exec[i].relocation_count *
1103                         sizeof(struct drm_i915_gem_relocation_entry);
1104                 /*
1105                  * We must check that the entire relocation array is safe
1106                  * to read, but since we may need to update the presumed
1107                  * offsets during execution, check for full write access.
1108                  */
1109                 if (!access_ok(VERIFY_WRITE, ptr, length))
1110                         return -EFAULT;
1111
1112                 if (likely(!i915.prefault_disable)) {
1113                         if (fault_in_multipages_readable(ptr, length))
1114                                 return -EFAULT;
1115                 }
1116         }
1117
1118         return 0;
1119 }
1120
1121 static struct i915_gem_context *
1122 i915_gem_validate_context(struct drm_device *dev, struct drm_file *file,
1123                           struct intel_engine_cs *engine, const u32 ctx_id)
1124 {
1125         struct i915_gem_context *ctx = NULL;
1126         struct i915_ctx_hang_stats *hs;
1127
1128         if (engine->id != RCS && ctx_id != DEFAULT_CONTEXT_HANDLE)
1129                 return ERR_PTR(-EINVAL);
1130
1131         ctx = i915_gem_context_lookup(file->driver_priv, ctx_id);
1132         if (IS_ERR(ctx))
1133                 return ctx;
1134
1135         hs = &ctx->hang_stats;
1136         if (hs->banned) {
1137                 DRM_DEBUG("Context %u tried to submit while banned\n", ctx_id);
1138                 return ERR_PTR(-EIO);
1139         }
1140
1141         return ctx;
1142 }
1143
1144 void i915_vma_move_to_active(struct i915_vma *vma,
1145                              struct drm_i915_gem_request *req,
1146                              unsigned int flags)
1147 {
1148         struct drm_i915_gem_object *obj = vma->obj;
1149         const unsigned int idx = req->engine->id;
1150
1151         GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1152
1153         obj->dirty = 1; /* be paranoid  */
1154
1155         /* Add a reference if we're newly entering the active list.
1156          * The order in which we add operations to the retirement queue is
1157          * vital here: mark_active adds to the start of the callback list,
1158          * such that subsequent callbacks are called first. Therefore we
1159          * add the active reference first and queue for it to be dropped
1160          * *last*.
1161          */
1162         if (obj->active == 0)
1163                 i915_gem_object_get(obj);
1164         obj->active |= 1 << idx;
1165         i915_gem_active_set(&obj->last_read[idx], req);
1166
1167         if (flags & EXEC_OBJECT_WRITE) {
1168                 i915_gem_active_set(&obj->last_write, req);
1169
1170                 intel_fb_obj_invalidate(obj, ORIGIN_CS);
1171
1172                 /* update for the implicit flush after a batch */
1173                 obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
1174         }
1175
1176         if (flags & EXEC_OBJECT_NEEDS_FENCE) {
1177                 i915_gem_active_set(&obj->last_fence, req);
1178                 if (flags & __EXEC_OBJECT_HAS_FENCE) {
1179                         struct drm_i915_private *dev_priv = req->i915;
1180
1181                         list_move_tail(&dev_priv->fence_regs[obj->fence_reg].lru_list,
1182                                        &dev_priv->mm.fence_list);
1183                 }
1184         }
1185
1186         i915_vma_set_active(vma, idx);
1187         i915_gem_active_set(&vma->last_read[idx], req);
1188         list_move_tail(&vma->vm_link, &vma->vm->active_list);
1189 }
1190
1191 static void
1192 i915_gem_execbuffer_move_to_active(struct list_head *vmas,
1193                                    struct drm_i915_gem_request *req)
1194 {
1195         struct i915_vma *vma;
1196
1197         list_for_each_entry(vma, vmas, exec_list) {
1198                 struct drm_i915_gem_object *obj = vma->obj;
1199                 u32 old_read = obj->base.read_domains;
1200                 u32 old_write = obj->base.write_domain;
1201
1202                 obj->base.write_domain = obj->base.pending_write_domain;
1203                 if (obj->base.write_domain)
1204                         vma->exec_entry->flags |= EXEC_OBJECT_WRITE;
1205                 else
1206                         obj->base.pending_read_domains |= obj->base.read_domains;
1207                 obj->base.read_domains = obj->base.pending_read_domains;
1208
1209                 i915_vma_move_to_active(vma, req, vma->exec_entry->flags);
1210                 trace_i915_gem_object_change_domain(obj, old_read, old_write);
1211         }
1212 }
1213
1214 static int
1215 i915_reset_gen7_sol_offsets(struct drm_i915_gem_request *req)
1216 {
1217         struct intel_ring *ring = req->ring;
1218         int ret, i;
1219
1220         if (!IS_GEN7(req->i915) || req->engine->id != RCS) {
1221                 DRM_DEBUG("sol reset is gen7/rcs only\n");
1222                 return -EINVAL;
1223         }
1224
1225         ret = intel_ring_begin(req, 4 * 3);
1226         if (ret)
1227                 return ret;
1228
1229         for (i = 0; i < 4; i++) {
1230                 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
1231                 intel_ring_emit_reg(ring, GEN7_SO_WRITE_OFFSET(i));
1232                 intel_ring_emit(ring, 0);
1233         }
1234
1235         intel_ring_advance(ring);
1236
1237         return 0;
1238 }
1239
1240 static struct drm_i915_gem_object*
1241 i915_gem_execbuffer_parse(struct intel_engine_cs *engine,
1242                           struct drm_i915_gem_exec_object2 *shadow_exec_entry,
1243                           struct eb_vmas *eb,
1244                           struct drm_i915_gem_object *batch_obj,
1245                           u32 batch_start_offset,
1246                           u32 batch_len,
1247                           bool is_master)
1248 {
1249         struct drm_i915_gem_object *shadow_batch_obj;
1250         struct i915_vma *vma;
1251         int ret;
1252
1253         shadow_batch_obj = i915_gem_batch_pool_get(&engine->batch_pool,
1254                                                    PAGE_ALIGN(batch_len));
1255         if (IS_ERR(shadow_batch_obj))
1256                 return shadow_batch_obj;
1257
1258         ret = intel_engine_cmd_parser(engine,
1259                                       batch_obj,
1260                                       shadow_batch_obj,
1261                                       batch_start_offset,
1262                                       batch_len,
1263                                       is_master);
1264         if (ret)
1265                 goto err;
1266
1267         ret = i915_gem_obj_ggtt_pin(shadow_batch_obj, 0, 0);
1268         if (ret)
1269                 goto err;
1270
1271         i915_gem_object_unpin_pages(shadow_batch_obj);
1272
1273         memset(shadow_exec_entry, 0, sizeof(*shadow_exec_entry));
1274
1275         vma = i915_gem_obj_to_ggtt(shadow_batch_obj);
1276         vma->exec_entry = shadow_exec_entry;
1277         vma->exec_entry->flags = __EXEC_OBJECT_HAS_PIN;
1278         i915_gem_object_get(shadow_batch_obj);
1279         list_add_tail(&vma->exec_list, &eb->vmas);
1280
1281         shadow_batch_obj->base.pending_read_domains = I915_GEM_DOMAIN_COMMAND;
1282
1283         return shadow_batch_obj;
1284
1285 err:
1286         i915_gem_object_unpin_pages(shadow_batch_obj);
1287         if (ret == -EACCES) /* unhandled chained batch */
1288                 return batch_obj;
1289         else
1290                 return ERR_PTR(ret);
1291 }
1292
1293 static int
1294 execbuf_submit(struct i915_execbuffer_params *params,
1295                struct drm_i915_gem_execbuffer2 *args,
1296                struct list_head *vmas)
1297 {
1298         struct drm_i915_private *dev_priv = params->request->i915;
1299         u64 exec_start, exec_len;
1300         int instp_mode;
1301         u32 instp_mask;
1302         int ret;
1303
1304         ret = i915_gem_execbuffer_move_to_gpu(params->request, vmas);
1305         if (ret)
1306                 return ret;
1307
1308         ret = i915_switch_context(params->request);
1309         if (ret)
1310                 return ret;
1311
1312         instp_mode = args->flags & I915_EXEC_CONSTANTS_MASK;
1313         instp_mask = I915_EXEC_CONSTANTS_MASK;
1314         switch (instp_mode) {
1315         case I915_EXEC_CONSTANTS_REL_GENERAL:
1316         case I915_EXEC_CONSTANTS_ABSOLUTE:
1317         case I915_EXEC_CONSTANTS_REL_SURFACE:
1318                 if (instp_mode != 0 && params->engine->id != RCS) {
1319                         DRM_DEBUG("non-0 rel constants mode on non-RCS\n");
1320                         return -EINVAL;
1321                 }
1322
1323                 if (instp_mode != dev_priv->relative_constants_mode) {
1324                         if (INTEL_INFO(dev_priv)->gen < 4) {
1325                                 DRM_DEBUG("no rel constants on pre-gen4\n");
1326                                 return -EINVAL;
1327                         }
1328
1329                         if (INTEL_INFO(dev_priv)->gen > 5 &&
1330                             instp_mode == I915_EXEC_CONSTANTS_REL_SURFACE) {
1331                                 DRM_DEBUG("rel surface constants mode invalid on gen5+\n");
1332                                 return -EINVAL;
1333                         }
1334
1335                         /* The HW changed the meaning on this bit on gen6 */
1336                         if (INTEL_INFO(dev_priv)->gen >= 6)
1337                                 instp_mask &= ~I915_EXEC_CONSTANTS_REL_SURFACE;
1338                 }
1339                 break;
1340         default:
1341                 DRM_DEBUG("execbuf with unknown constants: %d\n", instp_mode);
1342                 return -EINVAL;
1343         }
1344
1345         if (params->engine->id == RCS &&
1346             instp_mode != dev_priv->relative_constants_mode) {
1347                 struct intel_ring *ring = params->request->ring;
1348
1349                 ret = intel_ring_begin(params->request, 4);
1350                 if (ret)
1351                         return ret;
1352
1353                 intel_ring_emit(ring, MI_NOOP);
1354                 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
1355                 intel_ring_emit_reg(ring, INSTPM);
1356                 intel_ring_emit(ring, instp_mask << 16 | instp_mode);
1357                 intel_ring_advance(ring);
1358
1359                 dev_priv->relative_constants_mode = instp_mode;
1360         }
1361
1362         if (args->flags & I915_EXEC_GEN7_SOL_RESET) {
1363                 ret = i915_reset_gen7_sol_offsets(params->request);
1364                 if (ret)
1365                         return ret;
1366         }
1367
1368         exec_len   = args->batch_len;
1369         exec_start = params->batch_obj_vm_offset +
1370                      params->args_batch_start_offset;
1371
1372         if (exec_len == 0)
1373                 exec_len = params->batch_obj->base.size;
1374
1375         ret = params->engine->emit_bb_start(params->request,
1376                                             exec_start, exec_len,
1377                                             params->dispatch_flags);
1378         if (ret)
1379                 return ret;
1380
1381         trace_i915_gem_ring_dispatch(params->request, params->dispatch_flags);
1382
1383         i915_gem_execbuffer_move_to_active(vmas, params->request);
1384
1385         return 0;
1386 }
1387
1388 /**
1389  * Find one BSD ring to dispatch the corresponding BSD command.
1390  * The engine index is returned.
1391  */
1392 static unsigned int
1393 gen8_dispatch_bsd_engine(struct drm_i915_private *dev_priv,
1394                          struct drm_file *file)
1395 {
1396         struct drm_i915_file_private *file_priv = file->driver_priv;
1397
1398         /* Check whether the file_priv has already selected one ring. */
1399         if ((int)file_priv->bsd_engine < 0) {
1400                 /* If not, use the ping-pong mechanism to select one. */
1401                 mutex_lock(&dev_priv->drm.struct_mutex);
1402                 file_priv->bsd_engine = dev_priv->mm.bsd_engine_dispatch_index;
1403                 dev_priv->mm.bsd_engine_dispatch_index ^= 1;
1404                 mutex_unlock(&dev_priv->drm.struct_mutex);
1405         }
1406
1407         return file_priv->bsd_engine;
1408 }
1409
1410 #define I915_USER_RINGS (4)
1411
1412 static const enum intel_engine_id user_ring_map[I915_USER_RINGS + 1] = {
1413         [I915_EXEC_DEFAULT]     = RCS,
1414         [I915_EXEC_RENDER]      = RCS,
1415         [I915_EXEC_BLT]         = BCS,
1416         [I915_EXEC_BSD]         = VCS,
1417         [I915_EXEC_VEBOX]       = VECS
1418 };
1419
1420 static struct intel_engine_cs *
1421 eb_select_engine(struct drm_i915_private *dev_priv,
1422                  struct drm_file *file,
1423                  struct drm_i915_gem_execbuffer2 *args)
1424 {
1425         unsigned int user_ring_id = args->flags & I915_EXEC_RING_MASK;
1426         struct intel_engine_cs *engine;
1427
1428         if (user_ring_id > I915_USER_RINGS) {
1429                 DRM_DEBUG("execbuf with unknown ring: %u\n", user_ring_id);
1430                 return NULL;
1431         }
1432
1433         if ((user_ring_id != I915_EXEC_BSD) &&
1434             ((args->flags & I915_EXEC_BSD_MASK) != 0)) {
1435                 DRM_DEBUG("execbuf with non bsd ring but with invalid "
1436                           "bsd dispatch flags: %d\n", (int)(args->flags));
1437                 return NULL;
1438         }
1439
1440         if (user_ring_id == I915_EXEC_BSD && HAS_BSD2(dev_priv)) {
1441                 unsigned int bsd_idx = args->flags & I915_EXEC_BSD_MASK;
1442
1443                 if (bsd_idx == I915_EXEC_BSD_DEFAULT) {
1444                         bsd_idx = gen8_dispatch_bsd_engine(dev_priv, file);
1445                 } else if (bsd_idx >= I915_EXEC_BSD_RING1 &&
1446                            bsd_idx <= I915_EXEC_BSD_RING2) {
1447                         bsd_idx >>= I915_EXEC_BSD_SHIFT;
1448                         bsd_idx--;
1449                 } else {
1450                         DRM_DEBUG("execbuf with unknown bsd ring: %u\n",
1451                                   bsd_idx);
1452                         return NULL;
1453                 }
1454
1455                 engine = &dev_priv->engine[_VCS(bsd_idx)];
1456         } else {
1457                 engine = &dev_priv->engine[user_ring_map[user_ring_id]];
1458         }
1459
1460         if (!intel_engine_initialized(engine)) {
1461                 DRM_DEBUG("execbuf with invalid ring: %u\n", user_ring_id);
1462                 return NULL;
1463         }
1464
1465         return engine;
1466 }
1467
1468 static int
1469 i915_gem_do_execbuffer(struct drm_device *dev, void *data,
1470                        struct drm_file *file,
1471                        struct drm_i915_gem_execbuffer2 *args,
1472                        struct drm_i915_gem_exec_object2 *exec)
1473 {
1474         struct drm_i915_private *dev_priv = to_i915(dev);
1475         struct i915_ggtt *ggtt = &dev_priv->ggtt;
1476         struct eb_vmas *eb;
1477         struct drm_i915_gem_object *batch_obj;
1478         struct drm_i915_gem_exec_object2 shadow_exec_entry;
1479         struct intel_engine_cs *engine;
1480         struct i915_gem_context *ctx;
1481         struct i915_address_space *vm;
1482         struct i915_execbuffer_params params_master; /* XXX: will be removed later */
1483         struct i915_execbuffer_params *params = &params_master;
1484         const u32 ctx_id = i915_execbuffer2_get_context_id(*args);
1485         u32 dispatch_flags;
1486         int ret;
1487         bool need_relocs;
1488
1489         if (!i915_gem_check_execbuffer(args))
1490                 return -EINVAL;
1491
1492         ret = validate_exec_list(dev, exec, args->buffer_count);
1493         if (ret)
1494                 return ret;
1495
1496         dispatch_flags = 0;
1497         if (args->flags & I915_EXEC_SECURE) {
1498                 if (!drm_is_current_master(file) || !capable(CAP_SYS_ADMIN))
1499                     return -EPERM;
1500
1501                 dispatch_flags |= I915_DISPATCH_SECURE;
1502         }
1503         if (args->flags & I915_EXEC_IS_PINNED)
1504                 dispatch_flags |= I915_DISPATCH_PINNED;
1505
1506         engine = eb_select_engine(dev_priv, file, args);
1507         if (!engine)
1508                 return -EINVAL;
1509
1510         if (args->buffer_count < 1) {
1511                 DRM_DEBUG("execbuf with %d buffers\n", args->buffer_count);
1512                 return -EINVAL;
1513         }
1514
1515         if (args->flags & I915_EXEC_RESOURCE_STREAMER) {
1516                 if (!HAS_RESOURCE_STREAMER(dev)) {
1517                         DRM_DEBUG("RS is only allowed for Haswell, Gen8 and above\n");
1518                         return -EINVAL;
1519                 }
1520                 if (engine->id != RCS) {
1521                         DRM_DEBUG("RS is not available on %s\n",
1522                                  engine->name);
1523                         return -EINVAL;
1524                 }
1525
1526                 dispatch_flags |= I915_DISPATCH_RS;
1527         }
1528
1529         /* Take a local wakeref for preparing to dispatch the execbuf as
1530          * we expect to access the hardware fairly frequently in the
1531          * process. Upon first dispatch, we acquire another prolonged
1532          * wakeref that we hold until the GPU has been idle for at least
1533          * 100ms.
1534          */
1535         intel_runtime_pm_get(dev_priv);
1536
1537         ret = i915_mutex_lock_interruptible(dev);
1538         if (ret)
1539                 goto pre_mutex_err;
1540
1541         ctx = i915_gem_validate_context(dev, file, engine, ctx_id);
1542         if (IS_ERR(ctx)) {
1543                 mutex_unlock(&dev->struct_mutex);
1544                 ret = PTR_ERR(ctx);
1545                 goto pre_mutex_err;
1546         }
1547
1548         i915_gem_context_get(ctx);
1549
1550         if (ctx->ppgtt)
1551                 vm = &ctx->ppgtt->base;
1552         else
1553                 vm = &ggtt->base;
1554
1555         memset(&params_master, 0x00, sizeof(params_master));
1556
1557         eb = eb_create(args);
1558         if (eb == NULL) {
1559                 i915_gem_context_put(ctx);
1560                 mutex_unlock(&dev->struct_mutex);
1561                 ret = -ENOMEM;
1562                 goto pre_mutex_err;
1563         }
1564
1565         /* Look up object handles */
1566         ret = eb_lookup_vmas(eb, exec, args, vm, file);
1567         if (ret)
1568                 goto err;
1569
1570         /* take note of the batch buffer before we might reorder the lists */
1571         batch_obj = eb_get_batch(eb);
1572
1573         /* Move the objects en-masse into the GTT, evicting if necessary. */
1574         need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0;
1575         ret = i915_gem_execbuffer_reserve(engine, &eb->vmas, ctx,
1576                                           &need_relocs);
1577         if (ret)
1578                 goto err;
1579
1580         /* The objects are in their final locations, apply the relocations. */
1581         if (need_relocs)
1582                 ret = i915_gem_execbuffer_relocate(eb);
1583         if (ret) {
1584                 if (ret == -EFAULT) {
1585                         ret = i915_gem_execbuffer_relocate_slow(dev, args, file,
1586                                                                 engine,
1587                                                                 eb, exec, ctx);
1588                         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1589                 }
1590                 if (ret)
1591                         goto err;
1592         }
1593
1594         /* Set the pending read domains for the batch buffer to COMMAND */
1595         if (batch_obj->base.pending_write_domain) {
1596                 DRM_DEBUG("Attempting to use self-modifying batch buffer\n");
1597                 ret = -EINVAL;
1598                 goto err;
1599         }
1600
1601         params->args_batch_start_offset = args->batch_start_offset;
1602         if (intel_engine_needs_cmd_parser(engine) && args->batch_len) {
1603                 struct drm_i915_gem_object *parsed_batch_obj;
1604
1605                 parsed_batch_obj = i915_gem_execbuffer_parse(engine,
1606                                                              &shadow_exec_entry,
1607                                                              eb,
1608                                                              batch_obj,
1609                                                              args->batch_start_offset,
1610                                                              args->batch_len,
1611                                                              drm_is_current_master(file));
1612                 if (IS_ERR(parsed_batch_obj)) {
1613                         ret = PTR_ERR(parsed_batch_obj);
1614                         goto err;
1615                 }
1616
1617                 /*
1618                  * parsed_batch_obj == batch_obj means batch not fully parsed:
1619                  * Accept, but don't promote to secure.
1620                  */
1621
1622                 if (parsed_batch_obj != batch_obj) {
1623                         /*
1624                          * Batch parsed and accepted:
1625                          *
1626                          * Set the DISPATCH_SECURE bit to remove the NON_SECURE
1627                          * bit from MI_BATCH_BUFFER_START commands issued in
1628                          * the dispatch_execbuffer implementations. We
1629                          * specifically don't want that set on batches the
1630                          * command parser has accepted.
1631                          */
1632                         dispatch_flags |= I915_DISPATCH_SECURE;
1633                         params->args_batch_start_offset = 0;
1634                         batch_obj = parsed_batch_obj;
1635                 }
1636         }
1637
1638         batch_obj->base.pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
1639
1640         /* snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure
1641          * batch" bit. Hence we need to pin secure batches into the global gtt.
1642          * hsw should have this fixed, but bdw mucks it up again. */
1643         if (dispatch_flags & I915_DISPATCH_SECURE) {
1644                 /*
1645                  * So on first glance it looks freaky that we pin the batch here
1646                  * outside of the reservation loop. But:
1647                  * - The batch is already pinned into the relevant ppgtt, so we
1648                  *   already have the backing storage fully allocated.
1649                  * - No other BO uses the global gtt (well contexts, but meh),
1650                  *   so we don't really have issues with multiple objects not
1651                  *   fitting due to fragmentation.
1652                  * So this is actually safe.
1653                  */
1654                 ret = i915_gem_obj_ggtt_pin(batch_obj, 0, 0);
1655                 if (ret)
1656                         goto err;
1657
1658                 params->batch_obj_vm_offset = i915_gem_obj_ggtt_offset(batch_obj);
1659         } else
1660                 params->batch_obj_vm_offset = i915_gem_obj_offset(batch_obj, vm);
1661
1662         /* Allocate a request for this batch buffer nice and early. */
1663         params->request = i915_gem_request_alloc(engine, ctx);
1664         if (IS_ERR(params->request)) {
1665                 ret = PTR_ERR(params->request);
1666                 goto err_batch_unpin;
1667         }
1668
1669         ret = i915_gem_request_add_to_client(params->request, file);
1670         if (ret)
1671                 goto err_request;
1672
1673         /*
1674          * Save assorted stuff away to pass through to *_submission().
1675          * NB: This data should be 'persistent' and not local as it will
1676          * kept around beyond the duration of the IOCTL once the GPU
1677          * scheduler arrives.
1678          */
1679         params->dev                     = dev;
1680         params->file                    = file;
1681         params->engine                    = engine;
1682         params->dispatch_flags          = dispatch_flags;
1683         params->batch_obj               = batch_obj;
1684         params->ctx                     = ctx;
1685
1686         ret = execbuf_submit(params, args, &eb->vmas);
1687 err_request:
1688         __i915_add_request(params->request, params->batch_obj, ret == 0);
1689
1690 err_batch_unpin:
1691         /*
1692          * FIXME: We crucially rely upon the active tracking for the (ppgtt)
1693          * batch vma for correctness. For less ugly and less fragility this
1694          * needs to be adjusted to also track the ggtt batch vma properly as
1695          * active.
1696          */
1697         if (dispatch_flags & I915_DISPATCH_SECURE)
1698                 i915_gem_object_ggtt_unpin(batch_obj);
1699
1700 err:
1701         /* the request owns the ref now */
1702         i915_gem_context_put(ctx);
1703         eb_destroy(eb);
1704
1705         mutex_unlock(&dev->struct_mutex);
1706
1707 pre_mutex_err:
1708         /* intel_gpu_busy should also get a ref, so it will free when the device
1709          * is really idle. */
1710         intel_runtime_pm_put(dev_priv);
1711         return ret;
1712 }
1713
1714 /*
1715  * Legacy execbuffer just creates an exec2 list from the original exec object
1716  * list array and passes it to the real function.
1717  */
1718 int
1719 i915_gem_execbuffer(struct drm_device *dev, void *data,
1720                     struct drm_file *file)
1721 {
1722         struct drm_i915_gem_execbuffer *args = data;
1723         struct drm_i915_gem_execbuffer2 exec2;
1724         struct drm_i915_gem_exec_object *exec_list = NULL;
1725         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
1726         int ret, i;
1727
1728         if (args->buffer_count < 1) {
1729                 DRM_DEBUG("execbuf with %d buffers\n", args->buffer_count);
1730                 return -EINVAL;
1731         }
1732
1733         /* Copy in the exec list from userland */
1734         exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
1735         exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
1736         if (exec_list == NULL || exec2_list == NULL) {
1737                 DRM_DEBUG("Failed to allocate exec list for %d buffers\n",
1738                           args->buffer_count);
1739                 drm_free_large(exec_list);
1740                 drm_free_large(exec2_list);
1741                 return -ENOMEM;
1742         }
1743         ret = copy_from_user(exec_list,
1744                              u64_to_user_ptr(args->buffers_ptr),
1745                              sizeof(*exec_list) * args->buffer_count);
1746         if (ret != 0) {
1747                 DRM_DEBUG("copy %d exec entries failed %d\n",
1748                           args->buffer_count, ret);
1749                 drm_free_large(exec_list);
1750                 drm_free_large(exec2_list);
1751                 return -EFAULT;
1752         }
1753
1754         for (i = 0; i < args->buffer_count; i++) {
1755                 exec2_list[i].handle = exec_list[i].handle;
1756                 exec2_list[i].relocation_count = exec_list[i].relocation_count;
1757                 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
1758                 exec2_list[i].alignment = exec_list[i].alignment;
1759                 exec2_list[i].offset = exec_list[i].offset;
1760                 if (INTEL_INFO(dev)->gen < 4)
1761                         exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
1762                 else
1763                         exec2_list[i].flags = 0;
1764         }
1765
1766         exec2.buffers_ptr = args->buffers_ptr;
1767         exec2.buffer_count = args->buffer_count;
1768         exec2.batch_start_offset = args->batch_start_offset;
1769         exec2.batch_len = args->batch_len;
1770         exec2.DR1 = args->DR1;
1771         exec2.DR4 = args->DR4;
1772         exec2.num_cliprects = args->num_cliprects;
1773         exec2.cliprects_ptr = args->cliprects_ptr;
1774         exec2.flags = I915_EXEC_RENDER;
1775         i915_execbuffer2_set_context_id(exec2, 0);
1776
1777         ret = i915_gem_do_execbuffer(dev, data, file, &exec2, exec2_list);
1778         if (!ret) {
1779                 struct drm_i915_gem_exec_object __user *user_exec_list =
1780                         u64_to_user_ptr(args->buffers_ptr);
1781
1782                 /* Copy the new buffer offsets back to the user's exec list. */
1783                 for (i = 0; i < args->buffer_count; i++) {
1784                         exec2_list[i].offset =
1785                                 gen8_canonical_addr(exec2_list[i].offset);
1786                         ret = __copy_to_user(&user_exec_list[i].offset,
1787                                              &exec2_list[i].offset,
1788                                              sizeof(user_exec_list[i].offset));
1789                         if (ret) {
1790                                 ret = -EFAULT;
1791                                 DRM_DEBUG("failed to copy %d exec entries "
1792                                           "back to user (%d)\n",
1793                                           args->buffer_count, ret);
1794                                 break;
1795                         }
1796                 }
1797         }
1798
1799         drm_free_large(exec_list);
1800         drm_free_large(exec2_list);
1801         return ret;
1802 }
1803
1804 int
1805 i915_gem_execbuffer2(struct drm_device *dev, void *data,
1806                      struct drm_file *file)
1807 {
1808         struct drm_i915_gem_execbuffer2 *args = data;
1809         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
1810         int ret;
1811
1812         if (args->buffer_count < 1 ||
1813             args->buffer_count > UINT_MAX / sizeof(*exec2_list)) {
1814                 DRM_DEBUG("execbuf2 with %d buffers\n", args->buffer_count);
1815                 return -EINVAL;
1816         }
1817
1818         if (args->rsvd2 != 0) {
1819                 DRM_DEBUG("dirty rvsd2 field\n");
1820                 return -EINVAL;
1821         }
1822
1823         exec2_list = drm_malloc_gfp(args->buffer_count,
1824                                     sizeof(*exec2_list),
1825                                     GFP_TEMPORARY);
1826         if (exec2_list == NULL) {
1827                 DRM_DEBUG("Failed to allocate exec list for %d buffers\n",
1828                           args->buffer_count);
1829                 return -ENOMEM;
1830         }
1831         ret = copy_from_user(exec2_list,
1832                              u64_to_user_ptr(args->buffers_ptr),
1833                              sizeof(*exec2_list) * args->buffer_count);
1834         if (ret != 0) {
1835                 DRM_DEBUG("copy %d exec entries failed %d\n",
1836                           args->buffer_count, ret);
1837                 drm_free_large(exec2_list);
1838                 return -EFAULT;
1839         }
1840
1841         ret = i915_gem_do_execbuffer(dev, data, file, args, exec2_list);
1842         if (!ret) {
1843                 /* Copy the new buffer offsets back to the user's exec list. */
1844                 struct drm_i915_gem_exec_object2 __user *user_exec_list =
1845                                    u64_to_user_ptr(args->buffers_ptr);
1846                 int i;
1847
1848                 for (i = 0; i < args->buffer_count; i++) {
1849                         exec2_list[i].offset =
1850                                 gen8_canonical_addr(exec2_list[i].offset);
1851                         ret = __copy_to_user(&user_exec_list[i].offset,
1852                                              &exec2_list[i].offset,
1853                                              sizeof(user_exec_list[i].offset));
1854                         if (ret) {
1855                                 ret = -EFAULT;
1856                                 DRM_DEBUG("failed to copy %d exec entries "
1857                                           "back to user\n",
1858                                           args->buffer_count);
1859                                 break;
1860                         }
1861                 }
1862         }
1863
1864         drm_free_large(exec2_list);
1865         return ret;
1866 }