drivers/gpu/drm/i915/intel_ringbuffer.c

   1 /*
   2  * Copyright © 2008-2010 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *    Zou Nan hai <nanhai.zou@intel.com>
  26  *    Xiang Hai hao<haihao.xiang@intel.com>
  27  *
  28  */
  29
  30 #include <drm/drmP.h>
  31 #include "i915_drv.h"
  32 #include <drm/i915_drm.h>
  33 #include "i915_trace.h"
  34 #include "intel_drv.h"
  35
  36 bool
  37 intel_ring_initialized(struct intel_engine_cs *ring)
  38 {
  39         struct drm_device *dev = ring->dev;
  40
  41         if (!dev)
  42                 return false;
  43
  44         if (i915.enable_execlists) {
  45                 struct intel_context *dctx = ring->default_context;
  46                 struct intel_ringbuffer *ringbuf = dctx->engine[ring->id].ringbuf;
  47
  48                 return ringbuf->obj;
  49         } else
  50                 return ring->buffer && ring->buffer->obj;
  51 }
  52
  53 int __intel_ring_space(int head, int tail, int size)
  54 {
  55         int space = head - (tail + I915_RING_FREE_SPACE);
  56         if (space < 0)
  57                 space += size;
  58         return space;
  59 }
  60
  61 int intel_ring_space(struct intel_ringbuffer *ringbuf)
  62 {
  63         return __intel_ring_space(ringbuf->head & HEAD_ADDR,
  64                                   ringbuf->tail, ringbuf->size);
  65 }
  66
  67 bool intel_ring_stopped(struct intel_engine_cs *ring)
  68 {
  69         struct drm_i915_private *dev_priv = ring->dev->dev_private;
  70         return dev_priv->gpu_error.stop_rings & intel_ring_flag(ring);
  71 }
  72
  73 void __intel_ring_advance(struct intel_engine_cs *ring)
  74 {
  75         struct intel_ringbuffer *ringbuf = ring->buffer;
  76         ringbuf->tail &= ringbuf->size - 1;
  77         if (intel_ring_stopped(ring))
  78                 return;
  79         ring->write_tail(ring, ringbuf->tail);
  80 }
  81
  82 static int
  83 gen2_render_ring_flush(struct intel_engine_cs *ring,
  84                        u32      invalidate_domains,
  85                        u32      flush_domains)
  86 {
  87         u32 cmd;
  88         int ret;
  89
  90         cmd = MI_FLUSH;
  91         if (((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER) == 0)
  92                 cmd |= MI_NO_WRITE_FLUSH;
  93
  94         if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
  95                 cmd |= MI_READ_FLUSH;
  96
  97         ret = intel_ring_begin(ring, 2);
  98         if (ret)
  99                 return ret;
 100
 101         intel_ring_emit(ring, cmd);
 102         intel_ring_emit(ring, MI_NOOP);
 103         intel_ring_advance(ring);
 104
 105         return 0;
 106 }
 107
 108 static int
 109 gen4_render_ring_flush(struct intel_engine_cs *ring,
 110                        u32      invalidate_domains,
 111                        u32      flush_domains)
 112 {
 113         struct drm_device *dev = ring->dev;
 114         u32 cmd;
 115         int ret;
 116
 117         /*
 118          * read/write caches:
 119          *
 120          * I915_GEM_DOMAIN_RENDER is always invalidated, but is
 121          * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
 122          * also flushed at 2d versus 3d pipeline switches.
 123          *
 124          * read-only caches:
 125          *
 126          * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
 127          * MI_READ_FLUSH is set, and is always flushed on 965.
 128          *
 129          * I915_GEM_DOMAIN_COMMAND may not exist?
 130          *
 131          * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
 132          * invalidated when MI_EXE_FLUSH is set.
 133          *
 134          * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
 135          * invalidated with every MI_FLUSH.
 136          *
 137          * TLBs:
 138          *
 139          * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
 140          * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
 141          * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
 142          * are flushed at any MI_FLUSH.
 143          */
 144
 145         cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
 146         if ((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER)
 147                 cmd &= ~MI_NO_WRITE_FLUSH;
 148         if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
 149                 cmd |= MI_EXE_FLUSH;
 150
 151         if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
 152             (IS_G4X(dev) || IS_GEN5(dev)))
 153                 cmd |= MI_INVALIDATE_ISP;
 154
 155         ret = intel_ring_begin(ring, 2);
 156         if (ret)
 157                 return ret;
 158
 159         intel_ring_emit(ring, cmd);
 160         intel_ring_emit(ring, MI_NOOP);
 161         intel_ring_advance(ring);
 162
 163         return 0;
 164 }
 165
 166 /**
 167  * Emits a PIPE_CONTROL with a non-zero post-sync operation, for
 168  * implementing two workarounds on gen6.  From section 1.4.7.1
 169  * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1:
 170  *
 171  * [DevSNB-C+{W/A}] Before any depth stall flush (including those
 172  * produced by non-pipelined state commands), software needs to first
 173  * send a PIPE_CONTROL with no bits set except Post-Sync Operation !=
 174  * 0.
 175  *
 176  * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable
 177  * =1, a PIPE_CONTROL with any non-zero post-sync-op is required.
 178  *
 179  * And the workaround for these two requires this workaround first:
 180  *
 181  * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent
 182  * BEFORE the pipe-control with a post-sync op and no write-cache
 183  * flushes.
 184  *
 185  * And this last workaround is tricky because of the requirements on
 186  * that bit.  From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM
 187  * volume 2 part 1:
 188  *
 189  *     "1 of the following must also be set:
 190  *      - Render Target Cache Flush Enable ([12] of DW1)
 191  *      - Depth Cache Flush Enable ([0] of DW1)
 192  *      - Stall at Pixel Scoreboard ([1] of DW1)
 193  *      - Depth Stall ([13] of DW1)
 194  *      - Post-Sync Operation ([13] of DW1)
 195  *      - Notify Enable ([8] of DW1)"
 196  *
 197  * The cache flushes require the workaround flush that triggered this
 198  * one, so we can't use it.  Depth stall would trigger the same.
 199  * Post-sync nonzero is what triggered this second workaround, so we
 200  * can't use that one either.  Notify enable is IRQs, which aren't
 201  * really our business.  That leaves only stall at scoreboard.
 202  */
 203 static int
 204 intel_emit_post_sync_nonzero_flush(struct intel_engine_cs *ring)
 205 {
 206         u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
 207         int ret;
 208
 209
 210         ret = intel_ring_begin(ring, 6);
 211         if (ret)
 212                 return ret;
 213
 214         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
 215         intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |
 216                         PIPE_CONTROL_STALL_AT_SCOREBOARD);
 217         intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
 218         intel_ring_emit(ring, 0); /* low dword */
 219         intel_ring_emit(ring, 0); /* high dword */
 220         intel_ring_emit(ring, MI_NOOP);
 221         intel_ring_advance(ring);
 222
 223         ret = intel_ring_begin(ring, 6);
 224         if (ret)
 225                 return ret;
 226
 227         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
 228         intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE);
 229         intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
 230         intel_ring_emit(ring, 0);
 231         intel_ring_emit(ring, 0);
 232         intel_ring_emit(ring, MI_NOOP);
 233         intel_ring_advance(ring);
 234
 235         return 0;
 236 }
 237
 238 static int
 239 gen6_render_ring_flush(struct intel_engine_cs *ring,
 240                          u32 invalidate_domains, u32 flush_domains)
 241 {
 242         u32 flags = 0;
 243         u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
 244         int ret;
 245
 246         /* Force SNB workarounds for PIPE_CONTROL flushes */
 247         ret = intel_emit_post_sync_nonzero_flush(ring);
 248         if (ret)
 249                 return ret;
 250
 251         /* Just flush everything.  Experiments have shown that reducing the
 252          * number of bits based on the write domains has little performance
 253          * impact.
 254          */
 255         if (flush_domains) {
 256                 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
 257                 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
 258                 /*
 259                  * Ensure that any following seqno writes only happen
 260                  * when the render cache is indeed flushed.
 261                  */
 262                 flags |= PIPE_CONTROL_CS_STALL;
 263         }
 264         if (invalidate_domains) {
 265                 flags |= PIPE_CONTROL_TLB_INVALIDATE;
 266                 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
 267                 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
 268                 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
 269                 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
 270                 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
 271                 /*
 272                  * TLB invalidate requires a post-sync write.
 273                  */
 274                 flags |= PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_CS_STALL;
 275         }
 276
 277         ret = intel_ring_begin(ring, 4);
 278         if (ret)
 279                 return ret;
 280
 281         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
 282         intel_ring_emit(ring, flags);
 283         intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
 284         intel_ring_emit(ring, 0);
 285         intel_ring_advance(ring);
 286
 287         return 0;
 288 }
 289
 290 static int
 291 gen7_render_ring_cs_stall_wa(struct intel_engine_cs *ring)
 292 {
 293         int ret;
 294
 295         ret = intel_ring_begin(ring, 4);
 296         if (ret)
 297                 return ret;
 298
 299         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
 300         intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |
 301                               PIPE_CONTROL_STALL_AT_SCOREBOARD);
 302         intel_ring_emit(ring, 0);
 303         intel_ring_emit(ring, 0);
 304         intel_ring_advance(ring);
 305
 306         return 0;
 307 }
 308
 309 static int gen7_ring_fbc_flush(struct intel_engine_cs *ring, u32 value)
 310 {
 311         int ret;
 312
 313         if (!ring->fbc_dirty)
 314                 return 0;
 315
 316         ret = intel_ring_begin(ring, 6);
 317         if (ret)
 318                 return ret;
 319         /* WaFbcNukeOn3DBlt:ivb/hsw */
 320         intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
 321         intel_ring_emit(ring, MSG_FBC_REND_STATE);
 322         intel_ring_emit(ring, value);
 323         intel_ring_emit(ring, MI_STORE_REGISTER_MEM(1) | MI_SRM_LRM_GLOBAL_GTT);
 324         intel_ring_emit(ring, MSG_FBC_REND_STATE);
 325         intel_ring_emit(ring, ring->scratch.gtt_offset + 256);
 326         intel_ring_advance(ring);
 327
 328         ring->fbc_dirty = false;
 329         return 0;
 330 }
 331
 332 static int
 333 gen7_render_ring_flush(struct intel_engine_cs *ring,
 334                        u32 invalidate_domains, u32 flush_domains)
 335 {
 336         u32 flags = 0;
 337         u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
 338         int ret;
 339
 340         /*
 341          * Ensure that any following seqno writes only happen when the render
 342          * cache is indeed flushed.
 343          *
 344          * Workaround: 4th PIPE_CONTROL command (except the ones with only
 345          * read-cache invalidate bits set) must have the CS_STALL bit set. We
 346          * don't try to be clever and just set it unconditionally.
 347          */
 348         flags |= PIPE_CONTROL_CS_STALL;
 349
 350         /* Just flush everything.  Experiments have shown that reducing the
 351          * number of bits based on the write domains has little performance
 352          * impact.
 353          */
 354         if (flush_domains) {
 355                 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
 356                 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
 357         }
 358         if (invalidate_domains) {
 359                 flags |= PIPE_CONTROL_TLB_INVALIDATE;
 360                 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
 361                 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
 362                 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
 363                 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
 364                 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
 365                 flags |= PIPE_CONTROL_MEDIA_STATE_CLEAR;
 366                 /*
 367                  * TLB invalidate requires a post-sync write.
 368                  */
 369                 flags |= PIPE_CONTROL_QW_WRITE;
 370                 flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
 371
 372                 /* Workaround: we must issue a pipe_control with CS-stall bit
 373                  * set before a pipe_control command that has the state cache
 374                  * invalidate bit set. */
 375                 gen7_render_ring_cs_stall_wa(ring);
 376         }
 377
 378         ret = intel_ring_begin(ring, 4);
 379         if (ret)
 380                 return ret;
 381
 382         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
 383         intel_ring_emit(ring, flags);
 384         intel_ring_emit(ring, scratch_addr);
 385         intel_ring_emit(ring, 0);
 386         intel_ring_advance(ring);
 387
 388         if (!invalidate_domains && flush_domains)
 389                 return gen7_ring_fbc_flush(ring, FBC_REND_NUKE);
 390
 391         return 0;
 392 }
 393
 394 static int
 395 gen8_emit_pipe_control(struct intel_engine_cs *ring,
 396                        u32 flags, u32 scratch_addr)
 397 {
 398         int ret;
 399
 400         ret = intel_ring_begin(ring, 6);
 401         if (ret)
 402                 return ret;
 403
 404         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6));
 405         intel_ring_emit(ring, flags);
 406         intel_ring_emit(ring, scratch_addr);
 407         intel_ring_emit(ring, 0);
 408         intel_ring_emit(ring, 0);
 409         intel_ring_emit(ring, 0);
 410         intel_ring_advance(ring);
 411
 412         return 0;
 413 }
 414
 415 static int
 416 gen8_render_ring_flush(struct intel_engine_cs *ring,
 417                        u32 invalidate_domains, u32 flush_domains)
 418 {
 419         u32 flags = 0;
 420         u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
 421         int ret;
 422
 423         flags |= PIPE_CONTROL_CS_STALL;
 424
 425         if (flush_domains) {
 426                 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
 427                 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
 428         }
 429         if (invalidate_domains) {
 430                 flags |= PIPE_CONTROL_TLB_INVALIDATE;
 431                 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
 432                 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
 433                 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
 434                 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
 435                 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
 436                 flags |= PIPE_CONTROL_QW_WRITE;
 437                 flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
 438
 439                 /* WaCsStallBeforeStateCacheInvalidate:bdw,chv */
 440                 ret = gen8_emit_pipe_control(ring,
 441                                              PIPE_CONTROL_CS_STALL |
 442                                              PIPE_CONTROL_STALL_AT_SCOREBOARD,
 443                                              0);
 444                 if (ret)
 445                         return ret;
 446         }
 447
 448         ret = gen8_emit_pipe_control(ring, flags, scratch_addr);
 449         if (ret)
 450                 return ret;
 451
 452         if (!invalidate_domains && flush_domains)
 453                 return gen7_ring_fbc_flush(ring, FBC_REND_NUKE);
 454
 455         return 0;
 456 }
 457
 458 static void ring_write_tail(struct intel_engine_cs *ring,
 459                             u32 value)
 460 {
 461         struct drm_i915_private *dev_priv = ring->dev->dev_private;
 462         I915_WRITE_TAIL(ring, value);
 463 }
 464
 465 u64 intel_ring_get_active_head(struct intel_engine_cs *ring)
 466 {
 467         struct drm_i915_private *dev_priv = ring->dev->dev_private;
 468         u64 acthd;
 469
 470         if (INTEL_INFO(ring->dev)->gen >= 8)
 471                 acthd = I915_READ64_2x32(RING_ACTHD(ring->mmio_base),
 472                                          RING_ACTHD_UDW(ring->mmio_base));
 473         else if (INTEL_INFO(ring->dev)->gen >= 4)
 474                 acthd = I915_READ(RING_ACTHD(ring->mmio_base));
 475         else
 476                 acthd = I915_READ(ACTHD);
 477
 478         return acthd;
 479 }
 480
 481 static void ring_setup_phys_status_page(struct intel_engine_cs *ring)
 482 {
 483         struct drm_i915_private *dev_priv = ring->dev->dev_private;
 484         u32 addr;
 485
 486         addr = dev_priv->status_page_dmah->busaddr;
 487         if (INTEL_INFO(ring->dev)->gen >= 4)
 488                 addr |= (dev_priv->status_page_dmah->busaddr >> 28) & 0xf0;
 489         I915_WRITE(HWS_PGA, addr);
 490 }
 491
 492 static bool stop_ring(struct intel_engine_cs *ring)
 493 {
 494         struct drm_i915_private *dev_priv = to_i915(ring->dev);
 495
 496         if (!IS_GEN2(ring->dev)) {
 497                 I915_WRITE_MODE(ring, _MASKED_BIT_ENABLE(STOP_RING));
 498                 if (wait_for((I915_READ_MODE(ring) & MODE_IDLE) != 0, 1000)) {
 499                         DRM_ERROR("%s : timed out trying to stop ring\n", ring->name);
 500                         /* Sometimes we observe that the idle flag is not
 501                          * set even though the ring is empty. So double
 502                          * check before giving up.
 503                          */
 504                         if (I915_READ_HEAD(ring) != I915_READ_TAIL(ring))
 505                                 return false;
 506                 }
 507         }
 508
 509         I915_WRITE_CTL(ring, 0);
 510         I915_WRITE_HEAD(ring, 0);
 511         ring->write_tail(ring, 0);
 512
 513         if (!IS_GEN2(ring->dev)) {
 514                 (void)I915_READ_CTL(ring);
 515                 I915_WRITE_MODE(ring, _MASKED_BIT_DISABLE(STOP_RING));
 516         }
 517
 518         return (I915_READ_HEAD(ring) & HEAD_ADDR) == 0;
 519 }
 520
 521 static int init_ring_common(struct intel_engine_cs *ring)
 522 {
 523         struct drm_device *dev = ring->dev;
 524         struct drm_i915_private *dev_priv = dev->dev_private;
 525         struct intel_ringbuffer *ringbuf = ring->buffer;
 526         struct drm_i915_gem_object *obj = ringbuf->obj;
 527         int ret = 0;
 528
 529         gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL);
 530
 531         if (!stop_ring(ring)) {
 532                 /* G45 ring initialization often fails to reset head to zero */
 533                 DRM_DEBUG_KMS("%s head not reset to zero "
 534                               "ctl %08x head %08x tail %08x start %08x\n",
 535                               ring->name,
 536                               I915_READ_CTL(ring),
 537                               I915_READ_HEAD(ring),
 538                               I915_READ_TAIL(ring),
 539                               I915_READ_START(ring));
 540
 541                 if (!stop_ring(ring)) {
 542                         DRM_ERROR("failed to set %s head to zero "
 543                                   "ctl %08x head %08x tail %08x start %08x\n",
 544                                   ring->name,
 545                                   I915_READ_CTL(ring),
 546                                   I915_READ_HEAD(ring),
 547                                   I915_READ_TAIL(ring),
 548                                   I915_READ_START(ring));
 549                         ret = -EIO;
 550                         goto out;
 551                 }
 552         }
 553
 554         if (I915_NEED_GFX_HWS(dev))
 555                 intel_ring_setup_status_page(ring);
 556         else
 557                 ring_setup_phys_status_page(ring);
 558
 559         /* Enforce ordering by reading HEAD register back */
 560         I915_READ_HEAD(ring);
 561
 562         /* Initialize the ring. This must happen _after_ we've cleared the ring
 563          * registers with the above sequence (the readback of the HEAD registers
 564          * also enforces ordering), otherwise the hw might lose the new ring
 565          * register values. */
 566         I915_WRITE_START(ring, i915_gem_obj_ggtt_offset(obj));
 567
 568         /* WaClearRingBufHeadRegAtInit:ctg,elk */
 569         if (I915_READ_HEAD(ring))
 570                 DRM_DEBUG("%s initialization failed [head=%08x], fudging\n",
 571                           ring->name, I915_READ_HEAD(ring));
 572         I915_WRITE_HEAD(ring, 0);
 573         (void)I915_READ_HEAD(ring);
 574
 575         I915_WRITE_CTL(ring,
 576                         ((ringbuf->size - PAGE_SIZE) & RING_NR_PAGES)
 577                         | RING_VALID);
 578
 579         /* If the head is still not zero, the ring is dead */
 580         if (wait_for((I915_READ_CTL(ring) & RING_VALID) != 0 &&
 581                      I915_READ_START(ring) == i915_gem_obj_ggtt_offset(obj) &&
 582                      (I915_READ_HEAD(ring) & HEAD_ADDR) == 0, 50)) {
 583                 DRM_ERROR("%s initialization failed "
 584                           "ctl %08x (valid? %d) head %08x tail %08x start %08x [expected %08lx]\n",
 585                           ring->name,
 586                           I915_READ_CTL(ring), I915_READ_CTL(ring) & RING_VALID,
 587                           I915_READ_HEAD(ring), I915_READ_TAIL(ring),
 588                           I915_READ_START(ring), (unsigned long)i915_gem_obj_ggtt_offset(obj));
 589                 ret = -EIO;
 590                 goto out;
 591         }
 592
 593         ringbuf->head = I915_READ_HEAD(ring);
 594         ringbuf->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
 595         ringbuf->space = intel_ring_space(ringbuf);
 596         ringbuf->last_retired_head = -1;
 597
 598         memset(&ring->hangcheck, 0, sizeof(ring->hangcheck));
 599
 600 out:
 601         gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL);
 602
 603         return ret;
 604 }
 605
 606 void
 607 intel_fini_pipe_control(struct intel_engine_cs *ring)
 608 {
 609         struct drm_device *dev = ring->dev;
 610
 611         if (ring->scratch.obj == NULL)
 612                 return;
 613
 614         if (INTEL_INFO(dev)->gen >= 5) {
 615                 kunmap(sg_page(ring->scratch.obj->pages->sgl));
 616                 i915_gem_object_ggtt_unpin(ring->scratch.obj);
 617         }
 618
 619         drm_gem_object_unreference(&ring->scratch.obj->base);
 620         ring->scratch.obj = NULL;
 621 }
 622
 623 int
 624 intel_init_pipe_control(struct intel_engine_cs *ring)
 625 {
 626         int ret;
 627
 628         if (ring->scratch.obj)
 629                 return 0;
 630
 631         ring->scratch.obj = i915_gem_alloc_object(ring->dev, 4096);
 632         if (ring->scratch.obj == NULL) {
 633                 DRM_ERROR("Failed to allocate seqno page\n");
 634                 ret = -ENOMEM;
 635                 goto err;
 636         }
 637
 638         ret = i915_gem_object_set_cache_level(ring->scratch.obj, I915_CACHE_LLC);
 639         if (ret)
 640                 goto err_unref;
 641
 642         ret = i915_gem_obj_ggtt_pin(ring->scratch.obj, 4096, 0);
 643         if (ret)
 644                 goto err_unref;
 645
 646         ring->scratch.gtt_offset = i915_gem_obj_ggtt_offset(ring->scratch.obj);
 647         ring->scratch.cpu_page = kmap(sg_page(ring->scratch.obj->pages->sgl));
 648         if (ring->scratch.cpu_page == NULL) {
 649                 ret = -ENOMEM;
 650                 goto err_unpin;
 651         }
 652
 653         DRM_DEBUG_DRIVER("%s pipe control offset: 0x%08x\n",
 654                          ring->name, ring->scratch.gtt_offset);
 655         return 0;
 656
 657 err_unpin:
 658         i915_gem_object_ggtt_unpin(ring->scratch.obj);
 659 err_unref:
 660         drm_gem_object_unreference(&ring->scratch.obj->base);
 661 err:
 662         return ret;
 663 }
 664
 665 static int intel_ring_workarounds_emit(struct intel_engine_cs *ring,
 666                                        struct intel_context *ctx)
 667 {
 668         int ret, i;
 669         struct drm_device *dev = ring->dev;
 670         struct drm_i915_private *dev_priv = dev->dev_private;
 671         struct i915_workarounds *w = &dev_priv->workarounds;
 672
 673         if (WARN_ON(w->count == 0))
 674                 return 0;
 675
 676         ring->gpu_caches_dirty = true;
 677         ret = intel_ring_flush_all_caches(ring);
 678         if (ret)
 679                 return ret;
 680
 681         ret = intel_ring_begin(ring, (w->count * 2 + 2));
 682         if (ret)
 683                 return ret;
 684
 685         intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(w->count));
 686         for (i = 0; i < w->count; i++) {
 687                 intel_ring_emit(ring, w->reg[i].addr);
 688                 intel_ring_emit(ring, w->reg[i].value);
 689         }
 690         intel_ring_emit(ring, MI_NOOP);
 691
 692         intel_ring_advance(ring);
 693
 694         ring->gpu_caches_dirty = true;
 695         ret = intel_ring_flush_all_caches(ring);
 696         if (ret)
 697                 return ret;
 698
 699         DRM_DEBUG_DRIVER("Number of Workarounds emitted: %d\n", w->count);
 700
 701         return 0;
 702 }
 703
 704 static int wa_add(struct drm_i915_private *dev_priv,
 705                   const u32 addr, const u32 mask, const u32 val)
 706 {
 707         const u32 idx = dev_priv->workarounds.count;
 708
 709         if (WARN_ON(idx >= I915_MAX_WA_REGS))
 710                 return -ENOSPC;
 711
 712         dev_priv->workarounds.reg[idx].addr = addr;
 713         dev_priv->workarounds.reg[idx].value = val;
 714         dev_priv->workarounds.reg[idx].mask = mask;
 715
 716         dev_priv->workarounds.count++;
 717
 718         return 0;
 719 }
 720
 721 #define WA_REG(addr, mask, val) { \
 722                 const int r = wa_add(dev_priv, (addr), (mask), (val)); \
 723                 if (r) \
 724                         return r; \
 725         }
 726
 727 #define WA_SET_BIT_MASKED(addr, mask) \
 728         WA_REG(addr, (mask), _MASKED_BIT_ENABLE(mask))
 729
 730 #define WA_CLR_BIT_MASKED(addr, mask) \
 731         WA_REG(addr, (mask), _MASKED_BIT_DISABLE(mask))
 732
 733 #define WA_SET_FIELD_MASKED(addr, mask, value) \
 734         WA_REG(addr, mask, _MASKED_FIELD(mask, value))
 735
 736 #define WA_SET_BIT(addr, mask) WA_REG(addr, mask, I915_READ(addr) | (mask))
 737 #define WA_CLR_BIT(addr, mask) WA_REG(addr, mask, I915_READ(addr) & ~(mask))
 738
 739 #define WA_WRITE(addr, val) WA_REG(addr, 0xffffffff, val)
 740
 741 static int bdw_init_workarounds(struct intel_engine_cs *ring)
 742 {
 743         struct drm_device *dev = ring->dev;
 744         struct drm_i915_private *dev_priv = dev->dev_private;
 745
 746         /* WaDisablePartialInstShootdown:bdw */
 747         /* WaDisableThreadStallDopClockGating:bdw (pre-production) */
 748         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
 749                           PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE |
 750                           STALL_DOP_GATING_DISABLE);
 751
 752         /* WaDisableDopClockGating:bdw */
 753         WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
 754                           DOP_CLOCK_GATING_DISABLE);
 755
 756         WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
 757                           GEN8_SAMPLER_POWER_BYPASS_DIS);
 758
 759         /* Use Force Non-Coherent whenever executing a 3D context. This is a
 760          * workaround for for a possible hang in the unlikely event a TLB
 761          * invalidation occurs during a PSD flush.
 762          */
 763         /* WaDisableFenceDestinationToSLM:bdw (GT3 pre-production) */
 764         WA_SET_BIT_MASKED(HDC_CHICKEN0,
 765                           HDC_FORCE_NON_COHERENT |
 766                           (IS_BDW_GT3(dev) ? HDC_FENCE_DEST_SLM_DISABLE : 0));
 767
 768         /* Wa4x4STCOptimizationDisable:bdw */
 769         WA_SET_BIT_MASKED(CACHE_MODE_1,
 770                           GEN8_4x4_STC_OPTIMIZATION_DISABLE);
 771
 772         /*
 773          * BSpec recommends 8x4 when MSAA is used,
 774          * however in practice 16x4 seems fastest.
 775          *
 776          * Note that PS/WM thread counts depend on the WIZ hashing
 777          * disable bit, which we don't touch here, but it's good
 778          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
 779          */
 780         WA_SET_FIELD_MASKED(GEN7_GT_MODE,
 781                             GEN6_WIZ_HASHING_MASK,
 782                             GEN6_WIZ_HASHING_16x4);
 783
 784         return 0;
 785 }
 786
 787 static int chv_init_workarounds(struct intel_engine_cs *ring)
 788 {
 789         struct drm_device *dev = ring->dev;
 790         struct drm_i915_private *dev_priv = dev->dev_private;
 791
 792         /* WaDisablePartialInstShootdown:chv */
 793         /* WaDisableThreadStallDopClockGating:chv */
 794         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
 795                           PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE |
 796                           STALL_DOP_GATING_DISABLE);
 797
 798         /* Use Force Non-Coherent whenever executing a 3D context. This is a
 799          * workaround for a possible hang in the unlikely event a TLB
 800          * invalidation occurs during a PSD flush.
 801          */
 802         /* WaForceEnableNonCoherent:chv */
 803         /* WaHdcDisableFetchWhenMasked:chv */
 804         WA_SET_BIT_MASKED(HDC_CHICKEN0,
 805                           HDC_FORCE_NON_COHERENT |
 806                           HDC_DONOT_FETCH_MEM_WHEN_MASKED);
 807
 808         return 0;
 809 }
 810
 811 int init_workarounds_ring(struct intel_engine_cs *ring)
 812 {
 813         struct drm_device *dev = ring->dev;
 814         struct drm_i915_private *dev_priv = dev->dev_private;
 815
 816         WARN_ON(ring->id != RCS);
 817
 818         dev_priv->workarounds.count = 0;
 819
 820         if (IS_BROADWELL(dev))
 821                 return bdw_init_workarounds(ring);
 822
 823         if (IS_CHERRYVIEW(dev))
 824                 return chv_init_workarounds(ring);
 825
 826         return 0;
 827 }
 828
 829 static int init_render_ring(struct intel_engine_cs *ring)
 830 {
 831         struct drm_device *dev = ring->dev;
 832         struct drm_i915_private *dev_priv = dev->dev_private;
 833         int ret = init_ring_common(ring);
 834         if (ret)
 835                 return ret;
 836
 837         /* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */
 838         if (INTEL_INFO(dev)->gen >= 4 && INTEL_INFO(dev)->gen < 7)
 839                 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH));
 840
 841         /* We need to disable the AsyncFlip performance optimisations in order
 842          * to use MI_WAIT_FOR_EVENT within the CS. It should already be
 843          * programmed to '1' on all products.
 844          *
 845          * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv,bdw,chv
 846          */
 847         if (INTEL_INFO(dev)->gen >= 6 && INTEL_INFO(dev)->gen < 9)
 848                 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE));
 849
 850         /* Required for the hardware to program scanline values for waiting */
 851         /* WaEnableFlushTlbInvalidationMode:snb */
 852         if (INTEL_INFO(dev)->gen == 6)
 853                 I915_WRITE(GFX_MODE,
 854                            _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT));
 855
 856         /* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */
 857         if (IS_GEN7(dev))
 858                 I915_WRITE(GFX_MODE_GEN7,
 859                            _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT) |
 860                            _MASKED_BIT_ENABLE(GFX_REPLAY_MODE));
 861
 862         if (INTEL_INFO(dev)->gen >= 5) {
 863                 ret = intel_init_pipe_control(ring);
 864                 if (ret)
 865                         return ret;
 866         }
 867
 868         if (IS_GEN6(dev)) {
 869                 /* From the Sandybridge PRM, volume 1 part 3, page 24:
 870                  * "If this bit is set, STCunit will have LRA as replacement
 871                  *  policy. [...] This bit must be reset.  LRA replacement
 872                  *  policy is not supported."
 873                  */
 874                 I915_WRITE(CACHE_MODE_0,
 875                            _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
 876         }
 877
 878         if (INTEL_INFO(dev)->gen >= 6)
 879                 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
 880
 881         if (HAS_L3_DPF(dev))
 882                 I915_WRITE_IMR(ring, ~GT_PARITY_ERROR(dev));
 883
 884         return init_workarounds_ring(ring);
 885 }
 886
 887 static void render_ring_cleanup(struct intel_engine_cs *ring)
 888 {
 889         struct drm_device *dev = ring->dev;
 890         struct drm_i915_private *dev_priv = dev->dev_private;
 891
 892         if (dev_priv->semaphore_obj) {
 893                 i915_gem_object_ggtt_unpin(dev_priv->semaphore_obj);
 894                 drm_gem_object_unreference(&dev_priv->semaphore_obj->base);
 895                 dev_priv->semaphore_obj = NULL;
 896         }
 897
 898         intel_fini_pipe_control(ring);
 899 }
 900
 901 static int gen8_rcs_signal(struct intel_engine_cs *signaller,
 902                            unsigned int num_dwords)
 903 {
 904 #define MBOX_UPDATE_DWORDS 8
 905         struct drm_device *dev = signaller->dev;
 906         struct drm_i915_private *dev_priv = dev->dev_private;
 907         struct intel_engine_cs *waiter;
 908         int i, ret, num_rings;
 909
 910         num_rings = hweight32(INTEL_INFO(dev)->ring_mask);
 911         num_dwords += (num_rings-1) * MBOX_UPDATE_DWORDS;
 912 #undef MBOX_UPDATE_DWORDS
 913
 914         ret = intel_ring_begin(signaller, num_dwords);
 915         if (ret)
 916                 return ret;
 917
 918         for_each_ring(waiter, dev_priv, i) {
 919                 u64 gtt_offset = signaller->semaphore.signal_ggtt[i];
 920                 if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID)
 921                         continue;
 922
 923                 intel_ring_emit(signaller, GFX_OP_PIPE_CONTROL(6));
 924                 intel_ring_emit(signaller, PIPE_CONTROL_GLOBAL_GTT_IVB |
 925                                            PIPE_CONTROL_QW_WRITE |
 926                                            PIPE_CONTROL_FLUSH_ENABLE);
 927                 intel_ring_emit(signaller, lower_32_bits(gtt_offset));
 928                 intel_ring_emit(signaller, upper_32_bits(gtt_offset));
 929                 intel_ring_emit(signaller, signaller->outstanding_lazy_seqno);
 930                 intel_ring_emit(signaller, 0);
 931                 intel_ring_emit(signaller, MI_SEMAPHORE_SIGNAL |
 932                                            MI_SEMAPHORE_TARGET(waiter->id));
 933                 intel_ring_emit(signaller, 0);
 934         }
 935
 936         return 0;
 937 }
 938
 939 static int gen8_xcs_signal(struct intel_engine_cs *signaller,
 940                            unsigned int num_dwords)
 941 {
 942 #define MBOX_UPDATE_DWORDS 6
 943         struct drm_device *dev = signaller->dev;
 944         struct drm_i915_private *dev_priv = dev->dev_private;
 945         struct intel_engine_cs *waiter;
 946         int i, ret, num_rings;
 947
 948         num_rings = hweight32(INTEL_INFO(dev)->ring_mask);
 949         num_dwords += (num_rings-1) * MBOX_UPDATE_DWORDS;
 950 #undef MBOX_UPDATE_DWORDS
 951
 952         ret = intel_ring_begin(signaller, num_dwords);
 953         if (ret)
 954                 return ret;
 955
 956         for_each_ring(waiter, dev_priv, i) {
 957                 u64 gtt_offset = signaller->semaphore.signal_ggtt[i];
 958                 if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID)
 959                         continue;
 960
 961                 intel_ring_emit(signaller, (MI_FLUSH_DW + 1) |
 962                                            MI_FLUSH_DW_OP_STOREDW);
 963                 intel_ring_emit(signaller, lower_32_bits(gtt_offset) |
 964                                            MI_FLUSH_DW_USE_GTT);
 965                 intel_ring_emit(signaller, upper_32_bits(gtt_offset));
 966                 intel_ring_emit(signaller, signaller->outstanding_lazy_seqno);
 967                 intel_ring_emit(signaller, MI_SEMAPHORE_SIGNAL |
 968                                            MI_SEMAPHORE_TARGET(waiter->id));
 969                 intel_ring_emit(signaller, 0);
 970         }
 971
 972         return 0;
 973 }
 974
 975 static int gen6_signal(struct intel_engine_cs *signaller,
 976                        unsigned int num_dwords)
 977 {
 978         struct drm_device *dev = signaller->dev;
 979         struct drm_i915_private *dev_priv = dev->dev_private;
 980         struct intel_engine_cs *useless;
 981         int i, ret, num_rings;
 982
 983 #define MBOX_UPDATE_DWORDS 3
 984         num_rings = hweight32(INTEL_INFO(dev)->ring_mask);
 985         num_dwords += round_up((num_rings-1) * MBOX_UPDATE_DWORDS, 2);
 986 #undef MBOX_UPDATE_DWORDS
 987
 988         ret = intel_ring_begin(signaller, num_dwords);
 989         if (ret)
 990                 return ret;
 991
 992         for_each_ring(useless, dev_priv, i) {
 993                 u32 mbox_reg = signaller->semaphore.mbox.signal[i];
 994                 if (mbox_reg != GEN6_NOSYNC) {
 995                         intel_ring_emit(signaller, MI_LOAD_REGISTER_IMM(1));
 996                         intel_ring_emit(signaller, mbox_reg);
 997                         intel_ring_emit(signaller, signaller->outstanding_lazy_seqno);
 998                 }
 999         }
1000
1001         /* If num_dwords was rounded, make sure the tail pointer is correct */
1002         if (num_rings % 2 == 0)
1003                 intel_ring_emit(signaller, MI_NOOP);
1004
1005         return 0;
1006 }
1007
1008 /**
1009  * gen6_add_request - Update the semaphore mailbox registers
1010  *
1011  * @ring - ring that is adding a request
1012  * @seqno - return seqno stuck into the ring
1013  *
1014  * Update the mailbox registers in the *other* rings with the current seqno.
1015  * This acts like a signal in the canonical semaphore.
1016  */
1017 static int
1018 gen6_add_request(struct intel_engine_cs *ring)
1019 {
1020         int ret;
1021
1022         if (ring->semaphore.signal)
1023                 ret = ring->semaphore.signal(ring, 4);
1024         else
1025                 ret = intel_ring_begin(ring, 4);
1026
1027         if (ret)
1028                 return ret;
1029
1030         intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
1031         intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
1032         intel_ring_emit(ring, ring->outstanding_lazy_seqno);
1033         intel_ring_emit(ring, MI_USER_INTERRUPT);
1034         __intel_ring_advance(ring);
1035
1036         return 0;
1037 }
1038
1039 static inline bool i915_gem_has_seqno_wrapped(struct drm_device *dev,
1040                                               u32 seqno)
1041 {
1042         struct drm_i915_private *dev_priv = dev->dev_private;
1043         return dev_priv->last_seqno < seqno;
1044 }
1045
1046 /**
1047  * intel_ring_sync - sync the waiter to the signaller on seqno
1048  *
1049  * @waiter - ring that is waiting
1050  * @signaller - ring which has, or will signal
1051  * @seqno - seqno which the waiter will block on
1052  */
1053
1054 static int
1055 gen8_ring_sync(struct intel_engine_cs *waiter,
1056                struct intel_engine_cs *signaller,
1057                u32 seqno)
1058 {
1059         struct drm_i915_private *dev_priv = waiter->dev->dev_private;
1060         int ret;
1061
1062         ret = intel_ring_begin(waiter, 4);
1063         if (ret)
1064                 return ret;
1065
1066         intel_ring_emit(waiter, MI_SEMAPHORE_WAIT |
1067                                 MI_SEMAPHORE_GLOBAL_GTT |
1068                                 MI_SEMAPHORE_POLL |
1069                                 MI_SEMAPHORE_SAD_GTE_SDD);
1070         intel_ring_emit(waiter, seqno);
1071         intel_ring_emit(waiter,
1072                         lower_32_bits(GEN8_WAIT_OFFSET(waiter, signaller->id)));
1073         intel_ring_emit(waiter,
1074                         upper_32_bits(GEN8_WAIT_OFFSET(waiter, signaller->id)));
1075         intel_ring_advance(waiter);
1076         return 0;
1077 }
1078
1079 static int
1080 gen6_ring_sync(struct intel_engine_cs *waiter,
1081                struct intel_engine_cs *signaller,
1082                u32 seqno)
1083 {
1084         u32 dw1 = MI_SEMAPHORE_MBOX |
1085                   MI_SEMAPHORE_COMPARE |
1086                   MI_SEMAPHORE_REGISTER;
1087         u32 wait_mbox = signaller->semaphore.mbox.wait[waiter->id];
1088         int ret;
1089
1090         /* Throughout all of the GEM code, seqno passed implies our current
1091          * seqno is >= the last seqno executed. However for hardware the
1092          * comparison is strictly greater than.
1093          */
1094         seqno -= 1;
1095
1096         WARN_ON(wait_mbox == MI_SEMAPHORE_SYNC_INVALID);
1097
1098         ret = intel_ring_begin(waiter, 4);
1099         if (ret)
1100                 return ret;
1101
1102         /* If seqno wrap happened, omit the wait with no-ops */
1103         if (likely(!i915_gem_has_seqno_wrapped(waiter->dev, seqno))) {
1104                 intel_ring_emit(waiter, dw1 | wait_mbox);
1105                 intel_ring_emit(waiter, seqno);
1106                 intel_ring_emit(waiter, 0);
1107                 intel_ring_emit(waiter, MI_NOOP);
1108         } else {
1109                 intel_ring_emit(waiter, MI_NOOP);
1110                 intel_ring_emit(waiter, MI_NOOP);
1111                 intel_ring_emit(waiter, MI_NOOP);
1112                 intel_ring_emit(waiter, MI_NOOP);
1113         }
1114         intel_ring_advance(waiter);
1115
1116         return 0;
1117 }
1118
1119 #define PIPE_CONTROL_FLUSH(ring__, addr__)                                      \
1120 do {                                                                    \
1121         intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |                \
1122                  PIPE_CONTROL_DEPTH_STALL);                             \
1123         intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT);                    \
1124         intel_ring_emit(ring__, 0);                                                     \
1125         intel_ring_emit(ring__, 0);                                                     \
1126 } while (0)
1127
1128 static int
1129 pc_render_add_request(struct intel_engine_cs *ring)
1130 {
1131         u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
1132         int ret;
1133
1134         /* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently
1135          * incoherent with writes to memory, i.e. completely fubar,
1136          * so we need to use PIPE_NOTIFY instead.
1137          *
1138          * However, we also need to workaround the qword write
1139          * incoherence by flushing the 6 PIPE_NOTIFY buffers out to
1140          * memory before requesting an interrupt.
1141          */
1142         ret = intel_ring_begin(ring, 32);
1143         if (ret)
1144                 return ret;
1145
1146         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
1147                         PIPE_CONTROL_WRITE_FLUSH |
1148                         PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
1149         intel_ring_emit(ring, ring->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
1150         intel_ring_emit(ring, ring->outstanding_lazy_seqno);
1151         intel_ring_emit(ring, 0);
1152         PIPE_CONTROL_FLUSH(ring, scratch_addr);
1153         scratch_addr += 2 * CACHELINE_BYTES; /* write to separate cachelines */
1154         PIPE_CONTROL_FLUSH(ring, scratch_addr);
1155         scratch_addr += 2 * CACHELINE_BYTES;
1156         PIPE_CONTROL_FLUSH(ring, scratch_addr);
1157         scratch_addr += 2 * CACHELINE_BYTES;
1158         PIPE_CONTROL_FLUSH(ring, scratch_addr);
1159         scratch_addr += 2 * CACHELINE_BYTES;
1160         PIPE_CONTROL_FLUSH(ring, scratch_addr);
1161         scratch_addr += 2 * CACHELINE_BYTES;
1162         PIPE_CONTROL_FLUSH(ring, scratch_addr);
1163
1164         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
1165                         PIPE_CONTROL_WRITE_FLUSH |
1166                         PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
1167                         PIPE_CONTROL_NOTIFY);
1168         intel_ring_emit(ring, ring->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
1169         intel_ring_emit(ring, ring->outstanding_lazy_seqno);
1170         intel_ring_emit(ring, 0);
1171         __intel_ring_advance(ring);
1172
1173         return 0;
1174 }
1175
1176 static u32
1177 gen6_ring_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency)
1178 {
1179         /* Workaround to force correct ordering between irq and seqno writes on
1180          * ivb (and maybe also on snb) by reading from a CS register (like
1181          * ACTHD) before reading the status page. */
1182         if (!lazy_coherency) {
1183                 struct drm_i915_private *dev_priv = ring->dev->dev_private;
1184                 POSTING_READ(RING_ACTHD(ring->mmio_base));
1185         }
1186
1187         return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
1188 }
1189
1190 static u32
1191 ring_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency)
1192 {
1193         return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
1194 }
1195
1196 static void
1197 ring_set_seqno(struct intel_engine_cs *ring, u32 seqno)
1198 {
1199         intel_write_status_page(ring, I915_GEM_HWS_INDEX, seqno);
1200 }
1201
1202 static u32
1203 pc_render_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency)
1204 {
1205         return ring->scratch.cpu_page[0];
1206 }
1207
1208 static void
1209 pc_render_set_seqno(struct intel_engine_cs *ring, u32 seqno)
1210 {
1211         ring->scratch.cpu_page[0] = seqno;
1212 }
1213
1214 static bool
1215 gen5_ring_get_irq(struct intel_engine_cs *ring)
1216 {
1217         struct drm_device *dev = ring->dev;
1218         struct drm_i915_private *dev_priv = dev->dev_private;
1219         unsigned long flags;
1220
1221         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
1222                 return false;
1223
1224         spin_lock_irqsave(&dev_priv->irq_lock, flags);
1225         if (ring->irq_refcount++ == 0)
1226                 gen5_enable_gt_irq(dev_priv, ring->irq_enable_mask);
1227         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1228
1229         return true;
1230 }
1231
1232 static void
1233 gen5_ring_put_irq(struct intel_engine_cs *ring)
1234 {
1235         struct drm_device *dev = ring->dev;
1236         struct drm_i915_private *dev_priv = dev->dev_private;
1237         unsigned long flags;
1238
1239         spin_lock_irqsave(&dev_priv->irq_lock, flags);
1240         if (--ring->irq_refcount == 0)
1241                 gen5_disable_gt_irq(dev_priv, ring->irq_enable_mask);
1242         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1243 }
1244
1245 static bool
1246 i9xx_ring_get_irq(struct intel_engine_cs *ring)
1247 {
1248         struct drm_device *dev = ring->dev;
1249         struct drm_i915_private *dev_priv = dev->dev_private;
1250         unsigned long flags;
1251
1252         if (!intel_irqs_enabled(dev_priv))
1253                 return false;
1254
1255         spin_lock_irqsave(&dev_priv->irq_lock, flags);
1256         if (ring->irq_refcount++ == 0) {
1257                 dev_priv->irq_mask &= ~ring->irq_enable_mask;
1258                 I915_WRITE(IMR, dev_priv->irq_mask);
1259                 POSTING_READ(IMR);
1260         }
1261         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1262
1263         return true;
1264 }
1265
1266 static void
1267 i9xx_ring_put_irq(struct intel_engine_cs *ring)
1268 {
1269         struct drm_device *dev = ring->dev;
1270         struct drm_i915_private *dev_priv = dev->dev_private;
1271         unsigned long flags;
1272
1273         spin_lock_irqsave(&dev_priv->irq_lock, flags);
1274         if (--ring->irq_refcount == 0) {
1275                 dev_priv->irq_mask |= ring->irq_enable_mask;
1276                 I915_WRITE(IMR, dev_priv->irq_mask);
1277                 POSTING_READ(IMR);
1278         }
1279         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1280 }
1281
1282 static bool
1283 i8xx_ring_get_irq(struct intel_engine_cs *ring)
1284 {
1285         struct drm_device *dev = ring->dev;
1286         struct drm_i915_private *dev_priv = dev->dev_private;
1287         unsigned long flags;
1288
1289         if (!intel_irqs_enabled(dev_priv))
1290                 return false;
1291
1292         spin_lock_irqsave(&dev_priv->irq_lock, flags);
1293         if (ring->irq_refcount++ == 0) {
1294                 dev_priv->irq_mask &= ~ring->irq_enable_mask;
1295                 I915_WRITE16(IMR, dev_priv->irq_mask);
1296                 POSTING_READ16(IMR);
1297         }
1298         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1299
1300         return true;
1301 }
1302
1303 static void
1304 i8xx_ring_put_irq(struct intel_engine_cs *ring)
1305 {
1306         struct drm_device *dev = ring->dev;
1307         struct drm_i915_private *dev_priv = dev->dev_private;
1308         unsigned long flags;
1309
1310         spin_lock_irqsave(&dev_priv->irq_lock, flags);
1311         if (--ring->irq_refcount == 0) {
1312                 dev_priv->irq_mask |= ring->irq_enable_mask;
1313                 I915_WRITE16(IMR, dev_priv->irq_mask);
1314                 POSTING_READ16(IMR);
1315         }
1316         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1317 }
1318
1319 void intel_ring_setup_status_page(struct intel_engine_cs *ring)
1320 {
1321         struct drm_device *dev = ring->dev;
1322         struct drm_i915_private *dev_priv = ring->dev->dev_private;
1323         u32 mmio = 0;
1324
1325         /* The ring status page addresses are no longer next to the rest of
1326          * the ring registers as of gen7.
1327          */
1328         if (IS_GEN7(dev)) {
1329                 switch (ring->id) {
1330                 case RCS:
1331                         mmio = RENDER_HWS_PGA_GEN7;
1332                         break;
1333                 case BCS:
1334                         mmio = BLT_HWS_PGA_GEN7;
1335                         break;
1336                 /*
1337                  * VCS2 actually doesn't exist on Gen7. Only shut up
1338                  * gcc switch check warning
1339                  */
1340                 case VCS2:
1341                 case VCS:
1342                         mmio = BSD_HWS_PGA_GEN7;
1343                         break;
1344                 case VECS:
1345                         mmio = VEBOX_HWS_PGA_GEN7;
1346                         break;
1347                 }
1348         } else if (IS_GEN6(ring->dev)) {
1349                 mmio = RING_HWS_PGA_GEN6(ring->mmio_base);
1350         } else {
1351                 /* XXX: gen8 returns to sanity */
1352                 mmio = RING_HWS_PGA(ring->mmio_base);
1353         }
1354
1355         I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
1356         POSTING_READ(mmio);
1357
1358         /*
1359          * Flush the TLB for this page
1360          *
1361          * FIXME: These two bits have disappeared on gen8, so a question
1362          * arises: do we still need this and if so how should we go about
1363          * invalidating the TLB?
1364          */
1365         if (INTEL_INFO(dev)->gen >= 6 && INTEL_INFO(dev)->gen < 8) {
1366                 u32 reg = RING_INSTPM(ring->mmio_base);
1367
1368                 /* ring should be idle before issuing a sync flush*/
1369                 WARN_ON((I915_READ_MODE(ring) & MODE_IDLE) == 0);
1370
1371                 I915_WRITE(reg,
1372                            _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE |
1373                                               INSTPM_SYNC_FLUSH));
1374                 if (wait_for((I915_READ(reg) & INSTPM_SYNC_FLUSH) == 0,
1375                              1000))
1376                         DRM_ERROR("%s: wait for SyncFlush to complete for TLB invalidation timed out\n",
1377                                   ring->name);
1378         }
1379 }
1380
1381 static int
1382 bsd_ring_flush(struct intel_engine_cs *ring,
1383                u32     invalidate_domains,
1384                u32     flush_domains)
1385 {
1386         int ret;
1387
1388         ret = intel_ring_begin(ring, 2);
1389         if (ret)
1390                 return ret;
1391
1392         intel_ring_emit(ring, MI_FLUSH);
1393         intel_ring_emit(ring, MI_NOOP);
1394         intel_ring_advance(ring);
1395         return 0;
1396 }
1397
1398 static int
1399 i9xx_add_request(struct intel_engine_cs *ring)
1400 {
1401         int ret;
1402
1403         ret = intel_ring_begin(ring, 4);
1404         if (ret)
1405                 return ret;
1406
1407         intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
1408         intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
1409         intel_ring_emit(ring, ring->outstanding_lazy_seqno);
1410         intel_ring_emit(ring, MI_USER_INTERRUPT);
1411         __intel_ring_advance(ring);
1412
1413         return 0;
1414 }
1415
1416 static bool
1417 gen6_ring_get_irq(struct intel_engine_cs *ring)
1418 {
1419         struct drm_device *dev = ring->dev;
1420         struct drm_i915_private *dev_priv = dev->dev_private;
1421         unsigned long flags;
1422
1423         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
1424                 return false;
1425
1426         spin_lock_irqsave(&dev_priv->irq_lock, flags);
1427         if (ring->irq_refcount++ == 0) {
1428                 if (HAS_L3_DPF(dev) && ring->id == RCS)
1429                         I915_WRITE_IMR(ring,
1430                                        ~(ring->irq_enable_mask |
1431                                          GT_PARITY_ERROR(dev)));
1432                 else
1433                         I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
1434                 gen5_enable_gt_irq(dev_priv, ring->irq_enable_mask);
1435         }
1436         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1437
1438         return true;
1439 }
1440
1441 static void
1442 gen6_ring_put_irq(struct intel_engine_cs *ring)
1443 {
1444         struct drm_device *dev = ring->dev;
1445         struct drm_i915_private *dev_priv = dev->dev_private;
1446         unsigned long flags;
1447
1448         spin_lock_irqsave(&dev_priv->irq_lock, flags);
1449         if (--ring->irq_refcount == 0) {
1450                 if (HAS_L3_DPF(dev) && ring->id == RCS)
1451                         I915_WRITE_IMR(ring, ~GT_PARITY_ERROR(dev));
1452                 else
1453                         I915_WRITE_IMR(ring, ~0);
1454                 gen5_disable_gt_irq(dev_priv, ring->irq_enable_mask);
1455         }
1456         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1457 }
1458
1459 static bool
1460 hsw_vebox_get_irq(struct intel_engine_cs *ring)
1461 {
1462         struct drm_device *dev = ring->dev;
1463         struct drm_i915_private *dev_priv = dev->dev_private;
1464         unsigned long flags;
1465
1466         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
1467                 return false;
1468
1469         spin_lock_irqsave(&dev_priv->irq_lock, flags);
1470         if (ring->irq_refcount++ == 0) {
1471                 I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
1472                 gen6_enable_pm_irq(dev_priv, ring->irq_enable_mask);
1473         }
1474         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1475
1476         return true;
1477 }
1478
1479 static void
1480 hsw_vebox_put_irq(struct intel_engine_cs *ring)
1481 {
1482         struct drm_device *dev = ring->dev;
1483         struct drm_i915_private *dev_priv = dev->dev_private;
1484         unsigned long flags;
1485
1486         spin_lock_irqsave(&dev_priv->irq_lock, flags);
1487         if (--ring->irq_refcount == 0) {
1488                 I915_WRITE_IMR(ring, ~0);
1489                 gen6_disable_pm_irq(dev_priv, ring->irq_enable_mask);
1490         }
1491         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1492 }
1493
1494 static bool
1495 gen8_ring_get_irq(struct intel_engine_cs *ring)
1496 {
1497         struct drm_device *dev = ring->dev;
1498         struct drm_i915_private *dev_priv = dev->dev_private;
1499         unsigned long flags;
1500
1501         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
1502                 return false;
1503
1504         spin_lock_irqsave(&dev_priv->irq_lock, flags);
1505         if (ring->irq_refcount++ == 0) {
1506                 if (HAS_L3_DPF(dev) && ring->id == RCS) {
1507                         I915_WRITE_IMR(ring,
1508                                        ~(ring->irq_enable_mask |
1509                                          GT_RENDER_L3_PARITY_ERROR_INTERRUPT));
1510                 } else {
1511                         I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
1512                 }
1513                 POSTING_READ(RING_IMR(ring->mmio_base));
1514         }
1515         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1516
1517         return true;
1518 }
1519
1520 static void
1521 gen8_ring_put_irq(struct intel_engine_cs *ring)
1522 {
1523         struct drm_device *dev = ring->dev;
1524         struct drm_i915_private *dev_priv = dev->dev_private;
1525         unsigned long flags;
1526
1527         spin_lock_irqsave(&dev_priv->irq_lock, flags);
1528         if (--ring->irq_refcount == 0) {
1529                 if (HAS_L3_DPF(dev) && ring->id == RCS) {
1530                         I915_WRITE_IMR(ring,
1531                                        ~GT_RENDER_L3_PARITY_ERROR_INTERRUPT);
1532                 } else {
1533                         I915_WRITE_IMR(ring, ~0);
1534                 }
1535                 POSTING_READ(RING_IMR(ring->mmio_base));
1536         }
1537         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1538 }
1539
1540 static int
1541 i965_dispatch_execbuffer(struct intel_engine_cs *ring,
1542                          u64 offset, u32 length,
1543                          unsigned flags)
1544 {
1545         int ret;
1546
1547         ret = intel_ring_begin(ring, 2);
1548         if (ret)
1549                 return ret;
1550
1551         intel_ring_emit(ring,
1552                         MI_BATCH_BUFFER_START |
1553                         MI_BATCH_GTT |
1554                         (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_I965));
1555         intel_ring_emit(ring, offset);
1556         intel_ring_advance(ring);
1557
1558         return 0;
1559 }
1560
1561 /* Just userspace ABI convention to limit the wa batch bo to a resonable size */
1562 #define I830_BATCH_LIMIT (256*1024)
1563 #define I830_TLB_ENTRIES (2)
1564 #define I830_WA_SIZE max(I830_TLB_ENTRIES*4096, I830_BATCH_LIMIT)
1565 static int
1566 i830_dispatch_execbuffer(struct intel_engine_cs *ring,
1567                                 u64 offset, u32 len,
1568                                 unsigned flags)
1569 {
1570         u32 cs_offset = ring->scratch.gtt_offset;
1571         int ret;
1572
1573         ret = intel_ring_begin(ring, 6);
1574         if (ret)
1575                 return ret;
1576
1577         /* Evict the invalid PTE TLBs */
1578         intel_ring_emit(ring, COLOR_BLT_CMD | BLT_WRITE_RGBA);
1579         intel_ring_emit(ring, BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | 4096);
1580         intel_ring_emit(ring, I830_TLB_ENTRIES << 16 | 4); /* load each page */
1581         intel_ring_emit(ring, cs_offset);
1582         intel_ring_emit(ring, 0xdeadbeef);
1583         intel_ring_emit(ring, MI_NOOP);
1584         intel_ring_advance(ring);
1585
1586         if ((flags & I915_DISPATCH_PINNED) == 0) {
1587                 if (len > I830_BATCH_LIMIT)
1588                         return -ENOSPC;
1589
1590                 ret = intel_ring_begin(ring, 6 + 2);
1591                 if (ret)
1592                         return ret;
1593
1594                 /* Blit the batch (which has now all relocs applied) to the
1595                  * stable batch scratch bo area (so that the CS never
1596                  * stumbles over its tlb invalidation bug) ...
1597                  */
1598                 intel_ring_emit(ring, SRC_COPY_BLT_CMD | BLT_WRITE_RGBA);
1599                 intel_ring_emit(ring, BLT_DEPTH_32 | BLT_ROP_SRC_COPY | 4096);
1600                 intel_ring_emit(ring, DIV_ROUND_UP(len, 4096) << 16 | 4096);
1601                 intel_ring_emit(ring, cs_offset);
1602                 intel_ring_emit(ring, 4096);
1603                 intel_ring_emit(ring, offset);
1604
1605                 intel_ring_emit(ring, MI_FLUSH);
1606                 intel_ring_emit(ring, MI_NOOP);
1607                 intel_ring_advance(ring);
1608
1609                 /* ... and execute it. */
1610                 offset = cs_offset;
1611         }
1612
1613         ret = intel_ring_begin(ring, 4);
1614         if (ret)
1615                 return ret;
1616
1617         intel_ring_emit(ring, MI_BATCH_BUFFER);
1618         intel_ring_emit(ring, offset | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE));
1619         intel_ring_emit(ring, offset + len - 8);
1620         intel_ring_emit(ring, MI_NOOP);
1621         intel_ring_advance(ring);
1622
1623         return 0;
1624 }
1625
1626 static int
1627 i915_dispatch_execbuffer(struct intel_engine_cs *ring,
1628                          u64 offset, u32 len,
1629                          unsigned flags)
1630 {
1631         int ret;
1632
1633         ret = intel_ring_begin(ring, 2);
1634         if (ret)
1635                 return ret;
1636
1637         intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_GTT);
1638         intel_ring_emit(ring, offset | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE));
1639         intel_ring_advance(ring);
1640
1641         return 0;
1642 }
1643
1644 static void cleanup_status_page(struct intel_engine_cs *ring)
1645 {
1646         struct drm_i915_gem_object *obj;
1647
1648         obj = ring->status_page.obj;
1649         if (obj == NULL)
1650                 return;
1651
1652         kunmap(sg_page(obj->pages->sgl));
1653         i915_gem_object_ggtt_unpin(obj);
1654         drm_gem_object_unreference(&obj->base);
1655         ring->status_page.obj = NULL;
1656 }
1657
1658 static int init_status_page(struct intel_engine_cs *ring)
1659 {
1660         struct drm_i915_gem_object *obj;
1661
1662         if ((obj = ring->status_page.obj) == NULL) {
1663                 unsigned flags;
1664                 int ret;
1665
1666                 obj = i915_gem_alloc_object(ring->dev, 4096);
1667                 if (obj == NULL) {
1668                         DRM_ERROR("Failed to allocate status page\n");
1669                         return -ENOMEM;
1670                 }
1671
1672                 ret = i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
1673                 if (ret)
1674                         goto err_unref;
1675
1676                 flags = 0;
1677                 if (!HAS_LLC(ring->dev))
1678                         /* On g33, we cannot place HWS above 256MiB, so
1679                          * restrict its pinning to the low mappable arena.
1680                          * Though this restriction is not documented for
1681                          * gen4, gen5, or byt, they also behave similarly
1682                          * and hang if the HWS is placed at the top of the
1683                          * GTT. To generalise, it appears that all !llc
1684                          * platforms have issues with us placing the HWS
1685                          * above the mappable region (even though we never
1686                          * actualy map it).
1687                          */
1688                         flags |= PIN_MAPPABLE;
1689                 ret = i915_gem_obj_ggtt_pin(obj, 4096, flags);
1690                 if (ret) {
1691 err_unref:
1692                         drm_gem_object_unreference(&obj->base);
1693                         return ret;
1694                 }
1695
1696                 ring->status_page.obj = obj;
1697         }
1698
1699         ring->status_page.gfx_addr = i915_gem_obj_ggtt_offset(obj);
1700         ring->status_page.page_addr = kmap(sg_page(obj->pages->sgl));
1701         memset(ring->status_page.page_addr, 0, PAGE_SIZE);
1702
1703         DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
1704                         ring->name, ring->status_page.gfx_addr);
1705
1706         return 0;
1707 }
1708
1709 static int init_phys_status_page(struct intel_engine_cs *ring)
1710 {
1711         struct drm_i915_private *dev_priv = ring->dev->dev_private;
1712
1713         if (!dev_priv->status_page_dmah) {
1714                 dev_priv->status_page_dmah =
1715                         drm_pci_alloc(ring->dev, PAGE_SIZE, PAGE_SIZE);
1716                 if (!dev_priv->status_page_dmah)
1717                         return -ENOMEM;
1718         }
1719
1720         ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
1721         memset(ring->status_page.page_addr, 0, PAGE_SIZE);
1722
1723         return 0;
1724 }
1725
1726 void intel_unpin_ringbuffer_obj(struct intel_ringbuffer *ringbuf)
1727 {
1728         iounmap(ringbuf->virtual_start);
1729         ringbuf->virtual_start = NULL;
1730         i915_gem_object_ggtt_unpin(ringbuf->obj);
1731 }
1732
1733 int intel_pin_and_map_ringbuffer_obj(struct drm_device *dev,
1734                                      struct intel_ringbuffer *ringbuf)
1735 {
1736         struct drm_i915_private *dev_priv = to_i915(dev);
1737         struct drm_i915_gem_object *obj = ringbuf->obj;
1738         int ret;
1739
1740         ret = i915_gem_obj_ggtt_pin(obj, PAGE_SIZE, PIN_MAPPABLE);
1741         if (ret)
1742                 return ret;
1743
1744         ret = i915_gem_object_set_to_gtt_domain(obj, true);
1745         if (ret) {
1746                 i915_gem_object_ggtt_unpin(obj);
1747                 return ret;
1748         }
1749
1750         ringbuf->virtual_start = ioremap_wc(dev_priv->gtt.mappable_base +
1751                         i915_gem_obj_ggtt_offset(obj), ringbuf->size);
1752         if (ringbuf->virtual_start == NULL) {
1753                 i915_gem_object_ggtt_unpin(obj);
1754                 return -EINVAL;
1755         }
1756
1757         return 0;
1758 }
1759
1760 void intel_destroy_ringbuffer_obj(struct intel_ringbuffer *ringbuf)
1761 {
1762         drm_gem_object_unreference(&ringbuf->obj->base);
1763         ringbuf->obj = NULL;
1764 }
1765
1766 int intel_alloc_ringbuffer_obj(struct drm_device *dev,
1767                                struct intel_ringbuffer *ringbuf)
1768 {
1769         struct drm_i915_gem_object *obj;
1770
1771         obj = NULL;
1772         if (!HAS_LLC(dev))
1773                 obj = i915_gem_object_create_stolen(dev, ringbuf->size);
1774         if (obj == NULL)
1775                 obj = i915_gem_alloc_object(dev, ringbuf->size);
1776         if (obj == NULL)
1777                 return -ENOMEM;
1778
1779         /* mark ring buffers as read-only from GPU side by default */
1780         obj->gt_ro = 1;
1781
1782         ringbuf->obj = obj;
1783
1784         return 0;
1785 }
1786
1787 static int intel_init_ring_buffer(struct drm_device *dev,
1788                                   struct intel_engine_cs *ring)
1789 {
1790         struct intel_ringbuffer *ringbuf = ring->buffer;
1791         int ret;
1792
1793         if (ringbuf == NULL) {
1794                 ringbuf = kzalloc(sizeof(*ringbuf), GFP_KERNEL);
1795                 if (!ringbuf)
1796                         return -ENOMEM;
1797                 ring->buffer = ringbuf;
1798         }
1799
1800         ring->dev = dev;
1801         INIT_LIST_HEAD(&ring->active_list);
1802         INIT_LIST_HEAD(&ring->request_list);
1803         INIT_LIST_HEAD(&ring->execlist_queue);
1804         ringbuf->size = 32 * PAGE_SIZE;
1805         ringbuf->ring = ring;
1806         memset(ring->semaphore.sync_seqno, 0, sizeof(ring->semaphore.sync_seqno));
1807
1808         init_waitqueue_head(&ring->irq_queue);
1809
1810         if (I915_NEED_GFX_HWS(dev)) {
1811                 ret = init_status_page(ring);
1812                 if (ret)
1813                         goto error;
1814         } else {
1815                 BUG_ON(ring->id != RCS);
1816                 ret = init_phys_status_page(ring);
1817                 if (ret)
1818                         goto error;
1819         }
1820
1821         if (ringbuf->obj == NULL) {
1822                 ret = intel_alloc_ringbuffer_obj(dev, ringbuf);
1823                 if (ret) {
1824                         DRM_ERROR("Failed to allocate ringbuffer %s: %d\n",
1825                                         ring->name, ret);
1826                         goto error;
1827                 }
1828
1829                 ret = intel_pin_and_map_ringbuffer_obj(dev, ringbuf);
1830                 if (ret) {
1831                         DRM_ERROR("Failed to pin and map ringbuffer %s: %d\n",
1832                                         ring->name, ret);
1833                         intel_destroy_ringbuffer_obj(ringbuf);
1834                         goto error;
1835                 }
1836         }
1837
1838         /* Workaround an erratum on the i830 which causes a hang if
1839          * the TAIL pointer points to within the last 2 cachelines
1840          * of the buffer.
1841          */
1842         ringbuf->effective_size = ringbuf->size;
1843         if (IS_I830(dev) || IS_845G(dev))
1844                 ringbuf->effective_size -= 2 * CACHELINE_BYTES;
1845
1846         ret = i915_cmd_parser_init_ring(ring);
1847         if (ret)
1848                 goto error;
1849
1850         ret = ring->init(ring);
1851         if (ret)
1852                 goto error;
1853
1854         return 0;
1855
1856 error:
1857         kfree(ringbuf);
1858         ring->buffer = NULL;
1859         return ret;
1860 }
1861
1862 void intel_cleanup_ring_buffer(struct intel_engine_cs *ring)
1863 {
1864         struct drm_i915_private *dev_priv;
1865         struct intel_ringbuffer *ringbuf;
1866
1867         if (!intel_ring_initialized(ring))
1868                 return;
1869
1870         dev_priv = to_i915(ring->dev);
1871         ringbuf = ring->buffer;
1872
1873         intel_stop_ring_buffer(ring);
1874         WARN_ON(!IS_GEN2(ring->dev) && (I915_READ_MODE(ring) & MODE_IDLE) == 0);
1875
1876         intel_unpin_ringbuffer_obj(ringbuf);
1877         intel_destroy_ringbuffer_obj(ringbuf);
1878         ring->preallocated_lazy_request = NULL;
1879         ring->outstanding_lazy_seqno = 0;
1880
1881         if (ring->cleanup)
1882                 ring->cleanup(ring);
1883
1884         cleanup_status_page(ring);
1885
1886         i915_cmd_parser_fini_ring(ring);
1887
1888         kfree(ringbuf);
1889         ring->buffer = NULL;
1890 }
1891
1892 static int intel_ring_wait_request(struct intel_engine_cs *ring, int n)
1893 {
1894         struct intel_ringbuffer *ringbuf = ring->buffer;
1895         struct drm_i915_gem_request *request;
1896         u32 seqno = 0;
1897         int ret;
1898
1899         if (ringbuf->last_retired_head != -1) {
1900                 ringbuf->head = ringbuf->last_retired_head;
1901                 ringbuf->last_retired_head = -1;
1902
1903                 ringbuf->space = intel_ring_space(ringbuf);
1904                 if (ringbuf->space >= n)
1905                         return 0;
1906         }
1907
1908         list_for_each_entry(request, &ring->request_list, list) {
1909                 if (__intel_ring_space(request->tail, ringbuf->tail,
1910                                        ringbuf->size) >= n) {
1911                         seqno = request->seqno;
1912                         break;
1913                 }
1914         }
1915
1916         if (seqno == 0)
1917                 return -ENOSPC;
1918
1919         ret = i915_wait_seqno(ring, seqno);
1920         if (ret)
1921                 return ret;
1922
1923         i915_gem_retire_requests_ring(ring);
1924         ringbuf->head = ringbuf->last_retired_head;
1925         ringbuf->last_retired_head = -1;
1926
1927         ringbuf->space = intel_ring_space(ringbuf);
1928         return 0;
1929 }
1930
1931 static int ring_wait_for_space(struct intel_engine_cs *ring, int n)
1932 {
1933         struct drm_device *dev = ring->dev;
1934         struct drm_i915_private *dev_priv = dev->dev_private;
1935         struct intel_ringbuffer *ringbuf = ring->buffer;
1936         unsigned long end;
1937         int ret;
1938
1939         ret = intel_ring_wait_request(ring, n);
1940         if (ret != -ENOSPC)
1941                 return ret;
1942
1943         /* force the tail write in case we have been skipping them */
1944         __intel_ring_advance(ring);
1945
1946         /* With GEM the hangcheck timer should kick us out of the loop,
1947          * leaving it early runs the risk of corrupting GEM state (due
1948          * to running on almost untested codepaths). But on resume
1949          * timers don't work yet, so prevent a complete hang in that
1950          * case by choosing an insanely large timeout. */
1951         end = jiffies + 60 * HZ;
1952
1953         trace_i915_ring_wait_begin(ring);
1954         do {
1955                 ringbuf->head = I915_READ_HEAD(ring);
1956                 ringbuf->space = intel_ring_space(ringbuf);
1957                 if (ringbuf->space >= n) {
1958                         ret = 0;
1959                         break;
1960                 }
1961
1962                 msleep(1);
1963
1964                 if (dev_priv->mm.interruptible && signal_pending(current)) {
1965                         ret = -ERESTARTSYS;
1966                         break;
1967                 }
1968
1969                 ret = i915_gem_check_wedge(&dev_priv->gpu_error,
1970                                            dev_priv->mm.interruptible);
1971                 if (ret)
1972                         break;
1973
1974                 if (time_after(jiffies, end)) {
1975                         ret = -EBUSY;
1976                         break;
1977                 }
1978         } while (1);
1979         trace_i915_ring_wait_end(ring);
1980         return ret;
1981 }
1982
1983 static int intel_wrap_ring_buffer(struct intel_engine_cs *ring)
1984 {
1985         uint32_t __iomem *virt;
1986         struct intel_ringbuffer *ringbuf = ring->buffer;
1987         int rem = ringbuf->size - ringbuf->tail;
1988
1989         if (ringbuf->space < rem) {
1990                 int ret = ring_wait_for_space(ring, rem);
1991                 if (ret)
1992                         return ret;
1993         }
1994
1995         virt = ringbuf->virtual_start + ringbuf->tail;
1996         rem /= 4;
1997         while (rem--)
1998                 iowrite32(MI_NOOP, virt++);
1999
2000         ringbuf->tail = 0;
2001         ringbuf->space = intel_ring_space(ringbuf);
2002
2003         return 0;
2004 }
2005
2006 int intel_ring_idle(struct intel_engine_cs *ring)
2007 {
2008         u32 seqno;
2009         int ret;
2010
2011         /* We need to add any requests required to flush the objects and ring */
2012         if (ring->outstanding_lazy_seqno) {
2013                 ret = i915_add_request(ring, NULL);
2014                 if (ret)
2015                         return ret;
2016         }
2017
2018         /* Wait upon the last request to be completed */
2019         if (list_empty(&ring->request_list))
2020                 return 0;
2021
2022         seqno = list_entry(ring->request_list.prev,
2023                            struct drm_i915_gem_request,
2024                            list)->seqno;
2025
2026         return i915_wait_seqno(ring, seqno);
2027 }
2028
2029 static int
2030 intel_ring_alloc_seqno(struct intel_engine_cs *ring)
2031 {
2032         if (ring->outstanding_lazy_seqno)
2033                 return 0;
2034
2035         if (ring->preallocated_lazy_request == NULL) {
2036                 struct drm_i915_gem_request *request;
2037
2038                 request = kmalloc(sizeof(*request), GFP_KERNEL);
2039                 if (request == NULL)
2040                         return -ENOMEM;
2041
2042                 ring->preallocated_lazy_request = request;
2043         }
2044
2045         return i915_gem_get_seqno(ring->dev, &ring->outstanding_lazy_seqno);
2046 }
2047
2048 static int __intel_ring_prepare(struct intel_engine_cs *ring,
2049                                 int bytes)
2050 {
2051         struct intel_ringbuffer *ringbuf = ring->buffer;
2052         int ret;
2053
2054         if (unlikely(ringbuf->tail + bytes > ringbuf->effective_size)) {
2055                 ret = intel_wrap_ring_buffer(ring);
2056                 if (unlikely(ret))
2057                         return ret;
2058         }
2059
2060         if (unlikely(ringbuf->space < bytes)) {
2061                 ret = ring_wait_for_space(ring, bytes);
2062                 if (unlikely(ret))
2063                         return ret;
2064         }
2065
2066         return 0;
2067 }
2068
2069 int intel_ring_begin(struct intel_engine_cs *ring,
2070                      int num_dwords)
2071 {
2072         struct drm_i915_private *dev_priv = ring->dev->dev_private;
2073         int ret;
2074
2075         ret = i915_gem_check_wedge(&dev_priv->gpu_error,
2076                                    dev_priv->mm.interruptible);
2077         if (ret)
2078                 return ret;
2079
2080         ret = __intel_ring_prepare(ring, num_dwords * sizeof(uint32_t));
2081         if (ret)
2082                 return ret;
2083
2084         /* Preallocate the olr before touching the ring */
2085         ret = intel_ring_alloc_seqno(ring);
2086         if (ret)
2087                 return ret;
2088
2089         ring->buffer->space -= num_dwords * sizeof(uint32_t);
2090         return 0;
2091 }
2092
2093 /* Align the ring tail to a cacheline boundary */
2094 int intel_ring_cacheline_align(struct intel_engine_cs *ring)
2095 {
2096         int num_dwords = (ring->buffer->tail & (CACHELINE_BYTES - 1)) / sizeof(uint32_t);
2097         int ret;
2098
2099         if (num_dwords == 0)
2100                 return 0;
2101
2102         num_dwords = CACHELINE_BYTES / sizeof(uint32_t) - num_dwords;
2103         ret = intel_ring_begin(ring, num_dwords);
2104         if (ret)
2105                 return ret;
2106
2107         while (num_dwords--)
2108                 intel_ring_emit(ring, MI_NOOP);
2109
2110         intel_ring_advance(ring);
2111
2112         return 0;
2113 }
2114
2115 void intel_ring_init_seqno(struct intel_engine_cs *ring, u32 seqno)
2116 {
2117         struct drm_device *dev = ring->dev;
2118         struct drm_i915_private *dev_priv = dev->dev_private;
2119
2120         BUG_ON(ring->outstanding_lazy_seqno);
2121
2122         if (INTEL_INFO(dev)->gen == 6 || INTEL_INFO(dev)->gen == 7) {
2123                 I915_WRITE(RING_SYNC_0(ring->mmio_base), 0);
2124                 I915_WRITE(RING_SYNC_1(ring->mmio_base), 0);
2125                 if (HAS_VEBOX(dev))
2126                         I915_WRITE(RING_SYNC_2(ring->mmio_base), 0);
2127         }
2128
2129         ring->set_seqno(ring, seqno);
2130         ring->hangcheck.seqno = seqno;
2131 }
2132
2133 static void gen6_bsd_ring_write_tail(struct intel_engine_cs *ring,
2134                                      u32 value)
2135 {
2136         struct drm_i915_private *dev_priv = ring->dev->dev_private;
2137
2138        /* Every tail move must follow the sequence below */
2139
2140         /* Disable notification that the ring is IDLE. The GT
2141          * will then assume that it is busy and bring it out of rc6.
2142          */
2143         I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
2144                    _MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
2145
2146         /* Clear the context id. Here be magic! */
2147         I915_WRITE64(GEN6_BSD_RNCID, 0x0);
2148
2149         /* Wait for the ring not to be idle, i.e. for it to wake up. */
2150         if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
2151                       GEN6_BSD_SLEEP_INDICATOR) == 0,
2152                      50))
2153                 DRM_ERROR("timed out waiting for the BSD ring to wake up\n");
2154
2155         /* Now that the ring is fully powered up, update the tail */
2156         I915_WRITE_TAIL(ring, value);
2157         POSTING_READ(RING_TAIL(ring->mmio_base));
2158
2159         /* Let the ring send IDLE messages to the GT again,
2160          * and so let it sleep to conserve power when idle.
2161          */
2162         I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
2163                    _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
2164 }
2165
2166 static int gen6_bsd_ring_flush(struct intel_engine_cs *ring,
2167                                u32 invalidate, u32 flush)
2168 {
2169         uint32_t cmd;
2170         int ret;
2171
2172         ret = intel_ring_begin(ring, 4);
2173         if (ret)
2174                 return ret;
2175
2176         cmd = MI_FLUSH_DW;
2177         if (INTEL_INFO(ring->dev)->gen >= 8)
2178                 cmd += 1;
2179         /*
2180          * Bspec vol 1c.5 - video engine command streamer:
2181          * "If ENABLED, all TLBs will be invalidated once the flush
2182          * operation is complete. This bit is only valid when the
2183          * Post-Sync Operation field is a value of 1h or 3h."
2184          */
2185         if (invalidate & I915_GEM_GPU_DOMAINS)
2186                 cmd |= MI_INVALIDATE_TLB | MI_INVALIDATE_BSD |
2187                         MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW;
2188         intel_ring_emit(ring, cmd);
2189         intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
2190         if (INTEL_INFO(ring->dev)->gen >= 8) {
2191                 intel_ring_emit(ring, 0); /* upper addr */
2192                 intel_ring_emit(ring, 0); /* value */
2193         } else  {
2194                 intel_ring_emit(ring, 0);
2195                 intel_ring_emit(ring, MI_NOOP);
2196         }
2197         intel_ring_advance(ring);
2198         return 0;
2199 }
2200
2201 static int
2202 gen8_ring_dispatch_execbuffer(struct intel_engine_cs *ring,
2203                               u64 offset, u32 len,
2204                               unsigned flags)
2205 {
2206         bool ppgtt = USES_PPGTT(ring->dev) && !(flags & I915_DISPATCH_SECURE);
2207         int ret;
2208
2209         ret = intel_ring_begin(ring, 4);
2210         if (ret)
2211                 return ret;
2212
2213         /* FIXME(BDW): Address space and security selectors. */
2214         intel_ring_emit(ring, MI_BATCH_BUFFER_START_GEN8 | (ppgtt<<8));
2215         intel_ring_emit(ring, lower_32_bits(offset));
2216         intel_ring_emit(ring, upper_32_bits(offset));
2217         intel_ring_emit(ring, MI_NOOP);
2218         intel_ring_advance(ring);
2219
2220         return 0;
2221 }
2222
2223 static int
2224 hsw_ring_dispatch_execbuffer(struct intel_engine_cs *ring,
2225                               u64 offset, u32 len,
2226                               unsigned flags)
2227 {
2228         int ret;
2229
2230         ret = intel_ring_begin(ring, 2);
2231         if (ret)
2232                 return ret;
2233
2234         intel_ring_emit(ring,
2235                         MI_BATCH_BUFFER_START |
2236                         (flags & I915_DISPATCH_SECURE ?
2237                          0 : MI_BATCH_PPGTT_HSW | MI_BATCH_NON_SECURE_HSW));
2238         /* bit0-7 is the length on GEN6+ */
2239         intel_ring_emit(ring, offset);
2240         intel_ring_advance(ring);
2241
2242         return 0;
2243 }
2244
2245 static int
2246 gen6_ring_dispatch_execbuffer(struct intel_engine_cs *ring,
2247                               u64 offset, u32 len,
2248                               unsigned flags)
2249 {
2250         int ret;
2251
2252         ret = intel_ring_begin(ring, 2);
2253         if (ret)
2254                 return ret;
2255
2256         intel_ring_emit(ring,
2257                         MI_BATCH_BUFFER_START |
2258                         (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_I965));
2259         /* bit0-7 is the length on GEN6+ */
2260         intel_ring_emit(ring, offset);
2261         intel_ring_advance(ring);
2262
2263         return 0;
2264 }
2265
2266 /* Blitter support (SandyBridge+) */
2267
2268 static int gen6_ring_flush(struct intel_engine_cs *ring,
2269                            u32 invalidate, u32 flush)
2270 {
2271         struct drm_device *dev = ring->dev;
2272         struct drm_i915_private *dev_priv = dev->dev_private;
2273         uint32_t cmd;
2274         int ret;
2275
2276         ret = intel_ring_begin(ring, 4);
2277         if (ret)
2278                 return ret;
2279
2280         cmd = MI_FLUSH_DW;
2281         if (INTEL_INFO(ring->dev)->gen >= 8)
2282                 cmd += 1;
2283         /*
2284          * Bspec vol 1c.3 - blitter engine command streamer:
2285          * "If ENABLED, all TLBs will be invalidated once the flush
2286          * operation is complete. This bit is only valid when the
2287          * Post-Sync Operation field is a value of 1h or 3h."
2288          */
2289         if (invalidate & I915_GEM_DOMAIN_RENDER)
2290                 cmd |= MI_INVALIDATE_TLB | MI_FLUSH_DW_STORE_INDEX |
2291                         MI_FLUSH_DW_OP_STOREDW;
2292         intel_ring_emit(ring, cmd);
2293         intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
2294         if (INTEL_INFO(ring->dev)->gen >= 8) {
2295                 intel_ring_emit(ring, 0); /* upper addr */
2296                 intel_ring_emit(ring, 0); /* value */
2297         } else  {
2298                 intel_ring_emit(ring, 0);
2299                 intel_ring_emit(ring, MI_NOOP);
2300         }
2301         intel_ring_advance(ring);
2302
2303         if (!invalidate && flush) {
2304                 if (IS_GEN7(dev))
2305                         return gen7_ring_fbc_flush(ring, FBC_REND_CACHE_CLEAN);
2306                 else if (IS_BROADWELL(dev))
2307                         dev_priv->fbc.need_sw_cache_clean = true;
2308         }
2309
2310         return 0;
2311 }
2312
2313 int intel_init_render_ring_buffer(struct drm_device *dev)
2314 {
2315         struct drm_i915_private *dev_priv = dev->dev_private;
2316         struct intel_engine_cs *ring = &dev_priv->ring[RCS];
2317         struct drm_i915_gem_object *obj;
2318         int ret;
2319
2320         ring->name = "render ring";
2321         ring->id = RCS;
2322         ring->mmio_base = RENDER_RING_BASE;
2323
2324         if (INTEL_INFO(dev)->gen >= 8) {
2325                 if (i915_semaphore_is_enabled(dev)) {
2326                         obj = i915_gem_alloc_object(dev, 4096);
2327                         if (obj == NULL) {
2328                                 DRM_ERROR("Failed to allocate semaphore bo. Disabling semaphores\n");
2329                                 i915.semaphores = 0;
2330                         } else {
2331                                 i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
2332                                 ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_NONBLOCK);
2333                                 if (ret != 0) {
2334                                         drm_gem_object_unreference(&obj->base);
2335                                         DRM_ERROR("Failed to pin semaphore bo. Disabling semaphores\n");
2336                                         i915.semaphores = 0;
2337                                 } else
2338                                         dev_priv->semaphore_obj = obj;
2339                         }
2340                 }
2341
2342                 ring->init_context = intel_ring_workarounds_emit;
2343                 ring->add_request = gen6_add_request;
2344                 ring->flush = gen8_render_ring_flush;
2345                 ring->irq_get = gen8_ring_get_irq;
2346                 ring->irq_put = gen8_ring_put_irq;
2347                 ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
2348                 ring->get_seqno = gen6_ring_get_seqno;
2349                 ring->set_seqno = ring_set_seqno;
2350                 if (i915_semaphore_is_enabled(dev)) {
2351                         WARN_ON(!dev_priv->semaphore_obj);
2352                         ring->semaphore.sync_to = gen8_ring_sync;
2353                         ring->semaphore.signal = gen8_rcs_signal;
2354                         GEN8_RING_SEMAPHORE_INIT;
2355                 }
2356         } else if (INTEL_INFO(dev)->gen >= 6) {
2357                 ring->add_request = gen6_add_request;
2358                 ring->flush = gen7_render_ring_flush;
2359                 if (INTEL_INFO(dev)->gen == 6)
2360                         ring->flush = gen6_render_ring_flush;
2361                 ring->irq_get = gen6_ring_get_irq;
2362                 ring->irq_put = gen6_ring_put_irq;
2363                 ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
2364                 ring->get_seqno = gen6_ring_get_seqno;
2365                 ring->set_seqno = ring_set_seqno;
2366                 if (i915_semaphore_is_enabled(dev)) {
2367                         ring->semaphore.sync_to = gen6_ring_sync;
2368                         ring->semaphore.signal = gen6_signal;
2369                         /*
2370                          * The current semaphore is only applied on pre-gen8
2371                          * platform.  And there is no VCS2 ring on the pre-gen8
2372                          * platform. So the semaphore between RCS and VCS2 is
2373                          * initialized as INVALID.  Gen8 will initialize the
2374                          * sema between VCS2 and RCS later.
2375                          */
2376                         ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_INVALID;
2377                         ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_RV;
2378                         ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_RB;
2379                         ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_RVE;
2380                         ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
2381                         ring->semaphore.mbox.signal[RCS] = GEN6_NOSYNC;
2382                         ring->semaphore.mbox.signal[VCS] = GEN6_VRSYNC;
2383                         ring->semaphore.mbox.signal[BCS] = GEN6_BRSYNC;
2384                         ring->semaphore.mbox.signal[VECS] = GEN6_VERSYNC;
2385                         ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
2386                 }
2387         } else if (IS_GEN5(dev)) {
2388                 ring->add_request = pc_render_add_request;
2389                 ring->flush = gen4_render_ring_flush;
2390                 ring->get_seqno = pc_render_get_seqno;
2391                 ring->set_seqno = pc_render_set_seqno;
2392                 ring->irq_get = gen5_ring_get_irq;
2393                 ring->irq_put = gen5_ring_put_irq;
2394                 ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT |
2395                                         GT_RENDER_PIPECTL_NOTIFY_INTERRUPT;
2396         } else {
2397                 ring->add_request = i9xx_add_request;
2398                 if (INTEL_INFO(dev)->gen < 4)
2399                         ring->flush = gen2_render_ring_flush;
2400                 else
2401                         ring->flush = gen4_render_ring_flush;
2402                 ring->get_seqno = ring_get_seqno;
2403                 ring->set_seqno = ring_set_seqno;
2404                 if (IS_GEN2(dev)) {
2405                         ring->irq_get = i8xx_ring_get_irq;
2406                         ring->irq_put = i8xx_ring_put_irq;
2407                 } else {
2408                         ring->irq_get = i9xx_ring_get_irq;
2409                         ring->irq_put = i9xx_ring_put_irq;
2410                 }
2411                 ring->irq_enable_mask = I915_USER_INTERRUPT;
2412         }
2413         ring->write_tail = ring_write_tail;
2414
2415         if (IS_HASWELL(dev))
2416                 ring->dispatch_execbuffer = hsw_ring_dispatch_execbuffer;
2417         else if (IS_GEN8(dev))
2418                 ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
2419         else if (INTEL_INFO(dev)->gen >= 6)
2420                 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
2421         else if (INTEL_INFO(dev)->gen >= 4)
2422                 ring->dispatch_execbuffer = i965_dispatch_execbuffer;
2423         else if (IS_I830(dev) || IS_845G(dev))
2424                 ring->dispatch_execbuffer = i830_dispatch_execbuffer;
2425         else
2426                 ring->dispatch_execbuffer = i915_dispatch_execbuffer;
2427         ring->init = init_render_ring;
2428         ring->cleanup = render_ring_cleanup;
2429
2430         /* Workaround batchbuffer to combat CS tlb bug. */
2431         if (HAS_BROKEN_CS_TLB(dev)) {
2432                 obj = i915_gem_alloc_object(dev, I830_WA_SIZE);
2433                 if (obj == NULL) {
2434                         DRM_ERROR("Failed to allocate batch bo\n");
2435                         return -ENOMEM;
2436                 }
2437
2438                 ret = i915_gem_obj_ggtt_pin(obj, 0, 0);
2439                 if (ret != 0) {
2440                         drm_gem_object_unreference(&obj->base);
2441                         DRM_ERROR("Failed to ping batch bo\n");
2442                         return ret;
2443                 }
2444
2445                 ring->scratch.obj = obj;
2446                 ring->scratch.gtt_offset = i915_gem_obj_ggtt_offset(obj);
2447         }
2448
2449         return intel_init_ring_buffer(dev, ring);
2450 }
2451
2452 int intel_init_bsd_ring_buffer(struct drm_device *dev)
2453 {
2454         struct drm_i915_private *dev_priv = dev->dev_private;
2455         struct intel_engine_cs *ring = &dev_priv->ring[VCS];
2456
2457         ring->name = "bsd ring";
2458         ring->id = VCS;
2459
2460         ring->write_tail = ring_write_tail;
2461         if (INTEL_INFO(dev)->gen >= 6) {
2462                 ring->mmio_base = GEN6_BSD_RING_BASE;
2463                 /* gen6 bsd needs a special wa for tail updates */
2464                 if (IS_GEN6(dev))
2465                         ring->write_tail = gen6_bsd_ring_write_tail;
2466                 ring->flush = gen6_bsd_ring_flush;
2467                 ring->add_request = gen6_add_request;
2468                 ring->get_seqno = gen6_ring_get_seqno;
2469                 ring->set_seqno = ring_set_seqno;
2470                 if (INTEL_INFO(dev)->gen >= 8) {
2471                         ring->irq_enable_mask =
2472                                 GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT;
2473                         ring->irq_get = gen8_ring_get_irq;
2474                         ring->irq_put = gen8_ring_put_irq;
2475                         ring->dispatch_execbuffer =
2476                                 gen8_ring_dispatch_execbuffer;
2477                         if (i915_semaphore_is_enabled(dev)) {
2478                                 ring->semaphore.sync_to = gen8_ring_sync;
2479                                 ring->semaphore.signal = gen8_xcs_signal;
2480                                 GEN8_RING_SEMAPHORE_INIT;
2481                         }
2482                 } else {
2483                         ring->irq_enable_mask = GT_BSD_USER_INTERRUPT;
2484                         ring->irq_get = gen6_ring_get_irq;
2485                         ring->irq_put = gen6_ring_put_irq;
2486                         ring->dispatch_execbuffer =
2487                                 gen6_ring_dispatch_execbuffer;
2488                         if (i915_semaphore_is_enabled(dev)) {
2489                                 ring->semaphore.sync_to = gen6_ring_sync;
2490                                 ring->semaphore.signal = gen6_signal;
2491                                 ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VR;
2492                                 ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_INVALID;
2493                                 ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VB;
2494                                 ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_VVE;
2495                                 ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
2496                                 ring->semaphore.mbox.signal[RCS] = GEN6_RVSYNC;
2497                                 ring->semaphore.mbox.signal[VCS] = GEN6_NOSYNC;
2498                                 ring->semaphore.mbox.signal[BCS] = GEN6_BVSYNC;
2499                                 ring->semaphore.mbox.signal[VECS] = GEN6_VEVSYNC;
2500                                 ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
2501                         }
2502                 }
2503         } else {
2504                 ring->mmio_base = BSD_RING_BASE;
2505                 ring->flush = bsd_ring_flush;
2506                 ring->add_request = i9xx_add_request;
2507                 ring->get_seqno = ring_get_seqno;
2508                 ring->set_seqno = ring_set_seqno;
2509                 if (IS_GEN5(dev)) {
2510                         ring->irq_enable_mask = ILK_BSD_USER_INTERRUPT;
2511                         ring->irq_get = gen5_ring_get_irq;
2512                         ring->irq_put = gen5_ring_put_irq;
2513                 } else {
2514                         ring->irq_enable_mask = I915_BSD_USER_INTERRUPT;
2515                         ring->irq_get = i9xx_ring_get_irq;
2516                         ring->irq_put = i9xx_ring_put_irq;
2517                 }
2518                 ring->dispatch_execbuffer = i965_dispatch_execbuffer;
2519         }
2520         ring->init = init_ring_common;
2521
2522         return intel_init_ring_buffer(dev, ring);
2523 }
2524
2525 /**
2526  * Initialize the second BSD ring for Broadwell GT3.
2527  * It is noted that this only exists on Broadwell GT3.
2528  */
2529 int intel_init_bsd2_ring_buffer(struct drm_device *dev)
2530 {
2531         struct drm_i915_private *dev_priv = dev->dev_private;
2532         struct intel_engine_cs *ring = &dev_priv->ring[VCS2];
2533
2534         if ((INTEL_INFO(dev)->gen != 8)) {
2535                 DRM_ERROR("No dual-BSD ring on non-BDW machine\n");
2536                 return -EINVAL;
2537         }
2538
2539         ring->name = "bsd2 ring";
2540         ring->id = VCS2;
2541
2542         ring->write_tail = ring_write_tail;
2543         ring->mmio_base = GEN8_BSD2_RING_BASE;
2544         ring->flush = gen6_bsd_ring_flush;
2545         ring->add_request = gen6_add_request;
2546         ring->get_seqno = gen6_ring_get_seqno;
2547         ring->set_seqno = ring_set_seqno;
2548         ring->irq_enable_mask =
2549                         GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT;
2550         ring->irq_get = gen8_ring_get_irq;
2551         ring->irq_put = gen8_ring_put_irq;
2552         ring->dispatch_execbuffer =
2553                         gen8_ring_dispatch_execbuffer;
2554         if (i915_semaphore_is_enabled(dev)) {
2555                 ring->semaphore.sync_to = gen8_ring_sync;
2556                 ring->semaphore.signal = gen8_xcs_signal;
2557                 GEN8_RING_SEMAPHORE_INIT;
2558         }
2559         ring->init = init_ring_common;
2560
2561         return intel_init_ring_buffer(dev, ring);
2562 }
2563
2564 int intel_init_blt_ring_buffer(struct drm_device *dev)
2565 {
2566         struct drm_i915_private *dev_priv = dev->dev_private;
2567         struct intel_engine_cs *ring = &dev_priv->ring[BCS];
2568
2569         ring->name = "blitter ring";
2570         ring->id = BCS;
2571
2572         ring->mmio_base = BLT_RING_BASE;
2573         ring->write_tail = ring_write_tail;
2574         ring->flush = gen6_ring_flush;
2575         ring->add_request = gen6_add_request;
2576         ring->get_seqno = gen6_ring_get_seqno;
2577         ring->set_seqno = ring_set_seqno;
2578         if (INTEL_INFO(dev)->gen >= 8) {
2579                 ring->irq_enable_mask =
2580                         GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT;
2581                 ring->irq_get = gen8_ring_get_irq;
2582                 ring->irq_put = gen8_ring_put_irq;
2583                 ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
2584                 if (i915_semaphore_is_enabled(dev)) {
2585                         ring->semaphore.sync_to = gen8_ring_sync;
2586                         ring->semaphore.signal = gen8_xcs_signal;
2587                         GEN8_RING_SEMAPHORE_INIT;
2588                 }
2589         } else {
2590                 ring->irq_enable_mask = GT_BLT_USER_INTERRUPT;
2591                 ring->irq_get = gen6_ring_get_irq;
2592                 ring->irq_put = gen6_ring_put_irq;
2593                 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
2594                 if (i915_semaphore_is_enabled(dev)) {
2595                         ring->semaphore.signal = gen6_signal;
2596                         ring->semaphore.sync_to = gen6_ring_sync;
2597                         /*
2598                          * The current semaphore is only applied on pre-gen8
2599                          * platform.  And there is no VCS2 ring on the pre-gen8
2600                          * platform. So the semaphore between BCS and VCS2 is
2601                          * initialized as INVALID.  Gen8 will initialize the
2602                          * sema between BCS and VCS2 later.
2603                          */
2604                         ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_BR;
2605                         ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_BV;
2606                         ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_INVALID;
2607                         ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_BVE;
2608                         ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
2609                         ring->semaphore.mbox.signal[RCS] = GEN6_RBSYNC;
2610                         ring->semaphore.mbox.signal[VCS] = GEN6_VBSYNC;
2611                         ring->semaphore.mbox.signal[BCS] = GEN6_NOSYNC;
2612                         ring->semaphore.mbox.signal[VECS] = GEN6_VEBSYNC;
2613                         ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
2614                 }
2615         }
2616         ring->init = init_ring_common;
2617
2618         return intel_init_ring_buffer(dev, ring);
2619 }
2620
2621 int intel_init_vebox_ring_buffer(struct drm_device *dev)
2622 {
2623         struct drm_i915_private *dev_priv = dev->dev_private;
2624         struct intel_engine_cs *ring = &dev_priv->ring[VECS];
2625
2626         ring->name = "video enhancement ring";
2627         ring->id = VECS;
2628
2629         ring->mmio_base = VEBOX_RING_BASE;
2630         ring->write_tail = ring_write_tail;
2631         ring->flush = gen6_ring_flush;
2632         ring->add_request = gen6_add_request;
2633         ring->get_seqno = gen6_ring_get_seqno;
2634         ring->set_seqno = ring_set_seqno;
2635
2636         if (INTEL_INFO(dev)->gen >= 8) {
2637                 ring->irq_enable_mask =
2638                         GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT;
2639                 ring->irq_get = gen8_ring_get_irq;
2640                 ring->irq_put = gen8_ring_put_irq;
2641                 ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
2642                 if (i915_semaphore_is_enabled(dev)) {
2643                         ring->semaphore.sync_to = gen8_ring_sync;
2644                         ring->semaphore.signal = gen8_xcs_signal;
2645                         GEN8_RING_SEMAPHORE_INIT;
2646                 }
2647         } else {
2648                 ring->irq_enable_mask = PM_VEBOX_USER_INTERRUPT;
2649                 ring->irq_get = hsw_vebox_get_irq;
2650                 ring->irq_put = hsw_vebox_put_irq;
2651                 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
2652                 if (i915_semaphore_is_enabled(dev)) {
2653                         ring->semaphore.sync_to = gen6_ring_sync;
2654                         ring->semaphore.signal = gen6_signal;
2655                         ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VER;
2656                         ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_VEV;
2657                         ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VEB;
2658                         ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_INVALID;
2659                         ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
2660                         ring->semaphore.mbox.signal[RCS] = GEN6_RVESYNC;
2661                         ring->semaphore.mbox.signal[VCS] = GEN6_VVESYNC;
2662                         ring->semaphore.mbox.signal[BCS] = GEN6_BVESYNC;
2663                         ring->semaphore.mbox.signal[VECS] = GEN6_NOSYNC;
2664                         ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
2665                 }
2666         }
2667         ring->init = init_ring_common;
2668
2669         return intel_init_ring_buffer(dev, ring);
2670 }
2671
2672 int
2673 intel_ring_flush_all_caches(struct intel_engine_cs *ring)
2674 {
2675         int ret;
2676
2677         if (!ring->gpu_caches_dirty)
2678                 return 0;
2679
2680         ret = ring->flush(ring, 0, I915_GEM_GPU_DOMAINS);
2681         if (ret)
2682                 return ret;
2683
2684         trace_i915_gem_ring_flush(ring, 0, I915_GEM_GPU_DOMAINS);
2685
2686         ring->gpu_caches_dirty = false;
2687         return 0;
2688 }
2689
2690 int
2691 intel_ring_invalidate_all_caches(struct intel_engine_cs *ring)
2692 {
2693         uint32_t flush_domains;
2694         int ret;
2695
2696         flush_domains = 0;
2697         if (ring->gpu_caches_dirty)
2698                 flush_domains = I915_GEM_GPU_DOMAINS;
2699
2700         ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, flush_domains);
2701         if (ret)
2702                 return ret;
2703
2704         trace_i915_gem_ring_flush(ring, I915_GEM_GPU_DOMAINS, flush_domains);
2705
2706         ring->gpu_caches_dirty = false;
2707         return 0;
2708 }
2709
2710 void
2711 intel_stop_ring_buffer(struct intel_engine_cs *ring)
2712 {
2713         int ret;
2714
2715         if (!intel_ring_initialized(ring))
2716                 return;
2717
2718         ret = intel_ring_idle(ring);
2719         if (ret && !i915_reset_in_progress(&to_i915(ring->dev)->gpu_error))
2720                 DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
2721                           ring->name, ret);
2722
2723         stop_ring(ring);
2724 }