drivers/gpu/drm/i915/i915_gem_gtt.c

   1 /*
   2  * Copyright © 2010 Daniel Vetter
   3  * Copyright © 2011-2014 Intel Corporation
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  22  * IN THE SOFTWARE.
  23  *
  24  */
  25
  26 #include <linux/seq_file.h>
  27 #include <drm/drmP.h>
  28 #include <drm/i915_drm.h>
  29 #include "i915_drv.h"
  30 #include "i915_trace.h"
  31 #include "intel_drv.h"
  32
  33 static void bdw_setup_private_ppat(struct drm_i915_private *dev_priv);
  34 static void chv_setup_private_ppat(struct drm_i915_private *dev_priv);
  35
  36 static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt)
  37 {
  38         if (enable_ppgtt == 0 || !HAS_ALIASING_PPGTT(dev))
  39                 return 0;
  40
  41         if (enable_ppgtt == 1)
  42                 return 1;
  43
  44         if (enable_ppgtt == 2 && HAS_PPGTT(dev))
  45                 return 2;
  46
  47 #ifdef CONFIG_INTEL_IOMMU
  48         /* Disable ppgtt on SNB if VT-d is on. */
  49         if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) {
  50                 DRM_INFO("Disabling PPGTT because VT-d is on\n");
  51                 return 0;
  52         }
  53 #endif
  54
  55         /* Early VLV doesn't have this */
  56         if (IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev) &&
  57             dev->pdev->revision < 0xb) {
  58                 DRM_DEBUG_DRIVER("disabling PPGTT on pre-B3 step VLV\n");
  59                 return 0;
  60         }
  61
  62         return HAS_ALIASING_PPGTT(dev) ? 1 : 0;
  63 }
  64
  65
  66 static void ppgtt_bind_vma(struct i915_vma *vma,
  67                            enum i915_cache_level cache_level,
  68                            u32 flags);
  69 static void ppgtt_unbind_vma(struct i915_vma *vma);
  70
  71 static inline gen8_gtt_pte_t gen8_pte_encode(dma_addr_t addr,
  72                                              enum i915_cache_level level,
  73                                              bool valid)
  74 {
  75         gen8_gtt_pte_t pte = valid ? _PAGE_PRESENT | _PAGE_RW : 0;
  76         pte |= addr;
  77
  78         switch (level) {
  79         case I915_CACHE_NONE:
  80                 pte |= PPAT_UNCACHED_INDEX;
  81                 break;
  82         case I915_CACHE_WT:
  83                 pte |= PPAT_DISPLAY_ELLC_INDEX;
  84                 break;
  85         default:
  86                 pte |= PPAT_CACHED_INDEX;
  87                 break;
  88         }
  89
  90         return pte;
  91 }
  92
  93 static inline gen8_ppgtt_pde_t gen8_pde_encode(struct drm_device *dev,
  94                                              dma_addr_t addr,
  95                                              enum i915_cache_level level)
  96 {
  97         gen8_ppgtt_pde_t pde = _PAGE_PRESENT | _PAGE_RW;
  98         pde |= addr;
  99         if (level != I915_CACHE_NONE)
 100                 pde |= PPAT_CACHED_PDE_INDEX;
 101         else
 102                 pde |= PPAT_UNCACHED_INDEX;
 103         return pde;
 104 }
 105
 106 static gen6_gtt_pte_t snb_pte_encode(dma_addr_t addr,
 107                                      enum i915_cache_level level,
 108                                      bool valid, u32 unused)
 109 {
 110         gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
 111         pte |= GEN6_PTE_ADDR_ENCODE(addr);
 112
 113         switch (level) {
 114         case I915_CACHE_L3_LLC:
 115         case I915_CACHE_LLC:
 116                 pte |= GEN6_PTE_CACHE_LLC;
 117                 break;
 118         case I915_CACHE_NONE:
 119                 pte |= GEN6_PTE_UNCACHED;
 120                 break;
 121         default:
 122                 WARN_ON(1);
 123         }
 124
 125         return pte;
 126 }
 127
 128 static gen6_gtt_pte_t ivb_pte_encode(dma_addr_t addr,
 129                                      enum i915_cache_level level,
 130                                      bool valid, u32 unused)
 131 {
 132         gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
 133         pte |= GEN6_PTE_ADDR_ENCODE(addr);
 134
 135         switch (level) {
 136         case I915_CACHE_L3_LLC:
 137                 pte |= GEN7_PTE_CACHE_L3_LLC;
 138                 break;
 139         case I915_CACHE_LLC:
 140                 pte |= GEN6_PTE_CACHE_LLC;
 141                 break;
 142         case I915_CACHE_NONE:
 143                 pte |= GEN6_PTE_UNCACHED;
 144                 break;
 145         default:
 146                 WARN_ON(1);
 147         }
 148
 149         return pte;
 150 }
 151
 152 static gen6_gtt_pte_t byt_pte_encode(dma_addr_t addr,
 153                                      enum i915_cache_level level,
 154                                      bool valid, u32 flags)
 155 {
 156         gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
 157         pte |= GEN6_PTE_ADDR_ENCODE(addr);
 158
 159         /* Mark the page as writeable.  Other platforms don't have a
 160          * setting for read-only/writable, so this matches that behavior.
 161          */
 162         if (!(flags & PTE_READ_ONLY))
 163                 pte |= BYT_PTE_WRITEABLE;
 164
 165         if (level != I915_CACHE_NONE)
 166                 pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES;
 167
 168         return pte;
 169 }
 170
 171 static gen6_gtt_pte_t hsw_pte_encode(dma_addr_t addr,
 172                                      enum i915_cache_level level,
 173                                      bool valid, u32 unused)
 174 {
 175         gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
 176         pte |= HSW_PTE_ADDR_ENCODE(addr);
 177
 178         if (level != I915_CACHE_NONE)
 179                 pte |= HSW_WB_LLC_AGE3;
 180
 181         return pte;
 182 }
 183
 184 static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,
 185                                       enum i915_cache_level level,
 186                                       bool valid, u32 unused)
 187 {
 188         gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
 189         pte |= HSW_PTE_ADDR_ENCODE(addr);
 190
 191         switch (level) {
 192         case I915_CACHE_NONE:
 193                 break;
 194         case I915_CACHE_WT:
 195                 pte |= HSW_WT_ELLC_LLC_AGE3;
 196                 break;
 197         default:
 198                 pte |= HSW_WB_ELLC_LLC_AGE3;
 199                 break;
 200         }
 201
 202         return pte;
 203 }
 204
 205 /* Broadwell Page Directory Pointer Descriptors */
 206 static int gen8_write_pdp(struct intel_engine_cs *ring, unsigned entry,
 207                            uint64_t val)
 208 {
 209         int ret;
 210
 211         BUG_ON(entry >= 4);
 212
 213         ret = intel_ring_begin(ring, 6);
 214         if (ret)
 215                 return ret;
 216
 217         intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
 218         intel_ring_emit(ring, GEN8_RING_PDP_UDW(ring, entry));
 219         intel_ring_emit(ring, (u32)(val >> 32));
 220         intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
 221         intel_ring_emit(ring, GEN8_RING_PDP_LDW(ring, entry));
 222         intel_ring_emit(ring, (u32)(val));
 223         intel_ring_advance(ring);
 224
 225         return 0;
 226 }
 227
 228 static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt,
 229                           struct intel_engine_cs *ring)
 230 {
 231         int i, ret;
 232
 233         /* bit of a hack to find the actual last used pd */
 234         int used_pd = ppgtt->num_pd_entries / GEN8_PDES_PER_PAGE;
 235
 236         for (i = used_pd - 1; i >= 0; i--) {
 237                 dma_addr_t addr = ppgtt->pd_dma_addr[i];
 238                 ret = gen8_write_pdp(ring, i, addr);
 239                 if (ret)
 240                         return ret;
 241         }
 242
 243         return 0;
 244 }
 245
 246 static void gen8_ppgtt_clear_range(struct i915_address_space *vm,
 247                                    uint64_t start,
 248                                    uint64_t length,
 249                                    bool use_scratch)
 250 {
 251         struct i915_hw_ppgtt *ppgtt =
 252                 container_of(vm, struct i915_hw_ppgtt, base);
 253         gen8_gtt_pte_t *pt_vaddr, scratch_pte;
 254         unsigned pdpe = start >> GEN8_PDPE_SHIFT & GEN8_PDPE_MASK;
 255         unsigned pde = start >> GEN8_PDE_SHIFT & GEN8_PDE_MASK;
 256         unsigned pte = start >> GEN8_PTE_SHIFT & GEN8_PTE_MASK;
 257         unsigned num_entries = length >> PAGE_SHIFT;
 258         unsigned last_pte, i;
 259
 260         scratch_pte = gen8_pte_encode(ppgtt->base.scratch.addr,
 261                                       I915_CACHE_LLC, use_scratch);
 262
 263         while (num_entries) {
 264                 struct page *page_table = ppgtt->gen8_pt_pages[pdpe][pde];
 265
 266                 last_pte = pte + num_entries;
 267                 if (last_pte > GEN8_PTES_PER_PAGE)
 268                         last_pte = GEN8_PTES_PER_PAGE;
 269
 270                 pt_vaddr = kmap_atomic(page_table);
 271
 272                 for (i = pte; i < last_pte; i++) {
 273                         pt_vaddr[i] = scratch_pte;
 274                         num_entries--;
 275                 }
 276
 277                 if (!HAS_LLC(ppgtt->base.dev))
 278                         drm_clflush_virt_range(pt_vaddr, PAGE_SIZE);
 279                 kunmap_atomic(pt_vaddr);
 280
 281                 pte = 0;
 282                 if (++pde == GEN8_PDES_PER_PAGE) {
 283                         pdpe++;
 284                         pde = 0;
 285                 }
 286         }
 287 }
 288
 289 static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
 290                                       struct sg_table *pages,
 291                                       uint64_t start,
 292                                       enum i915_cache_level cache_level, u32 unused)
 293 {
 294         struct i915_hw_ppgtt *ppgtt =
 295                 container_of(vm, struct i915_hw_ppgtt, base);
 296         gen8_gtt_pte_t *pt_vaddr;
 297         unsigned pdpe = start >> GEN8_PDPE_SHIFT & GEN8_PDPE_MASK;
 298         unsigned pde = start >> GEN8_PDE_SHIFT & GEN8_PDE_MASK;
 299         unsigned pte = start >> GEN8_PTE_SHIFT & GEN8_PTE_MASK;
 300         struct sg_page_iter sg_iter;
 301
 302         pt_vaddr = NULL;
 303
 304         for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
 305                 if (WARN_ON(pdpe >= GEN8_LEGACY_PDPS))
 306                         break;
 307
 308                 if (pt_vaddr == NULL)
 309                         pt_vaddr = kmap_atomic(ppgtt->gen8_pt_pages[pdpe][pde]);
 310
 311                 pt_vaddr[pte] =
 312                         gen8_pte_encode(sg_page_iter_dma_address(&sg_iter),
 313                                         cache_level, true);
 314                 if (++pte == GEN8_PTES_PER_PAGE) {
 315                         if (!HAS_LLC(ppgtt->base.dev))
 316                                 drm_clflush_virt_range(pt_vaddr, PAGE_SIZE);
 317                         kunmap_atomic(pt_vaddr);
 318                         pt_vaddr = NULL;
 319                         if (++pde == GEN8_PDES_PER_PAGE) {
 320                                 pdpe++;
 321                                 pde = 0;
 322                         }
 323                         pte = 0;
 324                 }
 325         }
 326         if (pt_vaddr) {
 327                 if (!HAS_LLC(ppgtt->base.dev))
 328                         drm_clflush_virt_range(pt_vaddr, PAGE_SIZE);
 329                 kunmap_atomic(pt_vaddr);
 330         }
 331 }
 332
 333 static void gen8_free_page_tables(struct page **pt_pages)
 334 {
 335         int i;
 336
 337         if (pt_pages == NULL)
 338                 return;
 339
 340         for (i = 0; i < GEN8_PDES_PER_PAGE; i++)
 341                 if (pt_pages[i])
 342                         __free_pages(pt_pages[i], 0);
 343 }
 344
 345 static void gen8_ppgtt_free(const struct i915_hw_ppgtt *ppgtt)
 346 {
 347         int i;
 348
 349         for (i = 0; i < ppgtt->num_pd_pages; i++) {
 350                 gen8_free_page_tables(ppgtt->gen8_pt_pages[i]);
 351                 kfree(ppgtt->gen8_pt_pages[i]);
 352                 kfree(ppgtt->gen8_pt_dma_addr[i]);
 353         }
 354
 355         __free_pages(ppgtt->pd_pages, get_order(ppgtt->num_pd_pages << PAGE_SHIFT));
 356 }
 357
 358 static void gen8_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)
 359 {
 360         struct pci_dev *hwdev = ppgtt->base.dev->pdev;
 361         int i, j;
 362
 363         for (i = 0; i < ppgtt->num_pd_pages; i++) {
 364                 /* TODO: In the future we'll support sparse mappings, so this
 365                  * will have to change. */
 366                 if (!ppgtt->pd_dma_addr[i])
 367                         continue;
 368
 369                 pci_unmap_page(hwdev, ppgtt->pd_dma_addr[i], PAGE_SIZE,
 370                                PCI_DMA_BIDIRECTIONAL);
 371
 372                 for (j = 0; j < GEN8_PDES_PER_PAGE; j++) {
 373                         dma_addr_t addr = ppgtt->gen8_pt_dma_addr[i][j];
 374                         if (addr)
 375                                 pci_unmap_page(hwdev, addr, PAGE_SIZE,
 376                                                PCI_DMA_BIDIRECTIONAL);
 377                 }
 378         }
 379 }
 380
 381 static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
 382 {
 383         struct i915_hw_ppgtt *ppgtt =
 384                 container_of(vm, struct i915_hw_ppgtt, base);
 385
 386         gen8_ppgtt_unmap_pages(ppgtt);
 387         gen8_ppgtt_free(ppgtt);
 388 }
 389
 390 static struct page **__gen8_alloc_page_tables(void)
 391 {
 392         struct page **pt_pages;
 393         int i;
 394
 395         pt_pages = kcalloc(GEN8_PDES_PER_PAGE, sizeof(struct page *), GFP_KERNEL);
 396         if (!pt_pages)
 397                 return ERR_PTR(-ENOMEM);
 398
 399         for (i = 0; i < GEN8_PDES_PER_PAGE; i++) {
 400                 pt_pages[i] = alloc_page(GFP_KERNEL);
 401                 if (!pt_pages[i])
 402                         goto bail;
 403         }
 404
 405         return pt_pages;
 406
 407 bail:
 408         gen8_free_page_tables(pt_pages);
 409         kfree(pt_pages);
 410         return ERR_PTR(-ENOMEM);
 411 }
 412
 413 static int gen8_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt,
 414                                            const int max_pdp)
 415 {
 416         struct page **pt_pages[GEN8_LEGACY_PDPS];
 417         int i, ret;
 418
 419         for (i = 0; i < max_pdp; i++) {
 420                 pt_pages[i] = __gen8_alloc_page_tables();
 421                 if (IS_ERR(pt_pages[i])) {
 422                         ret = PTR_ERR(pt_pages[i]);
 423                         goto unwind_out;
 424                 }
 425         }
 426
 427         /* NB: Avoid touching gen8_pt_pages until last to keep the allocation,
 428          * "atomic" - for cleanup purposes.
 429          */
 430         for (i = 0; i < max_pdp; i++)
 431                 ppgtt->gen8_pt_pages[i] = pt_pages[i];
 432
 433         return 0;
 434
 435 unwind_out:
 436         while (i--) {
 437                 gen8_free_page_tables(pt_pages[i]);
 438                 kfree(pt_pages[i]);
 439         }
 440
 441         return ret;
 442 }
 443
 444 static int gen8_ppgtt_allocate_dma(struct i915_hw_ppgtt *ppgtt)
 445 {
 446         int i;
 447
 448         for (i = 0; i < ppgtt->num_pd_pages; i++) {
 449                 ppgtt->gen8_pt_dma_addr[i] = kcalloc(GEN8_PDES_PER_PAGE,
 450                                                      sizeof(dma_addr_t),
 451                                                      GFP_KERNEL);
 452                 if (!ppgtt->gen8_pt_dma_addr[i])
 453                         return -ENOMEM;
 454         }
 455
 456         return 0;
 457 }
 458
 459 static int gen8_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt,
 460                                                 const int max_pdp)
 461 {
 462         ppgtt->pd_pages = alloc_pages(GFP_KERNEL, get_order(max_pdp << PAGE_SHIFT));
 463         if (!ppgtt->pd_pages)
 464                 return -ENOMEM;
 465
 466         ppgtt->num_pd_pages = 1 << get_order(max_pdp << PAGE_SHIFT);
 467         BUG_ON(ppgtt->num_pd_pages > GEN8_LEGACY_PDPS);
 468
 469         return 0;
 470 }
 471
 472 static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt,
 473                             const int max_pdp)
 474 {
 475         int ret;
 476
 477         ret = gen8_ppgtt_allocate_page_directories(ppgtt, max_pdp);
 478         if (ret)
 479                 return ret;
 480
 481         ret = gen8_ppgtt_allocate_page_tables(ppgtt, max_pdp);
 482         if (ret) {
 483                 __free_pages(ppgtt->pd_pages, get_order(max_pdp << PAGE_SHIFT));
 484                 return ret;
 485         }
 486
 487         ppgtt->num_pd_entries = max_pdp * GEN8_PDES_PER_PAGE;
 488
 489         ret = gen8_ppgtt_allocate_dma(ppgtt);
 490         if (ret)
 491                 gen8_ppgtt_free(ppgtt);
 492
 493         return ret;
 494 }
 495
 496 static int gen8_ppgtt_setup_page_directories(struct i915_hw_ppgtt *ppgtt,
 497                                              const int pd)
 498 {
 499         dma_addr_t pd_addr;
 500         int ret;
 501
 502         pd_addr = pci_map_page(ppgtt->base.dev->pdev,
 503                                &ppgtt->pd_pages[pd], 0,
 504                                PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 505
 506         ret = pci_dma_mapping_error(ppgtt->base.dev->pdev, pd_addr);
 507         if (ret)
 508                 return ret;
 509
 510         ppgtt->pd_dma_addr[pd] = pd_addr;
 511
 512         return 0;
 513 }
 514
 515 static int gen8_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt,
 516                                         const int pd,
 517                                         const int pt)
 518 {
 519         dma_addr_t pt_addr;
 520         struct page *p;
 521         int ret;
 522
 523         p = ppgtt->gen8_pt_pages[pd][pt];
 524         pt_addr = pci_map_page(ppgtt->base.dev->pdev,
 525                                p, 0, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 526         ret = pci_dma_mapping_error(ppgtt->base.dev->pdev, pt_addr);
 527         if (ret)
 528                 return ret;
 529
 530         ppgtt->gen8_pt_dma_addr[pd][pt] = pt_addr;
 531
 532         return 0;
 533 }
 534
 535 /**
 536  * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers
 537  * with a net effect resembling a 2-level page table in normal x86 terms. Each
 538  * PDP represents 1GB of memory 4 * 512 * 512 * 4096 = 4GB legacy 32b address
 539  * space.
 540  *
 541  * FIXME: split allocation into smaller pieces. For now we only ever do this
 542  * once, but with full PPGTT, the multiple contiguous allocations will be bad.
 543  * TODO: Do something with the size parameter
 544  */
 545 static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)
 546 {
 547         const int max_pdp = DIV_ROUND_UP(size, 1 << 30);
 548         const int min_pt_pages = GEN8_PDES_PER_PAGE * max_pdp;
 549         int i, j, ret;
 550
 551         if (size % (1<<30))
 552                 DRM_INFO("Pages will be wasted unless GTT size (%llu) is divisible by 1GB\n", size);
 553
 554         /* 1. Do all our allocations for page directories and page tables. */
 555         ret = gen8_ppgtt_alloc(ppgtt, max_pdp);
 556         if (ret)
 557                 return ret;
 558
 559         /*
 560          * 2. Create DMA mappings for the page directories and page tables.
 561          */
 562         for (i = 0; i < max_pdp; i++) {
 563                 ret = gen8_ppgtt_setup_page_directories(ppgtt, i);
 564                 if (ret)
 565                         goto bail;
 566
 567                 for (j = 0; j < GEN8_PDES_PER_PAGE; j++) {
 568                         ret = gen8_ppgtt_setup_page_tables(ppgtt, i, j);
 569                         if (ret)
 570                                 goto bail;
 571                 }
 572         }
 573
 574         /*
 575          * 3. Map all the page directory entires to point to the page tables
 576          * we've allocated.
 577          *
 578          * For now, the PPGTT helper functions all require that the PDEs are
 579          * plugged in correctly. So we do that now/here. For aliasing PPGTT, we
 580          * will never need to touch the PDEs again.
 581          */
 582         for (i = 0; i < max_pdp; i++) {
 583                 gen8_ppgtt_pde_t *pd_vaddr;
 584                 pd_vaddr = kmap_atomic(&ppgtt->pd_pages[i]);
 585                 for (j = 0; j < GEN8_PDES_PER_PAGE; j++) {
 586                         dma_addr_t addr = ppgtt->gen8_pt_dma_addr[i][j];
 587                         pd_vaddr[j] = gen8_pde_encode(ppgtt->base.dev, addr,
 588                                                       I915_CACHE_LLC);
 589                 }
 590                 if (!HAS_LLC(ppgtt->base.dev))
 591                         drm_clflush_virt_range(pd_vaddr, PAGE_SIZE);
 592                 kunmap_atomic(pd_vaddr);
 593         }
 594
 595         ppgtt->switch_mm = gen8_mm_switch;
 596         ppgtt->base.clear_range = gen8_ppgtt_clear_range;
 597         ppgtt->base.insert_entries = gen8_ppgtt_insert_entries;
 598         ppgtt->base.cleanup = gen8_ppgtt_cleanup;
 599         ppgtt->base.start = 0;
 600         ppgtt->base.total = ppgtt->num_pd_entries * GEN8_PTES_PER_PAGE * PAGE_SIZE;
 601
 602         ppgtt->base.clear_range(&ppgtt->base, 0, ppgtt->base.total, true);
 603
 604         DRM_DEBUG_DRIVER("Allocated %d pages for page directories (%d wasted)\n",
 605                          ppgtt->num_pd_pages, ppgtt->num_pd_pages - max_pdp);
 606         DRM_DEBUG_DRIVER("Allocated %d pages for page tables (%lld wasted)\n",
 607                          ppgtt->num_pd_entries,
 608                          (ppgtt->num_pd_entries - min_pt_pages) + size % (1<<30));
 609         return 0;
 610
 611 bail:
 612         gen8_ppgtt_unmap_pages(ppgtt);
 613         gen8_ppgtt_free(ppgtt);
 614         return ret;
 615 }
 616
 617 static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
 618 {
 619         struct drm_i915_private *dev_priv = ppgtt->base.dev->dev_private;
 620         struct i915_address_space *vm = &ppgtt->base;
 621         gen6_gtt_pte_t __iomem *pd_addr;
 622         gen6_gtt_pte_t scratch_pte;
 623         uint32_t pd_entry;
 624         int pte, pde;
 625
 626         scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC, true, 0);
 627
 628         pd_addr = (gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm +
 629                 ppgtt->pd_offset / sizeof(gen6_gtt_pte_t);
 630
 631         seq_printf(m, "  VM %p (pd_offset %x-%x):\n", vm,
 632                    ppgtt->pd_offset, ppgtt->pd_offset + ppgtt->num_pd_entries);
 633         for (pde = 0; pde < ppgtt->num_pd_entries; pde++) {
 634                 u32 expected;
 635                 gen6_gtt_pte_t *pt_vaddr;
 636                 dma_addr_t pt_addr = ppgtt->pt_dma_addr[pde];
 637                 pd_entry = readl(pd_addr + pde);
 638                 expected = (GEN6_PDE_ADDR_ENCODE(pt_addr) | GEN6_PDE_VALID);
 639
 640                 if (pd_entry != expected)
 641                         seq_printf(m, "\tPDE #%d mismatch: Actual PDE: %x Expected PDE: %x\n",
 642                                    pde,
 643                                    pd_entry,
 644                                    expected);
 645                 seq_printf(m, "\tPDE: %x\n", pd_entry);
 646
 647                 pt_vaddr = kmap_atomic(ppgtt->pt_pages[pde]);
 648                 for (pte = 0; pte < I915_PPGTT_PT_ENTRIES; pte+=4) {
 649                         unsigned long va =
 650                                 (pde * PAGE_SIZE * I915_PPGTT_PT_ENTRIES) +
 651                                 (pte * PAGE_SIZE);
 652                         int i;
 653                         bool found = false;
 654                         for (i = 0; i < 4; i++)
 655                                 if (pt_vaddr[pte + i] != scratch_pte)
 656                                         found = true;
 657                         if (!found)
 658                                 continue;
 659
 660                         seq_printf(m, "\t\t0x%lx [%03d,%04d]: =", va, pde, pte);
 661                         for (i = 0; i < 4; i++) {
 662                                 if (pt_vaddr[pte + i] != scratch_pte)
 663                                         seq_printf(m, " %08x", pt_vaddr[pte + i]);
 664                                 else
 665                                         seq_puts(m, "  SCRATCH ");
 666                         }
 667                         seq_puts(m, "\n");
 668                 }
 669                 kunmap_atomic(pt_vaddr);
 670         }
 671 }
 672
 673 static void gen6_write_pdes(struct i915_hw_ppgtt *ppgtt)
 674 {
 675         struct drm_i915_private *dev_priv = ppgtt->base.dev->dev_private;
 676         gen6_gtt_pte_t __iomem *pd_addr;
 677         uint32_t pd_entry;
 678         int i;
 679
 680         WARN_ON(ppgtt->pd_offset & 0x3f);
 681         pd_addr = (gen6_gtt_pte_t __iomem*)dev_priv->gtt.gsm +
 682                 ppgtt->pd_offset / sizeof(gen6_gtt_pte_t);
 683         for (i = 0; i < ppgtt->num_pd_entries; i++) {
 684                 dma_addr_t pt_addr;
 685
 686                 pt_addr = ppgtt->pt_dma_addr[i];
 687                 pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr);
 688                 pd_entry |= GEN6_PDE_VALID;
 689
 690                 writel(pd_entry, pd_addr + i);
 691         }
 692         readl(pd_addr);
 693 }
 694
 695 static uint32_t get_pd_offset(struct i915_hw_ppgtt *ppgtt)
 696 {
 697         BUG_ON(ppgtt->pd_offset & 0x3f);
 698
 699         return (ppgtt->pd_offset / 64) << 16;
 700 }
 701
 702 static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt,
 703                          struct intel_engine_cs *ring)
 704 {
 705         int ret;
 706
 707         /* NB: TLBs must be flushed and invalidated before a switch */
 708         ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
 709         if (ret)
 710                 return ret;
 711
 712         ret = intel_ring_begin(ring, 6);
 713         if (ret)
 714                 return ret;
 715
 716         intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2));
 717         intel_ring_emit(ring, RING_PP_DIR_DCLV(ring));
 718         intel_ring_emit(ring, PP_DIR_DCLV_2G);
 719         intel_ring_emit(ring, RING_PP_DIR_BASE(ring));
 720         intel_ring_emit(ring, get_pd_offset(ppgtt));
 721         intel_ring_emit(ring, MI_NOOP);
 722         intel_ring_advance(ring);
 723
 724         return 0;
 725 }
 726
 727 static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt,
 728                           struct intel_engine_cs *ring)
 729 {
 730         int ret;
 731
 732         /* NB: TLBs must be flushed and invalidated before a switch */
 733         ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
 734         if (ret)
 735                 return ret;
 736
 737         ret = intel_ring_begin(ring, 6);
 738         if (ret)
 739                 return ret;
 740
 741         intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2));
 742         intel_ring_emit(ring, RING_PP_DIR_DCLV(ring));
 743         intel_ring_emit(ring, PP_DIR_DCLV_2G);
 744         intel_ring_emit(ring, RING_PP_DIR_BASE(ring));
 745         intel_ring_emit(ring, get_pd_offset(ppgtt));
 746         intel_ring_emit(ring, MI_NOOP);
 747         intel_ring_advance(ring);
 748
 749         /* XXX: RCS is the only one to auto invalidate the TLBs? */
 750         if (ring->id != RCS) {
 751                 ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
 752                 if (ret)
 753                         return ret;
 754         }
 755
 756         return 0;
 757 }
 758
 759 static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt,
 760                           struct intel_engine_cs *ring)
 761 {
 762         struct drm_device *dev = ppgtt->base.dev;
 763         struct drm_i915_private *dev_priv = dev->dev_private;
 764
 765
 766         I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
 767         I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt));
 768
 769         POSTING_READ(RING_PP_DIR_DCLV(ring));
 770
 771         return 0;
 772 }
 773
 774 static void gen8_ppgtt_enable(struct drm_device *dev)
 775 {
 776         struct drm_i915_private *dev_priv = dev->dev_private;
 777         struct intel_engine_cs *ring;
 778         int j;
 779
 780         for_each_ring(ring, dev_priv, j) {
 781                 I915_WRITE(RING_MODE_GEN7(ring),
 782                            _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
 783         }
 784 }
 785
 786 static void gen7_ppgtt_enable(struct drm_device *dev)
 787 {
 788         struct drm_i915_private *dev_priv = dev->dev_private;
 789         struct intel_engine_cs *ring;
 790         uint32_t ecochk, ecobits;
 791         int i;
 792
 793         ecobits = I915_READ(GAC_ECO_BITS);
 794         I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
 795
 796         ecochk = I915_READ(GAM_ECOCHK);
 797         if (IS_HASWELL(dev)) {
 798                 ecochk |= ECOCHK_PPGTT_WB_HSW;
 799         } else {
 800                 ecochk |= ECOCHK_PPGTT_LLC_IVB;
 801                 ecochk &= ~ECOCHK_PPGTT_GFDT_IVB;
 802         }
 803         I915_WRITE(GAM_ECOCHK, ecochk);
 804
 805         for_each_ring(ring, dev_priv, i) {
 806                 /* GFX_MODE is per-ring on gen7+ */
 807                 I915_WRITE(RING_MODE_GEN7(ring),
 808                            _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
 809         }
 810 }
 811
 812 static void gen6_ppgtt_enable(struct drm_device *dev)
 813 {
 814         struct drm_i915_private *dev_priv = dev->dev_private;
 815         uint32_t ecochk, gab_ctl, ecobits;
 816
 817         ecobits = I915_READ(GAC_ECO_BITS);
 818         I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT |
 819                    ECOBITS_PPGTT_CACHE64B);
 820
 821         gab_ctl = I915_READ(GAB_CTL);
 822         I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
 823
 824         ecochk = I915_READ(GAM_ECOCHK);
 825         I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | ECOCHK_PPGTT_CACHE64B);
 826
 827         I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
 828 }
 829
 830 /* PPGTT support for Sandybdrige/Gen6 and later */
 831 static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
 832                                    uint64_t start,
 833                                    uint64_t length,
 834                                    bool use_scratch)
 835 {
 836         struct i915_hw_ppgtt *ppgtt =
 837                 container_of(vm, struct i915_hw_ppgtt, base);
 838         gen6_gtt_pte_t *pt_vaddr, scratch_pte;
 839         unsigned first_entry = start >> PAGE_SHIFT;
 840         unsigned num_entries = length >> PAGE_SHIFT;
 841         unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;
 842         unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
 843         unsigned last_pte, i;
 844
 845         scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC, true, 0);
 846
 847         while (num_entries) {
 848                 last_pte = first_pte + num_entries;
 849                 if (last_pte > I915_PPGTT_PT_ENTRIES)
 850                         last_pte = I915_PPGTT_PT_ENTRIES;
 851
 852                 pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pt]);
 853
 854                 for (i = first_pte; i < last_pte; i++)
 855                         pt_vaddr[i] = scratch_pte;
 856
 857                 kunmap_atomic(pt_vaddr);
 858
 859                 num_entries -= last_pte - first_pte;
 860                 first_pte = 0;
 861                 act_pt++;
 862         }
 863 }
 864
 865 static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
 866                                       struct sg_table *pages,
 867                                       uint64_t start,
 868                                       enum i915_cache_level cache_level, u32 flags)
 869 {
 870         struct i915_hw_ppgtt *ppgtt =
 871                 container_of(vm, struct i915_hw_ppgtt, base);
 872         gen6_gtt_pte_t *pt_vaddr;
 873         unsigned first_entry = start >> PAGE_SHIFT;
 874         unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;
 875         unsigned act_pte = first_entry % I915_PPGTT_PT_ENTRIES;
 876         struct sg_page_iter sg_iter;
 877
 878         pt_vaddr = NULL;
 879         for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
 880                 if (pt_vaddr == NULL)
 881                         pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pt]);
 882
 883                 pt_vaddr[act_pte] =
 884                         vm->pte_encode(sg_page_iter_dma_address(&sg_iter),
 885                                        cache_level, true, flags);
 886
 887                 if (++act_pte == I915_PPGTT_PT_ENTRIES) {
 888                         kunmap_atomic(pt_vaddr);
 889                         pt_vaddr = NULL;
 890                         act_pt++;
 891                         act_pte = 0;
 892                 }
 893         }
 894         if (pt_vaddr)
 895                 kunmap_atomic(pt_vaddr);
 896 }
 897
 898 static void gen6_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)
 899 {
 900         int i;
 901
 902         if (ppgtt->pt_dma_addr) {
 903                 for (i = 0; i < ppgtt->num_pd_entries; i++)
 904                         pci_unmap_page(ppgtt->base.dev->pdev,
 905                                        ppgtt->pt_dma_addr[i],
 906                                        4096, PCI_DMA_BIDIRECTIONAL);
 907         }
 908 }
 909
 910 static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt)
 911 {
 912         int i;
 913
 914         kfree(ppgtt->pt_dma_addr);
 915         for (i = 0; i < ppgtt->num_pd_entries; i++)
 916                 __free_page(ppgtt->pt_pages[i]);
 917         kfree(ppgtt->pt_pages);
 918 }
 919
 920 static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
 921 {
 922         struct i915_hw_ppgtt *ppgtt =
 923                 container_of(vm, struct i915_hw_ppgtt, base);
 924
 925         drm_mm_remove_node(&ppgtt->node);
 926
 927         gen6_ppgtt_unmap_pages(ppgtt);
 928         gen6_ppgtt_free(ppgtt);
 929 }
 930
 931 static int gen6_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt)
 932 {
 933         struct drm_device *dev = ppgtt->base.dev;
 934         struct drm_i915_private *dev_priv = dev->dev_private;
 935         bool retried = false;
 936         int ret;
 937
 938         /* PPGTT PDEs reside in the GGTT and consists of 512 entries. The
 939          * allocator works in address space sizes, so it's multiplied by page
 940          * size. We allocate at the top of the GTT to avoid fragmentation.
 941          */
 942         BUG_ON(!drm_mm_initialized(&dev_priv->gtt.base.mm));
 943 alloc:
 944         ret = drm_mm_insert_node_in_range_generic(&dev_priv->gtt.base.mm,
 945                                                   &ppgtt->node, GEN6_PD_SIZE,
 946                                                   GEN6_PD_ALIGN, 0,
 947                                                   0, dev_priv->gtt.base.total,
 948                                                   DRM_MM_TOPDOWN);
 949         if (ret == -ENOSPC && !retried) {
 950                 ret = i915_gem_evict_something(dev, &dev_priv->gtt.base,
 951                                                GEN6_PD_SIZE, GEN6_PD_ALIGN,
 952                                                I915_CACHE_NONE,
 953                                                0, dev_priv->gtt.base.total,
 954                                                0);
 955                 if (ret)
 956                         return ret;
 957
 958                 retried = true;
 959                 goto alloc;
 960         }
 961
 962         if (ppgtt->node.start < dev_priv->gtt.mappable_end)
 963                 DRM_DEBUG("Forced to use aperture for PDEs\n");
 964
 965         ppgtt->num_pd_entries = GEN6_PPGTT_PD_ENTRIES;
 966         return ret;
 967 }
 968
 969 static int gen6_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt)
 970 {
 971         int i;
 972
 973         ppgtt->pt_pages = kcalloc(ppgtt->num_pd_entries, sizeof(struct page *),
 974                                   GFP_KERNEL);
 975
 976         if (!ppgtt->pt_pages)
 977                 return -ENOMEM;
 978
 979         for (i = 0; i < ppgtt->num_pd_entries; i++) {
 980                 ppgtt->pt_pages[i] = alloc_page(GFP_KERNEL);
 981                 if (!ppgtt->pt_pages[i]) {
 982                         gen6_ppgtt_free(ppgtt);
 983                         return -ENOMEM;
 984                 }
 985         }
 986
 987         return 0;
 988 }
 989
 990 static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)
 991 {
 992         int ret;
 993
 994         ret = gen6_ppgtt_allocate_page_directories(ppgtt);
 995         if (ret)
 996                 return ret;
 997
 998         ret = gen6_ppgtt_allocate_page_tables(ppgtt);
 999         if (ret) {
1000                 drm_mm_remove_node(&ppgtt->node);
1001                 return ret;
1002         }
1003
1004         ppgtt->pt_dma_addr = kcalloc(ppgtt->num_pd_entries, sizeof(dma_addr_t),
1005                                      GFP_KERNEL);
1006         if (!ppgtt->pt_dma_addr) {
1007                 drm_mm_remove_node(&ppgtt->node);
1008                 gen6_ppgtt_free(ppgtt);
1009                 return -ENOMEM;
1010         }
1011
1012         return 0;
1013 }
1014
1015 static int gen6_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt)
1016 {
1017         struct drm_device *dev = ppgtt->base.dev;
1018         int i;
1019
1020         for (i = 0; i < ppgtt->num_pd_entries; i++) {
1021                 dma_addr_t pt_addr;
1022
1023                 pt_addr = pci_map_page(dev->pdev, ppgtt->pt_pages[i], 0, 4096,
1024                                        PCI_DMA_BIDIRECTIONAL);
1025
1026                 if (pci_dma_mapping_error(dev->pdev, pt_addr)) {
1027                         gen6_ppgtt_unmap_pages(ppgtt);
1028                         return -EIO;
1029                 }
1030
1031                 ppgtt->pt_dma_addr[i] = pt_addr;
1032         }
1033
1034         return 0;
1035 }
1036
1037 static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
1038 {
1039         struct drm_device *dev = ppgtt->base.dev;
1040         struct drm_i915_private *dev_priv = dev->dev_private;
1041         int ret;
1042
1043         ppgtt->base.pte_encode = dev_priv->gtt.base.pte_encode;
1044         if (IS_GEN6(dev)) {
1045                 ppgtt->switch_mm = gen6_mm_switch;
1046         } else if (IS_HASWELL(dev)) {
1047                 ppgtt->switch_mm = hsw_mm_switch;
1048         } else if (IS_GEN7(dev)) {
1049                 ppgtt->switch_mm = gen7_mm_switch;
1050         } else
1051                 BUG();
1052
1053         ret = gen6_ppgtt_alloc(ppgtt);
1054         if (ret)
1055                 return ret;
1056
1057         ret = gen6_ppgtt_setup_page_tables(ppgtt);
1058         if (ret) {
1059                 gen6_ppgtt_free(ppgtt);
1060                 return ret;
1061         }
1062
1063         ppgtt->base.clear_range = gen6_ppgtt_clear_range;
1064         ppgtt->base.insert_entries = gen6_ppgtt_insert_entries;
1065         ppgtt->base.cleanup = gen6_ppgtt_cleanup;
1066         ppgtt->base.start = 0;
1067         ppgtt->base.total =  ppgtt->num_pd_entries * I915_PPGTT_PT_ENTRIES * PAGE_SIZE;
1068         ppgtt->debug_dump = gen6_dump_ppgtt;
1069
1070         ppgtt->pd_offset =
1071                 ppgtt->node.start / PAGE_SIZE * sizeof(gen6_gtt_pte_t);
1072
1073         ppgtt->base.clear_range(&ppgtt->base, 0, ppgtt->base.total, true);
1074
1075         DRM_DEBUG_DRIVER("Allocated pde space (%ldM) at GTT entry: %lx\n",
1076                          ppgtt->node.size >> 20,
1077                          ppgtt->node.start / PAGE_SIZE);
1078
1079         gen6_write_pdes(ppgtt);
1080         DRM_DEBUG("Adding PPGTT at offset %x\n",
1081                   ppgtt->pd_offset << 10);
1082
1083         return 0;
1084 }
1085
1086 static int __hw_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)
1087 {
1088         struct drm_i915_private *dev_priv = dev->dev_private;
1089
1090         ppgtt->base.dev = dev;
1091         ppgtt->base.scratch = dev_priv->gtt.base.scratch;
1092
1093         if (INTEL_INFO(dev)->gen < 8)
1094                 return gen6_ppgtt_init(ppgtt);
1095         else if (IS_GEN8(dev))
1096                 return gen8_ppgtt_init(ppgtt, dev_priv->gtt.base.total);
1097         else
1098                 BUG();
1099 }
1100 int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)
1101 {
1102         struct drm_i915_private *dev_priv = dev->dev_private;
1103         int ret = 0;
1104
1105         ret = __hw_ppgtt_init(dev, ppgtt);
1106         if (ret == 0) {
1107                 kref_init(&ppgtt->ref);
1108                 drm_mm_init(&ppgtt->base.mm, ppgtt->base.start,
1109                             ppgtt->base.total);
1110                 i915_init_vm(dev_priv, &ppgtt->base);
1111         }
1112
1113         return ret;
1114 }
1115
1116 int i915_ppgtt_init_hw(struct drm_device *dev)
1117 {
1118         struct drm_i915_private *dev_priv = dev->dev_private;
1119         struct intel_engine_cs *ring;
1120         struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
1121         int i, ret = 0;
1122
1123         /* In the case of execlists, PPGTT is enabled by the context descriptor
1124          * and the PDPs are contained within the context itself.  We don't
1125          * need to do anything here. */
1126         if (i915.enable_execlists)
1127                 return 0;
1128
1129         if (!USES_PPGTT(dev))
1130                 return 0;
1131
1132         if (IS_GEN6(dev))
1133                 gen6_ppgtt_enable(dev);
1134         else if (IS_GEN7(dev))
1135                 gen7_ppgtt_enable(dev);
1136         else if (INTEL_INFO(dev)->gen >= 8)
1137                 gen8_ppgtt_enable(dev);
1138         else
1139                 WARN_ON(1);
1140
1141         if (ppgtt) {
1142                 for_each_ring(ring, dev_priv, i) {
1143                         ret = ppgtt->switch_mm(ppgtt, ring);
1144                         if (ret != 0)
1145                                 return ret;
1146                 }
1147         }
1148
1149         return ret;
1150 }
1151 struct i915_hw_ppgtt *
1152 i915_ppgtt_create(struct drm_device *dev, struct drm_i915_file_private *fpriv)
1153 {
1154         struct i915_hw_ppgtt *ppgtt;
1155         int ret;
1156
1157         ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
1158         if (!ppgtt)
1159                 return ERR_PTR(-ENOMEM);
1160
1161         ret = i915_ppgtt_init(dev, ppgtt);
1162         if (ret) {
1163                 kfree(ppgtt);
1164                 return ERR_PTR(ret);
1165         }
1166
1167         ppgtt->file_priv = fpriv;
1168
1169         return ppgtt;
1170 }
1171
1172 void  i915_ppgtt_release(struct kref *kref)
1173 {
1174         struct i915_hw_ppgtt *ppgtt =
1175                 container_of(kref, struct i915_hw_ppgtt, ref);
1176
1177         /* vmas should already be unbound */
1178         WARN_ON(!list_empty(&ppgtt->base.active_list));
1179         WARN_ON(!list_empty(&ppgtt->base.inactive_list));
1180
1181         list_del(&ppgtt->base.global_link);
1182         drm_mm_takedown(&ppgtt->base.mm);
1183
1184         ppgtt->base.cleanup(&ppgtt->base);
1185         kfree(ppgtt);
1186 }
1187
1188 static void
1189 ppgtt_bind_vma(struct i915_vma *vma,
1190                enum i915_cache_level cache_level,
1191                u32 flags)
1192 {
1193         /* Currently applicable only to VLV */
1194         if (vma->obj->gt_ro)
1195                 flags |= PTE_READ_ONLY;
1196
1197         vma->vm->insert_entries(vma->vm, vma->obj->pages, vma->node.start,
1198                                 cache_level, flags);
1199 }
1200
1201 static void ppgtt_unbind_vma(struct i915_vma *vma)
1202 {
1203         vma->vm->clear_range(vma->vm,
1204                              vma->node.start,
1205                              vma->obj->base.size,
1206                              true);
1207 }
1208
1209 extern int intel_iommu_gfx_mapped;
1210 /* Certain Gen5 chipsets require require idling the GPU before
1211  * unmapping anything from the GTT when VT-d is enabled.
1212  */
1213 static inline bool needs_idle_maps(struct drm_device *dev)
1214 {
1215 #ifdef CONFIG_INTEL_IOMMU
1216         /* Query intel_iommu to see if we need the workaround. Presumably that
1217          * was loaded first.
1218          */
1219         if (IS_GEN5(dev) && IS_MOBILE(dev) && intel_iommu_gfx_mapped)
1220                 return true;
1221 #endif
1222         return false;
1223 }
1224
1225 static bool do_idling(struct drm_i915_private *dev_priv)
1226 {
1227         bool ret = dev_priv->mm.interruptible;
1228
1229         if (unlikely(dev_priv->gtt.do_idle_maps)) {
1230                 dev_priv->mm.interruptible = false;
1231                 if (i915_gpu_idle(dev_priv->dev)) {
1232                         DRM_ERROR("Couldn't idle GPU\n");
1233                         /* Wait a bit, in hopes it avoids the hang */
1234                         udelay(10);
1235                 }
1236         }
1237
1238         return ret;
1239 }
1240
1241 static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible)
1242 {
1243         if (unlikely(dev_priv->gtt.do_idle_maps))
1244                 dev_priv->mm.interruptible = interruptible;
1245 }
1246
1247 void i915_check_and_clear_faults(struct drm_device *dev)
1248 {
1249         struct drm_i915_private *dev_priv = dev->dev_private;
1250         struct intel_engine_cs *ring;
1251         int i;
1252
1253         if (INTEL_INFO(dev)->gen < 6)
1254                 return;
1255
1256         for_each_ring(ring, dev_priv, i) {
1257                 u32 fault_reg;
1258                 fault_reg = I915_READ(RING_FAULT_REG(ring));
1259                 if (fault_reg & RING_FAULT_VALID) {
1260                         DRM_DEBUG_DRIVER("Unexpected fault\n"
1261                                          "\tAddr: 0x%08lx\\n"
1262                                          "\tAddress space: %s\n"
1263                                          "\tSource ID: %d\n"
1264                                          "\tType: %d\n",
1265                                          fault_reg & PAGE_MASK,
1266                                          fault_reg & RING_FAULT_GTTSEL_MASK ? "GGTT" : "PPGTT",
1267                                          RING_FAULT_SRCID(fault_reg),
1268                                          RING_FAULT_FAULT_TYPE(fault_reg));
1269                         I915_WRITE(RING_FAULT_REG(ring),
1270                                    fault_reg & ~RING_FAULT_VALID);
1271                 }
1272         }
1273         POSTING_READ(RING_FAULT_REG(&dev_priv->ring[RCS]));
1274 }
1275
1276 static void i915_ggtt_flush(struct drm_i915_private *dev_priv)
1277 {
1278         if (INTEL_INFO(dev_priv->dev)->gen < 6) {
1279                 intel_gtt_chipset_flush();
1280         } else {
1281                 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
1282                 POSTING_READ(GFX_FLSH_CNTL_GEN6);
1283         }
1284 }
1285
1286 void i915_gem_suspend_gtt_mappings(struct drm_device *dev)
1287 {
1288         struct drm_i915_private *dev_priv = dev->dev_private;
1289
1290         /* Don't bother messing with faults pre GEN6 as we have little
1291          * documentation supporting that it's a good idea.
1292          */
1293         if (INTEL_INFO(dev)->gen < 6)
1294                 return;
1295
1296         i915_check_and_clear_faults(dev);
1297
1298         dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
1299                                        dev_priv->gtt.base.start,
1300                                        dev_priv->gtt.base.total,
1301                                        true);
1302
1303         i915_ggtt_flush(dev_priv);
1304 }
1305
1306 void i915_gem_restore_gtt_mappings(struct drm_device *dev)
1307 {
1308         struct drm_i915_private *dev_priv = dev->dev_private;
1309         struct drm_i915_gem_object *obj;
1310         struct i915_address_space *vm;
1311
1312         i915_check_and_clear_faults(dev);
1313
1314         /* First fill our portion of the GTT with scratch pages */
1315         dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
1316                                        dev_priv->gtt.base.start,
1317                                        dev_priv->gtt.base.total,
1318                                        true);
1319
1320         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
1321                 struct i915_vma *vma = i915_gem_obj_to_vma(obj,
1322                                                            &dev_priv->gtt.base);
1323                 if (!vma)
1324                         continue;
1325
1326                 i915_gem_clflush_object(obj, obj->pin_display);
1327                 /* The bind_vma code tries to be smart about tracking mappings.
1328                  * Unfortunately above, we've just wiped out the mappings
1329                  * without telling our object about it. So we need to fake it.
1330                  */
1331                 obj->has_global_gtt_mapping = 0;
1332                 vma->bind_vma(vma, obj->cache_level, GLOBAL_BIND);
1333         }
1334
1335
1336         if (INTEL_INFO(dev)->gen >= 8) {
1337                 if (IS_CHERRYVIEW(dev))
1338                         chv_setup_private_ppat(dev_priv);
1339                 else
1340                         bdw_setup_private_ppat(dev_priv);
1341
1342                 return;
1343         }
1344
1345         list_for_each_entry(vm, &dev_priv->vm_list, global_link) {
1346                 /* TODO: Perhaps it shouldn't be gen6 specific */
1347                 if (i915_is_ggtt(vm)) {
1348                         if (dev_priv->mm.aliasing_ppgtt)
1349                                 gen6_write_pdes(dev_priv->mm.aliasing_ppgtt);
1350                         continue;
1351                 }
1352
1353                 gen6_write_pdes(container_of(vm, struct i915_hw_ppgtt, base));
1354         }
1355
1356         i915_ggtt_flush(dev_priv);
1357 }
1358
1359 int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
1360 {
1361         if (obj->has_dma_mapping)
1362                 return 0;
1363
1364         if (!dma_map_sg(&obj->base.dev->pdev->dev,
1365                         obj->pages->sgl, obj->pages->nents,
1366                         PCI_DMA_BIDIRECTIONAL))
1367                 return -ENOSPC;
1368
1369         return 0;
1370 }
1371
1372 static inline void gen8_set_pte(void __iomem *addr, gen8_gtt_pte_t pte)
1373 {
1374 #ifdef writeq
1375         writeq(pte, addr);
1376 #else
1377         iowrite32((u32)pte, addr);
1378         iowrite32(pte >> 32, addr + 4);
1379 #endif
1380 }
1381
1382 static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
1383                                      struct sg_table *st,
1384                                      uint64_t start,
1385                                      enum i915_cache_level level, u32 unused)
1386 {
1387         struct drm_i915_private *dev_priv = vm->dev->dev_private;
1388         unsigned first_entry = start >> PAGE_SHIFT;
1389         gen8_gtt_pte_t __iomem *gtt_entries =
1390                 (gen8_gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;
1391         int i = 0;
1392         struct sg_page_iter sg_iter;
1393         dma_addr_t addr = 0; /* shut up gcc */
1394
1395         for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) {
1396                 addr = sg_dma_address(sg_iter.sg) +
1397                         (sg_iter.sg_pgoffset << PAGE_SHIFT);
1398                 gen8_set_pte(&gtt_entries[i],
1399                              gen8_pte_encode(addr, level, true));
1400                 i++;
1401         }
1402
1403         /*
1404          * XXX: This serves as a posting read to make sure that the PTE has
1405          * actually been updated. There is some concern that even though
1406          * registers and PTEs are within the same BAR that they are potentially
1407          * of NUMA access patterns. Therefore, even with the way we assume
1408          * hardware should work, we must keep this posting read for paranoia.
1409          */
1410         if (i != 0)
1411                 WARN_ON(readq(&gtt_entries[i-1])
1412                         != gen8_pte_encode(addr, level, true));
1413
1414         /* This next bit makes the above posting read even more important. We
1415          * want to flush the TLBs only after we're certain all the PTE updates
1416          * have finished.
1417          */
1418         I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
1419         POSTING_READ(GFX_FLSH_CNTL_GEN6);
1420 }
1421
1422 /*
1423  * Binds an object into the global gtt with the specified cache level. The object
1424  * will be accessible to the GPU via commands whose operands reference offsets
1425  * within the global GTT as well as accessible by the GPU through the GMADR
1426  * mapped BAR (dev_priv->mm.gtt->gtt).
1427  */
1428 static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
1429                                      struct sg_table *st,
1430                                      uint64_t start,
1431                                      enum i915_cache_level level, u32 flags)
1432 {
1433         struct drm_i915_private *dev_priv = vm->dev->dev_private;
1434         unsigned first_entry = start >> PAGE_SHIFT;
1435         gen6_gtt_pte_t __iomem *gtt_entries =
1436                 (gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;
1437         int i = 0;
1438         struct sg_page_iter sg_iter;
1439         dma_addr_t addr = 0;
1440
1441         for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) {
1442                 addr = sg_page_iter_dma_address(&sg_iter);
1443                 iowrite32(vm->pte_encode(addr, level, true, flags), &gtt_entries[i]);
1444                 i++;
1445         }
1446
1447         /* XXX: This serves as a posting read to make sure that the PTE has
1448          * actually been updated. There is some concern that even though
1449          * registers and PTEs are within the same BAR that they are potentially
1450          * of NUMA access patterns. Therefore, even with the way we assume
1451          * hardware should work, we must keep this posting read for paranoia.
1452          */
1453         if (i != 0) {
1454                 unsigned long gtt = readl(&gtt_entries[i-1]);
1455                 WARN_ON(gtt != vm->pte_encode(addr, level, true, flags));
1456         }
1457
1458         /* This next bit makes the above posting read even more important. We
1459          * want to flush the TLBs only after we're certain all the PTE updates
1460          * have finished.
1461          */
1462         I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
1463         POSTING_READ(GFX_FLSH_CNTL_GEN6);
1464 }
1465
1466 static void gen8_ggtt_clear_range(struct i915_address_space *vm,
1467                                   uint64_t start,
1468                                   uint64_t length,
1469                                   bool use_scratch)
1470 {
1471         struct drm_i915_private *dev_priv = vm->dev->dev_private;
1472         unsigned first_entry = start >> PAGE_SHIFT;
1473         unsigned num_entries = length >> PAGE_SHIFT;
1474         gen8_gtt_pte_t scratch_pte, __iomem *gtt_base =
1475                 (gen8_gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
1476         const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
1477         int i;
1478
1479         if (WARN(num_entries > max_entries,
1480                  "First entry = %d; Num entries = %d (max=%d)\n",
1481                  first_entry, num_entries, max_entries))
1482                 num_entries = max_entries;
1483
1484         scratch_pte = gen8_pte_encode(vm->scratch.addr,
1485                                       I915_CACHE_LLC,
1486                                       use_scratch);
1487         for (i = 0; i < num_entries; i++)
1488                 gen8_set_pte(&gtt_base[i], scratch_pte);
1489         readl(gtt_base);
1490 }
1491
1492 static void gen6_ggtt_clear_range(struct i915_address_space *vm,
1493                                   uint64_t start,
1494                                   uint64_t length,
1495                                   bool use_scratch)
1496 {
1497         struct drm_i915_private *dev_priv = vm->dev->dev_private;
1498         unsigned first_entry = start >> PAGE_SHIFT;
1499         unsigned num_entries = length >> PAGE_SHIFT;
1500         gen6_gtt_pte_t scratch_pte, __iomem *gtt_base =
1501                 (gen6_gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
1502         const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
1503         int i;
1504
1505         if (WARN(num_entries > max_entries,
1506                  "First entry = %d; Num entries = %d (max=%d)\n",
1507                  first_entry, num_entries, max_entries))
1508                 num_entries = max_entries;
1509
1510         scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC, use_scratch, 0);
1511
1512         for (i = 0; i < num_entries; i++)
1513                 iowrite32(scratch_pte, &gtt_base[i]);
1514         readl(gtt_base);
1515 }
1516
1517
1518 static void i915_ggtt_bind_vma(struct i915_vma *vma,
1519                                enum i915_cache_level cache_level,
1520                                u32 unused)
1521 {
1522         const unsigned long entry = vma->node.start >> PAGE_SHIFT;
1523         unsigned int flags = (cache_level == I915_CACHE_NONE) ?
1524                 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
1525
1526         BUG_ON(!i915_is_ggtt(vma->vm));
1527         intel_gtt_insert_sg_entries(vma->obj->pages, entry, flags);
1528         vma->obj->has_global_gtt_mapping = 1;
1529 }
1530
1531 static void i915_ggtt_clear_range(struct i915_address_space *vm,
1532                                   uint64_t start,
1533                                   uint64_t length,
1534                                   bool unused)
1535 {
1536         unsigned first_entry = start >> PAGE_SHIFT;
1537         unsigned num_entries = length >> PAGE_SHIFT;
1538         intel_gtt_clear_range(first_entry, num_entries);
1539 }
1540
1541 static void i915_ggtt_unbind_vma(struct i915_vma *vma)
1542 {
1543         const unsigned int first = vma->node.start >> PAGE_SHIFT;
1544         const unsigned int size = vma->obj->base.size >> PAGE_SHIFT;
1545
1546         BUG_ON(!i915_is_ggtt(vma->vm));
1547         vma->obj->has_global_gtt_mapping = 0;
1548         intel_gtt_clear_range(first, size);
1549 }
1550
1551 static void ggtt_bind_vma(struct i915_vma *vma,
1552                           enum i915_cache_level cache_level,
1553                           u32 flags)
1554 {
1555         struct drm_device *dev = vma->vm->dev;
1556         struct drm_i915_private *dev_priv = dev->dev_private;
1557         struct drm_i915_gem_object *obj = vma->obj;
1558
1559         /* Currently applicable only to VLV */
1560         if (obj->gt_ro)
1561                 flags |= PTE_READ_ONLY;
1562
1563         /* If there is no aliasing PPGTT, or the caller needs a global mapping,
1564          * or we have a global mapping already but the cacheability flags have
1565          * changed, set the global PTEs.
1566          *
1567          * If there is an aliasing PPGTT it is anecdotally faster, so use that
1568          * instead if none of the above hold true.
1569          *
1570          * NB: A global mapping should only be needed for special regions like
1571          * "gtt mappable", SNB errata, or if specified via special execbuf
1572          * flags. At all other times, the GPU will use the aliasing PPGTT.
1573          */
1574         if (!dev_priv->mm.aliasing_ppgtt || flags & GLOBAL_BIND) {
1575                 if (!obj->has_global_gtt_mapping ||
1576                     (cache_level != obj->cache_level)) {
1577                         vma->vm->insert_entries(vma->vm, obj->pages,
1578                                                 vma->node.start,
1579                                                 cache_level, flags);
1580                         obj->has_global_gtt_mapping = 1;
1581                 }
1582         }
1583
1584         if (dev_priv->mm.aliasing_ppgtt &&
1585             (!obj->has_aliasing_ppgtt_mapping ||
1586              (cache_level != obj->cache_level))) {
1587                 struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt;
1588                 appgtt->base.insert_entries(&appgtt->base,
1589                                             vma->obj->pages,
1590                                             vma->node.start,
1591                                             cache_level, flags);
1592                 vma->obj->has_aliasing_ppgtt_mapping = 1;
1593         }
1594 }
1595
1596 static void ggtt_unbind_vma(struct i915_vma *vma)
1597 {
1598         struct drm_device *dev = vma->vm->dev;
1599         struct drm_i915_private *dev_priv = dev->dev_private;
1600         struct drm_i915_gem_object *obj = vma->obj;
1601
1602         if (obj->has_global_gtt_mapping) {
1603                 vma->vm->clear_range(vma->vm,
1604                                      vma->node.start,
1605                                      obj->base.size,
1606                                      true);
1607                 obj->has_global_gtt_mapping = 0;
1608         }
1609
1610         if (obj->has_aliasing_ppgtt_mapping) {
1611                 struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt;
1612                 appgtt->base.clear_range(&appgtt->base,
1613                                          vma->node.start,
1614                                          obj->base.size,
1615                                          true);
1616                 obj->has_aliasing_ppgtt_mapping = 0;
1617         }
1618 }
1619
1620 void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
1621 {
1622         struct drm_device *dev = obj->base.dev;
1623         struct drm_i915_private *dev_priv = dev->dev_private;
1624         bool interruptible;
1625
1626         interruptible = do_idling(dev_priv);
1627
1628         if (!obj->has_dma_mapping)
1629                 dma_unmap_sg(&dev->pdev->dev,
1630                              obj->pages->sgl, obj->pages->nents,
1631                              PCI_DMA_BIDIRECTIONAL);
1632
1633         undo_idling(dev_priv, interruptible);
1634 }
1635
1636 static void i915_gtt_color_adjust(struct drm_mm_node *node,
1637                                   unsigned long color,
1638                                   unsigned long *start,
1639                                   unsigned long *end)
1640 {
1641         if (node->color != color)
1642                 *start += 4096;
1643
1644         if (!list_empty(&node->node_list)) {
1645                 node = list_entry(node->node_list.next,
1646                                   struct drm_mm_node,
1647                                   node_list);
1648                 if (node->allocated && node->color != color)
1649                         *end -= 4096;
1650         }
1651 }
1652
1653 int i915_gem_setup_global_gtt(struct drm_device *dev,
1654                               unsigned long start,
1655                               unsigned long mappable_end,
1656                               unsigned long end)
1657 {
1658         /* Let GEM Manage all of the aperture.
1659          *
1660          * However, leave one page at the end still bound to the scratch page.
1661          * There are a number of places where the hardware apparently prefetches
1662          * past the end of the object, and we've seen multiple hangs with the
1663          * GPU head pointer stuck in a batchbuffer bound at the last page of the
1664          * aperture.  One page should be enough to keep any prefetching inside
1665          * of the aperture.
1666          */
1667         struct drm_i915_private *dev_priv = dev->dev_private;
1668         struct i915_address_space *ggtt_vm = &dev_priv->gtt.base;
1669         struct drm_mm_node *entry;
1670         struct drm_i915_gem_object *obj;
1671         unsigned long hole_start, hole_end;
1672         int ret;
1673
1674         BUG_ON(mappable_end > end);
1675
1676         /* Subtract the guard page ... */
1677         drm_mm_init(&ggtt_vm->mm, start, end - start - PAGE_SIZE);
1678         if (!HAS_LLC(dev))
1679                 dev_priv->gtt.base.mm.color_adjust = i915_gtt_color_adjust;
1680
1681         /* Mark any preallocated objects as occupied */
1682         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
1683                 struct i915_vma *vma = i915_gem_obj_to_vma(obj, ggtt_vm);
1684
1685                 DRM_DEBUG_KMS("reserving preallocated space: %lx + %zx\n",
1686                               i915_gem_obj_ggtt_offset(obj), obj->base.size);
1687
1688                 WARN_ON(i915_gem_obj_ggtt_bound(obj));
1689                 ret = drm_mm_reserve_node(&ggtt_vm->mm, &vma->node);
1690                 if (ret) {
1691                         DRM_DEBUG_KMS("Reservation failed: %i\n", ret);
1692                         return ret;
1693                 }
1694                 obj->has_global_gtt_mapping = 1;
1695         }
1696
1697         dev_priv->gtt.base.start = start;
1698         dev_priv->gtt.base.total = end - start;
1699
1700         /* Clear any non-preallocated blocks */
1701         drm_mm_for_each_hole(entry, &ggtt_vm->mm, hole_start, hole_end) {
1702                 DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n",
1703                               hole_start, hole_end);
1704                 ggtt_vm->clear_range(ggtt_vm, hole_start,
1705                                      hole_end - hole_start, true);
1706         }
1707
1708         /* And finally clear the reserved guard page */
1709         ggtt_vm->clear_range(ggtt_vm, end - PAGE_SIZE, PAGE_SIZE, true);
1710
1711         if (USES_PPGTT(dev) && !USES_FULL_PPGTT(dev)) {
1712                 struct i915_hw_ppgtt *ppgtt;
1713
1714                 ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
1715                 if (!ppgtt)
1716                         return -ENOMEM;
1717
1718                 ret = __hw_ppgtt_init(dev, ppgtt);
1719                 if (ret != 0)
1720                         return ret;
1721
1722                 dev_priv->mm.aliasing_ppgtt = ppgtt;
1723         }
1724
1725         return 0;
1726 }
1727
1728 void i915_gem_init_global_gtt(struct drm_device *dev)
1729 {
1730         struct drm_i915_private *dev_priv = dev->dev_private;
1731         unsigned long gtt_size, mappable_size;
1732
1733         gtt_size = dev_priv->gtt.base.total;
1734         mappable_size = dev_priv->gtt.mappable_end;
1735
1736         i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size);
1737 }
1738
1739 void i915_global_gtt_cleanup(struct drm_device *dev)
1740 {
1741         struct drm_i915_private *dev_priv = dev->dev_private;
1742         struct i915_address_space *vm = &dev_priv->gtt.base;
1743
1744         if (dev_priv->mm.aliasing_ppgtt) {
1745                 struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
1746
1747                 ppgtt->base.cleanup(&ppgtt->base);
1748         }
1749
1750         if (drm_mm_initialized(&vm->mm)) {
1751                 drm_mm_takedown(&vm->mm);
1752                 list_del(&vm->global_link);
1753         }
1754
1755         vm->cleanup(vm);
1756 }
1757
1758 static int setup_scratch_page(struct drm_device *dev)
1759 {
1760         struct drm_i915_private *dev_priv = dev->dev_private;
1761         struct page *page;
1762         dma_addr_t dma_addr;
1763
1764         page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO);
1765         if (page == NULL)
1766                 return -ENOMEM;
1767         get_page(page);
1768         set_pages_uc(page, 1);
1769
1770 #ifdef CONFIG_INTEL_IOMMU
1771         dma_addr = pci_map_page(dev->pdev, page, 0, PAGE_SIZE,
1772                                 PCI_DMA_BIDIRECTIONAL);
1773         if (pci_dma_mapping_error(dev->pdev, dma_addr))
1774                 return -EINVAL;
1775 #else
1776         dma_addr = page_to_phys(page);
1777 #endif
1778         dev_priv->gtt.base.scratch.page = page;
1779         dev_priv->gtt.base.scratch.addr = dma_addr;
1780
1781         return 0;
1782 }
1783
1784 static void teardown_scratch_page(struct drm_device *dev)
1785 {
1786         struct drm_i915_private *dev_priv = dev->dev_private;
1787         struct page *page = dev_priv->gtt.base.scratch.page;
1788
1789         set_pages_wb(page, 1);
1790         pci_unmap_page(dev->pdev, dev_priv->gtt.base.scratch.addr,
1791                        PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
1792         put_page(page);
1793         __free_page(page);
1794 }
1795
1796 static inline unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
1797 {
1798         snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT;
1799         snb_gmch_ctl &= SNB_GMCH_GGMS_MASK;
1800         return snb_gmch_ctl << 20;
1801 }
1802
1803 static inline unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl)
1804 {
1805         bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT;
1806         bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK;
1807         if (bdw_gmch_ctl)
1808                 bdw_gmch_ctl = 1 << bdw_gmch_ctl;
1809
1810 #ifdef CONFIG_X86_32
1811         /* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * PAGE_SIZE */
1812         if (bdw_gmch_ctl > 4)
1813                 bdw_gmch_ctl = 4;
1814 #endif
1815
1816         return bdw_gmch_ctl << 20;
1817 }
1818
1819 static inline unsigned int chv_get_total_gtt_size(u16 gmch_ctrl)
1820 {
1821         gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT;
1822         gmch_ctrl &= SNB_GMCH_GGMS_MASK;
1823
1824         if (gmch_ctrl)
1825                 return 1 << (20 + gmch_ctrl);
1826
1827         return 0;
1828 }
1829
1830 static inline size_t gen6_get_stolen_size(u16 snb_gmch_ctl)
1831 {
1832         snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT;
1833         snb_gmch_ctl &= SNB_GMCH_GMS_MASK;
1834         return snb_gmch_ctl << 25; /* 32 MB units */
1835 }
1836
1837 static inline size_t gen8_get_stolen_size(u16 bdw_gmch_ctl)
1838 {
1839         bdw_gmch_ctl >>= BDW_GMCH_GMS_SHIFT;
1840         bdw_gmch_ctl &= BDW_GMCH_GMS_MASK;
1841         return bdw_gmch_ctl << 25; /* 32 MB units */
1842 }
1843
1844 static size_t chv_get_stolen_size(u16 gmch_ctrl)
1845 {
1846         gmch_ctrl >>= SNB_GMCH_GMS_SHIFT;
1847         gmch_ctrl &= SNB_GMCH_GMS_MASK;
1848
1849         /*
1850          * 0x0  to 0x10: 32MB increments starting at 0MB
1851          * 0x11 to 0x16: 4MB increments starting at 8MB
1852          * 0x17 to 0x1d: 4MB increments start at 36MB
1853          */
1854         if (gmch_ctrl < 0x11)
1855                 return gmch_ctrl << 25;
1856         else if (gmch_ctrl < 0x17)
1857                 return (gmch_ctrl - 0x11 + 2) << 22;
1858         else
1859                 return (gmch_ctrl - 0x17 + 9) << 22;
1860 }
1861
1862 static int ggtt_probe_common(struct drm_device *dev,
1863                              size_t gtt_size)
1864 {
1865         struct drm_i915_private *dev_priv = dev->dev_private;
1866         phys_addr_t gtt_phys_addr;
1867         int ret;
1868
1869         /* For Modern GENs the PTEs and register space are split in the BAR */
1870         gtt_phys_addr = pci_resource_start(dev->pdev, 0) +
1871                 (pci_resource_len(dev->pdev, 0) / 2);
1872
1873         dev_priv->gtt.gsm = ioremap_wc(gtt_phys_addr, gtt_size);
1874         if (!dev_priv->gtt.gsm) {
1875                 DRM_ERROR("Failed to map the gtt page table\n");
1876                 return -ENOMEM;
1877         }
1878
1879         ret = setup_scratch_page(dev);
1880         if (ret) {
1881                 DRM_ERROR("Scratch setup failed\n");
1882                 /* iounmap will also get called at remove, but meh */
1883                 iounmap(dev_priv->gtt.gsm);
1884         }
1885
1886         return ret;
1887 }
1888
1889 /* The GGTT and PPGTT need a private PPAT setup in order to handle cacheability
1890  * bits. When using advanced contexts each context stores its own PAT, but
1891  * writing this data shouldn't be harmful even in those cases. */
1892 static void bdw_setup_private_ppat(struct drm_i915_private *dev_priv)
1893 {
1894         uint64_t pat;
1895
1896         pat = GEN8_PPAT(0, GEN8_PPAT_WB | GEN8_PPAT_LLC)     | /* for normal objects, no eLLC */
1897               GEN8_PPAT(1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC) | /* for something pointing to ptes? */
1898               GEN8_PPAT(2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC) | /* for scanout with eLLC */
1899               GEN8_PPAT(3, GEN8_PPAT_UC)                     | /* Uncached objects, mostly for scanout */
1900               GEN8_PPAT(4, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0)) |
1901               GEN8_PPAT(5, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1)) |
1902               GEN8_PPAT(6, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2)) |
1903               GEN8_PPAT(7, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3));
1904
1905         /* XXX: spec defines this as 2 distinct registers. It's unclear if a 64b
1906          * write would work. */
1907         I915_WRITE(GEN8_PRIVATE_PAT, pat);
1908         I915_WRITE(GEN8_PRIVATE_PAT + 4, pat >> 32);
1909 }
1910
1911 static void chv_setup_private_ppat(struct drm_i915_private *dev_priv)
1912 {
1913         uint64_t pat;
1914
1915         /*
1916          * Map WB on BDW to snooped on CHV.
1917          *
1918          * Only the snoop bit has meaning for CHV, the rest is
1919          * ignored.
1920          *
1921          * Note that the harware enforces snooping for all page
1922          * table accesses. The snoop bit is actually ignored for
1923          * PDEs.
1924          */
1925         pat = GEN8_PPAT(0, CHV_PPAT_SNOOP) |
1926               GEN8_PPAT(1, 0) |
1927               GEN8_PPAT(2, 0) |
1928               GEN8_PPAT(3, 0) |
1929               GEN8_PPAT(4, CHV_PPAT_SNOOP) |
1930               GEN8_PPAT(5, CHV_PPAT_SNOOP) |
1931               GEN8_PPAT(6, CHV_PPAT_SNOOP) |
1932               GEN8_PPAT(7, CHV_PPAT_SNOOP);
1933
1934         I915_WRITE(GEN8_PRIVATE_PAT, pat);
1935         I915_WRITE(GEN8_PRIVATE_PAT + 4, pat >> 32);
1936 }
1937
1938 static int gen8_gmch_probe(struct drm_device *dev,
1939                            size_t *gtt_total,
1940                            size_t *stolen,
1941                            phys_addr_t *mappable_base,
1942                            unsigned long *mappable_end)
1943 {
1944         struct drm_i915_private *dev_priv = dev->dev_private;
1945         unsigned int gtt_size;
1946         u16 snb_gmch_ctl;
1947         int ret;
1948
1949         /* TODO: We're not aware of mappable constraints on gen8 yet */
1950         *mappable_base = pci_resource_start(dev->pdev, 2);
1951         *mappable_end = pci_resource_len(dev->pdev, 2);
1952
1953         if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(39)))
1954                 pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(39));
1955
1956         pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
1957
1958         if (IS_CHERRYVIEW(dev)) {
1959                 *stolen = chv_get_stolen_size(snb_gmch_ctl);
1960                 gtt_size = chv_get_total_gtt_size(snb_gmch_ctl);
1961         } else {
1962                 *stolen = gen8_get_stolen_size(snb_gmch_ctl);
1963                 gtt_size = gen8_get_total_gtt_size(snb_gmch_ctl);
1964         }
1965
1966         *gtt_total = (gtt_size / sizeof(gen8_gtt_pte_t)) << PAGE_SHIFT;
1967
1968         if (IS_CHERRYVIEW(dev))
1969                 chv_setup_private_ppat(dev_priv);
1970         else
1971                 bdw_setup_private_ppat(dev_priv);
1972
1973         ret = ggtt_probe_common(dev, gtt_size);
1974
1975         dev_priv->gtt.base.clear_range = gen8_ggtt_clear_range;
1976         dev_priv->gtt.base.insert_entries = gen8_ggtt_insert_entries;
1977
1978         return ret;
1979 }
1980
1981 static int gen6_gmch_probe(struct drm_device *dev,
1982                            size_t *gtt_total,
1983                            size_t *stolen,
1984                            phys_addr_t *mappable_base,
1985                            unsigned long *mappable_end)
1986 {
1987         struct drm_i915_private *dev_priv = dev->dev_private;
1988         unsigned int gtt_size;
1989         u16 snb_gmch_ctl;
1990         int ret;
1991
1992         *mappable_base = pci_resource_start(dev->pdev, 2);
1993         *mappable_end = pci_resource_len(dev->pdev, 2);
1994
1995         /* 64/512MB is the current min/max we actually know of, but this is just
1996          * a coarse sanity check.
1997          */
1998         if ((*mappable_end < (64<<20) || (*mappable_end > (512<<20)))) {
1999                 DRM_ERROR("Unknown GMADR size (%lx)\n",
2000                           dev_priv->gtt.mappable_end);
2001                 return -ENXIO;
2002         }
2003
2004         if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(40)))
2005                 pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(40));
2006         pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
2007
2008         *stolen = gen6_get_stolen_size(snb_gmch_ctl);
2009
2010         gtt_size = gen6_get_total_gtt_size(snb_gmch_ctl);
2011         *gtt_total = (gtt_size / sizeof(gen6_gtt_pte_t)) << PAGE_SHIFT;
2012
2013         ret = ggtt_probe_common(dev, gtt_size);
2014
2015         dev_priv->gtt.base.clear_range = gen6_ggtt_clear_range;
2016         dev_priv->gtt.base.insert_entries = gen6_ggtt_insert_entries;
2017
2018         return ret;
2019 }
2020
2021 static void gen6_gmch_remove(struct i915_address_space *vm)
2022 {
2023
2024         struct i915_gtt *gtt = container_of(vm, struct i915_gtt, base);
2025
2026         iounmap(gtt->gsm);
2027         teardown_scratch_page(vm->dev);
2028 }
2029
2030 static int i915_gmch_probe(struct drm_device *dev,
2031                            size_t *gtt_total,
2032                            size_t *stolen,
2033                            phys_addr_t *mappable_base,
2034                            unsigned long *mappable_end)
2035 {
2036         struct drm_i915_private *dev_priv = dev->dev_private;
2037         int ret;
2038
2039         ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->dev->pdev, NULL);
2040         if (!ret) {
2041                 DRM_ERROR("failed to set up gmch\n");
2042                 return -EIO;
2043         }
2044
2045         intel_gtt_get(gtt_total, stolen, mappable_base, mappable_end);
2046
2047         dev_priv->gtt.do_idle_maps = needs_idle_maps(dev_priv->dev);
2048         dev_priv->gtt.base.clear_range = i915_ggtt_clear_range;
2049
2050         if (unlikely(dev_priv->gtt.do_idle_maps))
2051                 DRM_INFO("applying Ironlake quirks for intel_iommu\n");
2052
2053         return 0;
2054 }
2055
2056 static void i915_gmch_remove(struct i915_address_space *vm)
2057 {
2058         intel_gmch_remove();
2059 }
2060
2061 int i915_gem_gtt_init(struct drm_device *dev)
2062 {
2063         struct drm_i915_private *dev_priv = dev->dev_private;
2064         struct i915_gtt *gtt = &dev_priv->gtt;
2065         int ret;
2066
2067         if (INTEL_INFO(dev)->gen <= 5) {
2068                 gtt->gtt_probe = i915_gmch_probe;
2069                 gtt->base.cleanup = i915_gmch_remove;
2070         } else if (INTEL_INFO(dev)->gen < 8) {
2071                 gtt->gtt_probe = gen6_gmch_probe;
2072                 gtt->base.cleanup = gen6_gmch_remove;
2073                 if (IS_HASWELL(dev) && dev_priv->ellc_size)
2074                         gtt->base.pte_encode = iris_pte_encode;
2075                 else if (IS_HASWELL(dev))
2076                         gtt->base.pte_encode = hsw_pte_encode;
2077                 else if (IS_VALLEYVIEW(dev))
2078                         gtt->base.pte_encode = byt_pte_encode;
2079                 else if (INTEL_INFO(dev)->gen >= 7)
2080                         gtt->base.pte_encode = ivb_pte_encode;
2081                 else
2082                         gtt->base.pte_encode = snb_pte_encode;
2083         } else {
2084                 dev_priv->gtt.gtt_probe = gen8_gmch_probe;
2085                 dev_priv->gtt.base.cleanup = gen6_gmch_remove;
2086         }
2087
2088         ret = gtt->gtt_probe(dev, &gtt->base.total, &gtt->stolen_size,
2089                              &gtt->mappable_base, &gtt->mappable_end);
2090         if (ret)
2091                 return ret;
2092
2093         gtt->base.dev = dev;
2094
2095         /* GMADR is the PCI mmio aperture into the global GTT. */
2096         DRM_INFO("Memory usable by graphics device = %zdM\n",
2097                  gtt->base.total >> 20);
2098         DRM_DEBUG_DRIVER("GMADR size = %ldM\n", gtt->mappable_end >> 20);
2099         DRM_DEBUG_DRIVER("GTT stolen size = %zdM\n", gtt->stolen_size >> 20);
2100 #ifdef CONFIG_INTEL_IOMMU
2101         if (intel_iommu_gfx_mapped)
2102                 DRM_INFO("VT-d active for gfx access\n");
2103 #endif
2104         /*
2105          * i915.enable_ppgtt is read-only, so do an early pass to validate the
2106          * user's requested state against the hardware/driver capabilities.  We
2107          * do this now so that we can print out any log messages once rather
2108          * than every time we check intel_enable_ppgtt().
2109          */
2110         i915.enable_ppgtt = sanitize_enable_ppgtt(dev, i915.enable_ppgtt);
2111         DRM_DEBUG_DRIVER("ppgtt mode: %i\n", i915.enable_ppgtt);
2112
2113         return 0;
2114 }
2115
2116 static struct i915_vma *__i915_gem_vma_create(struct drm_i915_gem_object *obj,
2117                                               struct i915_address_space *vm)
2118 {
2119         struct i915_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL);
2120         if (vma == NULL)
2121                 return ERR_PTR(-ENOMEM);
2122
2123         INIT_LIST_HEAD(&vma->vma_link);
2124         INIT_LIST_HEAD(&vma->mm_list);
2125         INIT_LIST_HEAD(&vma->exec_list);
2126         vma->vm = vm;
2127         vma->obj = obj;
2128
2129         switch (INTEL_INFO(vm->dev)->gen) {
2130         case 8:
2131         case 7:
2132         case 6:
2133                 if (i915_is_ggtt(vm)) {
2134                         vma->unbind_vma = ggtt_unbind_vma;
2135                         vma->bind_vma = ggtt_bind_vma;
2136                 } else {
2137                         vma->unbind_vma = ppgtt_unbind_vma;
2138                         vma->bind_vma = ppgtt_bind_vma;
2139                 }
2140                 break;
2141         case 5:
2142         case 4:
2143         case 3:
2144         case 2:
2145                 BUG_ON(!i915_is_ggtt(vm));
2146                 vma->unbind_vma = i915_ggtt_unbind_vma;
2147                 vma->bind_vma = i915_ggtt_bind_vma;
2148                 break;
2149         default:
2150                 BUG();
2151         }
2152
2153         /* Keep GGTT vmas first to make debug easier */
2154         if (i915_is_ggtt(vm))
2155                 list_add(&vma->vma_link, &obj->vma_list);
2156         else {
2157                 list_add_tail(&vma->vma_link, &obj->vma_list);
2158                 i915_ppgtt_get(i915_vm_to_ppgtt(vm));
2159         }
2160
2161         return vma;
2162 }
2163
2164 struct i915_vma *
2165 i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj,
2166                                   struct i915_address_space *vm)
2167 {
2168         struct i915_vma *vma;
2169
2170         vma = i915_gem_obj_to_vma(obj, vm);
2171         if (!vma)
2172                 vma = __i915_gem_vma_create(obj, vm);
2173
2174         return vma;
2175 }