drivers/char/agp/intel-gtt.c

   1 /*
   2  * Intel GTT (Graphics Translation Table) routines
   3  *
   4  * Caveat: This driver implements the linux agp interface, but this is far from
   5  * a agp driver! GTT support ended up here for purely historical reasons: The
   6  * old userspace intel graphics drivers needed an interface to map memory into
   7  * the GTT. And the drm provides a default interface for graphic devices sitting
   8  * on an agp port. So it made sense to fake the GTT support as an agp port to
   9  * avoid having to create a new api.
  10  *
  11  * With gem this does not make much sense anymore, just needlessly complicates
  12  * the code. But as long as the old graphics stack is still support, it's stuck
  13  * here.
  14  *
  15  * /fairy-tale-mode off
  16  */
  17
  18 #include <linux/module.h>
  19 #include <linux/pci.h>
  20 #include <linux/init.h>
  21 #include <linux/kernel.h>
  22 #include <linux/pagemap.h>
  23 #include <linux/agp_backend.h>
  24 #include <asm/smp.h>
  25 #include "agp.h"
  26 #include "intel-agp.h"
  27 #include <linux/intel-gtt.h>
  28 #include <drm/intel-gtt.h>
  29
  30 /*
  31  * If we have Intel graphics, we're not going to have anything other than
  32  * an Intel IOMMU. So make the correct use of the PCI DMA API contingent
  33  * on the Intel IOMMU support (CONFIG_DMAR).
  34  * Only newer chipsets need to bother with this, of course.
  35  */
  36 #ifdef CONFIG_DMAR
  37 #define USE_PCI_DMA_API 1
  38 #endif
  39
  40 /* Max amount of stolen space, anything above will be returned to Linux */
  41 int intel_max_stolen = 32 * 1024 * 1024;
  42 EXPORT_SYMBOL(intel_max_stolen);
  43
  44 static const struct aper_size_info_fixed intel_i810_sizes[] =
  45 {
  46         {64, 16384, 4},
  47         /* The 32M mode still requires a 64k gatt */
  48         {32, 8192, 4}
  49 };
  50
  51 #define AGP_DCACHE_MEMORY       1
  52 #define AGP_PHYS_MEMORY         2
  53 #define INTEL_AGP_CACHED_MEMORY 3
  54
  55 static struct gatt_mask intel_i810_masks[] =
  56 {
  57         {.mask = I810_PTE_VALID, .type = 0},
  58         {.mask = (I810_PTE_VALID | I810_PTE_LOCAL), .type = AGP_DCACHE_MEMORY},
  59         {.mask = I810_PTE_VALID, .type = 0},
  60         {.mask = I810_PTE_VALID | I830_PTE_SYSTEM_CACHED,
  61          .type = INTEL_AGP_CACHED_MEMORY}
  62 };
  63
  64 #define INTEL_AGP_UNCACHED_MEMORY              0
  65 #define INTEL_AGP_CACHED_MEMORY_LLC            1
  66 #define INTEL_AGP_CACHED_MEMORY_LLC_GFDT       2
  67 #define INTEL_AGP_CACHED_MEMORY_LLC_MLC        3
  68 #define INTEL_AGP_CACHED_MEMORY_LLC_MLC_GFDT   4
  69
  70 static struct gatt_mask intel_gen6_masks[] =
  71 {
  72         {.mask = I810_PTE_VALID | GEN6_PTE_UNCACHED,
  73          .type = INTEL_AGP_UNCACHED_MEMORY },
  74         {.mask = I810_PTE_VALID | GEN6_PTE_LLC,
  75          .type = INTEL_AGP_CACHED_MEMORY_LLC },
  76         {.mask = I810_PTE_VALID | GEN6_PTE_LLC | GEN6_PTE_GFDT,
  77          .type = INTEL_AGP_CACHED_MEMORY_LLC_GFDT },
  78         {.mask = I810_PTE_VALID | GEN6_PTE_LLC_MLC,
  79          .type = INTEL_AGP_CACHED_MEMORY_LLC_MLC },
  80         {.mask = I810_PTE_VALID | GEN6_PTE_LLC_MLC | GEN6_PTE_GFDT,
  81          .type = INTEL_AGP_CACHED_MEMORY_LLC_MLC_GFDT },
  82 };
  83
  84 struct intel_gtt_driver {
  85         unsigned int gen : 8;
  86         unsigned int is_g33 : 1;
  87         unsigned int is_pineview : 1;
  88         unsigned int is_ironlake : 1;
  89         /* Chipset specific GTT setup */
  90         int (*setup)(void);
  91 };
  92
  93 static struct _intel_private {
  94         struct intel_gtt base;
  95         const struct intel_gtt_driver *driver;
  96         struct pci_dev *pcidev; /* device one */
  97         struct pci_dev *bridge_dev;
  98         u8 __iomem *registers;
  99         phys_addr_t gtt_bus_addr;
 100         phys_addr_t gma_bus_addr;
 101         phys_addr_t pte_bus_addr;
 102         u32 __iomem *gtt;               /* I915G */
 103         int num_dcache_entries;
 104         union {
 105                 void __iomem *i9xx_flush_page;
 106                 void *i8xx_flush_page;
 107         };
 108         struct page *i8xx_page;
 109         struct resource ifp_resource;
 110         int resource_valid;
 111 } intel_private;
 112
 113 #define INTEL_GTT_GEN   intel_private.driver->gen
 114 #define IS_G33          intel_private.driver->is_g33
 115 #define IS_PINEVIEW     intel_private.driver->is_pineview
 116 #define IS_IRONLAKE     intel_private.driver->is_ironlake
 117
 118 #ifdef USE_PCI_DMA_API
 119 static int intel_agp_map_page(struct page *page, dma_addr_t *ret)
 120 {
 121         *ret = pci_map_page(intel_private.pcidev, page, 0,
 122                             PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 123         if (pci_dma_mapping_error(intel_private.pcidev, *ret))
 124                 return -EINVAL;
 125         return 0;
 126 }
 127
 128 static void intel_agp_unmap_page(struct page *page, dma_addr_t dma)
 129 {
 130         pci_unmap_page(intel_private.pcidev, dma,
 131                        PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 132 }
 133
 134 static void intel_agp_free_sglist(struct agp_memory *mem)
 135 {
 136         struct sg_table st;
 137
 138         st.sgl = mem->sg_list;
 139         st.orig_nents = st.nents = mem->page_count;
 140
 141         sg_free_table(&st);
 142
 143         mem->sg_list = NULL;
 144         mem->num_sg = 0;
 145 }
 146
 147 static int intel_agp_map_memory(struct agp_memory *mem)
 148 {
 149         struct sg_table st;
 150         struct scatterlist *sg;
 151         int i;
 152
 153         DBG("try mapping %lu pages\n", (unsigned long)mem->page_count);
 154
 155         if (sg_alloc_table(&st, mem->page_count, GFP_KERNEL))
 156                 goto err;
 157
 158         mem->sg_list = sg = st.sgl;
 159
 160         for (i = 0 ; i < mem->page_count; i++, sg = sg_next(sg))
 161                 sg_set_page(sg, mem->pages[i], PAGE_SIZE, 0);
 162
 163         mem->num_sg = pci_map_sg(intel_private.pcidev, mem->sg_list,
 164                                  mem->page_count, PCI_DMA_BIDIRECTIONAL);
 165         if (unlikely(!mem->num_sg))
 166                 goto err;
 167
 168         return 0;
 169
 170 err:
 171         sg_free_table(&st);
 172         return -ENOMEM;
 173 }
 174
 175 static void intel_agp_unmap_memory(struct agp_memory *mem)
 176 {
 177         DBG("try unmapping %lu pages\n", (unsigned long)mem->page_count);
 178
 179         pci_unmap_sg(intel_private.pcidev, mem->sg_list,
 180                      mem->page_count, PCI_DMA_BIDIRECTIONAL);
 181         intel_agp_free_sglist(mem);
 182 }
 183
 184 static void intel_agp_insert_sg_entries(struct agp_memory *mem,
 185                                         off_t pg_start, int mask_type)
 186 {
 187         struct scatterlist *sg;
 188         int i, j;
 189
 190         j = pg_start;
 191
 192         WARN_ON(!mem->num_sg);
 193
 194         if (mem->num_sg == mem->page_count) {
 195                 for_each_sg(mem->sg_list, sg, mem->page_count, i) {
 196                         writel(agp_bridge->driver->mask_memory(agp_bridge,
 197                                         sg_dma_address(sg), mask_type),
 198                                         intel_private.gtt+j);
 199                         j++;
 200                 }
 201         } else {
 202                 /* sg may merge pages, but we have to separate
 203                  * per-page addr for GTT */
 204                 unsigned int len, m;
 205
 206                 for_each_sg(mem->sg_list, sg, mem->num_sg, i) {
 207                         len = sg_dma_len(sg) / PAGE_SIZE;
 208                         for (m = 0; m < len; m++) {
 209                                 writel(agp_bridge->driver->mask_memory(agp_bridge,
 210                                                                        sg_dma_address(sg) + m * PAGE_SIZE,
 211                                                                        mask_type),
 212                                        intel_private.gtt+j);
 213                                 j++;
 214                         }
 215                 }
 216         }
 217         readl(intel_private.gtt+j-1);
 218 }
 219
 220 #else
 221
 222 static void intel_agp_insert_sg_entries(struct agp_memory *mem,
 223                                         off_t pg_start, int mask_type)
 224 {
 225         int i, j;
 226
 227         for (i = 0, j = pg_start; i < mem->page_count; i++, j++) {
 228                 writel(agp_bridge->driver->mask_memory(agp_bridge,
 229                                 page_to_phys(mem->pages[i]), mask_type),
 230                        intel_private.gtt+j);
 231         }
 232
 233         readl(intel_private.gtt+j-1);
 234 }
 235
 236 #endif
 237
 238 static int intel_i810_fetch_size(void)
 239 {
 240         u32 smram_miscc;
 241         struct aper_size_info_fixed *values;
 242
 243         pci_read_config_dword(intel_private.bridge_dev,
 244                               I810_SMRAM_MISCC, &smram_miscc);
 245         values = A_SIZE_FIX(agp_bridge->driver->aperture_sizes);
 246
 247         if ((smram_miscc & I810_GMS) == I810_GMS_DISABLE) {
 248                 dev_warn(&intel_private.bridge_dev->dev, "i810 is disabled\n");
 249                 return 0;
 250         }
 251         if ((smram_miscc & I810_GFX_MEM_WIN_SIZE) == I810_GFX_MEM_WIN_32M) {
 252                 agp_bridge->current_size = (void *) (values + 1);
 253                 agp_bridge->aperture_size_idx = 1;
 254                 return values[1].size;
 255         } else {
 256                 agp_bridge->current_size = (void *) (values);
 257                 agp_bridge->aperture_size_idx = 0;
 258                 return values[0].size;
 259         }
 260
 261         return 0;
 262 }
 263
 264 static int intel_i810_configure(void)
 265 {
 266         struct aper_size_info_fixed *current_size;
 267         u32 temp;
 268         int i;
 269
 270         current_size = A_SIZE_FIX(agp_bridge->current_size);
 271
 272         if (!intel_private.registers) {
 273                 pci_read_config_dword(intel_private.pcidev, I810_MMADDR, &temp);
 274                 temp &= 0xfff80000;
 275
 276                 intel_private.registers = ioremap(temp, 128 * 4096);
 277                 if (!intel_private.registers) {
 278                         dev_err(&intel_private.pcidev->dev,
 279                                 "can't remap memory\n");
 280                         return -ENOMEM;
 281                 }
 282         }
 283
 284         if ((readl(intel_private.registers+I810_DRAM_CTL)
 285                 & I810_DRAM_ROW_0) == I810_DRAM_ROW_0_SDRAM) {
 286                 /* This will need to be dynamically assigned */
 287                 dev_info(&intel_private.pcidev->dev,
 288                          "detected 4MB dedicated video ram\n");
 289                 intel_private.num_dcache_entries = 1024;
 290         }
 291         pci_read_config_dword(intel_private.pcidev, I810_GMADDR, &temp);
 292         agp_bridge->gart_bus_addr = (temp & PCI_BASE_ADDRESS_MEM_MASK);
 293         writel(agp_bridge->gatt_bus_addr | I810_PGETBL_ENABLED, intel_private.registers+I810_PGETBL_CTL);
 294         readl(intel_private.registers+I810_PGETBL_CTL); /* PCI Posting. */
 295
 296         if (agp_bridge->driver->needs_scratch_page) {
 297                 for (i = 0; i < current_size->num_entries; i++) {
 298                         writel(agp_bridge->scratch_page, intel_private.registers+I810_PTE_BASE+(i*4));
 299                 }
 300                 readl(intel_private.registers+I810_PTE_BASE+((i-1)*4)); /* PCI posting. */
 301         }
 302         global_cache_flush();
 303         return 0;
 304 }
 305
 306 static void intel_i810_cleanup(void)
 307 {
 308         writel(0, intel_private.registers+I810_PGETBL_CTL);
 309         readl(intel_private.registers); /* PCI Posting. */
 310         iounmap(intel_private.registers);
 311 }
 312
 313 static void intel_fake_agp_enable(struct agp_bridge_data *bridge, u32 mode)
 314 {
 315         return;
 316 }
 317
 318 /* Exists to support ARGB cursors */
 319 static struct page *i8xx_alloc_pages(void)
 320 {
 321         struct page *page;
 322
 323         page = alloc_pages(GFP_KERNEL | GFP_DMA32, 2);
 324         if (page == NULL)
 325                 return NULL;
 326
 327         if (set_pages_uc(page, 4) < 0) {
 328                 set_pages_wb(page, 4);
 329                 __free_pages(page, 2);
 330                 return NULL;
 331         }
 332         get_page(page);
 333         atomic_inc(&agp_bridge->current_memory_agp);
 334         return page;
 335 }
 336
 337 static void i8xx_destroy_pages(struct page *page)
 338 {
 339         if (page == NULL)
 340                 return;
 341
 342         set_pages_wb(page, 4);
 343         put_page(page);
 344         __free_pages(page, 2);
 345         atomic_dec(&agp_bridge->current_memory_agp);
 346 }
 347
 348 static int intel_i830_type_to_mask_type(struct agp_bridge_data *bridge,
 349                                         int type)
 350 {
 351         if (type < AGP_USER_TYPES)
 352                 return type;
 353         else if (type == AGP_USER_CACHED_MEMORY)
 354                 return INTEL_AGP_CACHED_MEMORY;
 355         else
 356                 return 0;
 357 }
 358
 359 static int intel_gen6_type_to_mask_type(struct agp_bridge_data *bridge,
 360                                         int type)
 361 {
 362         unsigned int type_mask = type & ~AGP_USER_CACHED_MEMORY_GFDT;
 363         unsigned int gfdt = type & AGP_USER_CACHED_MEMORY_GFDT;
 364
 365         if (type_mask == AGP_USER_UNCACHED_MEMORY)
 366                 return INTEL_AGP_UNCACHED_MEMORY;
 367         else if (type_mask == AGP_USER_CACHED_MEMORY_LLC_MLC)
 368                 return gfdt ? INTEL_AGP_CACHED_MEMORY_LLC_MLC_GFDT :
 369                               INTEL_AGP_CACHED_MEMORY_LLC_MLC;
 370         else /* set 'normal'/'cached' to LLC by default */
 371                 return gfdt ? INTEL_AGP_CACHED_MEMORY_LLC_GFDT :
 372                               INTEL_AGP_CACHED_MEMORY_LLC;
 373 }
 374
 375
 376 static int intel_i810_insert_entries(struct agp_memory *mem, off_t pg_start,
 377                                 int type)
 378 {
 379         int i, j, num_entries;
 380         void *temp;
 381         int ret = -EINVAL;
 382         int mask_type;
 383
 384         if (mem->page_count == 0)
 385                 goto out;
 386
 387         temp = agp_bridge->current_size;
 388         num_entries = A_SIZE_FIX(temp)->num_entries;
 389
 390         if ((pg_start + mem->page_count) > num_entries)
 391                 goto out_err;
 392
 393
 394         for (j = pg_start; j < (pg_start + mem->page_count); j++) {
 395                 if (!PGE_EMPTY(agp_bridge, readl(agp_bridge->gatt_table+j))) {
 396                         ret = -EBUSY;
 397                         goto out_err;
 398                 }
 399         }
 400
 401         if (type != mem->type)
 402                 goto out_err;
 403
 404         mask_type = agp_bridge->driver->agp_type_to_mask_type(agp_bridge, type);
 405
 406         switch (mask_type) {
 407         case AGP_DCACHE_MEMORY:
 408                 if (!mem->is_flushed)
 409                         global_cache_flush();
 410                 for (i = pg_start; i < (pg_start + mem->page_count); i++) {
 411                         writel((i*4096)|I810_PTE_LOCAL|I810_PTE_VALID,
 412                                intel_private.registers+I810_PTE_BASE+(i*4));
 413                 }
 414                 readl(intel_private.registers+I810_PTE_BASE+((i-1)*4));
 415                 break;
 416         case AGP_PHYS_MEMORY:
 417         case AGP_NORMAL_MEMORY:
 418                 if (!mem->is_flushed)
 419                         global_cache_flush();
 420                 for (i = 0, j = pg_start; i < mem->page_count; i++, j++) {
 421                         writel(agp_bridge->driver->mask_memory(agp_bridge,
 422                                         page_to_phys(mem->pages[i]), mask_type),
 423                                intel_private.registers+I810_PTE_BASE+(j*4));
 424                 }
 425                 readl(intel_private.registers+I810_PTE_BASE+((j-1)*4));
 426                 break;
 427         default:
 428                 goto out_err;
 429         }
 430
 431 out:
 432         ret = 0;
 433 out_err:
 434         mem->is_flushed = true;
 435         return ret;
 436 }
 437
 438 static int intel_i810_remove_entries(struct agp_memory *mem, off_t pg_start,
 439                                 int type)
 440 {
 441         int i;
 442
 443         if (mem->page_count == 0)
 444                 return 0;
 445
 446         for (i = pg_start; i < (mem->page_count + pg_start); i++) {
 447                 writel(agp_bridge->scratch_page, intel_private.registers+I810_PTE_BASE+(i*4));
 448         }
 449         readl(intel_private.registers+I810_PTE_BASE+((i-1)*4));
 450
 451         return 0;
 452 }
 453
 454 /*
 455  * The i810/i830 requires a physical address to program its mouse
 456  * pointer into hardware.
 457  * However the Xserver still writes to it through the agp aperture.
 458  */
 459 static struct agp_memory *alloc_agpphysmem_i8xx(size_t pg_count, int type)
 460 {
 461         struct agp_memory *new;
 462         struct page *page;
 463
 464         switch (pg_count) {
 465         case 1: page = agp_bridge->driver->agp_alloc_page(agp_bridge);
 466                 break;
 467         case 4:
 468                 /* kludge to get 4 physical pages for ARGB cursor */
 469                 page = i8xx_alloc_pages();
 470                 break;
 471         default:
 472                 return NULL;
 473         }
 474
 475         if (page == NULL)
 476                 return NULL;
 477
 478         new = agp_create_memory(pg_count);
 479         if (new == NULL)
 480                 return NULL;
 481
 482         new->pages[0] = page;
 483         if (pg_count == 4) {
 484                 /* kludge to get 4 physical pages for ARGB cursor */
 485                 new->pages[1] = new->pages[0] + 1;
 486                 new->pages[2] = new->pages[1] + 1;
 487                 new->pages[3] = new->pages[2] + 1;
 488         }
 489         new->page_count = pg_count;
 490         new->num_scratch_pages = pg_count;
 491         new->type = AGP_PHYS_MEMORY;
 492         new->physical = page_to_phys(new->pages[0]);
 493         return new;
 494 }
 495
 496 static struct agp_memory *intel_i810_alloc_by_type(size_t pg_count, int type)
 497 {
 498         struct agp_memory *new;
 499
 500         if (type == AGP_DCACHE_MEMORY) {
 501                 if (pg_count != intel_private.num_dcache_entries)
 502                         return NULL;
 503
 504                 new = agp_create_memory(1);
 505                 if (new == NULL)
 506                         return NULL;
 507
 508                 new->type = AGP_DCACHE_MEMORY;
 509                 new->page_count = pg_count;
 510                 new->num_scratch_pages = 0;
 511                 agp_free_page_array(new);
 512                 return new;
 513         }
 514         if (type == AGP_PHYS_MEMORY)
 515                 return alloc_agpphysmem_i8xx(pg_count, type);
 516         return NULL;
 517 }
 518
 519 static void intel_i810_free_by_type(struct agp_memory *curr)
 520 {
 521         agp_free_key(curr->key);
 522         if (curr->type == AGP_PHYS_MEMORY) {
 523                 if (curr->page_count == 4)
 524                         i8xx_destroy_pages(curr->pages[0]);
 525                 else {
 526                         agp_bridge->driver->agp_destroy_page(curr->pages[0],
 527                                                              AGP_PAGE_DESTROY_UNMAP);
 528                         agp_bridge->driver->agp_destroy_page(curr->pages[0],
 529                                                              AGP_PAGE_DESTROY_FREE);
 530                 }
 531                 agp_free_page_array(curr);
 532         }
 533         kfree(curr);
 534 }
 535
 536 static unsigned long intel_i810_mask_memory(struct agp_bridge_data *bridge,
 537                                             dma_addr_t addr, int type)
 538 {
 539         /* Type checking must be done elsewhere */
 540         return addr | bridge->driver->masks[type].mask;
 541 }
 542
 543 static const struct aper_size_info_fixed const intel_fake_agp_sizes[] = {
 544         {128, 32768, 5},
 545         /* The 64M mode still requires a 128k gatt */
 546         {64, 16384, 5},
 547         {256, 65536, 6},
 548         {512, 131072, 7},
 549 };
 550
 551 static unsigned int intel_gtt_stolen_entries(void)
 552 {
 553         u16 gmch_ctrl;
 554         u8 rdct;
 555         int local = 0;
 556         static const int ddt[4] = { 0, 16, 32, 64 };
 557         unsigned int overhead_entries, stolen_entries;
 558         unsigned int stolen_size = 0;
 559
 560         pci_read_config_word(intel_private.bridge_dev,
 561                              I830_GMCH_CTRL, &gmch_ctrl);
 562
 563         if (INTEL_GTT_GEN > 4 || IS_PINEVIEW)
 564                 overhead_entries = 0;
 565         else
 566                 overhead_entries = intel_private.base.gtt_mappable_entries
 567                         / 1024;
 568
 569         overhead_entries += 1; /* BIOS popup */
 570
 571         if (intel_private.bridge_dev->device == PCI_DEVICE_ID_INTEL_82830_HB ||
 572             intel_private.bridge_dev->device == PCI_DEVICE_ID_INTEL_82845G_HB) {
 573                 switch (gmch_ctrl & I830_GMCH_GMS_MASK) {
 574                 case I830_GMCH_GMS_STOLEN_512:
 575                         stolen_size = KB(512);
 576                         break;
 577                 case I830_GMCH_GMS_STOLEN_1024:
 578                         stolen_size = MB(1);
 579                         break;
 580                 case I830_GMCH_GMS_STOLEN_8192:
 581                         stolen_size = MB(8);
 582                         break;
 583                 case I830_GMCH_GMS_LOCAL:
 584                         rdct = readb(intel_private.registers+I830_RDRAM_CHANNEL_TYPE);
 585                         stolen_size = (I830_RDRAM_ND(rdct) + 1) *
 586                                         MB(ddt[I830_RDRAM_DDT(rdct)]);
 587                         local = 1;
 588                         break;
 589                 default:
 590                         stolen_size = 0;
 591                         break;
 592                 }
 593         } else if (INTEL_GTT_GEN == 6) {
 594                 /*
 595                  * SandyBridge has new memory control reg at 0x50.w
 596                  */
 597                 u16 snb_gmch_ctl;
 598                 pci_read_config_word(intel_private.pcidev, SNB_GMCH_CTRL, &snb_gmch_ctl);
 599                 switch (snb_gmch_ctl & SNB_GMCH_GMS_STOLEN_MASK) {
 600                 case SNB_GMCH_GMS_STOLEN_32M:
 601                         stolen_size = MB(32);
 602                         break;
 603                 case SNB_GMCH_GMS_STOLEN_64M:
 604                         stolen_size = MB(64);
 605                         break;
 606                 case SNB_GMCH_GMS_STOLEN_96M:
 607                         stolen_size = MB(96);
 608                         break;
 609                 case SNB_GMCH_GMS_STOLEN_128M:
 610                         stolen_size = MB(128);
 611                         break;
 612                 case SNB_GMCH_GMS_STOLEN_160M:
 613                         stolen_size = MB(160);
 614                         break;
 615                 case SNB_GMCH_GMS_STOLEN_192M:
 616                         stolen_size = MB(192);
 617                         break;
 618                 case SNB_GMCH_GMS_STOLEN_224M:
 619                         stolen_size = MB(224);
 620                         break;
 621                 case SNB_GMCH_GMS_STOLEN_256M:
 622                         stolen_size = MB(256);
 623                         break;
 624                 case SNB_GMCH_GMS_STOLEN_288M:
 625                         stolen_size = MB(288);
 626                         break;
 627                 case SNB_GMCH_GMS_STOLEN_320M:
 628                         stolen_size = MB(320);
 629                         break;
 630                 case SNB_GMCH_GMS_STOLEN_352M:
 631                         stolen_size = MB(352);
 632                         break;
 633                 case SNB_GMCH_GMS_STOLEN_384M:
 634                         stolen_size = MB(384);
 635                         break;
 636                 case SNB_GMCH_GMS_STOLEN_416M:
 637                         stolen_size = MB(416);
 638                         break;
 639                 case SNB_GMCH_GMS_STOLEN_448M:
 640                         stolen_size = MB(448);
 641                         break;
 642                 case SNB_GMCH_GMS_STOLEN_480M:
 643                         stolen_size = MB(480);
 644                         break;
 645                 case SNB_GMCH_GMS_STOLEN_512M:
 646                         stolen_size = MB(512);
 647                         break;
 648                 }
 649         } else {
 650                 switch (gmch_ctrl & I855_GMCH_GMS_MASK) {
 651                 case I855_GMCH_GMS_STOLEN_1M:
 652                         stolen_size = MB(1);
 653                         break;
 654                 case I855_GMCH_GMS_STOLEN_4M:
 655                         stolen_size = MB(4);
 656                         break;
 657                 case I855_GMCH_GMS_STOLEN_8M:
 658                         stolen_size = MB(8);
 659                         break;
 660                 case I855_GMCH_GMS_STOLEN_16M:
 661                         stolen_size = MB(16);
 662                         break;
 663                 case I855_GMCH_GMS_STOLEN_32M:
 664                         stolen_size = MB(32);
 665                         break;
 666                 case I915_GMCH_GMS_STOLEN_48M:
 667                         stolen_size = MB(48);
 668                         break;
 669                 case I915_GMCH_GMS_STOLEN_64M:
 670                         stolen_size = MB(64);
 671                         break;
 672                 case G33_GMCH_GMS_STOLEN_128M:
 673                         stolen_size = MB(128);
 674                         break;
 675                 case G33_GMCH_GMS_STOLEN_256M:
 676                         stolen_size = MB(256);
 677                         break;
 678                 case INTEL_GMCH_GMS_STOLEN_96M:
 679                         stolen_size = MB(96);
 680                         break;
 681                 case INTEL_GMCH_GMS_STOLEN_160M:
 682                         stolen_size = MB(160);
 683                         break;
 684                 case INTEL_GMCH_GMS_STOLEN_224M:
 685                         stolen_size = MB(224);
 686                         break;
 687                 case INTEL_GMCH_GMS_STOLEN_352M:
 688                         stolen_size = MB(352);
 689                         break;
 690                 default:
 691                         stolen_size = 0;
 692                         break;
 693                 }
 694         }
 695
 696         if (!local && stolen_size > intel_max_stolen) {
 697                 dev_info(&intel_private.bridge_dev->dev,
 698                          "detected %dK stolen memory, trimming to %dK\n",
 699                          stolen_size / KB(1), intel_max_stolen / KB(1));
 700                 stolen_size = intel_max_stolen;
 701         } else if (stolen_size > 0) {
 702                 dev_info(&intel_private.bridge_dev->dev, "detected %dK %s memory\n",
 703                        stolen_size / KB(1), local ? "local" : "stolen");
 704         } else {
 705                 dev_info(&intel_private.bridge_dev->dev,
 706                        "no pre-allocated video memory detected\n");
 707                 stolen_size = 0;
 708         }
 709
 710         stolen_entries = stolen_size/KB(4) - overhead_entries;
 711
 712         return stolen_entries;
 713 }
 714
 715 static unsigned int intel_gtt_total_entries(void)
 716 {
 717         int size;
 718
 719         if (IS_G33 || INTEL_GTT_GEN == 4 || INTEL_GTT_GEN == 5) {
 720                 u32 pgetbl_ctl;
 721                 pgetbl_ctl = readl(intel_private.registers+I810_PGETBL_CTL);
 722
 723                 switch (pgetbl_ctl & I965_PGETBL_SIZE_MASK) {
 724                 case I965_PGETBL_SIZE_128KB:
 725                         size = KB(128);
 726                         break;
 727                 case I965_PGETBL_SIZE_256KB:
 728                         size = KB(256);
 729                         break;
 730                 case I965_PGETBL_SIZE_512KB:
 731                         size = KB(512);
 732                         break;
 733                 case I965_PGETBL_SIZE_1MB:
 734                         size = KB(1024);
 735                         break;
 736                 case I965_PGETBL_SIZE_2MB:
 737                         size = KB(2048);
 738                         break;
 739                 case I965_PGETBL_SIZE_1_5MB:
 740                         size = KB(1024 + 512);
 741                         break;
 742                 default:
 743                         dev_info(&intel_private.pcidev->dev,
 744                                  "unknown page table size, assuming 512KB\n");
 745                         size = KB(512);
 746                 }
 747
 748                 return size/4;
 749         } else if (INTEL_GTT_GEN == 6) {
 750                 u16 snb_gmch_ctl;
 751
 752                 pci_read_config_word(intel_private.pcidev, SNB_GMCH_CTRL, &snb_gmch_ctl);
 753                 switch (snb_gmch_ctl & SNB_GTT_SIZE_MASK) {
 754                 default:
 755                 case SNB_GTT_SIZE_0M:
 756                         printk(KERN_ERR "Bad GTT size mask: 0x%04x.\n", snb_gmch_ctl);
 757                         size = MB(0);
 758                         break;
 759                 case SNB_GTT_SIZE_1M:
 760                         size = MB(1);
 761                         break;
 762                 case SNB_GTT_SIZE_2M:
 763                         size = MB(2);
 764                         break;
 765                 }
 766                 return size/4;
 767         } else {
 768                 /* On previous hardware, the GTT size was just what was
 769                  * required to map the aperture.
 770                  */
 771                 return intel_private.base.gtt_mappable_entries;
 772         }
 773 }
 774
 775 static unsigned int intel_gtt_mappable_entries(void)
 776 {
 777         unsigned int aperture_size;
 778
 779         if (INTEL_GTT_GEN == 2) {
 780                 u16 gmch_ctrl;
 781
 782                 pci_read_config_word(intel_private.bridge_dev,
 783                                      I830_GMCH_CTRL, &gmch_ctrl);
 784
 785                 if ((gmch_ctrl & I830_GMCH_MEM_MASK) == I830_GMCH_MEM_64M)
 786                         aperture_size = MB(64);
 787                 else
 788                         aperture_size = MB(128);
 789         } else {
 790                 /* 9xx supports large sizes, just look at the length */
 791                 aperture_size = pci_resource_len(intel_private.pcidev, 2);
 792         }
 793
 794         return aperture_size >> PAGE_SHIFT;
 795 }
 796
 797 static int intel_gtt_init(void)
 798 {
 799         u32 gtt_map_size;
 800         int ret;
 801
 802         ret = intel_private.driver->setup();
 803         if (ret != 0)
 804                 return ret;
 805
 806         intel_private.base.gtt_mappable_entries = intel_gtt_mappable_entries();
 807         intel_private.base.gtt_total_entries = intel_gtt_total_entries();
 808
 809         gtt_map_size = intel_private.base.gtt_total_entries * 4;
 810
 811         intel_private.gtt = ioremap(intel_private.gtt_bus_addr,
 812                                     gtt_map_size);
 813         if (!intel_private.gtt) {
 814                 iounmap(intel_private.registers);
 815                 return -ENOMEM;
 816         }
 817
 818         global_cache_flush();   /* FIXME: ? */
 819
 820         /* we have to call this as early as possible after the MMIO base address is known */
 821         intel_private.base.gtt_stolen_entries = intel_gtt_stolen_entries();
 822         if (intel_private.base.gtt_stolen_entries == 0) {
 823                 iounmap(intel_private.registers);
 824                 iounmap(intel_private.gtt);
 825                 return -ENOMEM;
 826         }
 827
 828         return 0;
 829 }
 830
 831 static int intel_fake_agp_fetch_size(void)
 832 {
 833         int num_sizes = ARRAY_SIZE(intel_fake_agp_sizes);
 834         unsigned int aper_size;
 835         int i;
 836
 837         aper_size = (intel_private.base.gtt_mappable_entries << PAGE_SHIFT)
 838                     / MB(1);
 839
 840         for (i = 0; i < num_sizes; i++) {
 841                 if (aper_size == intel_fake_agp_sizes[i].size) {
 842                         agp_bridge->current_size =
 843                                 (void *) (intel_fake_agp_sizes + i);
 844                         return aper_size;
 845                 }
 846         }
 847
 848         return 0;
 849 }
 850
 851 static void intel_i830_fini_flush(void)
 852 {
 853         kunmap(intel_private.i8xx_page);
 854         intel_private.i8xx_flush_page = NULL;
 855         unmap_page_from_agp(intel_private.i8xx_page);
 856
 857         __free_page(intel_private.i8xx_page);
 858         intel_private.i8xx_page = NULL;
 859 }
 860
 861 static void intel_i830_setup_flush(void)
 862 {
 863         /* return if we've already set the flush mechanism up */
 864         if (intel_private.i8xx_page)
 865                 return;
 866
 867         intel_private.i8xx_page = alloc_page(GFP_KERNEL | __GFP_ZERO | GFP_DMA32);
 868         if (!intel_private.i8xx_page)
 869                 return;
 870
 871         intel_private.i8xx_flush_page = kmap(intel_private.i8xx_page);
 872         if (!intel_private.i8xx_flush_page)
 873                 intel_i830_fini_flush();
 874 }
 875
 876 /* The chipset_flush interface needs to get data that has already been
 877  * flushed out of the CPU all the way out to main memory, because the GPU
 878  * doesn't snoop those buffers.
 879  *
 880  * The 8xx series doesn't have the same lovely interface for flushing the
 881  * chipset write buffers that the later chips do. According to the 865
 882  * specs, it's 64 octwords, or 1KB.  So, to get those previous things in
 883  * that buffer out, we just fill 1KB and clflush it out, on the assumption
 884  * that it'll push whatever was in there out.  It appears to work.
 885  */
 886 static void intel_i830_chipset_flush(struct agp_bridge_data *bridge)
 887 {
 888         unsigned int *pg = intel_private.i8xx_flush_page;
 889
 890         memset(pg, 0, 1024);
 891
 892         if (cpu_has_clflush)
 893                 clflush_cache_range(pg, 1024);
 894         else if (wbinvd_on_all_cpus() != 0)
 895                 printk(KERN_ERR "Timed out waiting for cache flush.\n");
 896 }
 897
 898 static void intel_enable_gtt(void)
 899 {
 900         u32 gma_addr;
 901         u16 gmch_ctrl;
 902
 903         if (INTEL_GTT_GEN == 2)
 904                 pci_read_config_dword(intel_private.pcidev, I810_GMADDR,
 905                                       &gma_addr);
 906         else
 907                 pci_read_config_dword(intel_private.pcidev, I915_GMADDR,
 908                                       &gma_addr);
 909
 910         intel_private.gma_bus_addr = (gma_addr & PCI_BASE_ADDRESS_MEM_MASK);
 911
 912         pci_read_config_word(intel_private.bridge_dev, I830_GMCH_CTRL, &gmch_ctrl);
 913         gmch_ctrl |= I830_GMCH_ENABLED;
 914         pci_write_config_word(intel_private.bridge_dev, I830_GMCH_CTRL, gmch_ctrl);
 915
 916         writel(intel_private.pte_bus_addr|I810_PGETBL_ENABLED,
 917                intel_private.registers+I810_PGETBL_CTL);
 918         readl(intel_private.registers+I810_PGETBL_CTL); /* PCI Posting. */
 919 }
 920
 921 static int i830_setup(void)
 922 {
 923         u32 reg_addr;
 924
 925         pci_read_config_dword(intel_private.pcidev, I810_MMADDR, &reg_addr);
 926         reg_addr &= 0xfff80000;
 927
 928         intel_private.registers = ioremap(reg_addr, KB(64));
 929         if (!intel_private.registers)
 930                 return -ENOMEM;
 931
 932         intel_private.gtt_bus_addr = reg_addr + I810_PTE_BASE;
 933         intel_private.pte_bus_addr =
 934                 readl(intel_private.registers+I810_PGETBL_CTL) & 0xfffff000;
 935
 936         intel_i830_setup_flush();
 937
 938         return 0;
 939 }
 940
 941 static int intel_fake_agp_create_gatt_table(struct agp_bridge_data *bridge)
 942 {
 943         agp_bridge->gatt_table_real = NULL;
 944         agp_bridge->gatt_table = NULL;
 945         agp_bridge->gatt_bus_addr = 0;
 946
 947         return 0;
 948 }
 949
 950 static int intel_fake_agp_free_gatt_table(struct agp_bridge_data *bridge)
 951 {
 952         return 0;
 953 }
 954
 955 static int intel_i830_configure(void)
 956 {
 957         int i;
 958
 959         intel_enable_gtt();
 960
 961         agp_bridge->gart_bus_addr = intel_private.gma_bus_addr;
 962
 963         if (agp_bridge->driver->needs_scratch_page) {
 964                 for (i = intel_private.base.gtt_stolen_entries;
 965                                 i < intel_private.base.gtt_total_entries; i++) {
 966                         writel(agp_bridge->scratch_page, intel_private.gtt+i);
 967                 }
 968                 readl(intel_private.gtt+i-1);   /* PCI Posting. */
 969         }
 970
 971         global_cache_flush();
 972
 973         return 0;
 974 }
 975
 976 static int intel_i830_insert_entries(struct agp_memory *mem, off_t pg_start,
 977                                      int type)
 978 {
 979         int i, j, num_entries;
 980         void *temp;
 981         int ret = -EINVAL;
 982         int mask_type;
 983
 984         if (mem->page_count == 0)
 985                 goto out;
 986
 987         temp = agp_bridge->current_size;
 988         num_entries = A_SIZE_FIX(temp)->num_entries;
 989
 990         if (pg_start < intel_private.base.gtt_stolen_entries) {
 991                 dev_printk(KERN_DEBUG, &intel_private.pcidev->dev,
 992                            "pg_start == 0x%.8lx, gtt_stolen_entries == 0x%.8x\n",
 993                            pg_start, intel_private.base.gtt_stolen_entries);
 994
 995                 dev_info(&intel_private.pcidev->dev,
 996                          "trying to insert into local/stolen memory\n");
 997                 goto out_err;
 998         }
 999
1000         if ((pg_start + mem->page_count) > num_entries)
1001                 goto out_err;
1002
1003         /* The i830 can't check the GTT for entries since its read only,
1004          * depend on the caller to make the correct offset decisions.
1005          */
1006
1007         if (type != mem->type)
1008                 goto out_err;
1009
1010         mask_type = agp_bridge->driver->agp_type_to_mask_type(agp_bridge, type);
1011
1012         if (mask_type != 0 && mask_type != AGP_PHYS_MEMORY &&
1013             mask_type != INTEL_AGP_CACHED_MEMORY)
1014                 goto out_err;
1015
1016         if (!mem->is_flushed)
1017                 global_cache_flush();
1018
1019         for (i = 0, j = pg_start; i < mem->page_count; i++, j++) {
1020                 writel(agp_bridge->driver->mask_memory(agp_bridge,
1021                                 page_to_phys(mem->pages[i]), mask_type),
1022                        intel_private.gtt+j);
1023         }
1024         readl(intel_private.gtt+j-1);
1025
1026 out:
1027         ret = 0;
1028 out_err:
1029         mem->is_flushed = true;
1030         return ret;
1031 }
1032
1033 static int intel_i830_remove_entries(struct agp_memory *mem, off_t pg_start,
1034                                      int type)
1035 {
1036         int i;
1037
1038         if (mem->page_count == 0)
1039                 return 0;
1040
1041         if (pg_start < intel_private.base.gtt_stolen_entries) {
1042                 dev_info(&intel_private.pcidev->dev,
1043                          "trying to disable local/stolen memory\n");
1044                 return -EINVAL;
1045         }
1046
1047         for (i = pg_start; i < (mem->page_count + pg_start); i++) {
1048                 writel(agp_bridge->scratch_page, intel_private.gtt+i);
1049         }
1050         readl(intel_private.gtt+i-1);
1051
1052         return 0;
1053 }
1054
1055 static struct agp_memory *intel_fake_agp_alloc_by_type(size_t pg_count,
1056                                                        int type)
1057 {
1058         if (type == AGP_PHYS_MEMORY)
1059                 return alloc_agpphysmem_i8xx(pg_count, type);
1060         /* always return NULL for other allocation types for now */
1061         return NULL;
1062 }
1063
1064 static int intel_alloc_chipset_flush_resource(void)
1065 {
1066         int ret;
1067         ret = pci_bus_alloc_resource(intel_private.bridge_dev->bus, &intel_private.ifp_resource, PAGE_SIZE,
1068                                      PAGE_SIZE, PCIBIOS_MIN_MEM, 0,
1069                                      pcibios_align_resource, intel_private.bridge_dev);
1070
1071         return ret;
1072 }
1073
1074 static void intel_i915_setup_chipset_flush(void)
1075 {
1076         int ret;
1077         u32 temp;
1078
1079         pci_read_config_dword(intel_private.bridge_dev, I915_IFPADDR, &temp);
1080         if (!(temp & 0x1)) {
1081                 intel_alloc_chipset_flush_resource();
1082                 intel_private.resource_valid = 1;
1083                 pci_write_config_dword(intel_private.bridge_dev, I915_IFPADDR, (intel_private.ifp_resource.start & 0xffffffff) | 0x1);
1084         } else {
1085                 temp &= ~1;
1086
1087                 intel_private.resource_valid = 1;
1088                 intel_private.ifp_resource.start = temp;
1089                 intel_private.ifp_resource.end = temp + PAGE_SIZE;
1090                 ret = request_resource(&iomem_resource, &intel_private.ifp_resource);
1091                 /* some BIOSes reserve this area in a pnp some don't */
1092                 if (ret)
1093                         intel_private.resource_valid = 0;
1094         }
1095 }
1096
1097 static void intel_i965_g33_setup_chipset_flush(void)
1098 {
1099         u32 temp_hi, temp_lo;
1100         int ret;
1101
1102         pci_read_config_dword(intel_private.bridge_dev, I965_IFPADDR + 4, &temp_hi);
1103         pci_read_config_dword(intel_private.bridge_dev, I965_IFPADDR, &temp_lo);
1104
1105         if (!(temp_lo & 0x1)) {
1106
1107                 intel_alloc_chipset_flush_resource();
1108
1109                 intel_private.resource_valid = 1;
1110                 pci_write_config_dword(intel_private.bridge_dev, I965_IFPADDR + 4,
1111                         upper_32_bits(intel_private.ifp_resource.start));
1112                 pci_write_config_dword(intel_private.bridge_dev, I965_IFPADDR, (intel_private.ifp_resource.start & 0xffffffff) | 0x1);
1113         } else {
1114                 u64 l64;
1115
1116                 temp_lo &= ~0x1;
1117                 l64 = ((u64)temp_hi << 32) | temp_lo;
1118
1119                 intel_private.resource_valid = 1;
1120                 intel_private.ifp_resource.start = l64;
1121                 intel_private.ifp_resource.end = l64 + PAGE_SIZE;
1122                 ret = request_resource(&iomem_resource, &intel_private.ifp_resource);
1123                 /* some BIOSes reserve this area in a pnp some don't */
1124                 if (ret)
1125                         intel_private.resource_valid = 0;
1126         }
1127 }
1128
1129 static void intel_i9xx_setup_flush(void)
1130 {
1131         /* return if already configured */
1132         if (intel_private.ifp_resource.start)
1133                 return;
1134
1135         if (INTEL_GTT_GEN == 6)
1136                 return;
1137
1138         /* setup a resource for this object */
1139         intel_private.ifp_resource.name = "Intel Flush Page";
1140         intel_private.ifp_resource.flags = IORESOURCE_MEM;
1141
1142         /* Setup chipset flush for 915 */
1143         if (IS_G33 || INTEL_GTT_GEN >= 4) {
1144                 intel_i965_g33_setup_chipset_flush();
1145         } else {
1146                 intel_i915_setup_chipset_flush();
1147         }
1148
1149         if (intel_private.ifp_resource.start)
1150                 intel_private.i9xx_flush_page = ioremap_nocache(intel_private.ifp_resource.start, PAGE_SIZE);
1151         if (!intel_private.i9xx_flush_page)
1152                 dev_err(&intel_private.pcidev->dev,
1153                         "can't ioremap flush page - no chipset flushing\n");
1154 }
1155
1156 static int intel_i9xx_configure(void)
1157 {
1158         int i;
1159
1160         intel_enable_gtt();
1161
1162         agp_bridge->gart_bus_addr = intel_private.gma_bus_addr;
1163
1164         if (agp_bridge->driver->needs_scratch_page) {
1165                 for (i = intel_private.base.gtt_stolen_entries; i <
1166                                 intel_private.base.gtt_total_entries; i++) {
1167                         writel(agp_bridge->scratch_page, intel_private.gtt+i);
1168                 }
1169                 readl(intel_private.gtt+i-1);   /* PCI Posting. */
1170         }
1171
1172         global_cache_flush();
1173
1174         return 0;
1175 }
1176
1177 static void intel_gtt_cleanup(void)
1178 {
1179         if (intel_private.i9xx_flush_page)
1180                 iounmap(intel_private.i9xx_flush_page);
1181         if (intel_private.resource_valid)
1182                 release_resource(&intel_private.ifp_resource);
1183         intel_private.ifp_resource.start = 0;
1184         intel_private.resource_valid = 0;
1185         iounmap(intel_private.gtt);
1186         iounmap(intel_private.registers);
1187 }
1188
1189 static void intel_i915_chipset_flush(struct agp_bridge_data *bridge)
1190 {
1191         if (intel_private.i9xx_flush_page)
1192                 writel(1, intel_private.i9xx_flush_page);
1193 }
1194
1195 static int intel_i915_insert_entries(struct agp_memory *mem, off_t pg_start,
1196                                      int type)
1197 {
1198         int num_entries;
1199         void *temp;
1200         int ret = -EINVAL;
1201         int mask_type;
1202
1203         if (mem->page_count == 0)
1204                 goto out;
1205
1206         temp = agp_bridge->current_size;
1207         num_entries = A_SIZE_FIX(temp)->num_entries;
1208
1209         if (pg_start < intel_private.base.gtt_stolen_entries) {
1210                 dev_printk(KERN_DEBUG, &intel_private.pcidev->dev,
1211                            "pg_start == 0x%.8lx, gtt_stolen_entries == 0x%.8x\n",
1212                            pg_start, intel_private.base.gtt_stolen_entries);
1213
1214                 dev_info(&intel_private.pcidev->dev,
1215                          "trying to insert into local/stolen memory\n");
1216                 goto out_err;
1217         }
1218
1219         if ((pg_start + mem->page_count) > num_entries)
1220                 goto out_err;
1221
1222         /* The i915 can't check the GTT for entries since it's read only;
1223          * depend on the caller to make the correct offset decisions.
1224          */
1225
1226         if (type != mem->type)
1227                 goto out_err;
1228
1229         mask_type = agp_bridge->driver->agp_type_to_mask_type(agp_bridge, type);
1230
1231         if (INTEL_GTT_GEN != 6 && mask_type != 0 &&
1232             mask_type != AGP_PHYS_MEMORY &&
1233             mask_type != INTEL_AGP_CACHED_MEMORY)
1234                 goto out_err;
1235
1236         if (!mem->is_flushed)
1237                 global_cache_flush();
1238
1239         intel_agp_insert_sg_entries(mem, pg_start, mask_type);
1240
1241  out:
1242         ret = 0;
1243  out_err:
1244         mem->is_flushed = true;
1245         return ret;
1246 }
1247
1248 static int intel_i915_remove_entries(struct agp_memory *mem, off_t pg_start,
1249                                      int type)
1250 {
1251         int i;
1252
1253         if (mem->page_count == 0)
1254                 return 0;
1255
1256         if (pg_start < intel_private.base.gtt_stolen_entries) {
1257                 dev_info(&intel_private.pcidev->dev,
1258                          "trying to disable local/stolen memory\n");
1259                 return -EINVAL;
1260         }
1261
1262         for (i = pg_start; i < (mem->page_count + pg_start); i++)
1263                 writel(agp_bridge->scratch_page, intel_private.gtt+i);
1264
1265         readl(intel_private.gtt+i-1);
1266
1267         return 0;
1268 }
1269
1270 static int i9xx_setup(void)
1271 {
1272         u32 reg_addr;
1273
1274         pci_read_config_dword(intel_private.pcidev, I915_MMADDR, &reg_addr);
1275
1276         reg_addr &= 0xfff80000;
1277
1278         intel_private.registers = ioremap(reg_addr, 128 * 4096);
1279         if (!intel_private.registers)
1280                 return -ENOMEM;
1281
1282         if (INTEL_GTT_GEN == 3) {
1283                 u32 gtt_addr;
1284
1285                 pci_read_config_dword(intel_private.pcidev,
1286                                       I915_PTEADDR, &gtt_addr);
1287                 intel_private.gtt_bus_addr = gtt_addr;
1288         } else {
1289                 u32 gtt_offset;
1290
1291                 switch (INTEL_GTT_GEN) {
1292                 case 5:
1293                 case 6:
1294                         gtt_offset = MB(2);
1295                         break;
1296                 case 4:
1297                 default:
1298                         gtt_offset =  KB(512);
1299                         break;
1300                 }
1301                 intel_private.gtt_bus_addr = reg_addr + gtt_offset;
1302         }
1303
1304         intel_private.pte_bus_addr =
1305                 readl(intel_private.registers+I810_PGETBL_CTL) & 0xfffff000;
1306
1307         intel_i9xx_setup_flush();
1308
1309         return 0;
1310 }
1311
1312 /*
1313  * The i965 supports 36-bit physical addresses, but to keep
1314  * the format of the GTT the same, the bits that don't fit
1315  * in a 32-bit word are shifted down to bits 4..7.
1316  *
1317  * Gcc is smart enough to notice that "(addr >> 28) & 0xf0"
1318  * is always zero on 32-bit architectures, so no need to make
1319  * this conditional.
1320  */
1321 static unsigned long intel_i965_mask_memory(struct agp_bridge_data *bridge,
1322                                             dma_addr_t addr, int type)
1323 {
1324         /* Shift high bits down */
1325         addr |= (addr >> 28) & 0xf0;
1326
1327         /* Type checking must be done elsewhere */
1328         return addr | bridge->driver->masks[type].mask;
1329 }
1330
1331 static unsigned long intel_gen6_mask_memory(struct agp_bridge_data *bridge,
1332                                             dma_addr_t addr, int type)
1333 {
1334         /* gen6 has bit11-4 for physical addr bit39-32 */
1335         addr |= (addr >> 28) & 0xff0;
1336
1337         /* Type checking must be done elsewhere */
1338         return addr | bridge->driver->masks[type].mask;
1339 }
1340
1341 static const struct agp_bridge_driver intel_810_driver = {
1342         .owner                  = THIS_MODULE,
1343         .aperture_sizes         = intel_i810_sizes,
1344         .size_type              = FIXED_APER_SIZE,
1345         .num_aperture_sizes     = 2,
1346         .needs_scratch_page     = true,
1347         .configure              = intel_i810_configure,
1348         .fetch_size             = intel_i810_fetch_size,
1349         .cleanup                = intel_i810_cleanup,
1350         .mask_memory            = intel_i810_mask_memory,
1351         .masks                  = intel_i810_masks,
1352         .agp_enable             = intel_fake_agp_enable,
1353         .cache_flush            = global_cache_flush,
1354         .create_gatt_table      = agp_generic_create_gatt_table,
1355         .free_gatt_table        = agp_generic_free_gatt_table,
1356         .insert_memory          = intel_i810_insert_entries,
1357         .remove_memory          = intel_i810_remove_entries,
1358         .alloc_by_type          = intel_i810_alloc_by_type,
1359         .free_by_type           = intel_i810_free_by_type,
1360         .agp_alloc_page         = agp_generic_alloc_page,
1361         .agp_alloc_pages        = agp_generic_alloc_pages,
1362         .agp_destroy_page       = agp_generic_destroy_page,
1363         .agp_destroy_pages      = agp_generic_destroy_pages,
1364         .agp_type_to_mask_type  = agp_generic_type_to_mask_type,
1365 };
1366
1367 static const struct agp_bridge_driver intel_830_driver = {
1368         .owner                  = THIS_MODULE,
1369         .size_type              = FIXED_APER_SIZE,
1370         .aperture_sizes         = intel_fake_agp_sizes,
1371         .num_aperture_sizes     = ARRAY_SIZE(intel_fake_agp_sizes),
1372         .needs_scratch_page     = true,
1373         .configure              = intel_i830_configure,
1374         .fetch_size             = intel_fake_agp_fetch_size,
1375         .cleanup                = intel_gtt_cleanup,
1376         .mask_memory            = intel_i810_mask_memory,
1377         .masks                  = intel_i810_masks,
1378         .agp_enable             = intel_fake_agp_enable,
1379         .cache_flush            = global_cache_flush,
1380         .create_gatt_table      = intel_fake_agp_create_gatt_table,
1381         .free_gatt_table        = intel_fake_agp_free_gatt_table,
1382         .insert_memory          = intel_i830_insert_entries,
1383         .remove_memory          = intel_i830_remove_entries,
1384         .alloc_by_type          = intel_fake_agp_alloc_by_type,
1385         .free_by_type           = intel_i810_free_by_type,
1386         .agp_alloc_page         = agp_generic_alloc_page,
1387         .agp_alloc_pages        = agp_generic_alloc_pages,
1388         .agp_destroy_page       = agp_generic_destroy_page,
1389         .agp_destroy_pages      = agp_generic_destroy_pages,
1390         .agp_type_to_mask_type  = intel_i830_type_to_mask_type,
1391         .chipset_flush          = intel_i830_chipset_flush,
1392 };
1393
1394 static const struct agp_bridge_driver intel_915_driver = {
1395         .owner                  = THIS_MODULE,
1396         .size_type              = FIXED_APER_SIZE,
1397         .aperture_sizes         = intel_fake_agp_sizes,
1398         .num_aperture_sizes     = ARRAY_SIZE(intel_fake_agp_sizes),
1399         .needs_scratch_page     = true,
1400         .configure              = intel_i9xx_configure,
1401         .fetch_size             = intel_fake_agp_fetch_size,
1402         .cleanup                = intel_gtt_cleanup,
1403         .mask_memory            = intel_i810_mask_memory,
1404         .masks                  = intel_i810_masks,
1405         .agp_enable             = intel_fake_agp_enable,
1406         .cache_flush            = global_cache_flush,
1407         .create_gatt_table      = intel_fake_agp_create_gatt_table,
1408         .free_gatt_table        = intel_fake_agp_free_gatt_table,
1409         .insert_memory          = intel_i915_insert_entries,
1410         .remove_memory          = intel_i915_remove_entries,
1411         .alloc_by_type          = intel_fake_agp_alloc_by_type,
1412         .free_by_type           = intel_i810_free_by_type,
1413         .agp_alloc_page         = agp_generic_alloc_page,
1414         .agp_alloc_pages        = agp_generic_alloc_pages,
1415         .agp_destroy_page       = agp_generic_destroy_page,
1416         .agp_destroy_pages      = agp_generic_destroy_pages,
1417         .agp_type_to_mask_type  = intel_i830_type_to_mask_type,
1418         .chipset_flush          = intel_i915_chipset_flush,
1419 #ifdef USE_PCI_DMA_API
1420         .agp_map_page           = intel_agp_map_page,
1421         .agp_unmap_page         = intel_agp_unmap_page,
1422         .agp_map_memory         = intel_agp_map_memory,
1423         .agp_unmap_memory       = intel_agp_unmap_memory,
1424 #endif
1425 };
1426
1427 static const struct agp_bridge_driver intel_i965_driver = {
1428         .owner                  = THIS_MODULE,
1429         .size_type              = FIXED_APER_SIZE,
1430         .aperture_sizes         = intel_fake_agp_sizes,
1431         .num_aperture_sizes     = ARRAY_SIZE(intel_fake_agp_sizes),
1432         .needs_scratch_page     = true,
1433         .configure              = intel_i9xx_configure,
1434         .fetch_size             = intel_fake_agp_fetch_size,
1435         .cleanup                = intel_gtt_cleanup,
1436         .mask_memory            = intel_i965_mask_memory,
1437         .masks                  = intel_i810_masks,
1438         .agp_enable             = intel_fake_agp_enable,
1439         .cache_flush            = global_cache_flush,
1440         .create_gatt_table      = intel_fake_agp_create_gatt_table,
1441         .free_gatt_table        = intel_fake_agp_free_gatt_table,
1442         .insert_memory          = intel_i915_insert_entries,
1443         .remove_memory          = intel_i915_remove_entries,
1444         .alloc_by_type          = intel_fake_agp_alloc_by_type,
1445         .free_by_type           = intel_i810_free_by_type,
1446         .agp_alloc_page         = agp_generic_alloc_page,
1447         .agp_alloc_pages        = agp_generic_alloc_pages,
1448         .agp_destroy_page       = agp_generic_destroy_page,
1449         .agp_destroy_pages      = agp_generic_destroy_pages,
1450         .agp_type_to_mask_type  = intel_i830_type_to_mask_type,
1451         .chipset_flush          = intel_i915_chipset_flush,
1452 #ifdef USE_PCI_DMA_API
1453         .agp_map_page           = intel_agp_map_page,
1454         .agp_unmap_page         = intel_agp_unmap_page,
1455         .agp_map_memory         = intel_agp_map_memory,
1456         .agp_unmap_memory       = intel_agp_unmap_memory,
1457 #endif
1458 };
1459
1460 static const struct agp_bridge_driver intel_gen6_driver = {
1461         .owner                  = THIS_MODULE,
1462         .size_type              = FIXED_APER_SIZE,
1463         .aperture_sizes         = intel_fake_agp_sizes,
1464         .num_aperture_sizes     = ARRAY_SIZE(intel_fake_agp_sizes),
1465         .needs_scratch_page     = true,
1466         .configure              = intel_i9xx_configure,
1467         .fetch_size             = intel_fake_agp_fetch_size,
1468         .cleanup                = intel_gtt_cleanup,
1469         .mask_memory            = intel_gen6_mask_memory,
1470         .masks                  = intel_gen6_masks,
1471         .agp_enable             = intel_fake_agp_enable,
1472         .cache_flush            = global_cache_flush,
1473         .create_gatt_table      = intel_fake_agp_create_gatt_table,
1474         .free_gatt_table        = intel_fake_agp_free_gatt_table,
1475         .insert_memory          = intel_i915_insert_entries,
1476         .remove_memory          = intel_i915_remove_entries,
1477         .alloc_by_type          = intel_fake_agp_alloc_by_type,
1478         .free_by_type           = intel_i810_free_by_type,
1479         .agp_alloc_page         = agp_generic_alloc_page,
1480         .agp_alloc_pages        = agp_generic_alloc_pages,
1481         .agp_destroy_page       = agp_generic_destroy_page,
1482         .agp_destroy_pages      = agp_generic_destroy_pages,
1483         .agp_type_to_mask_type  = intel_gen6_type_to_mask_type,
1484         .chipset_flush          = intel_i915_chipset_flush,
1485 #ifdef USE_PCI_DMA_API
1486         .agp_map_page           = intel_agp_map_page,
1487         .agp_unmap_page         = intel_agp_unmap_page,
1488         .agp_map_memory         = intel_agp_map_memory,
1489         .agp_unmap_memory       = intel_agp_unmap_memory,
1490 #endif
1491 };
1492
1493 static const struct agp_bridge_driver intel_g33_driver = {
1494         .owner                  = THIS_MODULE,
1495         .size_type              = FIXED_APER_SIZE,
1496         .aperture_sizes         = intel_fake_agp_sizes,
1497         .num_aperture_sizes     = ARRAY_SIZE(intel_fake_agp_sizes),
1498         .needs_scratch_page     = true,
1499         .configure              = intel_i9xx_configure,
1500         .fetch_size             = intel_fake_agp_fetch_size,
1501         .cleanup                = intel_gtt_cleanup,
1502         .mask_memory            = intel_i965_mask_memory,
1503         .masks                  = intel_i810_masks,
1504         .agp_enable             = intel_fake_agp_enable,
1505         .cache_flush            = global_cache_flush,
1506         .create_gatt_table      = intel_fake_agp_create_gatt_table,
1507         .free_gatt_table        = intel_fake_agp_free_gatt_table,
1508         .insert_memory          = intel_i915_insert_entries,
1509         .remove_memory          = intel_i915_remove_entries,
1510         .alloc_by_type          = intel_fake_agp_alloc_by_type,
1511         .free_by_type           = intel_i810_free_by_type,
1512         .agp_alloc_page         = agp_generic_alloc_page,
1513         .agp_alloc_pages        = agp_generic_alloc_pages,
1514         .agp_destroy_page       = agp_generic_destroy_page,
1515         .agp_destroy_pages      = agp_generic_destroy_pages,
1516         .agp_type_to_mask_type  = intel_i830_type_to_mask_type,
1517         .chipset_flush          = intel_i915_chipset_flush,
1518 #ifdef USE_PCI_DMA_API
1519         .agp_map_page           = intel_agp_map_page,
1520         .agp_unmap_page         = intel_agp_unmap_page,
1521         .agp_map_memory         = intel_agp_map_memory,
1522         .agp_unmap_memory       = intel_agp_unmap_memory,
1523 #endif
1524 };
1525
1526 static const struct intel_gtt_driver i8xx_gtt_driver = {
1527         .gen = 2,
1528         .setup = i830_setup,
1529 };
1530 static const struct intel_gtt_driver i915_gtt_driver = {
1531         .gen = 3,
1532         .setup = i9xx_setup,
1533 };
1534 static const struct intel_gtt_driver g33_gtt_driver = {
1535         .gen = 3,
1536         .is_g33 = 1,
1537         .setup = i9xx_setup,
1538 };
1539 static const struct intel_gtt_driver pineview_gtt_driver = {
1540         .gen = 3,
1541         .is_pineview = 1, .is_g33 = 1,
1542         .setup = i9xx_setup,
1543 };
1544 static const struct intel_gtt_driver i965_gtt_driver = {
1545         .gen = 4,
1546         .setup = i9xx_setup,
1547 };
1548 static const struct intel_gtt_driver g4x_gtt_driver = {
1549         .gen = 5,
1550         .setup = i9xx_setup,
1551 };
1552 static const struct intel_gtt_driver ironlake_gtt_driver = {
1553         .gen = 5,
1554         .is_ironlake = 1,
1555         .setup = i9xx_setup,
1556 };
1557 static const struct intel_gtt_driver sandybridge_gtt_driver = {
1558         .gen = 6,
1559         .setup = i9xx_setup,
1560 };
1561
1562 /* Table to describe Intel GMCH and AGP/PCIE GART drivers.  At least one of
1563  * driver and gmch_driver must be non-null, and find_gmch will determine
1564  * which one should be used if a gmch_chip_id is present.
1565  */
1566 static const struct intel_gtt_driver_description {
1567         unsigned int gmch_chip_id;
1568         char *name;
1569         const struct agp_bridge_driver *gmch_driver;
1570         const struct intel_gtt_driver *gtt_driver;
1571 } intel_gtt_chipsets[] = {
1572         { PCI_DEVICE_ID_INTEL_82810_IG1, "i810", &intel_810_driver , NULL},
1573         { PCI_DEVICE_ID_INTEL_82810_IG3, "i810", &intel_810_driver , NULL},
1574         { PCI_DEVICE_ID_INTEL_82810E_IG, "i810", &intel_810_driver , NULL},
1575         { PCI_DEVICE_ID_INTEL_82815_CGC, "i815", &intel_810_driver , NULL},
1576         { PCI_DEVICE_ID_INTEL_82830_CGC, "830M",
1577                 &intel_830_driver , &i8xx_gtt_driver},
1578         { PCI_DEVICE_ID_INTEL_82845G_IG, "830M",
1579                 &intel_830_driver , &i8xx_gtt_driver},
1580         { PCI_DEVICE_ID_INTEL_82854_IG, "854",
1581                 &intel_830_driver , &i8xx_gtt_driver},
1582         { PCI_DEVICE_ID_INTEL_82855GM_IG, "855GM",
1583                 &intel_830_driver , &i8xx_gtt_driver},
1584         { PCI_DEVICE_ID_INTEL_82865_IG, "865",
1585                 &intel_830_driver , &i8xx_gtt_driver},
1586         { PCI_DEVICE_ID_INTEL_E7221_IG, "E7221 (i915)",
1587                 &intel_915_driver , &i915_gtt_driver },
1588         { PCI_DEVICE_ID_INTEL_82915G_IG, "915G",
1589                 &intel_915_driver , &i915_gtt_driver },
1590         { PCI_DEVICE_ID_INTEL_82915GM_IG, "915GM",
1591                 &intel_915_driver , &i915_gtt_driver },
1592         { PCI_DEVICE_ID_INTEL_82945G_IG, "945G",
1593                 &intel_915_driver , &i915_gtt_driver },
1594         { PCI_DEVICE_ID_INTEL_82945GM_IG, "945GM",
1595                 &intel_915_driver , &i915_gtt_driver },
1596         { PCI_DEVICE_ID_INTEL_82945GME_IG, "945GME",
1597                 &intel_915_driver , &i915_gtt_driver },
1598         { PCI_DEVICE_ID_INTEL_82946GZ_IG, "946GZ",
1599                 &intel_i965_driver , &i965_gtt_driver },
1600         { PCI_DEVICE_ID_INTEL_82G35_IG, "G35",
1601                 &intel_i965_driver , &i965_gtt_driver },
1602         { PCI_DEVICE_ID_INTEL_82965Q_IG, "965Q",
1603                 &intel_i965_driver , &i965_gtt_driver },
1604         { PCI_DEVICE_ID_INTEL_82965G_IG, "965G",
1605                 &intel_i965_driver , &i965_gtt_driver },
1606         { PCI_DEVICE_ID_INTEL_82965GM_IG, "965GM",
1607                 &intel_i965_driver , &i965_gtt_driver },
1608         { PCI_DEVICE_ID_INTEL_82965GME_IG, "965GME/GLE",
1609                 &intel_i965_driver , &i965_gtt_driver },
1610         { PCI_DEVICE_ID_INTEL_G33_IG, "G33",
1611                 &intel_g33_driver , &g33_gtt_driver },
1612         { PCI_DEVICE_ID_INTEL_Q35_IG, "Q35",
1613                 &intel_g33_driver , &g33_gtt_driver },
1614         { PCI_DEVICE_ID_INTEL_Q33_IG, "Q33",
1615                 &intel_g33_driver , &g33_gtt_driver },
1616         { PCI_DEVICE_ID_INTEL_PINEVIEW_M_IG, "GMA3150",
1617                 &intel_g33_driver , &pineview_gtt_driver },
1618         { PCI_DEVICE_ID_INTEL_PINEVIEW_IG, "GMA3150",
1619                 &intel_g33_driver , &pineview_gtt_driver },
1620         { PCI_DEVICE_ID_INTEL_GM45_IG, "GM45",
1621                 &intel_i965_driver , &g4x_gtt_driver },
1622         { PCI_DEVICE_ID_INTEL_EAGLELAKE_IG, "Eaglelake",
1623                 &intel_i965_driver , &g4x_gtt_driver },
1624         { PCI_DEVICE_ID_INTEL_Q45_IG, "Q45/Q43",
1625                 &intel_i965_driver , &g4x_gtt_driver },
1626         { PCI_DEVICE_ID_INTEL_G45_IG, "G45/G43",
1627                 &intel_i965_driver , &g4x_gtt_driver },
1628         { PCI_DEVICE_ID_INTEL_B43_IG, "B43",
1629                 &intel_i965_driver , &g4x_gtt_driver },
1630         { PCI_DEVICE_ID_INTEL_B43_1_IG, "B43",
1631                 &intel_i965_driver , &g4x_gtt_driver },
1632         { PCI_DEVICE_ID_INTEL_G41_IG, "G41",
1633                 &intel_i965_driver , &g4x_gtt_driver },
1634         { PCI_DEVICE_ID_INTEL_IRONLAKE_D_IG,
1635             "HD Graphics", &intel_i965_driver , &ironlake_gtt_driver },
1636         { PCI_DEVICE_ID_INTEL_IRONLAKE_M_IG,
1637             "HD Graphics", &intel_i965_driver , &ironlake_gtt_driver },
1638         { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_GT1_IG,
1639             "Sandybridge", &intel_gen6_driver , &sandybridge_gtt_driver },
1640         { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_GT2_IG,
1641             "Sandybridge", &intel_gen6_driver , &sandybridge_gtt_driver },
1642         { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_GT2_PLUS_IG,
1643             "Sandybridge", &intel_gen6_driver , &sandybridge_gtt_driver },
1644         { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_M_GT1_IG,
1645             "Sandybridge", &intel_gen6_driver , &sandybridge_gtt_driver },
1646         { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_M_GT2_IG,
1647             "Sandybridge", &intel_gen6_driver , &sandybridge_gtt_driver },
1648         { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_M_GT2_PLUS_IG,
1649             "Sandybridge", &intel_gen6_driver , &sandybridge_gtt_driver },
1650         { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_S_IG,
1651             "Sandybridge", &intel_gen6_driver , &sandybridge_gtt_driver },
1652         { 0, NULL, NULL }
1653 };
1654
1655 static int find_gmch(u16 device)
1656 {
1657         struct pci_dev *gmch_device;
1658
1659         gmch_device = pci_get_device(PCI_VENDOR_ID_INTEL, device, NULL);
1660         if (gmch_device && PCI_FUNC(gmch_device->devfn) != 0) {
1661                 gmch_device = pci_get_device(PCI_VENDOR_ID_INTEL,
1662                                              device, gmch_device);
1663         }
1664
1665         if (!gmch_device)
1666                 return 0;
1667
1668         intel_private.pcidev = gmch_device;
1669         return 1;
1670 }
1671
1672 int intel_gmch_probe(struct pci_dev *pdev,
1673                                       struct agp_bridge_data *bridge)
1674 {
1675         int i, mask;
1676         bridge->driver = NULL;
1677
1678         for (i = 0; intel_gtt_chipsets[i].name != NULL; i++) {
1679                 if (find_gmch(intel_gtt_chipsets[i].gmch_chip_id)) {
1680                         bridge->driver =
1681                                 intel_gtt_chipsets[i].gmch_driver;
1682                         intel_private.driver =
1683                                 intel_gtt_chipsets[i].gtt_driver;
1684                         break;
1685                 }
1686         }
1687
1688         if (!bridge->driver)
1689                 return 0;
1690
1691         bridge->dev_private_data = &intel_private;
1692         bridge->dev = pdev;
1693
1694         intel_private.bridge_dev = pci_dev_get(pdev);
1695
1696         dev_info(&pdev->dev, "Intel %s Chipset\n", intel_gtt_chipsets[i].name);
1697
1698         if (bridge->driver->mask_memory == intel_gen6_mask_memory)
1699                 mask = 40;
1700         else if (bridge->driver->mask_memory == intel_i965_mask_memory)
1701                 mask = 36;
1702         else
1703                 mask = 32;
1704
1705         if (pci_set_dma_mask(intel_private.pcidev, DMA_BIT_MASK(mask)))
1706                 dev_err(&intel_private.pcidev->dev,
1707                         "set gfx device dma mask %d-bit failed!\n", mask);
1708         else
1709                 pci_set_consistent_dma_mask(intel_private.pcidev,
1710                                             DMA_BIT_MASK(mask));
1711
1712         if (bridge->driver == &intel_810_driver)
1713                 return 1;
1714
1715         if (intel_gtt_init() != 0)
1716                 return 0;
1717
1718         return 1;
1719 }
1720 EXPORT_SYMBOL(intel_gmch_probe);
1721
1722 struct intel_gtt *intel_gtt_get(void)
1723 {
1724         return &intel_private.base;
1725 }
1726 EXPORT_SYMBOL(intel_gtt_get);
1727
1728 void intel_gmch_remove(struct pci_dev *pdev)
1729 {
1730         if (intel_private.pcidev)
1731                 pci_dev_put(intel_private.pcidev);
1732         if (intel_private.bridge_dev)
1733                 pci_dev_put(intel_private.bridge_dev);
1734 }
1735 EXPORT_SYMBOL(intel_gmch_remove);
1736
1737 MODULE_AUTHOR("Dave Jones <davej@redhat.com>");
1738 MODULE_LICENSE("GPL and additional rights");