drivers/char/agp/intel-gtt.c

   1 /*
   2  * Intel GTT (Graphics Translation Table) routines
   3  *
   4  * Caveat: This driver implements the linux agp interface, but this is far from
   5  * a agp driver! GTT support ended up here for purely historical reasons: The
   6  * old userspace intel graphics drivers needed an interface to map memory into
   7  * the GTT. And the drm provides a default interface for graphic devices sitting
   8  * on an agp port. So it made sense to fake the GTT support as an agp port to
   9  * avoid having to create a new api.
  10  *
  11  * With gem this does not make much sense anymore, just needlessly complicates
  12  * the code. But as long as the old graphics stack is still support, it's stuck
  13  * here.
  14  *
  15  * /fairy-tale-mode off
  16  */
  17
  18 /*
  19  * If we have Intel graphics, we're not going to have anything other than
  20  * an Intel IOMMU. So make the correct use of the PCI DMA API contingent
  21  * on the Intel IOMMU support (CONFIG_DMAR).
  22  * Only newer chipsets need to bother with this, of course.
  23  */
  24 #ifdef CONFIG_DMAR
  25 #define USE_PCI_DMA_API 1
  26 #endif
  27
  28 /* Max amount of stolen space, anything above will be returned to Linux */
  29 int intel_max_stolen = 32 * 1024 * 1024;
  30 EXPORT_SYMBOL(intel_max_stolen);
  31
  32 static const struct aper_size_info_fixed intel_i810_sizes[] =
  33 {
  34         {64, 16384, 4},
  35         /* The 32M mode still requires a 64k gatt */
  36         {32, 8192, 4}
  37 };
  38
  39 #define AGP_DCACHE_MEMORY       1
  40 #define AGP_PHYS_MEMORY         2
  41 #define INTEL_AGP_CACHED_MEMORY 3
  42
  43 static struct gatt_mask intel_i810_masks[] =
  44 {
  45         {.mask = I810_PTE_VALID, .type = 0},
  46         {.mask = (I810_PTE_VALID | I810_PTE_LOCAL), .type = AGP_DCACHE_MEMORY},
  47         {.mask = I810_PTE_VALID, .type = 0},
  48         {.mask = I810_PTE_VALID | I830_PTE_SYSTEM_CACHED,
  49          .type = INTEL_AGP_CACHED_MEMORY}
  50 };
  51
  52 #define INTEL_AGP_UNCACHED_MEMORY              0
  53 #define INTEL_AGP_CACHED_MEMORY_LLC            1
  54 #define INTEL_AGP_CACHED_MEMORY_LLC_GFDT       2
  55 #define INTEL_AGP_CACHED_MEMORY_LLC_MLC        3
  56 #define INTEL_AGP_CACHED_MEMORY_LLC_MLC_GFDT   4
  57
  58 static struct gatt_mask intel_gen6_masks[] =
  59 {
  60         {.mask = I810_PTE_VALID | GEN6_PTE_UNCACHED,
  61          .type = INTEL_AGP_UNCACHED_MEMORY },
  62         {.mask = I810_PTE_VALID | GEN6_PTE_LLC,
  63          .type = INTEL_AGP_CACHED_MEMORY_LLC },
  64         {.mask = I810_PTE_VALID | GEN6_PTE_LLC | GEN6_PTE_GFDT,
  65          .type = INTEL_AGP_CACHED_MEMORY_LLC_GFDT },
  66         {.mask = I810_PTE_VALID | GEN6_PTE_LLC_MLC,
  67          .type = INTEL_AGP_CACHED_MEMORY_LLC_MLC },
  68         {.mask = I810_PTE_VALID | GEN6_PTE_LLC_MLC | GEN6_PTE_GFDT,
  69          .type = INTEL_AGP_CACHED_MEMORY_LLC_MLC_GFDT },
  70 };
  71
  72 static struct _intel_private {
  73         struct pci_dev *pcidev; /* device one */
  74         u8 __iomem *registers;
  75         u32 __iomem *gtt;               /* I915G */
  76         int num_dcache_entries;
  77         /* gtt_entries is the number of gtt entries that are already mapped
  78          * to stolen memory.  Stolen memory is larger than the memory mapped
  79          * through gtt_entries, as it includes some reserved space for the BIOS
  80          * popup and for the GTT.
  81          */
  82         int gtt_entries;                        /* i830+ */
  83         int gtt_total_size;
  84         union {
  85                 void __iomem *i9xx_flush_page;
  86                 void *i8xx_flush_page;
  87         };
  88         struct page *i8xx_page;
  89         struct resource ifp_resource;
  90         int resource_valid;
  91 } intel_private;
  92
  93 #ifdef USE_PCI_DMA_API
  94 static int intel_agp_map_page(struct page *page, dma_addr_t *ret)
  95 {
  96         *ret = pci_map_page(intel_private.pcidev, page, 0,
  97                             PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
  98         if (pci_dma_mapping_error(intel_private.pcidev, *ret))
  99                 return -EINVAL;
 100         return 0;
 101 }
 102
 103 static void intel_agp_unmap_page(struct page *page, dma_addr_t dma)
 104 {
 105         pci_unmap_page(intel_private.pcidev, dma,
 106                        PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 107 }
 108
 109 static void intel_agp_free_sglist(struct agp_memory *mem)
 110 {
 111         struct sg_table st;
 112
 113         st.sgl = mem->sg_list;
 114         st.orig_nents = st.nents = mem->page_count;
 115
 116         sg_free_table(&st);
 117
 118         mem->sg_list = NULL;
 119         mem->num_sg = 0;
 120 }
 121
 122 static int intel_agp_map_memory(struct agp_memory *mem)
 123 {
 124         struct sg_table st;
 125         struct scatterlist *sg;
 126         int i;
 127
 128         DBG("try mapping %lu pages\n", (unsigned long)mem->page_count);
 129
 130         if (sg_alloc_table(&st, mem->page_count, GFP_KERNEL))
 131                 goto err;
 132
 133         mem->sg_list = sg = st.sgl;
 134
 135         for (i = 0 ; i < mem->page_count; i++, sg = sg_next(sg))
 136                 sg_set_page(sg, mem->pages[i], PAGE_SIZE, 0);
 137
 138         mem->num_sg = pci_map_sg(intel_private.pcidev, mem->sg_list,
 139                                  mem->page_count, PCI_DMA_BIDIRECTIONAL);
 140         if (unlikely(!mem->num_sg))
 141                 goto err;
 142
 143         return 0;
 144
 145 err:
 146         sg_free_table(&st);
 147         return -ENOMEM;
 148 }
 149
 150 static void intel_agp_unmap_memory(struct agp_memory *mem)
 151 {
 152         DBG("try unmapping %lu pages\n", (unsigned long)mem->page_count);
 153
 154         pci_unmap_sg(intel_private.pcidev, mem->sg_list,
 155                      mem->page_count, PCI_DMA_BIDIRECTIONAL);
 156         intel_agp_free_sglist(mem);
 157 }
 158
 159 static void intel_agp_insert_sg_entries(struct agp_memory *mem,
 160                                         off_t pg_start, int mask_type)
 161 {
 162         struct scatterlist *sg;
 163         int i, j;
 164
 165         j = pg_start;
 166
 167         WARN_ON(!mem->num_sg);
 168
 169         if (mem->num_sg == mem->page_count) {
 170                 for_each_sg(mem->sg_list, sg, mem->page_count, i) {
 171                         writel(agp_bridge->driver->mask_memory(agp_bridge,
 172                                         sg_dma_address(sg), mask_type),
 173                                         intel_private.gtt+j);
 174                         j++;
 175                 }
 176         } else {
 177                 /* sg may merge pages, but we have to separate
 178                  * per-page addr for GTT */
 179                 unsigned int len, m;
 180
 181                 for_each_sg(mem->sg_list, sg, mem->num_sg, i) {
 182                         len = sg_dma_len(sg) / PAGE_SIZE;
 183                         for (m = 0; m < len; m++) {
 184                                 writel(agp_bridge->driver->mask_memory(agp_bridge,
 185                                                                        sg_dma_address(sg) + m * PAGE_SIZE,
 186                                                                        mask_type),
 187                                        intel_private.gtt+j);
 188                                 j++;
 189                         }
 190                 }
 191         }
 192         readl(intel_private.gtt+j-1);
 193 }
 194
 195 #else
 196
 197 static void intel_agp_insert_sg_entries(struct agp_memory *mem,
 198                                         off_t pg_start, int mask_type)
 199 {
 200         int i, j;
 201
 202         for (i = 0, j = pg_start; i < mem->page_count; i++, j++) {
 203                 writel(agp_bridge->driver->mask_memory(agp_bridge,
 204                                 page_to_phys(mem->pages[i]), mask_type),
 205                        intel_private.gtt+j);
 206         }
 207
 208         readl(intel_private.gtt+j-1);
 209 }
 210
 211 #endif
 212
 213 static int intel_i810_fetch_size(void)
 214 {
 215         u32 smram_miscc;
 216         struct aper_size_info_fixed *values;
 217
 218         pci_read_config_dword(agp_bridge->dev, I810_SMRAM_MISCC, &smram_miscc);
 219         values = A_SIZE_FIX(agp_bridge->driver->aperture_sizes);
 220
 221         if ((smram_miscc & I810_GMS) == I810_GMS_DISABLE) {
 222                 dev_warn(&agp_bridge->dev->dev, "i810 is disabled\n");
 223                 return 0;
 224         }
 225         if ((smram_miscc & I810_GFX_MEM_WIN_SIZE) == I810_GFX_MEM_WIN_32M) {
 226                 agp_bridge->current_size = (void *) (values + 1);
 227                 agp_bridge->aperture_size_idx = 1;
 228                 return values[1].size;
 229         } else {
 230                 agp_bridge->current_size = (void *) (values);
 231                 agp_bridge->aperture_size_idx = 0;
 232                 return values[0].size;
 233         }
 234
 235         return 0;
 236 }
 237
 238 static int intel_i810_configure(void)
 239 {
 240         struct aper_size_info_fixed *current_size;
 241         u32 temp;
 242         int i;
 243
 244         current_size = A_SIZE_FIX(agp_bridge->current_size);
 245
 246         if (!intel_private.registers) {
 247                 pci_read_config_dword(intel_private.pcidev, I810_MMADDR, &temp);
 248                 temp &= 0xfff80000;
 249
 250                 intel_private.registers = ioremap(temp, 128 * 4096);
 251                 if (!intel_private.registers) {
 252                         dev_err(&intel_private.pcidev->dev,
 253                                 "can't remap memory\n");
 254                         return -ENOMEM;
 255                 }
 256         }
 257
 258         if ((readl(intel_private.registers+I810_DRAM_CTL)
 259                 & I810_DRAM_ROW_0) == I810_DRAM_ROW_0_SDRAM) {
 260                 /* This will need to be dynamically assigned */
 261                 dev_info(&intel_private.pcidev->dev,
 262                          "detected 4MB dedicated video ram\n");
 263                 intel_private.num_dcache_entries = 1024;
 264         }
 265         pci_read_config_dword(intel_private.pcidev, I810_GMADDR, &temp);
 266         agp_bridge->gart_bus_addr = (temp & PCI_BASE_ADDRESS_MEM_MASK);
 267         writel(agp_bridge->gatt_bus_addr | I810_PGETBL_ENABLED, intel_private.registers+I810_PGETBL_CTL);
 268         readl(intel_private.registers+I810_PGETBL_CTL); /* PCI Posting. */
 269
 270         if (agp_bridge->driver->needs_scratch_page) {
 271                 for (i = 0; i < current_size->num_entries; i++) {
 272                         writel(agp_bridge->scratch_page, intel_private.registers+I810_PTE_BASE+(i*4));
 273                 }
 274                 readl(intel_private.registers+I810_PTE_BASE+((i-1)*4)); /* PCI posting. */
 275         }
 276         global_cache_flush();
 277         return 0;
 278 }
 279
 280 static void intel_i810_cleanup(void)
 281 {
 282         writel(0, intel_private.registers+I810_PGETBL_CTL);
 283         readl(intel_private.registers); /* PCI Posting. */
 284         iounmap(intel_private.registers);
 285 }
 286
 287 static void intel_i810_agp_enable(struct agp_bridge_data *bridge, u32 mode)
 288 {
 289         return;
 290 }
 291
 292 /* Exists to support ARGB cursors */
 293 static struct page *i8xx_alloc_pages(void)
 294 {
 295         struct page *page;
 296
 297         page = alloc_pages(GFP_KERNEL | GFP_DMA32, 2);
 298         if (page == NULL)
 299                 return NULL;
 300
 301         if (set_pages_uc(page, 4) < 0) {
 302                 set_pages_wb(page, 4);
 303                 __free_pages(page, 2);
 304                 return NULL;
 305         }
 306         get_page(page);
 307         atomic_inc(&agp_bridge->current_memory_agp);
 308         return page;
 309 }
 310
 311 static void i8xx_destroy_pages(struct page *page)
 312 {
 313         if (page == NULL)
 314                 return;
 315
 316         set_pages_wb(page, 4);
 317         put_page(page);
 318         __free_pages(page, 2);
 319         atomic_dec(&agp_bridge->current_memory_agp);
 320 }
 321
 322 static int intel_i830_type_to_mask_type(struct agp_bridge_data *bridge,
 323                                         int type)
 324 {
 325         if (type < AGP_USER_TYPES)
 326                 return type;
 327         else if (type == AGP_USER_CACHED_MEMORY)
 328                 return INTEL_AGP_CACHED_MEMORY;
 329         else
 330                 return 0;
 331 }
 332
 333 static int intel_gen6_type_to_mask_type(struct agp_bridge_data *bridge,
 334                                         int type)
 335 {
 336         unsigned int type_mask = type & ~AGP_USER_CACHED_MEMORY_GFDT;
 337         unsigned int gfdt = type & AGP_USER_CACHED_MEMORY_GFDT;
 338
 339         if (type_mask == AGP_USER_UNCACHED_MEMORY)
 340                 return INTEL_AGP_UNCACHED_MEMORY;
 341         else if (type_mask == AGP_USER_CACHED_MEMORY_LLC_MLC)
 342                 return gfdt ? INTEL_AGP_CACHED_MEMORY_LLC_MLC_GFDT :
 343                               INTEL_AGP_CACHED_MEMORY_LLC_MLC;
 344         else /* set 'normal'/'cached' to LLC by default */
 345                 return gfdt ? INTEL_AGP_CACHED_MEMORY_LLC_GFDT :
 346                               INTEL_AGP_CACHED_MEMORY_LLC;
 347 }
 348
 349
 350 static int intel_i810_insert_entries(struct agp_memory *mem, off_t pg_start,
 351                                 int type)
 352 {
 353         int i, j, num_entries;
 354         void *temp;
 355         int ret = -EINVAL;
 356         int mask_type;
 357
 358         if (mem->page_count == 0)
 359                 goto out;
 360
 361         temp = agp_bridge->current_size;
 362         num_entries = A_SIZE_FIX(temp)->num_entries;
 363
 364         if ((pg_start + mem->page_count) > num_entries)
 365                 goto out_err;
 366
 367
 368         for (j = pg_start; j < (pg_start + mem->page_count); j++) {
 369                 if (!PGE_EMPTY(agp_bridge, readl(agp_bridge->gatt_table+j))) {
 370                         ret = -EBUSY;
 371                         goto out_err;
 372                 }
 373         }
 374
 375         if (type != mem->type)
 376                 goto out_err;
 377
 378         mask_type = agp_bridge->driver->agp_type_to_mask_type(agp_bridge, type);
 379
 380         switch (mask_type) {
 381         case AGP_DCACHE_MEMORY:
 382                 if (!mem->is_flushed)
 383                         global_cache_flush();
 384                 for (i = pg_start; i < (pg_start + mem->page_count); i++) {
 385                         writel((i*4096)|I810_PTE_LOCAL|I810_PTE_VALID,
 386                                intel_private.registers+I810_PTE_BASE+(i*4));
 387                 }
 388                 readl(intel_private.registers+I810_PTE_BASE+((i-1)*4));
 389                 break;
 390         case AGP_PHYS_MEMORY:
 391         case AGP_NORMAL_MEMORY:
 392                 if (!mem->is_flushed)
 393                         global_cache_flush();
 394                 for (i = 0, j = pg_start; i < mem->page_count; i++, j++) {
 395                         writel(agp_bridge->driver->mask_memory(agp_bridge,
 396                                         page_to_phys(mem->pages[i]), mask_type),
 397                                intel_private.registers+I810_PTE_BASE+(j*4));
 398                 }
 399                 readl(intel_private.registers+I810_PTE_BASE+((j-1)*4));
 400                 break;
 401         default:
 402                 goto out_err;
 403         }
 404
 405 out:
 406         ret = 0;
 407 out_err:
 408         mem->is_flushed = true;
 409         return ret;
 410 }
 411
 412 static int intel_i810_remove_entries(struct agp_memory *mem, off_t pg_start,
 413                                 int type)
 414 {
 415         int i;
 416
 417         if (mem->page_count == 0)
 418                 return 0;
 419
 420         for (i = pg_start; i < (mem->page_count + pg_start); i++) {
 421                 writel(agp_bridge->scratch_page, intel_private.registers+I810_PTE_BASE+(i*4));
 422         }
 423         readl(intel_private.registers+I810_PTE_BASE+((i-1)*4));
 424
 425         return 0;
 426 }
 427
 428 /*
 429  * The i810/i830 requires a physical address to program its mouse
 430  * pointer into hardware.
 431  * However the Xserver still writes to it through the agp aperture.
 432  */
 433 static struct agp_memory *alloc_agpphysmem_i8xx(size_t pg_count, int type)
 434 {
 435         struct agp_memory *new;
 436         struct page *page;
 437
 438         switch (pg_count) {
 439         case 1: page = agp_bridge->driver->agp_alloc_page(agp_bridge);
 440                 break;
 441         case 4:
 442                 /* kludge to get 4 physical pages for ARGB cursor */
 443                 page = i8xx_alloc_pages();
 444                 break;
 445         default:
 446                 return NULL;
 447         }
 448
 449         if (page == NULL)
 450                 return NULL;
 451
 452         new = agp_create_memory(pg_count);
 453         if (new == NULL)
 454                 return NULL;
 455
 456         new->pages[0] = page;
 457         if (pg_count == 4) {
 458                 /* kludge to get 4 physical pages for ARGB cursor */
 459                 new->pages[1] = new->pages[0] + 1;
 460                 new->pages[2] = new->pages[1] + 1;
 461                 new->pages[3] = new->pages[2] + 1;
 462         }
 463         new->page_count = pg_count;
 464         new->num_scratch_pages = pg_count;
 465         new->type = AGP_PHYS_MEMORY;
 466         new->physical = page_to_phys(new->pages[0]);
 467         return new;
 468 }
 469
 470 static struct agp_memory *intel_i810_alloc_by_type(size_t pg_count, int type)
 471 {
 472         struct agp_memory *new;
 473
 474         if (type == AGP_DCACHE_MEMORY) {
 475                 if (pg_count != intel_private.num_dcache_entries)
 476                         return NULL;
 477
 478                 new = agp_create_memory(1);
 479                 if (new == NULL)
 480                         return NULL;
 481
 482                 new->type = AGP_DCACHE_MEMORY;
 483                 new->page_count = pg_count;
 484                 new->num_scratch_pages = 0;
 485                 agp_free_page_array(new);
 486                 return new;
 487         }
 488         if (type == AGP_PHYS_MEMORY)
 489                 return alloc_agpphysmem_i8xx(pg_count, type);
 490         return NULL;
 491 }
 492
 493 static void intel_i810_free_by_type(struct agp_memory *curr)
 494 {
 495         agp_free_key(curr->key);
 496         if (curr->type == AGP_PHYS_MEMORY) {
 497                 if (curr->page_count == 4)
 498                         i8xx_destroy_pages(curr->pages[0]);
 499                 else {
 500                         agp_bridge->driver->agp_destroy_page(curr->pages[0],
 501                                                              AGP_PAGE_DESTROY_UNMAP);
 502                         agp_bridge->driver->agp_destroy_page(curr->pages[0],
 503                                                              AGP_PAGE_DESTROY_FREE);
 504                 }
 505                 agp_free_page_array(curr);
 506         }
 507         kfree(curr);
 508 }
 509
 510 static unsigned long intel_i810_mask_memory(struct agp_bridge_data *bridge,
 511                                             dma_addr_t addr, int type)
 512 {
 513         /* Type checking must be done elsewhere */
 514         return addr | bridge->driver->masks[type].mask;
 515 }
 516
 517 static struct aper_size_info_fixed intel_i830_sizes[] =
 518 {
 519         {128, 32768, 5},
 520         /* The 64M mode still requires a 128k gatt */
 521         {64, 16384, 5},
 522         {256, 65536, 6},
 523         {512, 131072, 7},
 524 };
 525
 526 static void intel_i830_init_gtt_entries(void)
 527 {
 528         u16 gmch_ctrl;
 529         int gtt_entries = 0;
 530         u8 rdct;
 531         int local = 0;
 532         static const int ddt[4] = { 0, 16, 32, 64 };
 533         int size; /* reserved space (in kb) at the top of stolen memory */
 534
 535         pci_read_config_word(agp_bridge->dev, I830_GMCH_CTRL, &gmch_ctrl);
 536
 537         if (IS_I965) {
 538                 u32 pgetbl_ctl;
 539                 pgetbl_ctl = readl(intel_private.registers+I810_PGETBL_CTL);
 540
 541                 /* The 965 has a field telling us the size of the GTT,
 542                  * which may be larger than what is necessary to map the
 543                  * aperture.
 544                  */
 545                 switch (pgetbl_ctl & I965_PGETBL_SIZE_MASK) {
 546                 case I965_PGETBL_SIZE_128KB:
 547                         size = 128;
 548                         break;
 549                 case I965_PGETBL_SIZE_256KB:
 550                         size = 256;
 551                         break;
 552                 case I965_PGETBL_SIZE_512KB:
 553                         size = 512;
 554                         break;
 555                 case I965_PGETBL_SIZE_1MB:
 556                         size = 1024;
 557                         break;
 558                 case I965_PGETBL_SIZE_2MB:
 559                         size = 2048;
 560                         break;
 561                 case I965_PGETBL_SIZE_1_5MB:
 562                         size = 1024 + 512;
 563                         break;
 564                 default:
 565                         dev_info(&intel_private.pcidev->dev,
 566                                  "unknown page table size, assuming 512KB\n");
 567                         size = 512;
 568                 }
 569                 size += 4; /* add in BIOS popup space */
 570         } else if (IS_G33 && !IS_PINEVIEW) {
 571         /* G33's GTT size defined in gmch_ctrl */
 572                 switch (gmch_ctrl & G33_PGETBL_SIZE_MASK) {
 573                 case G33_PGETBL_SIZE_1M:
 574                         size = 1024;
 575                         break;
 576                 case G33_PGETBL_SIZE_2M:
 577                         size = 2048;
 578                         break;
 579                 default:
 580                         dev_info(&agp_bridge->dev->dev,
 581                                  "unknown page table size 0x%x, assuming 512KB\n",
 582                                 (gmch_ctrl & G33_PGETBL_SIZE_MASK));
 583                         size = 512;
 584                 }
 585                 size += 4;
 586         } else if (IS_G4X || IS_PINEVIEW) {
 587                 /* On 4 series hardware, GTT stolen is separate from graphics
 588                  * stolen, ignore it in stolen gtt entries counting.  However,
 589                  * 4KB of the stolen memory doesn't get mapped to the GTT.
 590                  */
 591                 size = 4;
 592         } else {
 593                 /* On previous hardware, the GTT size was just what was
 594                  * required to map the aperture.
 595                  */
 596                 size = agp_bridge->driver->fetch_size() + 4;
 597         }
 598
 599         if (agp_bridge->dev->device == PCI_DEVICE_ID_INTEL_82830_HB ||
 600             agp_bridge->dev->device == PCI_DEVICE_ID_INTEL_82845G_HB) {
 601                 switch (gmch_ctrl & I830_GMCH_GMS_MASK) {
 602                 case I830_GMCH_GMS_STOLEN_512:
 603                         gtt_entries = KB(512) - KB(size);
 604                         break;
 605                 case I830_GMCH_GMS_STOLEN_1024:
 606                         gtt_entries = MB(1) - KB(size);
 607                         break;
 608                 case I830_GMCH_GMS_STOLEN_8192:
 609                         gtt_entries = MB(8) - KB(size);
 610                         break;
 611                 case I830_GMCH_GMS_LOCAL:
 612                         rdct = readb(intel_private.registers+I830_RDRAM_CHANNEL_TYPE);
 613                         gtt_entries = (I830_RDRAM_ND(rdct) + 1) *
 614                                         MB(ddt[I830_RDRAM_DDT(rdct)]);
 615                         local = 1;
 616                         break;
 617                 default:
 618                         gtt_entries = 0;
 619                         break;
 620                 }
 621         } else if (IS_SNB) {
 622                 /*
 623                  * SandyBridge has new memory control reg at 0x50.w
 624                  */
 625                 u16 snb_gmch_ctl;
 626                 pci_read_config_word(intel_private.pcidev, SNB_GMCH_CTRL, &snb_gmch_ctl);
 627                 switch (snb_gmch_ctl & SNB_GMCH_GMS_STOLEN_MASK) {
 628                 case SNB_GMCH_GMS_STOLEN_32M:
 629                         gtt_entries = MB(32) - KB(size);
 630                         break;
 631                 case SNB_GMCH_GMS_STOLEN_64M:
 632                         gtt_entries = MB(64) - KB(size);
 633                         break;
 634                 case SNB_GMCH_GMS_STOLEN_96M:
 635                         gtt_entries = MB(96) - KB(size);
 636                         break;
 637                 case SNB_GMCH_GMS_STOLEN_128M:
 638                         gtt_entries = MB(128) - KB(size);
 639                         break;
 640                 case SNB_GMCH_GMS_STOLEN_160M:
 641                         gtt_entries = MB(160) - KB(size);
 642                         break;
 643                 case SNB_GMCH_GMS_STOLEN_192M:
 644                         gtt_entries = MB(192) - KB(size);
 645                         break;
 646                 case SNB_GMCH_GMS_STOLEN_224M:
 647                         gtt_entries = MB(224) - KB(size);
 648                         break;
 649                 case SNB_GMCH_GMS_STOLEN_256M:
 650                         gtt_entries = MB(256) - KB(size);
 651                         break;
 652                 case SNB_GMCH_GMS_STOLEN_288M:
 653                         gtt_entries = MB(288) - KB(size);
 654                         break;
 655                 case SNB_GMCH_GMS_STOLEN_320M:
 656                         gtt_entries = MB(320) - KB(size);
 657                         break;
 658                 case SNB_GMCH_GMS_STOLEN_352M:
 659                         gtt_entries = MB(352) - KB(size);
 660                         break;
 661                 case SNB_GMCH_GMS_STOLEN_384M:
 662                         gtt_entries = MB(384) - KB(size);
 663                         break;
 664                 case SNB_GMCH_GMS_STOLEN_416M:
 665                         gtt_entries = MB(416) - KB(size);
 666                         break;
 667                 case SNB_GMCH_GMS_STOLEN_448M:
 668                         gtt_entries = MB(448) - KB(size);
 669                         break;
 670                 case SNB_GMCH_GMS_STOLEN_480M:
 671                         gtt_entries = MB(480) - KB(size);
 672                         break;
 673                 case SNB_GMCH_GMS_STOLEN_512M:
 674                         gtt_entries = MB(512) - KB(size);
 675                         break;
 676                 }
 677         } else {
 678                 switch (gmch_ctrl & I855_GMCH_GMS_MASK) {
 679                 case I855_GMCH_GMS_STOLEN_1M:
 680                         gtt_entries = MB(1) - KB(size);
 681                         break;
 682                 case I855_GMCH_GMS_STOLEN_4M:
 683                         gtt_entries = MB(4) - KB(size);
 684                         break;
 685                 case I855_GMCH_GMS_STOLEN_8M:
 686                         gtt_entries = MB(8) - KB(size);
 687                         break;
 688                 case I855_GMCH_GMS_STOLEN_16M:
 689                         gtt_entries = MB(16) - KB(size);
 690                         break;
 691                 case I855_GMCH_GMS_STOLEN_32M:
 692                         gtt_entries = MB(32) - KB(size);
 693                         break;
 694                 case I915_GMCH_GMS_STOLEN_48M:
 695                         /* Check it's really I915G */
 696                         if (IS_I915 || IS_I965 || IS_G33 || IS_G4X)
 697                                 gtt_entries = MB(48) - KB(size);
 698                         else
 699                                 gtt_entries = 0;
 700                         break;
 701                 case I915_GMCH_GMS_STOLEN_64M:
 702                         /* Check it's really I915G */
 703                         if (IS_I915 || IS_I965 || IS_G33 || IS_G4X)
 704                                 gtt_entries = MB(64) - KB(size);
 705                         else
 706                                 gtt_entries = 0;
 707                         break;
 708                 case G33_GMCH_GMS_STOLEN_128M:
 709                         if (IS_G33 || IS_I965 || IS_G4X)
 710                                 gtt_entries = MB(128) - KB(size);
 711                         else
 712                                 gtt_entries = 0;
 713                         break;
 714                 case G33_GMCH_GMS_STOLEN_256M:
 715                         if (IS_G33 || IS_I965 || IS_G4X)
 716                                 gtt_entries = MB(256) - KB(size);
 717                         else
 718                                 gtt_entries = 0;
 719                         break;
 720                 case INTEL_GMCH_GMS_STOLEN_96M:
 721                         if (IS_I965 || IS_G4X)
 722                                 gtt_entries = MB(96) - KB(size);
 723                         else
 724                                 gtt_entries = 0;
 725                         break;
 726                 case INTEL_GMCH_GMS_STOLEN_160M:
 727                         if (IS_I965 || IS_G4X)
 728                                 gtt_entries = MB(160) - KB(size);
 729                         else
 730                                 gtt_entries = 0;
 731                         break;
 732                 case INTEL_GMCH_GMS_STOLEN_224M:
 733                         if (IS_I965 || IS_G4X)
 734                                 gtt_entries = MB(224) - KB(size);
 735                         else
 736                                 gtt_entries = 0;
 737                         break;
 738                 case INTEL_GMCH_GMS_STOLEN_352M:
 739                         if (IS_I965 || IS_G4X)
 740                                 gtt_entries = MB(352) - KB(size);
 741                         else
 742                                 gtt_entries = 0;
 743                         break;
 744                 default:
 745                         gtt_entries = 0;
 746                         break;
 747                 }
 748         }
 749         if (!local && gtt_entries > intel_max_stolen) {
 750                 dev_info(&agp_bridge->dev->dev,
 751                          "detected %dK stolen memory, trimming to %dK\n",
 752                          gtt_entries / KB(1), intel_max_stolen / KB(1));
 753                 gtt_entries = intel_max_stolen / KB(4);
 754         } else if (gtt_entries > 0) {
 755                 dev_info(&agp_bridge->dev->dev, "detected %dK %s memory\n",
 756                        gtt_entries / KB(1), local ? "local" : "stolen");
 757                 gtt_entries /= KB(4);
 758         } else {
 759                 dev_info(&agp_bridge->dev->dev,
 760                        "no pre-allocated video memory detected\n");
 761                 gtt_entries = 0;
 762         }
 763
 764         intel_private.gtt_entries = gtt_entries;
 765 }
 766
 767 static void intel_i830_fini_flush(void)
 768 {
 769         kunmap(intel_private.i8xx_page);
 770         intel_private.i8xx_flush_page = NULL;
 771         unmap_page_from_agp(intel_private.i8xx_page);
 772
 773         __free_page(intel_private.i8xx_page);
 774         intel_private.i8xx_page = NULL;
 775 }
 776
 777 static void intel_i830_setup_flush(void)
 778 {
 779         /* return if we've already set the flush mechanism up */
 780         if (intel_private.i8xx_page)
 781                 return;
 782
 783         intel_private.i8xx_page = alloc_page(GFP_KERNEL | __GFP_ZERO | GFP_DMA32);
 784         if (!intel_private.i8xx_page)
 785                 return;
 786
 787         intel_private.i8xx_flush_page = kmap(intel_private.i8xx_page);
 788         if (!intel_private.i8xx_flush_page)
 789                 intel_i830_fini_flush();
 790 }
 791
 792 /* The chipset_flush interface needs to get data that has already been
 793  * flushed out of the CPU all the way out to main memory, because the GPU
 794  * doesn't snoop those buffers.
 795  *
 796  * The 8xx series doesn't have the same lovely interface for flushing the
 797  * chipset write buffers that the later chips do. According to the 865
 798  * specs, it's 64 octwords, or 1KB.  So, to get those previous things in
 799  * that buffer out, we just fill 1KB and clflush it out, on the assumption
 800  * that it'll push whatever was in there out.  It appears to work.
 801  */
 802 static void intel_i830_chipset_flush(struct agp_bridge_data *bridge)
 803 {
 804         unsigned int *pg = intel_private.i8xx_flush_page;
 805
 806         memset(pg, 0, 1024);
 807
 808         if (cpu_has_clflush)
 809                 clflush_cache_range(pg, 1024);
 810         else if (wbinvd_on_all_cpus() != 0)
 811                 printk(KERN_ERR "Timed out waiting for cache flush.\n");
 812 }
 813
 814 /* The intel i830 automatically initializes the agp aperture during POST.
 815  * Use the memory already set aside for in the GTT.
 816  */
 817 static int intel_i830_create_gatt_table(struct agp_bridge_data *bridge)
 818 {
 819         int page_order;
 820         struct aper_size_info_fixed *size;
 821         int num_entries;
 822         u32 temp;
 823
 824         size = agp_bridge->current_size;
 825         page_order = size->page_order;
 826         num_entries = size->num_entries;
 827         agp_bridge->gatt_table_real = NULL;
 828
 829         pci_read_config_dword(intel_private.pcidev, I810_MMADDR, &temp);
 830         temp &= 0xfff80000;
 831
 832         intel_private.registers = ioremap(temp, 128 * 4096);
 833         if (!intel_private.registers)
 834                 return -ENOMEM;
 835
 836         temp = readl(intel_private.registers+I810_PGETBL_CTL) & 0xfffff000;
 837         global_cache_flush();   /* FIXME: ?? */
 838
 839         /* we have to call this as early as possible after the MMIO base address is known */
 840         intel_i830_init_gtt_entries();
 841         if (intel_private.gtt_entries == 0) {
 842                 iounmap(intel_private.registers);
 843                 return -ENOMEM;
 844         }
 845
 846         agp_bridge->gatt_table = NULL;
 847
 848         agp_bridge->gatt_bus_addr = temp;
 849
 850         return 0;
 851 }
 852
 853 /* Return the gatt table to a sane state. Use the top of stolen
 854  * memory for the GTT.
 855  */
 856 static int intel_i830_free_gatt_table(struct agp_bridge_data *bridge)
 857 {
 858         return 0;
 859 }
 860
 861 static int intel_i830_fetch_size(void)
 862 {
 863         u16 gmch_ctrl;
 864         struct aper_size_info_fixed *values;
 865
 866         values = A_SIZE_FIX(agp_bridge->driver->aperture_sizes);
 867
 868         if (agp_bridge->dev->device != PCI_DEVICE_ID_INTEL_82830_HB &&
 869             agp_bridge->dev->device != PCI_DEVICE_ID_INTEL_82845G_HB) {
 870                 /* 855GM/852GM/865G has 128MB aperture size */
 871                 agp_bridge->current_size = (void *) values;
 872                 agp_bridge->aperture_size_idx = 0;
 873                 return values[0].size;
 874         }
 875
 876         pci_read_config_word(agp_bridge->dev, I830_GMCH_CTRL, &gmch_ctrl);
 877
 878         if ((gmch_ctrl & I830_GMCH_MEM_MASK) == I830_GMCH_MEM_128M) {
 879                 agp_bridge->current_size = (void *) values;
 880                 agp_bridge->aperture_size_idx = 0;
 881                 return values[0].size;
 882         } else {
 883                 agp_bridge->current_size = (void *) (values + 1);
 884                 agp_bridge->aperture_size_idx = 1;
 885                 return values[1].size;
 886         }
 887
 888         return 0;
 889 }
 890
 891 static int intel_i830_configure(void)
 892 {
 893         struct aper_size_info_fixed *current_size;
 894         u32 temp;
 895         u16 gmch_ctrl;
 896         int i;
 897
 898         current_size = A_SIZE_FIX(agp_bridge->current_size);
 899
 900         pci_read_config_dword(intel_private.pcidev, I810_GMADDR, &temp);
 901         agp_bridge->gart_bus_addr = (temp & PCI_BASE_ADDRESS_MEM_MASK);
 902
 903         pci_read_config_word(agp_bridge->dev, I830_GMCH_CTRL, &gmch_ctrl);
 904         gmch_ctrl |= I830_GMCH_ENABLED;
 905         pci_write_config_word(agp_bridge->dev, I830_GMCH_CTRL, gmch_ctrl);
 906
 907         writel(agp_bridge->gatt_bus_addr|I810_PGETBL_ENABLED, intel_private.registers+I810_PGETBL_CTL);
 908         readl(intel_private.registers+I810_PGETBL_CTL); /* PCI Posting. */
 909
 910         if (agp_bridge->driver->needs_scratch_page) {
 911                 for (i = intel_private.gtt_entries; i < current_size->num_entries; i++) {
 912                         writel(agp_bridge->scratch_page, intel_private.registers+I810_PTE_BASE+(i*4));
 913                 }
 914                 readl(intel_private.registers+I810_PTE_BASE+((i-1)*4)); /* PCI Posting. */
 915         }
 916
 917         global_cache_flush();
 918
 919         intel_i830_setup_flush();
 920         return 0;
 921 }
 922
 923 static void intel_i830_cleanup(void)
 924 {
 925         iounmap(intel_private.registers);
 926 }
 927
 928 static int intel_i830_insert_entries(struct agp_memory *mem, off_t pg_start,
 929                                      int type)
 930 {
 931         int i, j, num_entries;
 932         void *temp;
 933         int ret = -EINVAL;
 934         int mask_type;
 935
 936         if (mem->page_count == 0)
 937                 goto out;
 938
 939         temp = agp_bridge->current_size;
 940         num_entries = A_SIZE_FIX(temp)->num_entries;
 941
 942         if (pg_start < intel_private.gtt_entries) {
 943                 dev_printk(KERN_DEBUG, &intel_private.pcidev->dev,
 944                            "pg_start == 0x%.8lx, intel_private.gtt_entries == 0x%.8x\n",
 945                            pg_start, intel_private.gtt_entries);
 946
 947                 dev_info(&intel_private.pcidev->dev,
 948                          "trying to insert into local/stolen memory\n");
 949                 goto out_err;
 950         }
 951
 952         if ((pg_start + mem->page_count) > num_entries)
 953                 goto out_err;
 954
 955         /* The i830 can't check the GTT for entries since its read only,
 956          * depend on the caller to make the correct offset decisions.
 957          */
 958
 959         if (type != mem->type)
 960                 goto out_err;
 961
 962         mask_type = agp_bridge->driver->agp_type_to_mask_type(agp_bridge, type);
 963
 964         if (mask_type != 0 && mask_type != AGP_PHYS_MEMORY &&
 965             mask_type != INTEL_AGP_CACHED_MEMORY)
 966                 goto out_err;
 967
 968         if (!mem->is_flushed)
 969                 global_cache_flush();
 970
 971         for (i = 0, j = pg_start; i < mem->page_count; i++, j++) {
 972                 writel(agp_bridge->driver->mask_memory(agp_bridge,
 973                                 page_to_phys(mem->pages[i]), mask_type),
 974                        intel_private.registers+I810_PTE_BASE+(j*4));
 975         }
 976         readl(intel_private.registers+I810_PTE_BASE+((j-1)*4));
 977
 978 out:
 979         ret = 0;
 980 out_err:
 981         mem->is_flushed = true;
 982         return ret;
 983 }
 984
 985 static int intel_i830_remove_entries(struct agp_memory *mem, off_t pg_start,
 986                                      int type)
 987 {
 988         int i;
 989
 990         if (mem->page_count == 0)
 991                 return 0;
 992
 993         if (pg_start < intel_private.gtt_entries) {
 994                 dev_info(&intel_private.pcidev->dev,
 995                          "trying to disable local/stolen memory\n");
 996                 return -EINVAL;
 997         }
 998
 999         for (i = pg_start; i < (mem->page_count + pg_start); i++) {
1000                 writel(agp_bridge->scratch_page, intel_private.registers+I810_PTE_BASE+(i*4));
1001         }
1002         readl(intel_private.registers+I810_PTE_BASE+((i-1)*4));
1003
1004         return 0;
1005 }
1006
1007 static struct agp_memory *intel_i830_alloc_by_type(size_t pg_count, int type)
1008 {
1009         if (type == AGP_PHYS_MEMORY)
1010                 return alloc_agpphysmem_i8xx(pg_count, type);
1011         /* always return NULL for other allocation types for now */
1012         return NULL;
1013 }
1014
1015 static int intel_alloc_chipset_flush_resource(void)
1016 {
1017         int ret;
1018         ret = pci_bus_alloc_resource(agp_bridge->dev->bus, &intel_private.ifp_resource, PAGE_SIZE,
1019                                      PAGE_SIZE, PCIBIOS_MIN_MEM, 0,
1020                                      pcibios_align_resource, agp_bridge->dev);
1021
1022         return ret;
1023 }
1024
1025 static void intel_i915_setup_chipset_flush(void)
1026 {
1027         int ret;
1028         u32 temp;
1029
1030         pci_read_config_dword(agp_bridge->dev, I915_IFPADDR, &temp);
1031         if (!(temp & 0x1)) {
1032                 intel_alloc_chipset_flush_resource();
1033                 intel_private.resource_valid = 1;
1034                 pci_write_config_dword(agp_bridge->dev, I915_IFPADDR, (intel_private.ifp_resource.start & 0xffffffff) | 0x1);
1035         } else {
1036                 temp &= ~1;
1037
1038                 intel_private.resource_valid = 1;
1039                 intel_private.ifp_resource.start = temp;
1040                 intel_private.ifp_resource.end = temp + PAGE_SIZE;
1041                 ret = request_resource(&iomem_resource, &intel_private.ifp_resource);
1042                 /* some BIOSes reserve this area in a pnp some don't */
1043                 if (ret)
1044                         intel_private.resource_valid = 0;
1045         }
1046 }
1047
1048 static void intel_i965_g33_setup_chipset_flush(void)
1049 {
1050         u32 temp_hi, temp_lo;
1051         int ret;
1052
1053         pci_read_config_dword(agp_bridge->dev, I965_IFPADDR + 4, &temp_hi);
1054         pci_read_config_dword(agp_bridge->dev, I965_IFPADDR, &temp_lo);
1055
1056         if (!(temp_lo & 0x1)) {
1057
1058                 intel_alloc_chipset_flush_resource();
1059
1060                 intel_private.resource_valid = 1;
1061                 pci_write_config_dword(agp_bridge->dev, I965_IFPADDR + 4,
1062                         upper_32_bits(intel_private.ifp_resource.start));
1063                 pci_write_config_dword(agp_bridge->dev, I965_IFPADDR, (intel_private.ifp_resource.start & 0xffffffff) | 0x1);
1064         } else {
1065                 u64 l64;
1066
1067                 temp_lo &= ~0x1;
1068                 l64 = ((u64)temp_hi << 32) | temp_lo;
1069
1070                 intel_private.resource_valid = 1;
1071                 intel_private.ifp_resource.start = l64;
1072                 intel_private.ifp_resource.end = l64 + PAGE_SIZE;
1073                 ret = request_resource(&iomem_resource, &intel_private.ifp_resource);
1074                 /* some BIOSes reserve this area in a pnp some don't */
1075                 if (ret)
1076                         intel_private.resource_valid = 0;
1077         }
1078 }
1079
1080 static void intel_i9xx_setup_flush(void)
1081 {
1082         /* return if already configured */
1083         if (intel_private.ifp_resource.start)
1084                 return;
1085
1086         if (IS_SNB)
1087                 return;
1088
1089         /* setup a resource for this object */
1090         intel_private.ifp_resource.name = "Intel Flush Page";
1091         intel_private.ifp_resource.flags = IORESOURCE_MEM;
1092
1093         /* Setup chipset flush for 915 */
1094         if (IS_I965 || IS_G33 || IS_G4X) {
1095                 intel_i965_g33_setup_chipset_flush();
1096         } else {
1097                 intel_i915_setup_chipset_flush();
1098         }
1099
1100         if (intel_private.ifp_resource.start)
1101                 intel_private.i9xx_flush_page = ioremap_nocache(intel_private.ifp_resource.start, PAGE_SIZE);
1102         if (!intel_private.i9xx_flush_page)
1103                 dev_err(&intel_private.pcidev->dev,
1104                         "can't ioremap flush page - no chipset flushing\n");
1105 }
1106
1107 static int intel_i9xx_configure(void)
1108 {
1109         struct aper_size_info_fixed *current_size;
1110         u32 temp;
1111         u16 gmch_ctrl;
1112         int i;
1113
1114         current_size = A_SIZE_FIX(agp_bridge->current_size);
1115
1116         pci_read_config_dword(intel_private.pcidev, I915_GMADDR, &temp);
1117
1118         agp_bridge->gart_bus_addr = (temp & PCI_BASE_ADDRESS_MEM_MASK);
1119
1120         pci_read_config_word(agp_bridge->dev, I830_GMCH_CTRL, &gmch_ctrl);
1121         gmch_ctrl |= I830_GMCH_ENABLED;
1122         pci_write_config_word(agp_bridge->dev, I830_GMCH_CTRL, gmch_ctrl);
1123
1124         writel(agp_bridge->gatt_bus_addr|I810_PGETBL_ENABLED, intel_private.registers+I810_PGETBL_CTL);
1125         readl(intel_private.registers+I810_PGETBL_CTL); /* PCI Posting. */
1126
1127         if (agp_bridge->driver->needs_scratch_page) {
1128                 for (i = intel_private.gtt_entries; i < intel_private.gtt_total_size; i++) {
1129                         writel(agp_bridge->scratch_page, intel_private.gtt+i);
1130                 }
1131                 readl(intel_private.gtt+i-1);   /* PCI Posting. */
1132         }
1133
1134         global_cache_flush();
1135
1136         intel_i9xx_setup_flush();
1137
1138         return 0;
1139 }
1140
1141 static void intel_i915_cleanup(void)
1142 {
1143         if (intel_private.i9xx_flush_page)
1144                 iounmap(intel_private.i9xx_flush_page);
1145         if (intel_private.resource_valid)
1146                 release_resource(&intel_private.ifp_resource);
1147         intel_private.ifp_resource.start = 0;
1148         intel_private.resource_valid = 0;
1149         iounmap(intel_private.gtt);
1150         iounmap(intel_private.registers);
1151 }
1152
1153 static void intel_i915_chipset_flush(struct agp_bridge_data *bridge)
1154 {
1155         if (intel_private.i9xx_flush_page)
1156                 writel(1, intel_private.i9xx_flush_page);
1157 }
1158
1159 static int intel_i915_insert_entries(struct agp_memory *mem, off_t pg_start,
1160                                      int type)
1161 {
1162         int num_entries;
1163         void *temp;
1164         int ret = -EINVAL;
1165         int mask_type;
1166
1167         if (mem->page_count == 0)
1168                 goto out;
1169
1170         temp = agp_bridge->current_size;
1171         num_entries = A_SIZE_FIX(temp)->num_entries;
1172
1173         if (pg_start < intel_private.gtt_entries) {
1174                 dev_printk(KERN_DEBUG, &intel_private.pcidev->dev,
1175                            "pg_start == 0x%.8lx, intel_private.gtt_entries == 0x%.8x\n",
1176                            pg_start, intel_private.gtt_entries);
1177
1178                 dev_info(&intel_private.pcidev->dev,
1179                          "trying to insert into local/stolen memory\n");
1180                 goto out_err;
1181         }
1182
1183         if ((pg_start + mem->page_count) > num_entries)
1184                 goto out_err;
1185
1186         /* The i915 can't check the GTT for entries since it's read only;
1187          * depend on the caller to make the correct offset decisions.
1188          */
1189
1190         if (type != mem->type)
1191                 goto out_err;
1192
1193         mask_type = agp_bridge->driver->agp_type_to_mask_type(agp_bridge, type);
1194
1195         if (!IS_SNB && mask_type != 0 && mask_type != AGP_PHYS_MEMORY &&
1196             mask_type != INTEL_AGP_CACHED_MEMORY)
1197                 goto out_err;
1198
1199         if (!mem->is_flushed)
1200                 global_cache_flush();
1201
1202         intel_agp_insert_sg_entries(mem, pg_start, mask_type);
1203
1204  out:
1205         ret = 0;
1206  out_err:
1207         mem->is_flushed = true;
1208         return ret;
1209 }
1210
1211 static int intel_i915_remove_entries(struct agp_memory *mem, off_t pg_start,
1212                                      int type)
1213 {
1214         int i;
1215
1216         if (mem->page_count == 0)
1217                 return 0;
1218
1219         if (pg_start < intel_private.gtt_entries) {
1220                 dev_info(&intel_private.pcidev->dev,
1221                          "trying to disable local/stolen memory\n");
1222                 return -EINVAL;
1223         }
1224
1225         for (i = pg_start; i < (mem->page_count + pg_start); i++)
1226                 writel(agp_bridge->scratch_page, intel_private.gtt+i);
1227
1228         readl(intel_private.gtt+i-1);
1229
1230         return 0;
1231 }
1232
1233 /* Return the aperture size by just checking the resource length.  The effect
1234  * described in the spec of the MSAC registers is just changing of the
1235  * resource size.
1236  */
1237 static int intel_i9xx_fetch_size(void)
1238 {
1239         int num_sizes = ARRAY_SIZE(intel_i830_sizes);
1240         int aper_size; /* size in megabytes */
1241         int i;
1242
1243         aper_size = pci_resource_len(intel_private.pcidev, 2) / MB(1);
1244
1245         for (i = 0; i < num_sizes; i++) {
1246                 if (aper_size == intel_i830_sizes[i].size) {
1247                         agp_bridge->current_size = intel_i830_sizes + i;
1248                         return aper_size;
1249                 }
1250         }
1251
1252         return 0;
1253 }
1254
1255 static int intel_i915_get_gtt_size(void)
1256 {
1257         int size;
1258
1259         if (IS_G33) {
1260                 u16 gmch_ctrl;
1261
1262                 /* G33's GTT size defined in gmch_ctrl */
1263                 pci_read_config_word(agp_bridge->dev, I830_GMCH_CTRL, &gmch_ctrl);
1264                 switch (gmch_ctrl & I830_GMCH_GMS_MASK) {
1265                 case I830_GMCH_GMS_STOLEN_512:
1266                         size = 512;
1267                         break;
1268                 case I830_GMCH_GMS_STOLEN_1024:
1269                         size = 1024;
1270                         break;
1271                 case I830_GMCH_GMS_STOLEN_8192:
1272                         size = 8*1024;
1273                         break;
1274                 default:
1275                         dev_info(&agp_bridge->dev->dev,
1276                                  "unknown page table size 0x%x, assuming 512KB\n",
1277                                 (gmch_ctrl & I830_GMCH_GMS_MASK));
1278                         size = 512;
1279                 }
1280         } else {
1281                 /* On previous hardware, the GTT size was just what was
1282                  * required to map the aperture.
1283                  */
1284                 size = agp_bridge->driver->fetch_size();
1285         }
1286
1287         return KB(size);
1288 }
1289
1290 /* The intel i915 automatically initializes the agp aperture during POST.
1291  * Use the memory already set aside for in the GTT.
1292  */
1293 static int intel_i915_create_gatt_table(struct agp_bridge_data *bridge)
1294 {
1295         int page_order;
1296         struct aper_size_info_fixed *size;
1297         int num_entries;
1298         u32 temp, temp2;
1299         int gtt_map_size;
1300
1301         size = agp_bridge->current_size;
1302         page_order = size->page_order;
1303         num_entries = size->num_entries;
1304         agp_bridge->gatt_table_real = NULL;
1305
1306         pci_read_config_dword(intel_private.pcidev, I915_MMADDR, &temp);
1307         pci_read_config_dword(intel_private.pcidev, I915_PTEADDR, &temp2);
1308
1309         gtt_map_size = intel_i915_get_gtt_size();
1310
1311         intel_private.gtt = ioremap(temp2, gtt_map_size);
1312         if (!intel_private.gtt)
1313                 return -ENOMEM;
1314
1315         intel_private.gtt_total_size = gtt_map_size / 4;
1316
1317         temp &= 0xfff80000;
1318
1319         intel_private.registers = ioremap(temp, 128 * 4096);
1320         if (!intel_private.registers) {
1321                 iounmap(intel_private.gtt);
1322                 return -ENOMEM;
1323         }
1324
1325         temp = readl(intel_private.registers+I810_PGETBL_CTL) & 0xfffff000;
1326         global_cache_flush();   /* FIXME: ? */
1327
1328         /* we have to call this as early as possible after the MMIO base address is known */
1329         intel_i830_init_gtt_entries();
1330         if (intel_private.gtt_entries == 0) {
1331                 iounmap(intel_private.gtt);
1332                 iounmap(intel_private.registers);
1333                 return -ENOMEM;
1334         }
1335
1336         agp_bridge->gatt_table = NULL;
1337
1338         agp_bridge->gatt_bus_addr = temp;
1339
1340         return 0;
1341 }
1342
1343 /*
1344  * The i965 supports 36-bit physical addresses, but to keep
1345  * the format of the GTT the same, the bits that don't fit
1346  * in a 32-bit word are shifted down to bits 4..7.
1347  *
1348  * Gcc is smart enough to notice that "(addr >> 28) & 0xf0"
1349  * is always zero on 32-bit architectures, so no need to make
1350  * this conditional.
1351  */
1352 static unsigned long intel_i965_mask_memory(struct agp_bridge_data *bridge,
1353                                             dma_addr_t addr, int type)
1354 {
1355         /* Shift high bits down */
1356         addr |= (addr >> 28) & 0xf0;
1357
1358         /* Type checking must be done elsewhere */
1359         return addr | bridge->driver->masks[type].mask;
1360 }
1361
1362 static unsigned long intel_gen6_mask_memory(struct agp_bridge_data *bridge,
1363                                             dma_addr_t addr, int type)
1364 {
1365         /* gen6 has bit11-4 for physical addr bit39-32 */
1366         addr |= (addr >> 28) & 0xff0;
1367
1368         /* Type checking must be done elsewhere */
1369         return addr | bridge->driver->masks[type].mask;
1370 }
1371
1372 static void intel_i965_get_gtt_range(int *gtt_offset, int *gtt_size)
1373 {
1374         u16 snb_gmch_ctl;
1375
1376         switch (agp_bridge->dev->device) {
1377         case PCI_DEVICE_ID_INTEL_GM45_HB:
1378         case PCI_DEVICE_ID_INTEL_EAGLELAKE_HB:
1379         case PCI_DEVICE_ID_INTEL_Q45_HB:
1380         case PCI_DEVICE_ID_INTEL_G45_HB:
1381         case PCI_DEVICE_ID_INTEL_G41_HB:
1382         case PCI_DEVICE_ID_INTEL_B43_HB:
1383         case PCI_DEVICE_ID_INTEL_IRONLAKE_D_HB:
1384         case PCI_DEVICE_ID_INTEL_IRONLAKE_M_HB:
1385         case PCI_DEVICE_ID_INTEL_IRONLAKE_MA_HB:
1386         case PCI_DEVICE_ID_INTEL_IRONLAKE_MC2_HB:
1387                 *gtt_offset = *gtt_size = MB(2);
1388                 break;
1389         case PCI_DEVICE_ID_INTEL_SANDYBRIDGE_HB:
1390         case PCI_DEVICE_ID_INTEL_SANDYBRIDGE_M_HB:
1391         case PCI_DEVICE_ID_INTEL_SANDYBRIDGE_S_HB:
1392                 *gtt_offset = MB(2);
1393
1394                 pci_read_config_word(intel_private.pcidev, SNB_GMCH_CTRL, &snb_gmch_ctl);
1395                 switch (snb_gmch_ctl & SNB_GTT_SIZE_MASK) {
1396                 default:
1397                 case SNB_GTT_SIZE_0M:
1398                         printk(KERN_ERR "Bad GTT size mask: 0x%04x.\n", snb_gmch_ctl);
1399                         *gtt_size = MB(0);
1400                         break;
1401                 case SNB_GTT_SIZE_1M:
1402                         *gtt_size = MB(1);
1403                         break;
1404                 case SNB_GTT_SIZE_2M:
1405                         *gtt_size = MB(2);
1406                         break;
1407                 }
1408                 break;
1409         default:
1410                 *gtt_offset = *gtt_size = KB(512);
1411         }
1412 }
1413
1414 /* The intel i965 automatically initializes the agp aperture during POST.
1415  * Use the memory already set aside for in the GTT.
1416  */
1417 static int intel_i965_create_gatt_table(struct agp_bridge_data *bridge)
1418 {
1419         int page_order;
1420         struct aper_size_info_fixed *size;
1421         int num_entries;
1422         u32 temp;
1423         int gtt_offset, gtt_size;
1424
1425         size = agp_bridge->current_size;
1426         page_order = size->page_order;
1427         num_entries = size->num_entries;
1428         agp_bridge->gatt_table_real = NULL;
1429
1430         pci_read_config_dword(intel_private.pcidev, I915_MMADDR, &temp);
1431
1432         temp &= 0xfff00000;
1433
1434         intel_i965_get_gtt_range(&gtt_offset, &gtt_size);
1435
1436         intel_private.gtt = ioremap((temp + gtt_offset) , gtt_size);
1437
1438         if (!intel_private.gtt)
1439                 return -ENOMEM;
1440
1441         intel_private.gtt_total_size = gtt_size / 4;
1442
1443         intel_private.registers = ioremap(temp, 128 * 4096);
1444         if (!intel_private.registers) {
1445                 iounmap(intel_private.gtt);
1446                 return -ENOMEM;
1447         }
1448
1449         temp = readl(intel_private.registers+I810_PGETBL_CTL) & 0xfffff000;
1450         global_cache_flush();   /* FIXME: ? */
1451
1452         /* we have to call this as early as possible after the MMIO base address is known */
1453         intel_i830_init_gtt_entries();
1454         if (intel_private.gtt_entries == 0) {
1455                 iounmap(intel_private.gtt);
1456                 iounmap(intel_private.registers);
1457                 return -ENOMEM;
1458         }
1459
1460         agp_bridge->gatt_table = NULL;
1461
1462         agp_bridge->gatt_bus_addr = temp;
1463
1464         return 0;
1465 }
1466
1467 static const struct agp_bridge_driver intel_810_driver = {
1468         .owner                  = THIS_MODULE,
1469         .aperture_sizes         = intel_i810_sizes,
1470         .size_type              = FIXED_APER_SIZE,
1471         .num_aperture_sizes     = 2,
1472         .needs_scratch_page     = true,
1473         .configure              = intel_i810_configure,
1474         .fetch_size             = intel_i810_fetch_size,
1475         .cleanup                = intel_i810_cleanup,
1476         .mask_memory            = intel_i810_mask_memory,
1477         .masks                  = intel_i810_masks,
1478         .agp_enable             = intel_i810_agp_enable,
1479         .cache_flush            = global_cache_flush,
1480         .create_gatt_table      = agp_generic_create_gatt_table,
1481         .free_gatt_table        = agp_generic_free_gatt_table,
1482         .insert_memory          = intel_i810_insert_entries,
1483         .remove_memory          = intel_i810_remove_entries,
1484         .alloc_by_type          = intel_i810_alloc_by_type,
1485         .free_by_type           = intel_i810_free_by_type,
1486         .agp_alloc_page         = agp_generic_alloc_page,
1487         .agp_alloc_pages        = agp_generic_alloc_pages,
1488         .agp_destroy_page       = agp_generic_destroy_page,
1489         .agp_destroy_pages      = agp_generic_destroy_pages,
1490         .agp_type_to_mask_type  = agp_generic_type_to_mask_type,
1491 };
1492
1493 static const struct agp_bridge_driver intel_830_driver = {
1494         .owner                  = THIS_MODULE,
1495         .aperture_sizes         = intel_i830_sizes,
1496         .size_type              = FIXED_APER_SIZE,
1497         .num_aperture_sizes     = 4,
1498         .needs_scratch_page     = true,
1499         .configure              = intel_i830_configure,
1500         .fetch_size             = intel_i830_fetch_size,
1501         .cleanup                = intel_i830_cleanup,
1502         .mask_memory            = intel_i810_mask_memory,
1503         .masks                  = intel_i810_masks,
1504         .agp_enable             = intel_i810_agp_enable,
1505         .cache_flush            = global_cache_flush,
1506         .create_gatt_table      = intel_i830_create_gatt_table,
1507         .free_gatt_table        = intel_i830_free_gatt_table,
1508         .insert_memory          = intel_i830_insert_entries,
1509         .remove_memory          = intel_i830_remove_entries,
1510         .alloc_by_type          = intel_i830_alloc_by_type,
1511         .free_by_type           = intel_i810_free_by_type,
1512         .agp_alloc_page         = agp_generic_alloc_page,
1513         .agp_alloc_pages        = agp_generic_alloc_pages,
1514         .agp_destroy_page       = agp_generic_destroy_page,
1515         .agp_destroy_pages      = agp_generic_destroy_pages,
1516         .agp_type_to_mask_type  = intel_i830_type_to_mask_type,
1517         .chipset_flush          = intel_i830_chipset_flush,
1518 };
1519
1520 static const struct agp_bridge_driver intel_915_driver = {
1521         .owner                  = THIS_MODULE,
1522         .aperture_sizes         = intel_i830_sizes,
1523         .size_type              = FIXED_APER_SIZE,
1524         .num_aperture_sizes     = 4,
1525         .needs_scratch_page     = true,
1526         .configure              = intel_i9xx_configure,
1527         .fetch_size             = intel_i9xx_fetch_size,
1528         .cleanup                = intel_i915_cleanup,
1529         .mask_memory            = intel_i810_mask_memory,
1530         .masks                  = intel_i810_masks,
1531         .agp_enable             = intel_i810_agp_enable,
1532         .cache_flush            = global_cache_flush,
1533         .create_gatt_table      = intel_i915_create_gatt_table,
1534         .free_gatt_table        = intel_i830_free_gatt_table,
1535         .insert_memory          = intel_i915_insert_entries,
1536         .remove_memory          = intel_i915_remove_entries,
1537         .alloc_by_type          = intel_i830_alloc_by_type,
1538         .free_by_type           = intel_i810_free_by_type,
1539         .agp_alloc_page         = agp_generic_alloc_page,
1540         .agp_alloc_pages        = agp_generic_alloc_pages,
1541         .agp_destroy_page       = agp_generic_destroy_page,
1542         .agp_destroy_pages      = agp_generic_destroy_pages,
1543         .agp_type_to_mask_type  = intel_i830_type_to_mask_type,
1544         .chipset_flush          = intel_i915_chipset_flush,
1545 #ifdef USE_PCI_DMA_API
1546         .agp_map_page           = intel_agp_map_page,
1547         .agp_unmap_page         = intel_agp_unmap_page,
1548         .agp_map_memory         = intel_agp_map_memory,
1549         .agp_unmap_memory       = intel_agp_unmap_memory,
1550 #endif
1551 };
1552
1553 static const struct agp_bridge_driver intel_i965_driver = {
1554         .owner                  = THIS_MODULE,
1555         .aperture_sizes         = intel_i830_sizes,
1556         .size_type              = FIXED_APER_SIZE,
1557         .num_aperture_sizes     = 4,
1558         .needs_scratch_page     = true,
1559         .configure              = intel_i9xx_configure,
1560         .fetch_size             = intel_i9xx_fetch_size,
1561         .cleanup                = intel_i915_cleanup,
1562         .mask_memory            = intel_i965_mask_memory,
1563         .masks                  = intel_i810_masks,
1564         .agp_enable             = intel_i810_agp_enable,
1565         .cache_flush            = global_cache_flush,
1566         .create_gatt_table      = intel_i965_create_gatt_table,
1567         .free_gatt_table        = intel_i830_free_gatt_table,
1568         .insert_memory          = intel_i915_insert_entries,
1569         .remove_memory          = intel_i915_remove_entries,
1570         .alloc_by_type          = intel_i830_alloc_by_type,
1571         .free_by_type           = intel_i810_free_by_type,
1572         .agp_alloc_page         = agp_generic_alloc_page,
1573         .agp_alloc_pages        = agp_generic_alloc_pages,
1574         .agp_destroy_page       = agp_generic_destroy_page,
1575         .agp_destroy_pages      = agp_generic_destroy_pages,
1576         .agp_type_to_mask_type  = intel_i830_type_to_mask_type,
1577         .chipset_flush          = intel_i915_chipset_flush,
1578 #ifdef USE_PCI_DMA_API
1579         .agp_map_page           = intel_agp_map_page,
1580         .agp_unmap_page         = intel_agp_unmap_page,
1581         .agp_map_memory         = intel_agp_map_memory,
1582         .agp_unmap_memory       = intel_agp_unmap_memory,
1583 #endif
1584 };
1585
1586 static const struct agp_bridge_driver intel_gen6_driver = {
1587         .owner                  = THIS_MODULE,
1588         .aperture_sizes         = intel_i830_sizes,
1589         .size_type              = FIXED_APER_SIZE,
1590         .num_aperture_sizes     = 4,
1591         .needs_scratch_page     = true,
1592         .configure              = intel_i9xx_configure,
1593         .fetch_size             = intel_i9xx_fetch_size,
1594         .cleanup                = intel_i915_cleanup,
1595         .mask_memory            = intel_gen6_mask_memory,
1596         .masks                  = intel_gen6_masks,
1597         .agp_enable             = intel_i810_agp_enable,
1598         .cache_flush            = global_cache_flush,
1599         .create_gatt_table      = intel_i965_create_gatt_table,
1600         .free_gatt_table        = intel_i830_free_gatt_table,
1601         .insert_memory          = intel_i915_insert_entries,
1602         .remove_memory          = intel_i915_remove_entries,
1603         .alloc_by_type          = intel_i830_alloc_by_type,
1604         .free_by_type           = intel_i810_free_by_type,
1605         .agp_alloc_page         = agp_generic_alloc_page,
1606         .agp_alloc_pages        = agp_generic_alloc_pages,
1607         .agp_destroy_page       = agp_generic_destroy_page,
1608         .agp_destroy_pages      = agp_generic_destroy_pages,
1609         .agp_type_to_mask_type  = intel_gen6_type_to_mask_type,
1610         .chipset_flush          = intel_i915_chipset_flush,
1611 #ifdef USE_PCI_DMA_API
1612         .agp_map_page           = intel_agp_map_page,
1613         .agp_unmap_page         = intel_agp_unmap_page,
1614         .agp_map_memory         = intel_agp_map_memory,
1615         .agp_unmap_memory       = intel_agp_unmap_memory,
1616 #endif
1617 };
1618
1619 static const struct agp_bridge_driver intel_g33_driver = {
1620         .owner                  = THIS_MODULE,
1621         .aperture_sizes         = intel_i830_sizes,
1622         .size_type              = FIXED_APER_SIZE,
1623         .num_aperture_sizes     = 4,
1624         .needs_scratch_page     = true,
1625         .configure              = intel_i9xx_configure,
1626         .fetch_size             = intel_i9xx_fetch_size,
1627         .cleanup                = intel_i915_cleanup,
1628         .mask_memory            = intel_i965_mask_memory,
1629         .masks                  = intel_i810_masks,
1630         .agp_enable             = intel_i810_agp_enable,
1631         .cache_flush            = global_cache_flush,
1632         .create_gatt_table      = intel_i915_create_gatt_table,
1633         .free_gatt_table        = intel_i830_free_gatt_table,
1634         .insert_memory          = intel_i915_insert_entries,
1635         .remove_memory          = intel_i915_remove_entries,
1636         .alloc_by_type          = intel_i830_alloc_by_type,
1637         .free_by_type           = intel_i810_free_by_type,
1638         .agp_alloc_page         = agp_generic_alloc_page,
1639         .agp_alloc_pages        = agp_generic_alloc_pages,
1640         .agp_destroy_page       = agp_generic_destroy_page,
1641         .agp_destroy_pages      = agp_generic_destroy_pages,
1642         .agp_type_to_mask_type  = intel_i830_type_to_mask_type,
1643         .chipset_flush          = intel_i915_chipset_flush,
1644 #ifdef USE_PCI_DMA_API
1645         .agp_map_page           = intel_agp_map_page,
1646         .agp_unmap_page         = intel_agp_unmap_page,
1647         .agp_map_memory         = intel_agp_map_memory,
1648         .agp_unmap_memory       = intel_agp_unmap_memory,
1649 #endif
1650 };