drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  */
  27
  28 #include "drmP.h"
  29 #include "drm.h"
  30 #include "i915_drm.h"
  31 #include "i915_drv.h"
  32 #include "i915_trace.h"
  33 #include "intel_drv.h"
  34 #include <linux/slab.h>
  35 #include <linux/swap.h>
  36 #include <linux/pci.h>
  37
  38 static void i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj);
  39 static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
  40 static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
  41 static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
  42                                              int write);
  43 static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
  44                                                      uint64_t offset,
  45                                                      uint64_t size);
  46 static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
  47 static int i915_gem_object_wait_rendering(struct drm_gem_object *obj);
  48 static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
  49                                            unsigned alignment);
  50 static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
  51 static int i915_gem_evict_something(struct drm_device *dev, int min_size);
  52 static int i915_gem_evict_from_inactive_list(struct drm_device *dev);
  53 static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
  54                                 struct drm_i915_gem_pwrite *args,
  55                                 struct drm_file *file_priv);
  56
  57 static LIST_HEAD(shrink_list);
  58 static DEFINE_SPINLOCK(shrink_list_lock);
  59
  60 int i915_gem_do_init(struct drm_device *dev, unsigned long start,
  61                      unsigned long end)
  62 {
  63         drm_i915_private_t *dev_priv = dev->dev_private;
  64
  65         if (start >= end ||
  66             (start & (PAGE_SIZE - 1)) != 0 ||
  67             (end & (PAGE_SIZE - 1)) != 0) {
  68                 return -EINVAL;
  69         }
  70
  71         drm_mm_init(&dev_priv->mm.gtt_space, start,
  72                     end - start);
  73
  74         dev->gtt_total = (uint32_t) (end - start);
  75
  76         return 0;
  77 }
  78
  79 int
  80 i915_gem_init_ioctl(struct drm_device *dev, void *data,
  81                     struct drm_file *file_priv)
  82 {
  83         struct drm_i915_gem_init *args = data;
  84         int ret;
  85
  86         mutex_lock(&dev->struct_mutex);
  87         ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end);
  88         mutex_unlock(&dev->struct_mutex);
  89
  90         return ret;
  91 }
  92
  93 int
  94 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
  95                             struct drm_file *file_priv)
  96 {
  97         struct drm_i915_gem_get_aperture *args = data;
  98
  99         if (!(dev->driver->driver_features & DRIVER_GEM))
 100                 return -ENODEV;
 101
 102         args->aper_size = dev->gtt_total;
 103         args->aper_available_size = (args->aper_size -
 104                                      atomic_read(&dev->pin_memory));
 105
 106         return 0;
 107 }
 108
 109
 110 /**
 111  * Creates a new mm object and returns a handle to it.
 112  */
 113 int
 114 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 115                       struct drm_file *file_priv)
 116 {
 117         struct drm_i915_gem_create *args = data;
 118         struct drm_gem_object *obj;
 119         int ret;
 120         u32 handle;
 121
 122         args->size = roundup(args->size, PAGE_SIZE);
 123
 124         /* Allocate the new object */
 125         obj = i915_gem_alloc_object(dev, args->size);
 126         if (obj == NULL)
 127                 return -ENOMEM;
 128
 129         ret = drm_gem_handle_create(file_priv, obj, &handle);
 130         drm_gem_object_handle_unreference_unlocked(obj);
 131
 132         if (ret)
 133                 return ret;
 134
 135         args->handle = handle;
 136
 137         return 0;
 138 }
 139
 140 static inline int
 141 fast_shmem_read(struct page **pages,
 142                 loff_t page_base, int page_offset,
 143                 char __user *data,
 144                 int length)
 145 {
 146         char __iomem *vaddr;
 147         int unwritten;
 148
 149         vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
 150         if (vaddr == NULL)
 151                 return -ENOMEM;
 152         unwritten = __copy_to_user_inatomic(data, vaddr + page_offset, length);
 153         kunmap_atomic(vaddr, KM_USER0);
 154
 155         if (unwritten)
 156                 return -EFAULT;
 157
 158         return 0;
 159 }
 160
 161 static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
 162 {
 163         drm_i915_private_t *dev_priv = obj->dev->dev_private;
 164         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 165
 166         return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
 167                 obj_priv->tiling_mode != I915_TILING_NONE;
 168 }
 169
 170 static inline void
 171 slow_shmem_copy(struct page *dst_page,
 172                 int dst_offset,
 173                 struct page *src_page,
 174                 int src_offset,
 175                 int length)
 176 {
 177         char *dst_vaddr, *src_vaddr;
 178
 179         dst_vaddr = kmap(dst_page);
 180         src_vaddr = kmap(src_page);
 181
 182         memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
 183
 184         kunmap(src_page);
 185         kunmap(dst_page);
 186 }
 187
 188 static inline void
 189 slow_shmem_bit17_copy(struct page *gpu_page,
 190                       int gpu_offset,
 191                       struct page *cpu_page,
 192                       int cpu_offset,
 193                       int length,
 194                       int is_read)
 195 {
 196         char *gpu_vaddr, *cpu_vaddr;
 197
 198         /* Use the unswizzled path if this page isn't affected. */
 199         if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
 200                 if (is_read)
 201                         return slow_shmem_copy(cpu_page, cpu_offset,
 202                                                gpu_page, gpu_offset, length);
 203                 else
 204                         return slow_shmem_copy(gpu_page, gpu_offset,
 205                                                cpu_page, cpu_offset, length);
 206         }
 207
 208         gpu_vaddr = kmap(gpu_page);
 209         cpu_vaddr = kmap(cpu_page);
 210
 211         /* Copy the data, XORing A6 with A17 (1). The user already knows he's
 212          * XORing with the other bits (A9 for Y, A9 and A10 for X)
 213          */
 214         while (length > 0) {
 215                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 216                 int this_length = min(cacheline_end - gpu_offset, length);
 217                 int swizzled_gpu_offset = gpu_offset ^ 64;
 218
 219                 if (is_read) {
 220                         memcpy(cpu_vaddr + cpu_offset,
 221                                gpu_vaddr + swizzled_gpu_offset,
 222                                this_length);
 223                 } else {
 224                         memcpy(gpu_vaddr + swizzled_gpu_offset,
 225                                cpu_vaddr + cpu_offset,
 226                                this_length);
 227                 }
 228                 cpu_offset += this_length;
 229                 gpu_offset += this_length;
 230                 length -= this_length;
 231         }
 232
 233         kunmap(cpu_page);
 234         kunmap(gpu_page);
 235 }
 236
 237 /**
 238  * This is the fast shmem pread path, which attempts to copy_from_user directly
 239  * from the backing pages of the object to the user's address space.  On a
 240  * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
 241  */
 242 static int
 243 i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
 244                           struct drm_i915_gem_pread *args,
 245                           struct drm_file *file_priv)
 246 {
 247         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 248         ssize_t remain;
 249         loff_t offset, page_base;
 250         char __user *user_data;
 251         int page_offset, page_length;
 252         int ret;
 253
 254         user_data = (char __user *) (uintptr_t) args->data_ptr;
 255         remain = args->size;
 256
 257         mutex_lock(&dev->struct_mutex);
 258
 259         ret = i915_gem_object_get_pages(obj, 0);
 260         if (ret != 0)
 261                 goto fail_unlock;
 262
 263         ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
 264                                                         args->size);
 265         if (ret != 0)
 266                 goto fail_put_pages;
 267
 268         obj_priv = to_intel_bo(obj);
 269         offset = args->offset;
 270
 271         while (remain > 0) {
 272                 /* Operation in this page
 273                  *
 274                  * page_base = page offset within aperture
 275                  * page_offset = offset within page
 276                  * page_length = bytes to copy for this page
 277                  */
 278                 page_base = (offset & ~(PAGE_SIZE-1));
 279                 page_offset = offset & (PAGE_SIZE-1);
 280                 page_length = remain;
 281                 if ((page_offset + remain) > PAGE_SIZE)
 282                         page_length = PAGE_SIZE - page_offset;
 283
 284                 ret = fast_shmem_read(obj_priv->pages,
 285                                       page_base, page_offset,
 286                                       user_data, page_length);
 287                 if (ret)
 288                         goto fail_put_pages;
 289
 290                 remain -= page_length;
 291                 user_data += page_length;
 292                 offset += page_length;
 293         }
 294
 295 fail_put_pages:
 296         i915_gem_object_put_pages(obj);
 297 fail_unlock:
 298         mutex_unlock(&dev->struct_mutex);
 299
 300         return ret;
 301 }
 302
 303 static int
 304 i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
 305 {
 306         int ret;
 307
 308         ret = i915_gem_object_get_pages(obj, __GFP_NORETRY | __GFP_NOWARN);
 309
 310         /* If we've insufficient memory to map in the pages, attempt
 311          * to make some space by throwing out some old buffers.
 312          */
 313         if (ret == -ENOMEM) {
 314                 struct drm_device *dev = obj->dev;
 315
 316                 ret = i915_gem_evict_something(dev, obj->size);
 317                 if (ret)
 318                         return ret;
 319
 320                 ret = i915_gem_object_get_pages(obj, 0);
 321         }
 322
 323         return ret;
 324 }
 325
 326 /**
 327  * This is the fallback shmem pread path, which allocates temporary storage
 328  * in kernel space to copy_to_user into outside of the struct_mutex, so we
 329  * can copy out of the object's backing pages while holding the struct mutex
 330  * and not take page faults.
 331  */
 332 static int
 333 i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
 334                           struct drm_i915_gem_pread *args,
 335                           struct drm_file *file_priv)
 336 {
 337         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 338         struct mm_struct *mm = current->mm;
 339         struct page **user_pages;
 340         ssize_t remain;
 341         loff_t offset, pinned_pages, i;
 342         loff_t first_data_page, last_data_page, num_pages;
 343         int shmem_page_index, shmem_page_offset;
 344         int data_page_index,  data_page_offset;
 345         int page_length;
 346         int ret;
 347         uint64_t data_ptr = args->data_ptr;
 348         int do_bit17_swizzling;
 349
 350         remain = args->size;
 351
 352         /* Pin the user pages containing the data.  We can't fault while
 353          * holding the struct mutex, yet we want to hold it while
 354          * dereferencing the user data.
 355          */
 356         first_data_page = data_ptr / PAGE_SIZE;
 357         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 358         num_pages = last_data_page - first_data_page + 1;
 359
 360         user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
 361         if (user_pages == NULL)
 362                 return -ENOMEM;
 363
 364         down_read(&mm->mmap_sem);
 365         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 366                                       num_pages, 1, 0, user_pages, NULL);
 367         up_read(&mm->mmap_sem);
 368         if (pinned_pages < num_pages) {
 369                 ret = -EFAULT;
 370                 goto fail_put_user_pages;
 371         }
 372
 373         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 374
 375         mutex_lock(&dev->struct_mutex);
 376
 377         ret = i915_gem_object_get_pages_or_evict(obj);
 378         if (ret)
 379                 goto fail_unlock;
 380
 381         ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
 382                                                         args->size);
 383         if (ret != 0)
 384                 goto fail_put_pages;
 385
 386         obj_priv = to_intel_bo(obj);
 387         offset = args->offset;
 388
 389         while (remain > 0) {
 390                 /* Operation in this page
 391                  *
 392                  * shmem_page_index = page number within shmem file
 393                  * shmem_page_offset = offset within page in shmem file
 394                  * data_page_index = page number in get_user_pages return
 395                  * data_page_offset = offset with data_page_index page.
 396                  * page_length = bytes to copy for this page
 397                  */
 398                 shmem_page_index = offset / PAGE_SIZE;
 399                 shmem_page_offset = offset & ~PAGE_MASK;
 400                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 401                 data_page_offset = data_ptr & ~PAGE_MASK;
 402
 403                 page_length = remain;
 404                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 405                         page_length = PAGE_SIZE - shmem_page_offset;
 406                 if ((data_page_offset + page_length) > PAGE_SIZE)
 407                         page_length = PAGE_SIZE - data_page_offset;
 408
 409                 if (do_bit17_swizzling) {
 410                         slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
 411                                               shmem_page_offset,
 412                                               user_pages[data_page_index],
 413                                               data_page_offset,
 414                                               page_length,
 415                                               1);
 416                 } else {
 417                         slow_shmem_copy(user_pages[data_page_index],
 418                                         data_page_offset,
 419                                         obj_priv->pages[shmem_page_index],
 420                                         shmem_page_offset,
 421                                         page_length);
 422                 }
 423
 424                 remain -= page_length;
 425                 data_ptr += page_length;
 426                 offset += page_length;
 427         }
 428
 429 fail_put_pages:
 430         i915_gem_object_put_pages(obj);
 431 fail_unlock:
 432         mutex_unlock(&dev->struct_mutex);
 433 fail_put_user_pages:
 434         for (i = 0; i < pinned_pages; i++) {
 435                 SetPageDirty(user_pages[i]);
 436                 page_cache_release(user_pages[i]);
 437         }
 438         drm_free_large(user_pages);
 439
 440         return ret;
 441 }
 442
 443 /**
 444  * Reads data from the object referenced by handle.
 445  *
 446  * On error, the contents of *data are undefined.
 447  */
 448 int
 449 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 450                      struct drm_file *file_priv)
 451 {
 452         struct drm_i915_gem_pread *args = data;
 453         struct drm_gem_object *obj;
 454         struct drm_i915_gem_object *obj_priv;
 455         int ret;
 456
 457         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 458         if (obj == NULL)
 459                 return -EBADF;
 460         obj_priv = to_intel_bo(obj);
 461
 462         /* Bounds check source.
 463          *
 464          * XXX: This could use review for overflow issues...
 465          */
 466         if (args->offset > obj->size || args->size > obj->size ||
 467             args->offset + args->size > obj->size) {
 468                 drm_gem_object_unreference_unlocked(obj);
 469                 return -EINVAL;
 470         }
 471
 472         if (i915_gem_object_needs_bit17_swizzle(obj)) {
 473                 ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv);
 474         } else {
 475                 ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
 476                 if (ret != 0)
 477                         ret = i915_gem_shmem_pread_slow(dev, obj, args,
 478                                                         file_priv);
 479         }
 480
 481         drm_gem_object_unreference_unlocked(obj);
 482
 483         return ret;
 484 }
 485
 486 /* This is the fast write path which cannot handle
 487  * page faults in the source data
 488  */
 489
 490 static inline int
 491 fast_user_write(struct io_mapping *mapping,
 492                 loff_t page_base, int page_offset,
 493                 char __user *user_data,
 494                 int length)
 495 {
 496         char *vaddr_atomic;
 497         unsigned long unwritten;
 498
 499         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
 500         unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
 501                                                       user_data, length);
 502         io_mapping_unmap_atomic(vaddr_atomic);
 503         if (unwritten)
 504                 return -EFAULT;
 505         return 0;
 506 }
 507
 508 /* Here's the write path which can sleep for
 509  * page faults
 510  */
 511
 512 static inline int
 513 slow_kernel_write(struct io_mapping *mapping,
 514                   loff_t gtt_base, int gtt_offset,
 515                   struct page *user_page, int user_offset,
 516                   int length)
 517 {
 518         char *src_vaddr, *dst_vaddr;
 519         unsigned long unwritten;
 520
 521         dst_vaddr = io_mapping_map_atomic_wc(mapping, gtt_base);
 522         src_vaddr = kmap_atomic(user_page, KM_USER1);
 523         unwritten = __copy_from_user_inatomic_nocache(dst_vaddr + gtt_offset,
 524                                                       src_vaddr + user_offset,
 525                                                       length);
 526         kunmap_atomic(src_vaddr, KM_USER1);
 527         io_mapping_unmap_atomic(dst_vaddr);
 528         if (unwritten)
 529                 return -EFAULT;
 530         return 0;
 531 }
 532
 533 static inline int
 534 fast_shmem_write(struct page **pages,
 535                  loff_t page_base, int page_offset,
 536                  char __user *data,
 537                  int length)
 538 {
 539         char __iomem *vaddr;
 540         unsigned long unwritten;
 541
 542         vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
 543         if (vaddr == NULL)
 544                 return -ENOMEM;
 545         unwritten = __copy_from_user_inatomic(vaddr + page_offset, data, length);
 546         kunmap_atomic(vaddr, KM_USER0);
 547
 548         if (unwritten)
 549                 return -EFAULT;
 550         return 0;
 551 }
 552
 553 /**
 554  * This is the fast pwrite path, where we copy the data directly from the
 555  * user into the GTT, uncached.
 556  */
 557 static int
 558 i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
 559                          struct drm_i915_gem_pwrite *args,
 560                          struct drm_file *file_priv)
 561 {
 562         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 563         drm_i915_private_t *dev_priv = dev->dev_private;
 564         ssize_t remain;
 565         loff_t offset, page_base;
 566         char __user *user_data;
 567         int page_offset, page_length;
 568         int ret;
 569
 570         user_data = (char __user *) (uintptr_t) args->data_ptr;
 571         remain = args->size;
 572         if (!access_ok(VERIFY_READ, user_data, remain))
 573                 return -EFAULT;
 574
 575
 576         mutex_lock(&dev->struct_mutex);
 577         ret = i915_gem_object_pin(obj, 0);
 578         if (ret) {
 579                 mutex_unlock(&dev->struct_mutex);
 580                 return ret;
 581         }
 582         ret = i915_gem_object_set_to_gtt_domain(obj, 1);
 583         if (ret)
 584                 goto fail;
 585
 586         obj_priv = to_intel_bo(obj);
 587         offset = obj_priv->gtt_offset + args->offset;
 588
 589         while (remain > 0) {
 590                 /* Operation in this page
 591                  *
 592                  * page_base = page offset within aperture
 593                  * page_offset = offset within page
 594                  * page_length = bytes to copy for this page
 595                  */
 596                 page_base = (offset & ~(PAGE_SIZE-1));
 597                 page_offset = offset & (PAGE_SIZE-1);
 598                 page_length = remain;
 599                 if ((page_offset + remain) > PAGE_SIZE)
 600                         page_length = PAGE_SIZE - page_offset;
 601
 602                 ret = fast_user_write (dev_priv->mm.gtt_mapping, page_base,
 603                                        page_offset, user_data, page_length);
 604
 605                 /* If we get a fault while copying data, then (presumably) our
 606                  * source page isn't available.  Return the error and we'll
 607                  * retry in the slow path.
 608                  */
 609                 if (ret)
 610                         goto fail;
 611
 612                 remain -= page_length;
 613                 user_data += page_length;
 614                 offset += page_length;
 615         }
 616
 617 fail:
 618         i915_gem_object_unpin(obj);
 619         mutex_unlock(&dev->struct_mutex);
 620
 621         return ret;
 622 }
 623
 624 /**
 625  * This is the fallback GTT pwrite path, which uses get_user_pages to pin
 626  * the memory and maps it using kmap_atomic for copying.
 627  *
 628  * This code resulted in x11perf -rgb10text consuming about 10% more CPU
 629  * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
 630  */
 631 static int
 632 i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
 633                          struct drm_i915_gem_pwrite *args,
 634                          struct drm_file *file_priv)
 635 {
 636         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 637         drm_i915_private_t *dev_priv = dev->dev_private;
 638         ssize_t remain;
 639         loff_t gtt_page_base, offset;
 640         loff_t first_data_page, last_data_page, num_pages;
 641         loff_t pinned_pages, i;
 642         struct page **user_pages;
 643         struct mm_struct *mm = current->mm;
 644         int gtt_page_offset, data_page_offset, data_page_index, page_length;
 645         int ret;
 646         uint64_t data_ptr = args->data_ptr;
 647
 648         remain = args->size;
 649
 650         /* Pin the user pages containing the data.  We can't fault while
 651          * holding the struct mutex, and all of the pwrite implementations
 652          * want to hold it while dereferencing the user data.
 653          */
 654         first_data_page = data_ptr / PAGE_SIZE;
 655         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 656         num_pages = last_data_page - first_data_page + 1;
 657
 658         user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
 659         if (user_pages == NULL)
 660                 return -ENOMEM;
 661
 662         down_read(&mm->mmap_sem);
 663         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 664                                       num_pages, 0, 0, user_pages, NULL);
 665         up_read(&mm->mmap_sem);
 666         if (pinned_pages < num_pages) {
 667                 ret = -EFAULT;
 668                 goto out_unpin_pages;
 669         }
 670
 671         mutex_lock(&dev->struct_mutex);
 672         ret = i915_gem_object_pin(obj, 0);
 673         if (ret)
 674                 goto out_unlock;
 675
 676         ret = i915_gem_object_set_to_gtt_domain(obj, 1);
 677         if (ret)
 678                 goto out_unpin_object;
 679
 680         obj_priv = to_intel_bo(obj);
 681         offset = obj_priv->gtt_offset + args->offset;
 682
 683         while (remain > 0) {
 684                 /* Operation in this page
 685                  *
 686                  * gtt_page_base = page offset within aperture
 687                  * gtt_page_offset = offset within page in aperture
 688                  * data_page_index = page number in get_user_pages return
 689                  * data_page_offset = offset with data_page_index page.
 690                  * page_length = bytes to copy for this page
 691                  */
 692                 gtt_page_base = offset & PAGE_MASK;
 693                 gtt_page_offset = offset & ~PAGE_MASK;
 694                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 695                 data_page_offset = data_ptr & ~PAGE_MASK;
 696
 697                 page_length = remain;
 698                 if ((gtt_page_offset + page_length) > PAGE_SIZE)
 699                         page_length = PAGE_SIZE - gtt_page_offset;
 700                 if ((data_page_offset + page_length) > PAGE_SIZE)
 701                         page_length = PAGE_SIZE - data_page_offset;
 702
 703                 ret = slow_kernel_write(dev_priv->mm.gtt_mapping,
 704                                         gtt_page_base, gtt_page_offset,
 705                                         user_pages[data_page_index],
 706                                         data_page_offset,
 707                                         page_length);
 708
 709                 /* If we get a fault while copying data, then (presumably) our
 710                  * source page isn't available.  Return the error and we'll
 711                  * retry in the slow path.
 712                  */
 713                 if (ret)
 714                         goto out_unpin_object;
 715
 716                 remain -= page_length;
 717                 offset += page_length;
 718                 data_ptr += page_length;
 719         }
 720
 721 out_unpin_object:
 722         i915_gem_object_unpin(obj);
 723 out_unlock:
 724         mutex_unlock(&dev->struct_mutex);
 725 out_unpin_pages:
 726         for (i = 0; i < pinned_pages; i++)
 727                 page_cache_release(user_pages[i]);
 728         drm_free_large(user_pages);
 729
 730         return ret;
 731 }
 732
 733 /**
 734  * This is the fast shmem pwrite path, which attempts to directly
 735  * copy_from_user into the kmapped pages backing the object.
 736  */
 737 static int
 738 i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
 739                            struct drm_i915_gem_pwrite *args,
 740                            struct drm_file *file_priv)
 741 {
 742         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 743         ssize_t remain;
 744         loff_t offset, page_base;
 745         char __user *user_data;
 746         int page_offset, page_length;
 747         int ret;
 748
 749         user_data = (char __user *) (uintptr_t) args->data_ptr;
 750         remain = args->size;
 751
 752         mutex_lock(&dev->struct_mutex);
 753
 754         ret = i915_gem_object_get_pages(obj, 0);
 755         if (ret != 0)
 756                 goto fail_unlock;
 757
 758         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
 759         if (ret != 0)
 760                 goto fail_put_pages;
 761
 762         obj_priv = to_intel_bo(obj);
 763         offset = args->offset;
 764         obj_priv->dirty = 1;
 765
 766         while (remain > 0) {
 767                 /* Operation in this page
 768                  *
 769                  * page_base = page offset within aperture
 770                  * page_offset = offset within page
 771                  * page_length = bytes to copy for this page
 772                  */
 773                 page_base = (offset & ~(PAGE_SIZE-1));
 774                 page_offset = offset & (PAGE_SIZE-1);
 775                 page_length = remain;
 776                 if ((page_offset + remain) > PAGE_SIZE)
 777                         page_length = PAGE_SIZE - page_offset;
 778
 779                 ret = fast_shmem_write(obj_priv->pages,
 780                                        page_base, page_offset,
 781                                        user_data, page_length);
 782                 if (ret)
 783                         goto fail_put_pages;
 784
 785                 remain -= page_length;
 786                 user_data += page_length;
 787                 offset += page_length;
 788         }
 789
 790 fail_put_pages:
 791         i915_gem_object_put_pages(obj);
 792 fail_unlock:
 793         mutex_unlock(&dev->struct_mutex);
 794
 795         return ret;
 796 }
 797
 798 /**
 799  * This is the fallback shmem pwrite path, which uses get_user_pages to pin
 800  * the memory and maps it using kmap_atomic for copying.
 801  *
 802  * This avoids taking mmap_sem for faulting on the user's address while the
 803  * struct_mutex is held.
 804  */
 805 static int
 806 i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
 807                            struct drm_i915_gem_pwrite *args,
 808                            struct drm_file *file_priv)
 809 {
 810         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 811         struct mm_struct *mm = current->mm;
 812         struct page **user_pages;
 813         ssize_t remain;
 814         loff_t offset, pinned_pages, i;
 815         loff_t first_data_page, last_data_page, num_pages;
 816         int shmem_page_index, shmem_page_offset;
 817         int data_page_index,  data_page_offset;
 818         int page_length;
 819         int ret;
 820         uint64_t data_ptr = args->data_ptr;
 821         int do_bit17_swizzling;
 822
 823         remain = args->size;
 824
 825         /* Pin the user pages containing the data.  We can't fault while
 826          * holding the struct mutex, and all of the pwrite implementations
 827          * want to hold it while dereferencing the user data.
 828          */
 829         first_data_page = data_ptr / PAGE_SIZE;
 830         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 831         num_pages = last_data_page - first_data_page + 1;
 832
 833         user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
 834         if (user_pages == NULL)
 835                 return -ENOMEM;
 836
 837         down_read(&mm->mmap_sem);
 838         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 839                                       num_pages, 0, 0, user_pages, NULL);
 840         up_read(&mm->mmap_sem);
 841         if (pinned_pages < num_pages) {
 842                 ret = -EFAULT;
 843                 goto fail_put_user_pages;
 844         }
 845
 846         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 847
 848         mutex_lock(&dev->struct_mutex);
 849
 850         ret = i915_gem_object_get_pages_or_evict(obj);
 851         if (ret)
 852                 goto fail_unlock;
 853
 854         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
 855         if (ret != 0)
 856                 goto fail_put_pages;
 857
 858         obj_priv = to_intel_bo(obj);
 859         offset = args->offset;
 860         obj_priv->dirty = 1;
 861
 862         while (remain > 0) {
 863                 /* Operation in this page
 864                  *
 865                  * shmem_page_index = page number within shmem file
 866                  * shmem_page_offset = offset within page in shmem file
 867                  * data_page_index = page number in get_user_pages return
 868                  * data_page_offset = offset with data_page_index page.
 869                  * page_length = bytes to copy for this page
 870                  */
 871                 shmem_page_index = offset / PAGE_SIZE;
 872                 shmem_page_offset = offset & ~PAGE_MASK;
 873                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 874                 data_page_offset = data_ptr & ~PAGE_MASK;
 875
 876                 page_length = remain;
 877                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 878                         page_length = PAGE_SIZE - shmem_page_offset;
 879                 if ((data_page_offset + page_length) > PAGE_SIZE)
 880                         page_length = PAGE_SIZE - data_page_offset;
 881
 882                 if (do_bit17_swizzling) {
 883                         slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
 884                                               shmem_page_offset,
 885                                               user_pages[data_page_index],
 886                                               data_page_offset,
 887                                               page_length,
 888                                               0);
 889                 } else {
 890                         slow_shmem_copy(obj_priv->pages[shmem_page_index],
 891                                         shmem_page_offset,
 892                                         user_pages[data_page_index],
 893                                         data_page_offset,
 894                                         page_length);
 895                 }
 896
 897                 remain -= page_length;
 898                 data_ptr += page_length;
 899                 offset += page_length;
 900         }
 901
 902 fail_put_pages:
 903         i915_gem_object_put_pages(obj);
 904 fail_unlock:
 905         mutex_unlock(&dev->struct_mutex);
 906 fail_put_user_pages:
 907         for (i = 0; i < pinned_pages; i++)
 908                 page_cache_release(user_pages[i]);
 909         drm_free_large(user_pages);
 910
 911         return ret;
 912 }
 913
 914 /**
 915  * Writes data to the object referenced by handle.
 916  *
 917  * On error, the contents of the buffer that were to be modified are undefined.
 918  */
 919 int
 920 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 921                       struct drm_file *file_priv)
 922 {
 923         struct drm_i915_gem_pwrite *args = data;
 924         struct drm_gem_object *obj;
 925         struct drm_i915_gem_object *obj_priv;
 926         int ret = 0;
 927
 928         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 929         if (obj == NULL)
 930                 return -EBADF;
 931         obj_priv = to_intel_bo(obj);
 932
 933         /* Bounds check destination.
 934          *
 935          * XXX: This could use review for overflow issues...
 936          */
 937         if (args->offset > obj->size || args->size > obj->size ||
 938             args->offset + args->size > obj->size) {
 939                 drm_gem_object_unreference_unlocked(obj);
 940                 return -EINVAL;
 941         }
 942
 943         /* We can only do the GTT pwrite on untiled buffers, as otherwise
 944          * it would end up going through the fenced access, and we'll get
 945          * different detiling behavior between reading and writing.
 946          * pread/pwrite currently are reading and writing from the CPU
 947          * perspective, requiring manual detiling by the client.
 948          */
 949         if (obj_priv->phys_obj)
 950                 ret = i915_gem_phys_pwrite(dev, obj, args, file_priv);
 951         else if (obj_priv->tiling_mode == I915_TILING_NONE &&
 952                  dev->gtt_total != 0 &&
 953                  obj->write_domain != I915_GEM_DOMAIN_CPU) {
 954                 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file_priv);
 955                 if (ret == -EFAULT) {
 956                         ret = i915_gem_gtt_pwrite_slow(dev, obj, args,
 957                                                        file_priv);
 958                 }
 959         } else if (i915_gem_object_needs_bit17_swizzle(obj)) {
 960                 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file_priv);
 961         } else {
 962                 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file_priv);
 963                 if (ret == -EFAULT) {
 964                         ret = i915_gem_shmem_pwrite_slow(dev, obj, args,
 965                                                          file_priv);
 966                 }
 967         }
 968
 969 #if WATCH_PWRITE
 970         if (ret)
 971                 DRM_INFO("pwrite failed %d\n", ret);
 972 #endif
 973
 974         drm_gem_object_unreference_unlocked(obj);
 975
 976         return ret;
 977 }
 978
 979 /**
 980  * Called when user space prepares to use an object with the CPU, either
 981  * through the mmap ioctl's mapping or a GTT mapping.
 982  */
 983 int
 984 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 985                           struct drm_file *file_priv)
 986 {
 987         struct drm_i915_private *dev_priv = dev->dev_private;
 988         struct drm_i915_gem_set_domain *args = data;
 989         struct drm_gem_object *obj;
 990         struct drm_i915_gem_object *obj_priv;
 991         uint32_t read_domains = args->read_domains;
 992         uint32_t write_domain = args->write_domain;
 993         int ret;
 994
 995         if (!(dev->driver->driver_features & DRIVER_GEM))
 996                 return -ENODEV;
 997
 998         /* Only handle setting domains to types used by the CPU. */
 999         if (write_domain & I915_GEM_GPU_DOMAINS)
1000                 return -EINVAL;
1001
1002         if (read_domains & I915_GEM_GPU_DOMAINS)
1003                 return -EINVAL;
1004
1005         /* Having something in the write domain implies it's in the read
1006          * domain, and only that read domain.  Enforce that in the request.
1007          */
1008         if (write_domain != 0 && read_domains != write_domain)
1009                 return -EINVAL;
1010
1011         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1012         if (obj == NULL)
1013                 return -EBADF;
1014         obj_priv = to_intel_bo(obj);
1015
1016         mutex_lock(&dev->struct_mutex);
1017
1018         intel_mark_busy(dev, obj);
1019
1020 #if WATCH_BUF
1021         DRM_INFO("set_domain_ioctl %p(%zd), %08x %08x\n",
1022                  obj, obj->size, read_domains, write_domain);
1023 #endif
1024         if (read_domains & I915_GEM_DOMAIN_GTT) {
1025                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1026
1027                 /* Update the LRU on the fence for the CPU access that's
1028                  * about to occur.
1029                  */
1030                 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1031                         struct drm_i915_fence_reg *reg =
1032                                 &dev_priv->fence_regs[obj_priv->fence_reg];
1033                         list_move_tail(&reg->lru_list,
1034                                        &dev_priv->mm.fence_list);
1035                 }
1036
1037                 /* Silently promote "you're not bound, there was nothing to do"
1038                  * to success, since the client was just asking us to
1039                  * make sure everything was done.
1040                  */
1041                 if (ret == -EINVAL)
1042                         ret = 0;
1043         } else {
1044                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1045         }
1046
1047         drm_gem_object_unreference(obj);
1048         mutex_unlock(&dev->struct_mutex);
1049         return ret;
1050 }
1051
1052 /**
1053  * Called when user space has done writes to this buffer
1054  */
1055 int
1056 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1057                       struct drm_file *file_priv)
1058 {
1059         struct drm_i915_gem_sw_finish *args = data;
1060         struct drm_gem_object *obj;
1061         struct drm_i915_gem_object *obj_priv;
1062         int ret = 0;
1063
1064         if (!(dev->driver->driver_features & DRIVER_GEM))
1065                 return -ENODEV;
1066
1067         mutex_lock(&dev->struct_mutex);
1068         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1069         if (obj == NULL) {
1070                 mutex_unlock(&dev->struct_mutex);
1071                 return -EBADF;
1072         }
1073
1074 #if WATCH_BUF
1075         DRM_INFO("%s: sw_finish %d (%p %zd)\n",
1076                  __func__, args->handle, obj, obj->size);
1077 #endif
1078         obj_priv = to_intel_bo(obj);
1079
1080         /* Pinned buffers may be scanout, so flush the cache */
1081         if (obj_priv->pin_count)
1082                 i915_gem_object_flush_cpu_write_domain(obj);
1083
1084         drm_gem_object_unreference(obj);
1085         mutex_unlock(&dev->struct_mutex);
1086         return ret;
1087 }
1088
1089 /**
1090  * Maps the contents of an object, returning the address it is mapped
1091  * into.
1092  *
1093  * While the mapping holds a reference on the contents of the object, it doesn't
1094  * imply a ref on the object itself.
1095  */
1096 int
1097 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1098                    struct drm_file *file_priv)
1099 {
1100         struct drm_i915_gem_mmap *args = data;
1101         struct drm_gem_object *obj;
1102         loff_t offset;
1103         unsigned long addr;
1104
1105         if (!(dev->driver->driver_features & DRIVER_GEM))
1106                 return -ENODEV;
1107
1108         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1109         if (obj == NULL)
1110                 return -EBADF;
1111
1112         offset = args->offset;
1113
1114         down_write(&current->mm->mmap_sem);
1115         addr = do_mmap(obj->filp, 0, args->size,
1116                        PROT_READ | PROT_WRITE, MAP_SHARED,
1117                        args->offset);
1118         up_write(&current->mm->mmap_sem);
1119         drm_gem_object_unreference_unlocked(obj);
1120         if (IS_ERR((void *)addr))
1121                 return addr;
1122
1123         args->addr_ptr = (uint64_t) addr;
1124
1125         return 0;
1126 }
1127
1128 /**
1129  * i915_gem_fault - fault a page into the GTT
1130  * vma: VMA in question
1131  * vmf: fault info
1132  *
1133  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1134  * from userspace.  The fault handler takes care of binding the object to
1135  * the GTT (if needed), allocating and programming a fence register (again,
1136  * only if needed based on whether the old reg is still valid or the object
1137  * is tiled) and inserting a new PTE into the faulting process.
1138  *
1139  * Note that the faulting process may involve evicting existing objects
1140  * from the GTT and/or fence registers to make room.  So performance may
1141  * suffer if the GTT working set is large or there are few fence registers
1142  * left.
1143  */
1144 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1145 {
1146         struct drm_gem_object *obj = vma->vm_private_data;
1147         struct drm_device *dev = obj->dev;
1148         struct drm_i915_private *dev_priv = dev->dev_private;
1149         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1150         pgoff_t page_offset;
1151         unsigned long pfn;
1152         int ret = 0;
1153         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1154
1155         /* We don't use vmf->pgoff since that has the fake offset */
1156         page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1157                 PAGE_SHIFT;
1158
1159         /* Now bind it into the GTT if needed */
1160         mutex_lock(&dev->struct_mutex);
1161         if (!obj_priv->gtt_space) {
1162                 ret = i915_gem_object_bind_to_gtt(obj, 0);
1163                 if (ret)
1164                         goto unlock;
1165
1166                 list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1167
1168                 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1169                 if (ret)
1170                         goto unlock;
1171         }
1172
1173         /* Need a new fence register? */
1174         if (obj_priv->tiling_mode != I915_TILING_NONE) {
1175                 ret = i915_gem_object_get_fence_reg(obj);
1176                 if (ret)
1177                         goto unlock;
1178         }
1179
1180         pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
1181                 page_offset;
1182
1183         /* Finally, remap it using the new GTT offset */
1184         ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1185 unlock:
1186         mutex_unlock(&dev->struct_mutex);
1187
1188         switch (ret) {
1189         case 0:
1190         case -ERESTARTSYS:
1191                 return VM_FAULT_NOPAGE;
1192         case -ENOMEM:
1193         case -EAGAIN:
1194                 return VM_FAULT_OOM;
1195         default:
1196                 return VM_FAULT_SIGBUS;
1197         }
1198 }
1199
1200 /**
1201  * i915_gem_create_mmap_offset - create a fake mmap offset for an object
1202  * @obj: obj in question
1203  *
1204  * GEM memory mapping works by handing back to userspace a fake mmap offset
1205  * it can use in a subsequent mmap(2) call.  The DRM core code then looks
1206  * up the object based on the offset and sets up the various memory mapping
1207  * structures.
1208  *
1209  * This routine allocates and attaches a fake offset for @obj.
1210  */
1211 static int
1212 i915_gem_create_mmap_offset(struct drm_gem_object *obj)
1213 {
1214         struct drm_device *dev = obj->dev;
1215         struct drm_gem_mm *mm = dev->mm_private;
1216         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1217         struct drm_map_list *list;
1218         struct drm_local_map *map;
1219         int ret = 0;
1220
1221         /* Set the object up for mmap'ing */
1222         list = &obj->map_list;
1223         list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
1224         if (!list->map)
1225                 return -ENOMEM;
1226
1227         map = list->map;
1228         map->type = _DRM_GEM;
1229         map->size = obj->size;
1230         map->handle = obj;
1231
1232         /* Get a DRM GEM mmap offset allocated... */
1233         list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
1234                                                     obj->size / PAGE_SIZE, 0, 0);
1235         if (!list->file_offset_node) {
1236                 DRM_ERROR("failed to allocate offset for bo %d\n", obj->name);
1237                 ret = -ENOMEM;
1238                 goto out_free_list;
1239         }
1240
1241         list->file_offset_node = drm_mm_get_block(list->file_offset_node,
1242                                                   obj->size / PAGE_SIZE, 0);
1243         if (!list->file_offset_node) {
1244                 ret = -ENOMEM;
1245                 goto out_free_list;
1246         }
1247
1248         list->hash.key = list->file_offset_node->start;
1249         if (drm_ht_insert_item(&mm->offset_hash, &list->hash)) {
1250                 DRM_ERROR("failed to add to map hash\n");
1251                 ret = -ENOMEM;
1252                 goto out_free_mm;
1253         }
1254
1255         /* By now we should be all set, any drm_mmap request on the offset
1256          * below will get to our mmap & fault handler */
1257         obj_priv->mmap_offset = ((uint64_t) list->hash.key) << PAGE_SHIFT;
1258
1259         return 0;
1260
1261 out_free_mm:
1262         drm_mm_put_block(list->file_offset_node);
1263 out_free_list:
1264         kfree(list->map);
1265
1266         return ret;
1267 }
1268
1269 /**
1270  * i915_gem_release_mmap - remove physical page mappings
1271  * @obj: obj in question
1272  *
1273  * Preserve the reservation of the mmapping with the DRM core code, but
1274  * relinquish ownership of the pages back to the system.
1275  *
1276  * It is vital that we remove the page mapping if we have mapped a tiled
1277  * object through the GTT and then lose the fence register due to
1278  * resource pressure. Similarly if the object has been moved out of the
1279  * aperture, than pages mapped into userspace must be revoked. Removing the
1280  * mapping will then trigger a page fault on the next user access, allowing
1281  * fixup by i915_gem_fault().
1282  */
1283 void
1284 i915_gem_release_mmap(struct drm_gem_object *obj)
1285 {
1286         struct drm_device *dev = obj->dev;
1287         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1288
1289         if (dev->dev_mapping)
1290                 unmap_mapping_range(dev->dev_mapping,
1291                                     obj_priv->mmap_offset, obj->size, 1);
1292 }
1293
1294 static void
1295 i915_gem_free_mmap_offset(struct drm_gem_object *obj)
1296 {
1297         struct drm_device *dev = obj->dev;
1298         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1299         struct drm_gem_mm *mm = dev->mm_private;
1300         struct drm_map_list *list;
1301
1302         list = &obj->map_list;
1303         drm_ht_remove_item(&mm->offset_hash, &list->hash);
1304
1305         if (list->file_offset_node) {
1306                 drm_mm_put_block(list->file_offset_node);
1307                 list->file_offset_node = NULL;
1308         }
1309
1310         if (list->map) {
1311                 kfree(list->map);
1312                 list->map = NULL;
1313         }
1314
1315         obj_priv->mmap_offset = 0;
1316 }
1317
1318 /**
1319  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1320  * @obj: object to check
1321  *
1322  * Return the required GTT alignment for an object, taking into account
1323  * potential fence register mapping if needed.
1324  */
1325 static uint32_t
1326 i915_gem_get_gtt_alignment(struct drm_gem_object *obj)
1327 {
1328         struct drm_device *dev = obj->dev;
1329         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1330         int start, i;
1331
1332         /*
1333          * Minimum alignment is 4k (GTT page size), but might be greater
1334          * if a fence register is needed for the object.
1335          */
1336         if (IS_I965G(dev) || obj_priv->tiling_mode == I915_TILING_NONE)
1337                 return 4096;
1338
1339         /*
1340          * Previous chips need to be aligned to the size of the smallest
1341          * fence register that can contain the object.
1342          */
1343         if (IS_I9XX(dev))
1344                 start = 1024*1024;
1345         else
1346                 start = 512*1024;
1347
1348         for (i = start; i < obj->size; i <<= 1)
1349                 ;
1350
1351         return i;
1352 }
1353
1354 /**
1355  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1356  * @dev: DRM device
1357  * @data: GTT mapping ioctl data
1358  * @file_priv: GEM object info
1359  *
1360  * Simply returns the fake offset to userspace so it can mmap it.
1361  * The mmap call will end up in drm_gem_mmap(), which will set things
1362  * up so we can get faults in the handler above.
1363  *
1364  * The fault handler will take care of binding the object into the GTT
1365  * (since it may have been evicted to make room for something), allocating
1366  * a fence register, and mapping the appropriate aperture address into
1367  * userspace.
1368  */
1369 int
1370 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1371                         struct drm_file *file_priv)
1372 {
1373         struct drm_i915_gem_mmap_gtt *args = data;
1374         struct drm_i915_private *dev_priv = dev->dev_private;
1375         struct drm_gem_object *obj;
1376         struct drm_i915_gem_object *obj_priv;
1377         int ret;
1378
1379         if (!(dev->driver->driver_features & DRIVER_GEM))
1380                 return -ENODEV;
1381
1382         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1383         if (obj == NULL)
1384                 return -EBADF;
1385
1386         mutex_lock(&dev->struct_mutex);
1387
1388         obj_priv = to_intel_bo(obj);
1389
1390         if (obj_priv->madv != I915_MADV_WILLNEED) {
1391                 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1392                 drm_gem_object_unreference(obj);
1393                 mutex_unlock(&dev->struct_mutex);
1394                 return -EINVAL;
1395         }
1396
1397
1398         if (!obj_priv->mmap_offset) {
1399                 ret = i915_gem_create_mmap_offset(obj);
1400                 if (ret) {
1401                         drm_gem_object_unreference(obj);
1402                         mutex_unlock(&dev->struct_mutex);
1403                         return ret;
1404                 }
1405         }
1406
1407         args->offset = obj_priv->mmap_offset;
1408
1409         /*
1410          * Pull it into the GTT so that we have a page list (makes the
1411          * initial fault faster and any subsequent flushing possible).
1412          */
1413         if (!obj_priv->agp_mem) {
1414                 ret = i915_gem_object_bind_to_gtt(obj, 0);
1415                 if (ret) {
1416                         drm_gem_object_unreference(obj);
1417                         mutex_unlock(&dev->struct_mutex);
1418                         return ret;
1419                 }
1420                 list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1421         }
1422
1423         drm_gem_object_unreference(obj);
1424         mutex_unlock(&dev->struct_mutex);
1425
1426         return 0;
1427 }
1428
1429 void
1430 i915_gem_object_put_pages(struct drm_gem_object *obj)
1431 {
1432         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1433         int page_count = obj->size / PAGE_SIZE;
1434         int i;
1435
1436         BUG_ON(obj_priv->pages_refcount == 0);
1437         BUG_ON(obj_priv->madv == __I915_MADV_PURGED);
1438
1439         if (--obj_priv->pages_refcount != 0)
1440                 return;
1441
1442         if (obj_priv->tiling_mode != I915_TILING_NONE)
1443                 i915_gem_object_save_bit_17_swizzle(obj);
1444
1445         if (obj_priv->madv == I915_MADV_DONTNEED)
1446                 obj_priv->dirty = 0;
1447
1448         for (i = 0; i < page_count; i++) {
1449                 if (obj_priv->dirty)
1450                         set_page_dirty(obj_priv->pages[i]);
1451
1452                 if (obj_priv->madv == I915_MADV_WILLNEED)
1453                         mark_page_accessed(obj_priv->pages[i]);
1454
1455                 page_cache_release(obj_priv->pages[i]);
1456         }
1457         obj_priv->dirty = 0;
1458
1459         drm_free_large(obj_priv->pages);
1460         obj_priv->pages = NULL;
1461 }
1462
1463 static void
1464 i915_gem_object_move_to_active(struct drm_gem_object *obj, uint32_t seqno,
1465                                struct intel_ring_buffer *ring)
1466 {
1467         struct drm_device *dev = obj->dev;
1468         drm_i915_private_t *dev_priv = dev->dev_private;
1469         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1470         BUG_ON(ring == NULL);
1471         obj_priv->ring = ring;
1472
1473         /* Add a reference if we're newly entering the active list. */
1474         if (!obj_priv->active) {
1475                 drm_gem_object_reference(obj);
1476                 obj_priv->active = 1;
1477         }
1478         /* Move from whatever list we were on to the tail of execution. */
1479         spin_lock(&dev_priv->mm.active_list_lock);
1480         list_move_tail(&obj_priv->list, &ring->active_list);
1481         spin_unlock(&dev_priv->mm.active_list_lock);
1482         obj_priv->last_rendering_seqno = seqno;
1483 }
1484
1485 static void
1486 i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
1487 {
1488         struct drm_device *dev = obj->dev;
1489         drm_i915_private_t *dev_priv = dev->dev_private;
1490         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1491
1492         BUG_ON(!obj_priv->active);
1493         list_move_tail(&obj_priv->list, &dev_priv->mm.flushing_list);
1494         obj_priv->last_rendering_seqno = 0;
1495 }
1496
1497 /* Immediately discard the backing storage */
1498 static void
1499 i915_gem_object_truncate(struct drm_gem_object *obj)
1500 {
1501         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1502         struct inode *inode;
1503
1504         inode = obj->filp->f_path.dentry->d_inode;
1505         if (inode->i_op->truncate)
1506                 inode->i_op->truncate (inode);
1507
1508         obj_priv->madv = __I915_MADV_PURGED;
1509 }
1510
1511 static inline int
1512 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj_priv)
1513 {
1514         return obj_priv->madv == I915_MADV_DONTNEED;
1515 }
1516
1517 static void
1518 i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
1519 {
1520         struct drm_device *dev = obj->dev;
1521         drm_i915_private_t *dev_priv = dev->dev_private;
1522         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1523
1524         i915_verify_inactive(dev, __FILE__, __LINE__);
1525         if (obj_priv->pin_count != 0)
1526                 list_del_init(&obj_priv->list);
1527         else
1528                 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1529
1530         BUG_ON(!list_empty(&obj_priv->gpu_write_list));
1531
1532         obj_priv->last_rendering_seqno = 0;
1533         obj_priv->ring = NULL;
1534         if (obj_priv->active) {
1535                 obj_priv->active = 0;
1536                 drm_gem_object_unreference(obj);
1537         }
1538         i915_verify_inactive(dev, __FILE__, __LINE__);
1539 }
1540
1541 static void
1542 i915_gem_process_flushing_list(struct drm_device *dev,
1543                                uint32_t flush_domains, uint32_t seqno,
1544                                struct intel_ring_buffer *ring)
1545 {
1546         drm_i915_private_t *dev_priv = dev->dev_private;
1547         struct drm_i915_gem_object *obj_priv, *next;
1548
1549         list_for_each_entry_safe(obj_priv, next,
1550                                  &dev_priv->mm.gpu_write_list,
1551                                  gpu_write_list) {
1552                 struct drm_gem_object *obj = &obj_priv->base;
1553
1554                 if ((obj->write_domain & flush_domains) ==
1555                     obj->write_domain &&
1556                     obj_priv->ring->ring_flag == ring->ring_flag) {
1557                         uint32_t old_write_domain = obj->write_domain;
1558
1559                         obj->write_domain = 0;
1560                         list_del_init(&obj_priv->gpu_write_list);
1561                         i915_gem_object_move_to_active(obj, seqno, ring);
1562
1563                         /* update the fence lru list */
1564                         if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1565                                 struct drm_i915_fence_reg *reg =
1566                                         &dev_priv->fence_regs[obj_priv->fence_reg];
1567                                 list_move_tail(&reg->lru_list,
1568                                                 &dev_priv->mm.fence_list);
1569                         }
1570
1571                         trace_i915_gem_object_change_domain(obj,
1572                                                             obj->read_domains,
1573                                                             old_write_domain);
1574                 }
1575         }
1576 }
1577
1578 uint32_t
1579 i915_add_request(struct drm_device *dev, struct drm_file *file_priv,
1580                  uint32_t flush_domains, struct intel_ring_buffer *ring)
1581 {
1582         drm_i915_private_t *dev_priv = dev->dev_private;
1583         struct drm_i915_file_private *i915_file_priv = NULL;
1584         struct drm_i915_gem_request *request;
1585         uint32_t seqno;
1586         int was_empty;
1587
1588         if (file_priv != NULL)
1589                 i915_file_priv = file_priv->driver_priv;
1590
1591         request = kzalloc(sizeof(*request), GFP_KERNEL);
1592         if (request == NULL)
1593                 return 0;
1594
1595         seqno = ring->add_request(dev, ring, file_priv, flush_domains);
1596
1597         request->seqno = seqno;
1598         request->ring = ring;
1599         request->emitted_jiffies = jiffies;
1600         was_empty = list_empty(&ring->request_list);
1601         list_add_tail(&request->list, &ring->request_list);
1602
1603         if (i915_file_priv) {
1604                 list_add_tail(&request->client_list,
1605                               &i915_file_priv->mm.request_list);
1606         } else {
1607                 INIT_LIST_HEAD(&request->client_list);
1608         }
1609
1610         /* Associate any objects on the flushing list matching the write
1611          * domain we're flushing with our flush.
1612          */
1613         if (flush_domains != 0)
1614                 i915_gem_process_flushing_list(dev, flush_domains, seqno, ring);
1615
1616         if (!dev_priv->mm.suspended) {
1617                 mod_timer(&dev_priv->hangcheck_timer, jiffies + DRM_I915_HANGCHECK_PERIOD);
1618                 if (was_empty)
1619                         queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1620         }
1621         return seqno;
1622 }
1623
1624 /**
1625  * Command execution barrier
1626  *
1627  * Ensures that all commands in the ring are finished
1628  * before signalling the CPU
1629  */
1630 static uint32_t
1631 i915_retire_commands(struct drm_device *dev, struct intel_ring_buffer *ring)
1632 {
1633         uint32_t flush_domains = 0;
1634
1635         /* The sampler always gets flushed on i965 (sigh) */
1636         if (IS_I965G(dev))
1637                 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
1638
1639         ring->flush(dev, ring,
1640                         I915_GEM_DOMAIN_COMMAND, flush_domains);
1641         return flush_domains;
1642 }
1643
1644 /**
1645  * Moves buffers associated only with the given active seqno from the active
1646  * to inactive list, potentially freeing them.
1647  */
1648 static void
1649 i915_gem_retire_request(struct drm_device *dev,
1650                         struct drm_i915_gem_request *request)
1651 {
1652         drm_i915_private_t *dev_priv = dev->dev_private;
1653
1654         trace_i915_gem_request_retire(dev, request->seqno);
1655
1656         /* Move any buffers on the active list that are no longer referenced
1657          * by the ringbuffer to the flushing/inactive lists as appropriate.
1658          */
1659         spin_lock(&dev_priv->mm.active_list_lock);
1660         while (!list_empty(&request->ring->active_list)) {
1661                 struct drm_gem_object *obj;
1662                 struct drm_i915_gem_object *obj_priv;
1663
1664                 obj_priv = list_first_entry(&request->ring->active_list,
1665                                             struct drm_i915_gem_object,
1666                                             list);
1667                 obj = &obj_priv->base;
1668
1669                 /* If the seqno being retired doesn't match the oldest in the
1670                  * list, then the oldest in the list must still be newer than
1671                  * this seqno.
1672                  */
1673                 if (obj_priv->last_rendering_seqno != request->seqno)
1674                         goto out;
1675
1676 #if WATCH_LRU
1677                 DRM_INFO("%s: retire %d moves to inactive list %p\n",
1678                          __func__, request->seqno, obj);
1679 #endif
1680
1681                 if (obj->write_domain != 0)
1682                         i915_gem_object_move_to_flushing(obj);
1683                 else {
1684                         /* Take a reference on the object so it won't be
1685                          * freed while the spinlock is held.  The list
1686                          * protection for this spinlock is safe when breaking
1687                          * the lock like this since the next thing we do
1688                          * is just get the head of the list again.
1689                          */
1690                         drm_gem_object_reference(obj);
1691                         i915_gem_object_move_to_inactive(obj);
1692                         spin_unlock(&dev_priv->mm.active_list_lock);
1693                         drm_gem_object_unreference(obj);
1694                         spin_lock(&dev_priv->mm.active_list_lock);
1695                 }
1696         }
1697 out:
1698         spin_unlock(&dev_priv->mm.active_list_lock);
1699 }
1700
1701 /**
1702  * Returns true if seq1 is later than seq2.
1703  */
1704 bool
1705 i915_seqno_passed(uint32_t seq1, uint32_t seq2)
1706 {
1707         return (int32_t)(seq1 - seq2) >= 0;
1708 }
1709
1710 uint32_t
1711 i915_get_gem_seqno(struct drm_device *dev,
1712                    struct intel_ring_buffer *ring)
1713 {
1714         return ring->get_gem_seqno(dev, ring);
1715 }
1716
1717 /**
1718  * This function clears the request list as sequence numbers are passed.
1719  */
1720 void
1721 i915_gem_retire_requests(struct drm_device *dev,
1722                 struct intel_ring_buffer *ring)
1723 {
1724         drm_i915_private_t *dev_priv = dev->dev_private;
1725         uint32_t seqno;
1726
1727         if (!ring->status_page.page_addr
1728                         || list_empty(&ring->request_list))
1729                 return;
1730
1731         seqno = i915_get_gem_seqno(dev, ring);
1732
1733         while (!list_empty(&ring->request_list)) {
1734                 struct drm_i915_gem_request *request;
1735                 uint32_t retiring_seqno;
1736
1737                 request = list_first_entry(&ring->request_list,
1738                                            struct drm_i915_gem_request,
1739                                            list);
1740                 retiring_seqno = request->seqno;
1741
1742                 if (i915_seqno_passed(seqno, retiring_seqno) ||
1743                     atomic_read(&dev_priv->mm.wedged)) {
1744                         i915_gem_retire_request(dev, request);
1745
1746                         list_del(&request->list);
1747                         list_del(&request->client_list);
1748                         kfree(request);
1749                 } else
1750                         break;
1751         }
1752
1753         if (unlikely (dev_priv->trace_irq_seqno &&
1754                       i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) {
1755
1756                 ring->user_irq_put(dev, ring);
1757                 dev_priv->trace_irq_seqno = 0;
1758         }
1759 }
1760
1761 void
1762 i915_gem_retire_work_handler(struct work_struct *work)
1763 {
1764         drm_i915_private_t *dev_priv;
1765         struct drm_device *dev;
1766
1767         dev_priv = container_of(work, drm_i915_private_t,
1768                                 mm.retire_work.work);
1769         dev = dev_priv->dev;
1770
1771         mutex_lock(&dev->struct_mutex);
1772         i915_gem_retire_requests(dev, &dev_priv->render_ring);
1773
1774         if (HAS_BSD(dev))
1775                 i915_gem_retire_requests(dev, &dev_priv->bsd_ring);
1776
1777         if (!dev_priv->mm.suspended &&
1778                 (!list_empty(&dev_priv->render_ring.request_list) ||
1779                         (HAS_BSD(dev) &&
1780                          !list_empty(&dev_priv->bsd_ring.request_list))))
1781                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1782         mutex_unlock(&dev->struct_mutex);
1783 }
1784
1785 int
1786 i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
1787                 int interruptible, struct intel_ring_buffer *ring)
1788 {
1789         drm_i915_private_t *dev_priv = dev->dev_private;
1790         u32 ier;
1791         int ret = 0;
1792
1793         BUG_ON(seqno == 0);
1794
1795         if (atomic_read(&dev_priv->mm.wedged))
1796                 return -EIO;
1797
1798         if (!i915_seqno_passed(ring->get_gem_seqno(dev, ring), seqno)) {
1799                 if (HAS_PCH_SPLIT(dev))
1800                         ier = I915_READ(DEIER) | I915_READ(GTIER);
1801                 else
1802                         ier = I915_READ(IER);
1803                 if (!ier) {
1804                         DRM_ERROR("something (likely vbetool) disabled "
1805                                   "interrupts, re-enabling\n");
1806                         i915_driver_irq_preinstall(dev);
1807                         i915_driver_irq_postinstall(dev);
1808                 }
1809
1810                 trace_i915_gem_request_wait_begin(dev, seqno);
1811
1812                 ring->waiting_gem_seqno = seqno;
1813                 ring->user_irq_get(dev, ring);
1814                 if (interruptible)
1815                         ret = wait_event_interruptible(ring->irq_queue,
1816                                 i915_seqno_passed(
1817                                         ring->get_gem_seqno(dev, ring), seqno)
1818                                 || atomic_read(&dev_priv->mm.wedged));
1819                 else
1820                         wait_event(ring->irq_queue,
1821                                 i915_seqno_passed(
1822                                         ring->get_gem_seqno(dev, ring), seqno)
1823                                 || atomic_read(&dev_priv->mm.wedged));
1824
1825                 ring->user_irq_put(dev, ring);
1826                 ring->waiting_gem_seqno = 0;
1827
1828                 trace_i915_gem_request_wait_end(dev, seqno);
1829         }
1830         if (atomic_read(&dev_priv->mm.wedged))
1831                 ret = -EIO;
1832
1833         if (ret && ret != -ERESTARTSYS)
1834                 DRM_ERROR("%s returns %d (awaiting %d at %d)\n",
1835                           __func__, ret, seqno, ring->get_gem_seqno(dev, ring));
1836
1837         /* Directly dispatch request retiring.  While we have the work queue
1838          * to handle this, the waiter on a request often wants an associated
1839          * buffer to have made it to the inactive list, and we would need
1840          * a separate wait queue to handle that.
1841          */
1842         if (ret == 0)
1843                 i915_gem_retire_requests(dev, ring);
1844
1845         return ret;
1846 }
1847
1848 /**
1849  * Waits for a sequence number to be signaled, and cleans up the
1850  * request and object lists appropriately for that event.
1851  */
1852 static int
1853 i915_wait_request(struct drm_device *dev, uint32_t seqno,
1854                 struct intel_ring_buffer *ring)
1855 {
1856         return i915_do_wait_request(dev, seqno, 1, ring);
1857 }
1858
1859 static void
1860 i915_gem_flush(struct drm_device *dev,
1861                uint32_t invalidate_domains,
1862                uint32_t flush_domains)
1863 {
1864         drm_i915_private_t *dev_priv = dev->dev_private;
1865         if (flush_domains & I915_GEM_DOMAIN_CPU)
1866                 drm_agp_chipset_flush(dev);
1867         dev_priv->render_ring.flush(dev, &dev_priv->render_ring,
1868                         invalidate_domains,
1869                         flush_domains);
1870
1871         if (HAS_BSD(dev))
1872                 dev_priv->bsd_ring.flush(dev, &dev_priv->bsd_ring,
1873                                 invalidate_domains,
1874                                 flush_domains);
1875 }
1876
1877 static void
1878 i915_gem_flush_ring(struct drm_device *dev,
1879                uint32_t invalidate_domains,
1880                uint32_t flush_domains,
1881                struct intel_ring_buffer *ring)
1882 {
1883         if (flush_domains & I915_GEM_DOMAIN_CPU)
1884                 drm_agp_chipset_flush(dev);
1885         ring->flush(dev, ring,
1886                         invalidate_domains,
1887                         flush_domains);
1888 }
1889
1890 /**
1891  * Ensures that all rendering to the object has completed and the object is
1892  * safe to unbind from the GTT or access from the CPU.
1893  */
1894 static int
1895 i915_gem_object_wait_rendering(struct drm_gem_object *obj)
1896 {
1897         struct drm_device *dev = obj->dev;
1898         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1899         int ret;
1900
1901         /* This function only exists to support waiting for existing rendering,
1902          * not for emitting required flushes.
1903          */
1904         BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0);
1905
1906         /* If there is rendering queued on the buffer being evicted, wait for
1907          * it.
1908          */
1909         if (obj_priv->active) {
1910 #if WATCH_BUF
1911                 DRM_INFO("%s: object %p wait for seqno %08x\n",
1912                           __func__, obj, obj_priv->last_rendering_seqno);
1913 #endif
1914                 ret = i915_wait_request(dev,
1915                                 obj_priv->last_rendering_seqno, obj_priv->ring);
1916                 if (ret != 0)
1917                         return ret;
1918         }
1919
1920         return 0;
1921 }
1922
1923 /**
1924  * Unbinds an object from the GTT aperture.
1925  */
1926 int
1927 i915_gem_object_unbind(struct drm_gem_object *obj)
1928 {
1929         struct drm_device *dev = obj->dev;
1930         drm_i915_private_t *dev_priv = dev->dev_private;
1931         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1932         int ret = 0;
1933
1934 #if WATCH_BUF
1935         DRM_INFO("%s:%d %p\n", __func__, __LINE__, obj);
1936         DRM_INFO("gtt_space %p\n", obj_priv->gtt_space);
1937 #endif
1938         if (obj_priv->gtt_space == NULL)
1939                 return 0;
1940
1941         if (obj_priv->pin_count != 0) {
1942                 DRM_ERROR("Attempting to unbind pinned buffer\n");
1943                 return -EINVAL;
1944         }
1945
1946         /* blow away mappings if mapped through GTT */
1947         i915_gem_release_mmap(obj);
1948
1949         /* Move the object to the CPU domain to ensure that
1950          * any possible CPU writes while it's not in the GTT
1951          * are flushed when we go to remap it. This will
1952          * also ensure that all pending GPU writes are finished
1953          * before we unbind.
1954          */
1955         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
1956         if (ret) {
1957                 if (ret != -ERESTARTSYS)
1958                         DRM_ERROR("set_domain failed: %d\n", ret);
1959                 return ret;
1960         }
1961
1962         BUG_ON(obj_priv->active);
1963
1964         /* release the fence reg _after_ flushing */
1965         if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
1966                 i915_gem_clear_fence_reg(obj);
1967
1968         if (obj_priv->agp_mem != NULL) {
1969                 drm_unbind_agp(obj_priv->agp_mem);
1970                 drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
1971                 obj_priv->agp_mem = NULL;
1972         }
1973
1974         i915_gem_object_put_pages(obj);
1975         BUG_ON(obj_priv->pages_refcount);
1976
1977         if (obj_priv->gtt_space) {
1978                 atomic_dec(&dev->gtt_count);
1979                 atomic_sub(obj->size, &dev->gtt_memory);
1980
1981                 drm_mm_put_block(obj_priv->gtt_space);
1982                 obj_priv->gtt_space = NULL;
1983         }
1984
1985         /* Remove ourselves from the LRU list if present. */
1986         spin_lock(&dev_priv->mm.active_list_lock);
1987         if (!list_empty(&obj_priv->list))
1988                 list_del_init(&obj_priv->list);
1989         spin_unlock(&dev_priv->mm.active_list_lock);
1990
1991         if (i915_gem_object_is_purgeable(obj_priv))
1992                 i915_gem_object_truncate(obj);
1993
1994         trace_i915_gem_object_unbind(obj);
1995
1996         return 0;
1997 }
1998
1999 static struct drm_gem_object *
2000 i915_gem_find_inactive_object(struct drm_device *dev, int min_size)
2001 {
2002         drm_i915_private_t *dev_priv = dev->dev_private;
2003         struct drm_i915_gem_object *obj_priv;
2004         struct drm_gem_object *best = NULL;
2005         struct drm_gem_object *first = NULL;
2006
2007         /* Try to find the smallest clean object */
2008         list_for_each_entry(obj_priv, &dev_priv->mm.inactive_list, list) {
2009                 struct drm_gem_object *obj = &obj_priv->base;
2010                 if (obj->size >= min_size) {
2011                         if ((!obj_priv->dirty ||
2012                              i915_gem_object_is_purgeable(obj_priv)) &&
2013                             (!best || obj->size < best->size)) {
2014                                 best = obj;
2015                                 if (best->size == min_size)
2016                                         return best;
2017                         }
2018                         if (!first)
2019                             first = obj;
2020                 }
2021         }
2022
2023         return best ? best : first;
2024 }
2025
2026 static int
2027 i915_gpu_idle(struct drm_device *dev)
2028 {
2029         drm_i915_private_t *dev_priv = dev->dev_private;
2030         bool lists_empty;
2031         uint32_t seqno1, seqno2;
2032         int ret;
2033
2034         spin_lock(&dev_priv->mm.active_list_lock);
2035         lists_empty = (list_empty(&dev_priv->mm.flushing_list) &&
2036                        list_empty(&dev_priv->render_ring.active_list) &&
2037                        (!HAS_BSD(dev) ||
2038                         list_empty(&dev_priv->bsd_ring.active_list)));
2039         spin_unlock(&dev_priv->mm.active_list_lock);
2040
2041         if (lists_empty)
2042                 return 0;
2043
2044         /* Flush everything onto the inactive list. */
2045         i915_gem_flush(dev, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2046         seqno1 = i915_add_request(dev, NULL, I915_GEM_GPU_DOMAINS,
2047                         &dev_priv->render_ring);
2048         if (seqno1 == 0)
2049                 return -ENOMEM;
2050         ret = i915_wait_request(dev, seqno1, &dev_priv->render_ring);
2051
2052         if (HAS_BSD(dev)) {
2053                 seqno2 = i915_add_request(dev, NULL, I915_GEM_GPU_DOMAINS,
2054                                 &dev_priv->bsd_ring);
2055                 if (seqno2 == 0)
2056                         return -ENOMEM;
2057
2058                 ret = i915_wait_request(dev, seqno2, &dev_priv->bsd_ring);
2059                 if (ret)
2060                         return ret;
2061         }
2062
2063
2064         return ret;
2065 }
2066
2067 static int
2068 i915_gem_evict_everything(struct drm_device *dev)
2069 {
2070         drm_i915_private_t *dev_priv = dev->dev_private;
2071         int ret;
2072         bool lists_empty;
2073
2074         spin_lock(&dev_priv->mm.active_list_lock);
2075         lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
2076                        list_empty(&dev_priv->mm.flushing_list) &&
2077                        list_empty(&dev_priv->render_ring.active_list) &&
2078                        (!HAS_BSD(dev)
2079                         || list_empty(&dev_priv->bsd_ring.active_list)));
2080         spin_unlock(&dev_priv->mm.active_list_lock);
2081
2082         if (lists_empty)
2083                 return -ENOSPC;
2084
2085         /* Flush everything (on to the inactive lists) and evict */
2086         ret = i915_gpu_idle(dev);
2087         if (ret)
2088                 return ret;
2089
2090         BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
2091
2092         ret = i915_gem_evict_from_inactive_list(dev);
2093         if (ret)
2094                 return ret;
2095
2096         spin_lock(&dev_priv->mm.active_list_lock);
2097         lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
2098                        list_empty(&dev_priv->mm.flushing_list) &&
2099                        list_empty(&dev_priv->render_ring.active_list) &&
2100                        (!HAS_BSD(dev)
2101                         || list_empty(&dev_priv->bsd_ring.active_list)));
2102         spin_unlock(&dev_priv->mm.active_list_lock);
2103         BUG_ON(!lists_empty);
2104
2105         return 0;
2106 }
2107
2108 static int
2109 i915_gem_evict_something(struct drm_device *dev, int min_size)
2110 {
2111         drm_i915_private_t *dev_priv = dev->dev_private;
2112         struct drm_gem_object *obj;
2113         int ret;
2114
2115         struct intel_ring_buffer *render_ring = &dev_priv->render_ring;
2116         struct intel_ring_buffer *bsd_ring = &dev_priv->bsd_ring;
2117         for (;;) {
2118                 i915_gem_retire_requests(dev, render_ring);
2119
2120                 if (HAS_BSD(dev))
2121                         i915_gem_retire_requests(dev, bsd_ring);
2122
2123                 /* If there's an inactive buffer available now, grab it
2124                  * and be done.
2125                  */
2126                 obj = i915_gem_find_inactive_object(dev, min_size);
2127                 if (obj) {
2128                         struct drm_i915_gem_object *obj_priv;
2129
2130 #if WATCH_LRU
2131                         DRM_INFO("%s: evicting %p\n", __func__, obj);
2132 #endif
2133                         obj_priv = to_intel_bo(obj);
2134                         BUG_ON(obj_priv->pin_count != 0);
2135                         BUG_ON(obj_priv->active);
2136
2137                         /* Wait on the rendering and unbind the buffer. */
2138                         return i915_gem_object_unbind(obj);
2139                 }
2140
2141                 /* If we didn't get anything, but the ring is still processing
2142                  * things, wait for the next to finish and hopefully leave us
2143                  * a buffer to evict.
2144                  */
2145                 if (!list_empty(&render_ring->request_list)) {
2146                         struct drm_i915_gem_request *request;
2147
2148                         request = list_first_entry(&render_ring->request_list,
2149                                                    struct drm_i915_gem_request,
2150                                                    list);
2151
2152                         ret = i915_wait_request(dev,
2153                                         request->seqno, request->ring);
2154                         if (ret)
2155                                 return ret;
2156
2157                         continue;
2158                 }
2159
2160                 if (HAS_BSD(dev) && !list_empty(&bsd_ring->request_list)) {
2161                         struct drm_i915_gem_request *request;
2162
2163                         request = list_first_entry(&bsd_ring->request_list,
2164                                                    struct drm_i915_gem_request,
2165                                                    list);
2166
2167                         ret = i915_wait_request(dev,
2168                                         request->seqno, request->ring);
2169                         if (ret)
2170                                 return ret;
2171
2172                         continue;
2173                 }
2174
2175                 /* If we didn't have anything on the request list but there
2176                  * are buffers awaiting a flush, emit one and try again.
2177                  * When we wait on it, those buffers waiting for that flush
2178                  * will get moved to inactive.
2179                  */
2180                 if (!list_empty(&dev_priv->mm.flushing_list)) {
2181                         struct drm_i915_gem_object *obj_priv;
2182
2183                         /* Find an object that we can immediately reuse */
2184                         list_for_each_entry(obj_priv, &dev_priv->mm.flushing_list, list) {
2185                                 obj = &obj_priv->base;
2186                                 if (obj->size >= min_size)
2187                                         break;
2188
2189                                 obj = NULL;
2190                         }
2191
2192                         if (obj != NULL) {
2193                                 uint32_t seqno;
2194
2195                                 i915_gem_flush_ring(dev,
2196                                                obj->write_domain,
2197                                                obj->write_domain,
2198                                                obj_priv->ring);
2199                                 seqno = i915_add_request(dev, NULL,
2200                                                 obj->write_domain,
2201                                                 obj_priv->ring);
2202                                 if (seqno == 0)
2203                                         return -ENOMEM;
2204                                 continue;
2205                         }
2206                 }
2207
2208                 /* If we didn't do any of the above, there's no single buffer
2209                  * large enough to swap out for the new one, so just evict
2210                  * everything and start again. (This should be rare.)
2211                  */
2212                 if (!list_empty (&dev_priv->mm.inactive_list))
2213                         return i915_gem_evict_from_inactive_list(dev);
2214                 else
2215                         return i915_gem_evict_everything(dev);
2216         }
2217 }
2218
2219 int
2220 i915_gem_object_get_pages(struct drm_gem_object *obj,
2221                           gfp_t gfpmask)
2222 {
2223         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2224         int page_count, i;
2225         struct address_space *mapping;
2226         struct inode *inode;
2227         struct page *page;
2228
2229         BUG_ON(obj_priv->pages_refcount
2230                         == DRM_I915_GEM_OBJECT_MAX_PAGES_REFCOUNT);
2231
2232         if (obj_priv->pages_refcount++ != 0)
2233                 return 0;
2234
2235         /* Get the list of pages out of our struct file.  They'll be pinned
2236          * at this point until we release them.
2237          */
2238         page_count = obj->size / PAGE_SIZE;
2239         BUG_ON(obj_priv->pages != NULL);
2240         obj_priv->pages = drm_calloc_large(page_count, sizeof(struct page *));
2241         if (obj_priv->pages == NULL) {
2242                 obj_priv->pages_refcount--;
2243                 return -ENOMEM;
2244         }
2245
2246         inode = obj->filp->f_path.dentry->d_inode;
2247         mapping = inode->i_mapping;
2248         for (i = 0; i < page_count; i++) {
2249                 page = read_cache_page_gfp(mapping, i,
2250                                            mapping_gfp_mask (mapping) |
2251                                            __GFP_COLD |
2252                                            gfpmask);
2253                 if (IS_ERR(page))
2254                         goto err_pages;
2255
2256                 obj_priv->pages[i] = page;
2257         }
2258
2259         if (obj_priv->tiling_mode != I915_TILING_NONE)
2260                 i915_gem_object_do_bit_17_swizzle(obj);
2261
2262         return 0;
2263
2264 err_pages:
2265         while (i--)
2266                 page_cache_release(obj_priv->pages[i]);
2267
2268         drm_free_large(obj_priv->pages);
2269         obj_priv->pages = NULL;
2270         obj_priv->pages_refcount--;
2271         return PTR_ERR(page);
2272 }
2273
2274 static void sandybridge_write_fence_reg(struct drm_i915_fence_reg *reg)
2275 {
2276         struct drm_gem_object *obj = reg->obj;
2277         struct drm_device *dev = obj->dev;
2278         drm_i915_private_t *dev_priv = dev->dev_private;
2279         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2280         int regnum = obj_priv->fence_reg;
2281         uint64_t val;
2282
2283         val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2284                     0xfffff000) << 32;
2285         val |= obj_priv->gtt_offset & 0xfffff000;
2286         val |= (uint64_t)((obj_priv->stride / 128) - 1) <<
2287                 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2288
2289         if (obj_priv->tiling_mode == I915_TILING_Y)
2290                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2291         val |= I965_FENCE_REG_VALID;
2292
2293         I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (regnum * 8), val);
2294 }
2295
2296 static void i965_write_fence_reg(struct drm_i915_fence_reg *reg)
2297 {
2298         struct drm_gem_object *obj = reg->obj;
2299         struct drm_device *dev = obj->dev;
2300         drm_i915_private_t *dev_priv = dev->dev_private;
2301         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2302         int regnum = obj_priv->fence_reg;
2303         uint64_t val;
2304
2305         val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2306                     0xfffff000) << 32;
2307         val |= obj_priv->gtt_offset & 0xfffff000;
2308         val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2309         if (obj_priv->tiling_mode == I915_TILING_Y)
2310                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2311         val |= I965_FENCE_REG_VALID;
2312
2313         I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val);
2314 }
2315
2316 static void i915_write_fence_reg(struct drm_i915_fence_reg *reg)
2317 {
2318         struct drm_gem_object *obj = reg->obj;
2319         struct drm_device *dev = obj->dev;
2320         drm_i915_private_t *dev_priv = dev->dev_private;
2321         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2322         int regnum = obj_priv->fence_reg;
2323         int tile_width;
2324         uint32_t fence_reg, val;
2325         uint32_t pitch_val;
2326
2327         if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) ||
2328             (obj_priv->gtt_offset & (obj->size - 1))) {
2329                 WARN(1, "%s: object 0x%08x not 1M or size (0x%zx) aligned\n",
2330                      __func__, obj_priv->gtt_offset, obj->size);
2331                 return;
2332         }
2333
2334         if (obj_priv->tiling_mode == I915_TILING_Y &&
2335             HAS_128_BYTE_Y_TILING(dev))
2336                 tile_width = 128;
2337         else
2338                 tile_width = 512;
2339
2340         /* Note: pitch better be a power of two tile widths */
2341         pitch_val = obj_priv->stride / tile_width;
2342         pitch_val = ffs(pitch_val) - 1;
2343
2344         if (obj_priv->tiling_mode == I915_TILING_Y &&
2345             HAS_128_BYTE_Y_TILING(dev))
2346                 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2347         else
2348                 WARN_ON(pitch_val > I915_FENCE_MAX_PITCH_VAL);
2349
2350         val = obj_priv->gtt_offset;
2351         if (obj_priv->tiling_mode == I915_TILING_Y)
2352                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2353         val |= I915_FENCE_SIZE_BITS(obj->size);
2354         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2355         val |= I830_FENCE_REG_VALID;
2356
2357         if (regnum < 8)
2358                 fence_reg = FENCE_REG_830_0 + (regnum * 4);
2359         else
2360                 fence_reg = FENCE_REG_945_8 + ((regnum - 8) * 4);
2361         I915_WRITE(fence_reg, val);
2362 }
2363
2364 static void i830_write_fence_reg(struct drm_i915_fence_reg *reg)
2365 {
2366         struct drm_gem_object *obj = reg->obj;
2367         struct drm_device *dev = obj->dev;
2368         drm_i915_private_t *dev_priv = dev->dev_private;
2369         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2370         int regnum = obj_priv->fence_reg;
2371         uint32_t val;
2372         uint32_t pitch_val;
2373         uint32_t fence_size_bits;
2374
2375         if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) ||
2376             (obj_priv->gtt_offset & (obj->size - 1))) {
2377                 WARN(1, "%s: object 0x%08x not 512K or size aligned\n",
2378                      __func__, obj_priv->gtt_offset);
2379                 return;
2380         }
2381
2382         pitch_val = obj_priv->stride / 128;
2383         pitch_val = ffs(pitch_val) - 1;
2384         WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2385
2386         val = obj_priv->gtt_offset;
2387         if (obj_priv->tiling_mode == I915_TILING_Y)
2388                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2389         fence_size_bits = I830_FENCE_SIZE_BITS(obj->size);
2390         WARN_ON(fence_size_bits & ~0x00000f00);
2391         val |= fence_size_bits;
2392         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2393         val |= I830_FENCE_REG_VALID;
2394
2395         I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val);
2396 }
2397
2398 static int i915_find_fence_reg(struct drm_device *dev)
2399 {
2400         struct drm_i915_fence_reg *reg = NULL;
2401         struct drm_i915_gem_object *obj_priv = NULL;
2402         struct drm_i915_private *dev_priv = dev->dev_private;
2403         struct drm_gem_object *obj = NULL;
2404         int i, avail, ret;
2405
2406         /* First try to find a free reg */
2407         avail = 0;
2408         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2409                 reg = &dev_priv->fence_regs[i];
2410                 if (!reg->obj)
2411                         return i;
2412
2413                 obj_priv = to_intel_bo(reg->obj);
2414                 if (!obj_priv->pin_count)
2415                     avail++;
2416         }
2417
2418         if (avail == 0)
2419                 return -ENOSPC;
2420
2421         /* None available, try to steal one or wait for a user to finish */
2422         i = I915_FENCE_REG_NONE;
2423         list_for_each_entry(reg, &dev_priv->mm.fence_list,
2424                             lru_list) {
2425                 obj = reg->obj;
2426                 obj_priv = to_intel_bo(obj);
2427
2428                 if (obj_priv->pin_count)
2429                         continue;
2430
2431                 /* found one! */
2432                 i = obj_priv->fence_reg;
2433                 break;
2434         }
2435
2436         BUG_ON(i == I915_FENCE_REG_NONE);
2437
2438         /* We only have a reference on obj from the active list. put_fence_reg
2439          * might drop that one, causing a use-after-free in it. So hold a
2440          * private reference to obj like the other callers of put_fence_reg
2441          * (set_tiling ioctl) do. */
2442         drm_gem_object_reference(obj);
2443         ret = i915_gem_object_put_fence_reg(obj);
2444         drm_gem_object_unreference(obj);
2445         if (ret != 0)
2446                 return ret;
2447
2448         return i;
2449 }
2450
2451 /**
2452  * i915_gem_object_get_fence_reg - set up a fence reg for an object
2453  * @obj: object to map through a fence reg
2454  *
2455  * When mapping objects through the GTT, userspace wants to be able to write
2456  * to them without having to worry about swizzling if the object is tiled.
2457  *
2458  * This function walks the fence regs looking for a free one for @obj,
2459  * stealing one if it can't find any.
2460  *
2461  * It then sets up the reg based on the object's properties: address, pitch
2462  * and tiling format.
2463  */
2464 int
2465 i915_gem_object_get_fence_reg(struct drm_gem_object *obj)
2466 {
2467         struct drm_device *dev = obj->dev;
2468         struct drm_i915_private *dev_priv = dev->dev_private;
2469         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2470         struct drm_i915_fence_reg *reg = NULL;
2471         int ret;
2472
2473         /* Just update our place in the LRU if our fence is getting used. */
2474         if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
2475                 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2476                 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2477                 return 0;
2478         }
2479
2480         switch (obj_priv->tiling_mode) {
2481         case I915_TILING_NONE:
2482                 WARN(1, "allocating a fence for non-tiled object?\n");
2483                 break;
2484         case I915_TILING_X:
2485                 if (!obj_priv->stride)
2486                         return -EINVAL;
2487                 WARN((obj_priv->stride & (512 - 1)),
2488                      "object 0x%08x is X tiled but has non-512B pitch\n",
2489                      obj_priv->gtt_offset);
2490                 break;
2491         case I915_TILING_Y:
2492                 if (!obj_priv->stride)
2493                         return -EINVAL;
2494                 WARN((obj_priv->stride & (128 - 1)),
2495                      "object 0x%08x is Y tiled but has non-128B pitch\n",
2496                      obj_priv->gtt_offset);
2497                 break;
2498         }
2499
2500         ret = i915_find_fence_reg(dev);
2501         if (ret < 0)
2502                 return ret;
2503
2504         obj_priv->fence_reg = ret;
2505         reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2506         list_add_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2507
2508         reg->obj = obj;
2509
2510         if (IS_GEN6(dev))
2511                 sandybridge_write_fence_reg(reg);
2512         else if (IS_I965G(dev))
2513                 i965_write_fence_reg(reg);
2514         else if (IS_I9XX(dev))
2515                 i915_write_fence_reg(reg);
2516         else
2517                 i830_write_fence_reg(reg);
2518
2519         trace_i915_gem_object_get_fence(obj, obj_priv->fence_reg,
2520                         obj_priv->tiling_mode);
2521
2522         return 0;
2523 }
2524
2525 /**
2526  * i915_gem_clear_fence_reg - clear out fence register info
2527  * @obj: object to clear
2528  *
2529  * Zeroes out the fence register itself and clears out the associated
2530  * data structures in dev_priv and obj_priv.
2531  */
2532 static void
2533 i915_gem_clear_fence_reg(struct drm_gem_object *obj)
2534 {
2535         struct drm_device *dev = obj->dev;
2536         drm_i915_private_t *dev_priv = dev->dev_private;
2537         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2538         struct drm_i915_fence_reg *reg =
2539                 &dev_priv->fence_regs[obj_priv->fence_reg];
2540
2541         if (IS_GEN6(dev)) {
2542                 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 +
2543                              (obj_priv->fence_reg * 8), 0);
2544         } else if (IS_I965G(dev)) {
2545                 I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0);
2546         } else {
2547                 uint32_t fence_reg;
2548
2549                 if (obj_priv->fence_reg < 8)
2550                         fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4;
2551                 else
2552                         fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg -
2553                                                        8) * 4;
2554
2555                 I915_WRITE(fence_reg, 0);
2556         }
2557
2558         reg->obj = NULL;
2559         obj_priv->fence_reg = I915_FENCE_REG_NONE;
2560         list_del_init(&reg->lru_list);
2561 }
2562
2563 /**
2564  * i915_gem_object_put_fence_reg - waits on outstanding fenced access
2565  * to the buffer to finish, and then resets the fence register.
2566  * @obj: tiled object holding a fence register.
2567  *
2568  * Zeroes out the fence register itself and clears out the associated
2569  * data structures in dev_priv and obj_priv.
2570  */
2571 int
2572 i915_gem_object_put_fence_reg(struct drm_gem_object *obj)
2573 {
2574         struct drm_device *dev = obj->dev;
2575         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2576
2577         if (obj_priv->fence_reg == I915_FENCE_REG_NONE)
2578                 return 0;
2579
2580         /* If we've changed tiling, GTT-mappings of the object
2581          * need to re-fault to ensure that the correct fence register
2582          * setup is in place.
2583          */
2584         i915_gem_release_mmap(obj);
2585
2586         /* On the i915, GPU access to tiled buffers is via a fence,
2587          * therefore we must wait for any outstanding access to complete
2588          * before clearing the fence.
2589          */
2590         if (!IS_I965G(dev)) {
2591                 int ret;
2592
2593                 i915_gem_object_flush_gpu_write_domain(obj);
2594                 ret = i915_gem_object_wait_rendering(obj);
2595                 if (ret != 0)
2596                         return ret;
2597         }
2598
2599         i915_gem_object_flush_gtt_write_domain(obj);
2600         i915_gem_clear_fence_reg (obj);
2601
2602         return 0;
2603 }
2604
2605 /**
2606  * Finds free space in the GTT aperture and binds the object there.
2607  */
2608 static int
2609 i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
2610 {
2611         struct drm_device *dev = obj->dev;
2612         drm_i915_private_t *dev_priv = dev->dev_private;
2613         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2614         struct drm_mm_node *free_space;
2615         gfp_t gfpmask =  __GFP_NORETRY | __GFP_NOWARN;
2616         int ret;
2617
2618         if (obj_priv->madv != I915_MADV_WILLNEED) {
2619                 DRM_ERROR("Attempting to bind a purgeable object\n");
2620                 return -EINVAL;
2621         }
2622
2623         if (alignment == 0)
2624                 alignment = i915_gem_get_gtt_alignment(obj);
2625         if (alignment & (i915_gem_get_gtt_alignment(obj) - 1)) {
2626                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2627                 return -EINVAL;
2628         }
2629
2630         /* If the object is bigger than the entire aperture, reject it early
2631          * before evicting everything in a vain attempt to find space.
2632          */
2633         if (obj->size > dev->gtt_total) {
2634                 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
2635                 return -E2BIG;
2636         }
2637
2638  search_free:
2639         free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2640                                         obj->size, alignment, 0);
2641         if (free_space != NULL) {
2642                 obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size,
2643                                                        alignment);
2644                 if (obj_priv->gtt_space != NULL) {
2645                         obj_priv->gtt_space->private = obj;
2646                         obj_priv->gtt_offset = obj_priv->gtt_space->start;
2647                 }
2648         }
2649         if (obj_priv->gtt_space == NULL) {
2650                 /* If the gtt is empty and we're still having trouble
2651                  * fitting our object in, we're out of memory.
2652                  */
2653 #if WATCH_LRU
2654                 DRM_INFO("%s: GTT full, evicting something\n", __func__);
2655 #endif
2656                 ret = i915_gem_evict_something(dev, obj->size);
2657                 if (ret)
2658                         return ret;
2659
2660                 goto search_free;
2661         }
2662
2663 #if WATCH_BUF
2664         DRM_INFO("Binding object of size %zd at 0x%08x\n",
2665                  obj->size, obj_priv->gtt_offset);
2666 #endif
2667         ret = i915_gem_object_get_pages(obj, gfpmask);
2668         if (ret) {
2669                 drm_mm_put_block(obj_priv->gtt_space);
2670                 obj_priv->gtt_space = NULL;
2671
2672                 if (ret == -ENOMEM) {
2673                         /* first try to clear up some space from the GTT */
2674                         ret = i915_gem_evict_something(dev, obj->size);
2675                         if (ret) {
2676                                 /* now try to shrink everyone else */
2677                                 if (gfpmask) {
2678                                         gfpmask = 0;
2679                                         goto search_free;
2680                                 }
2681
2682                                 return ret;
2683                         }
2684
2685                         goto search_free;
2686                 }
2687
2688                 return ret;
2689         }
2690
2691         /* Create an AGP memory structure pointing at our pages, and bind it
2692          * into the GTT.
2693          */
2694         obj_priv->agp_mem = drm_agp_bind_pages(dev,
2695                                                obj_priv->pages,
2696                                                obj->size >> PAGE_SHIFT,
2697                                                obj_priv->gtt_offset,
2698                                                obj_priv->agp_type);
2699         if (obj_priv->agp_mem == NULL) {
2700                 i915_gem_object_put_pages(obj);
2701                 drm_mm_put_block(obj_priv->gtt_space);
2702                 obj_priv->gtt_space = NULL;
2703
2704                 ret = i915_gem_evict_something(dev, obj->size);
2705                 if (ret)
2706                         return ret;
2707
2708                 goto search_free;
2709         }
2710         atomic_inc(&dev->gtt_count);
2711         atomic_add(obj->size, &dev->gtt_memory);
2712
2713         /* Assert that the object is not currently in any GPU domain. As it
2714          * wasn't in the GTT, there shouldn't be any way it could have been in
2715          * a GPU cache
2716          */
2717         BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
2718         BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
2719
2720         trace_i915_gem_object_bind(obj, obj_priv->gtt_offset);
2721
2722         return 0;
2723 }
2724
2725 void
2726 i915_gem_clflush_object(struct drm_gem_object *obj)
2727 {
2728         struct drm_i915_gem_object      *obj_priv = to_intel_bo(obj);
2729
2730         /* If we don't have a page list set up, then we're not pinned
2731          * to GPU, and we can ignore the cache flush because it'll happen
2732          * again at bind time.
2733          */
2734         if (obj_priv->pages == NULL)
2735                 return;
2736
2737         trace_i915_gem_object_clflush(obj);
2738
2739         drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE);
2740 }
2741
2742 /** Flushes any GPU write domain for the object if it's dirty. */
2743 static void
2744 i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj)
2745 {
2746         struct drm_device *dev = obj->dev;
2747         uint32_t old_write_domain;
2748         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2749
2750         if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
2751                 return;
2752
2753         /* Queue the GPU write cache flushing we need. */
2754         old_write_domain = obj->write_domain;
2755         i915_gem_flush(dev, 0, obj->write_domain);
2756         (void) i915_add_request(dev, NULL, obj->write_domain, obj_priv->ring);
2757         BUG_ON(obj->write_domain);
2758
2759         trace_i915_gem_object_change_domain(obj,
2760                                             obj->read_domains,
2761                                             old_write_domain);
2762 }
2763
2764 /** Flushes the GTT write domain for the object if it's dirty. */
2765 static void
2766 i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
2767 {
2768         uint32_t old_write_domain;
2769
2770         if (obj->write_domain != I915_GEM_DOMAIN_GTT)
2771                 return;
2772
2773         /* No actual flushing is required for the GTT write domain.   Writes
2774          * to it immediately go to main memory as far as we know, so there's
2775          * no chipset flush.  It also doesn't land in render cache.
2776          */
2777         old_write_domain = obj->write_domain;
2778         obj->write_domain = 0;
2779
2780         trace_i915_gem_object_change_domain(obj,
2781                                             obj->read_domains,
2782                                             old_write_domain);
2783 }
2784
2785 /** Flushes the CPU write domain for the object if it's dirty. */
2786 static void
2787 i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
2788 {
2789         struct drm_device *dev = obj->dev;
2790         uint32_t old_write_domain;
2791
2792         if (obj->write_domain != I915_GEM_DOMAIN_CPU)
2793                 return;
2794
2795         i915_gem_clflush_object(obj);
2796         drm_agp_chipset_flush(dev);
2797         old_write_domain = obj->write_domain;
2798         obj->write_domain = 0;
2799
2800         trace_i915_gem_object_change_domain(obj,
2801                                             obj->read_domains,
2802                                             old_write_domain);
2803 }
2804
2805 void
2806 i915_gem_object_flush_write_domain(struct drm_gem_object *obj)
2807 {
2808         switch (obj->write_domain) {
2809         case I915_GEM_DOMAIN_GTT:
2810                 i915_gem_object_flush_gtt_write_domain(obj);
2811                 break;
2812         case I915_GEM_DOMAIN_CPU:
2813                 i915_gem_object_flush_cpu_write_domain(obj);
2814                 break;
2815         default:
2816                 i915_gem_object_flush_gpu_write_domain(obj);
2817                 break;
2818         }
2819 }
2820
2821 /**
2822  * Moves a single object to the GTT read, and possibly write domain.
2823  *
2824  * This function returns when the move is complete, including waiting on
2825  * flushes to occur.
2826  */
2827 int
2828 i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
2829 {
2830         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2831         uint32_t old_write_domain, old_read_domains;
2832         int ret;
2833
2834         /* Not valid to be called on unbound objects. */
2835         if (obj_priv->gtt_space == NULL)
2836                 return -EINVAL;
2837
2838         i915_gem_object_flush_gpu_write_domain(obj);
2839         /* Wait on any GPU rendering and flushing to occur. */
2840         ret = i915_gem_object_wait_rendering(obj);
2841         if (ret != 0)
2842                 return ret;
2843
2844         old_write_domain = obj->write_domain;
2845         old_read_domains = obj->read_domains;
2846
2847         /* If we're writing through the GTT domain, then CPU and GPU caches
2848          * will need to be invalidated at next use.
2849          */
2850         if (write)
2851                 obj->read_domains &= I915_GEM_DOMAIN_GTT;
2852
2853         i915_gem_object_flush_cpu_write_domain(obj);
2854
2855         /* It should now be out of any other write domains, and we can update
2856          * the domain values for our changes.
2857          */
2858         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2859         obj->read_domains |= I915_GEM_DOMAIN_GTT;
2860         if (write) {
2861                 obj->write_domain = I915_GEM_DOMAIN_GTT;
2862                 obj_priv->dirty = 1;
2863         }
2864
2865         trace_i915_gem_object_change_domain(obj,
2866                                             old_read_domains,
2867                                             old_write_domain);
2868
2869         return 0;
2870 }
2871
2872 /*
2873  * Prepare buffer for display plane. Use uninterruptible for possible flush
2874  * wait, as in modesetting process we're not supposed to be interrupted.
2875  */
2876 int
2877 i915_gem_object_set_to_display_plane(struct drm_gem_object *obj)
2878 {
2879         struct drm_device *dev = obj->dev;
2880         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2881         uint32_t old_write_domain, old_read_domains;
2882         int ret;
2883
2884         /* Not valid to be called on unbound objects. */
2885         if (obj_priv->gtt_space == NULL)
2886                 return -EINVAL;
2887
2888         i915_gem_object_flush_gpu_write_domain(obj);
2889
2890         /* Wait on any GPU rendering and flushing to occur. */
2891         if (obj_priv->active) {
2892 #if WATCH_BUF
2893                 DRM_INFO("%s: object %p wait for seqno %08x\n",
2894                           __func__, obj, obj_priv->last_rendering_seqno);
2895 #endif
2896                 ret = i915_do_wait_request(dev,
2897                                 obj_priv->last_rendering_seqno,
2898                                 0,
2899                                 obj_priv->ring);
2900                 if (ret != 0)
2901                         return ret;
2902         }
2903
2904         i915_gem_object_flush_cpu_write_domain(obj);
2905
2906         old_write_domain = obj->write_domain;
2907         old_read_domains = obj->read_domains;
2908
2909         /* It should now be out of any other write domains, and we can update
2910          * the domain values for our changes.
2911          */
2912         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2913         obj->read_domains = I915_GEM_DOMAIN_GTT;
2914         obj->write_domain = I915_GEM_DOMAIN_GTT;
2915         obj_priv->dirty = 1;
2916
2917         trace_i915_gem_object_change_domain(obj,
2918                                             old_read_domains,
2919                                             old_write_domain);
2920
2921         return 0;
2922 }
2923
2924 /**
2925  * Moves a single object to the CPU read, and possibly write domain.
2926  *
2927  * This function returns when the move is complete, including waiting on
2928  * flushes to occur.
2929  */
2930 static int
2931 i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
2932 {
2933         uint32_t old_write_domain, old_read_domains;
2934         int ret;
2935
2936         i915_gem_object_flush_gpu_write_domain(obj);
2937         /* Wait on any GPU rendering and flushing to occur. */
2938         ret = i915_gem_object_wait_rendering(obj);
2939         if (ret != 0)
2940                 return ret;
2941
2942         i915_gem_object_flush_gtt_write_domain(obj);
2943
2944         /* If we have a partially-valid cache of the object in the CPU,
2945          * finish invalidating it and free the per-page flags.
2946          */
2947         i915_gem_object_set_to_full_cpu_read_domain(obj);
2948
2949         old_write_domain = obj->write_domain;
2950         old_read_domains = obj->read_domains;
2951
2952         /* Flush the CPU cache if it's still invalid. */
2953         if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2954                 i915_gem_clflush_object(obj);
2955
2956                 obj->read_domains |= I915_GEM_DOMAIN_CPU;
2957         }
2958
2959         /* It should now be out of any other write domains, and we can update
2960          * the domain values for our changes.
2961          */
2962         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2963
2964         /* If we're writing through the CPU, then the GPU read domains will
2965          * need to be invalidated at next use.
2966          */
2967         if (write) {
2968                 obj->read_domains &= I915_GEM_DOMAIN_CPU;
2969                 obj->write_domain = I915_GEM_DOMAIN_CPU;
2970         }
2971
2972         trace_i915_gem_object_change_domain(obj,
2973                                             old_read_domains,
2974                                             old_write_domain);
2975
2976         return 0;
2977 }
2978
2979 /*
2980  * Set the next domain for the specified object. This
2981  * may not actually perform the necessary flushing/invaliding though,
2982  * as that may want to be batched with other set_domain operations
2983  *
2984  * This is (we hope) the only really tricky part of gem. The goal
2985  * is fairly simple -- track which caches hold bits of the object
2986  * and make sure they remain coherent. A few concrete examples may
2987  * help to explain how it works. For shorthand, we use the notation
2988  * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
2989  * a pair of read and write domain masks.
2990  *
2991  * Case 1: the batch buffer
2992  *
2993  *      1. Allocated
2994  *      2. Written by CPU
2995  *      3. Mapped to GTT
2996  *      4. Read by GPU
2997  *      5. Unmapped from GTT
2998  *      6. Freed
2999  *
3000  *      Let's take these a step at a time
3001  *
3002  *      1. Allocated
3003  *              Pages allocated from the kernel may still have
3004  *              cache contents, so we set them to (CPU, CPU) always.
3005  *      2. Written by CPU (using pwrite)
3006  *              The pwrite function calls set_domain (CPU, CPU) and
3007  *              this function does nothing (as nothing changes)
3008  *      3. Mapped by GTT
3009  *              This function asserts that the object is not
3010  *              currently in any GPU-based read or write domains
3011  *      4. Read by GPU
3012  *              i915_gem_execbuffer calls set_domain (COMMAND, 0).
3013  *              As write_domain is zero, this function adds in the
3014  *              current read domains (CPU+COMMAND, 0).
3015  *              flush_domains is set to CPU.
3016  *              invalidate_domains is set to COMMAND
3017  *              clflush is run to get data out of the CPU caches
3018  *              then i915_dev_set_domain calls i915_gem_flush to
3019  *              emit an MI_FLUSH and drm_agp_chipset_flush
3020  *      5. Unmapped from GTT
3021  *              i915_gem_object_unbind calls set_domain (CPU, CPU)
3022  *              flush_domains and invalidate_domains end up both zero
3023  *              so no flushing/invalidating happens
3024  *      6. Freed
3025  *              yay, done
3026  *
3027  * Case 2: The shared render buffer
3028  *
3029  *      1. Allocated
3030  *      2. Mapped to GTT
3031  *      3. Read/written by GPU
3032  *      4. set_domain to (CPU,CPU)
3033  *      5. Read/written by CPU
3034  *      6. Read/written by GPU
3035  *
3036  *      1. Allocated
3037  *              Same as last example, (CPU, CPU)
3038  *      2. Mapped to GTT
3039  *              Nothing changes (assertions find that it is not in the GPU)
3040  *      3. Read/written by GPU
3041  *              execbuffer calls set_domain (RENDER, RENDER)
3042  *              flush_domains gets CPU
3043  *              invalidate_domains gets GPU
3044  *              clflush (obj)
3045  *              MI_FLUSH and drm_agp_chipset_flush
3046  *      4. set_domain (CPU, CPU)
3047  *              flush_domains gets GPU
3048  *              invalidate_domains gets CPU
3049  *              wait_rendering (obj) to make sure all drawing is complete.
3050  *              This will include an MI_FLUSH to get the data from GPU
3051  *              to memory
3052  *              clflush (obj) to invalidate the CPU cache
3053  *              Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
3054  *      5. Read/written by CPU
3055  *              cache lines are loaded and dirtied
3056  *      6. Read written by GPU
3057  *              Same as last GPU access
3058  *
3059  * Case 3: The constant buffer
3060  *
3061  *      1. Allocated
3062  *      2. Written by CPU
3063  *      3. Read by GPU
3064  *      4. Updated (written) by CPU again
3065  *      5. Read by GPU
3066  *
3067  *      1. Allocated
3068  *              (CPU, CPU)
3069  *      2. Written by CPU
3070  *              (CPU, CPU)
3071  *      3. Read by GPU
3072  *              (CPU+RENDER, 0)
3073  *              flush_domains = CPU
3074  *              invalidate_domains = RENDER
3075  *              clflush (obj)
3076  *              MI_FLUSH
3077  *              drm_agp_chipset_flush
3078  *      4. Updated (written) by CPU again
3079  *              (CPU, CPU)
3080  *              flush_domains = 0 (no previous write domain)
3081  *              invalidate_domains = 0 (no new read domains)
3082  *      5. Read by GPU
3083  *              (CPU+RENDER, 0)
3084  *              flush_domains = CPU
3085  *              invalidate_domains = RENDER
3086  *              clflush (obj)
3087  *              MI_FLUSH
3088  *              drm_agp_chipset_flush
3089  */
3090 static void
3091 i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
3092 {
3093         struct drm_device               *dev = obj->dev;
3094         struct drm_i915_gem_object      *obj_priv = to_intel_bo(obj);
3095         uint32_t                        invalidate_domains = 0;
3096         uint32_t                        flush_domains = 0;
3097         uint32_t                        old_read_domains;
3098
3099         BUG_ON(obj->pending_read_domains & I915_GEM_DOMAIN_CPU);
3100         BUG_ON(obj->pending_write_domain == I915_GEM_DOMAIN_CPU);
3101
3102         intel_mark_busy(dev, obj);
3103
3104 #if WATCH_BUF
3105         DRM_INFO("%s: object %p read %08x -> %08x write %08x -> %08x\n",
3106                  __func__, obj,
3107                  obj->read_domains, obj->pending_read_domains,
3108                  obj->write_domain, obj->pending_write_domain);
3109 #endif
3110         /*
3111          * If the object isn't moving to a new write domain,
3112          * let the object stay in multiple read domains
3113          */
3114         if (obj->pending_write_domain == 0)
3115                 obj->pending_read_domains |= obj->read_domains;
3116         else
3117                 obj_priv->dirty = 1;
3118
3119         /*
3120          * Flush the current write domain if
3121          * the new read domains don't match. Invalidate
3122          * any read domains which differ from the old
3123          * write domain
3124          */
3125         if (obj->write_domain &&
3126             obj->write_domain != obj->pending_read_domains) {
3127                 flush_domains |= obj->write_domain;
3128                 invalidate_domains |=
3129                         obj->pending_read_domains & ~obj->write_domain;
3130         }
3131         /*
3132          * Invalidate any read caches which may have
3133          * stale data. That is, any new read domains.
3134          */
3135         invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
3136         if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) {
3137 #if WATCH_BUF
3138                 DRM_INFO("%s: CPU domain flush %08x invalidate %08x\n",
3139                          __func__, flush_domains, invalidate_domains);
3140 #endif
3141                 i915_gem_clflush_object(obj);
3142         }
3143
3144         old_read_domains = obj->read_domains;
3145
3146         /* The actual obj->write_domain will be updated with
3147          * pending_write_domain after we emit the accumulated flush for all
3148          * of our domain changes in execbuffers (which clears objects'
3149          * write_domains).  So if we have a current write domain that we
3150          * aren't changing, set pending_write_domain to that.
3151          */
3152         if (flush_domains == 0 && obj->pending_write_domain == 0)
3153                 obj->pending_write_domain = obj->write_domain;
3154         obj->read_domains = obj->pending_read_domains;
3155
3156         dev->invalidate_domains |= invalidate_domains;
3157         dev->flush_domains |= flush_domains;
3158 #if WATCH_BUF
3159         DRM_INFO("%s: read %08x write %08x invalidate %08x flush %08x\n",
3160                  __func__,
3161                  obj->read_domains, obj->write_domain,
3162                  dev->invalidate_domains, dev->flush_domains);
3163 #endif
3164
3165         trace_i915_gem_object_change_domain(obj,
3166                                             old_read_domains,
3167                                             obj->write_domain);
3168 }
3169
3170 /**
3171  * Moves the object from a partially CPU read to a full one.
3172  *
3173  * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
3174  * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
3175  */
3176 static void
3177 i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
3178 {
3179         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3180
3181         if (!obj_priv->page_cpu_valid)
3182                 return;
3183
3184         /* If we're partially in the CPU read domain, finish moving it in.
3185          */
3186         if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
3187                 int i;
3188
3189                 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
3190                         if (obj_priv->page_cpu_valid[i])
3191                                 continue;
3192                         drm_clflush_pages(obj_priv->pages + i, 1);
3193                 }
3194         }
3195
3196         /* Free the page_cpu_valid mappings which are now stale, whether
3197          * or not we've got I915_GEM_DOMAIN_CPU.
3198          */
3199         kfree(obj_priv->page_cpu_valid);
3200         obj_priv->page_cpu_valid = NULL;
3201 }
3202
3203 /**
3204  * Set the CPU read domain on a range of the object.
3205  *
3206  * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
3207  * not entirely valid.  The page_cpu_valid member of the object flags which
3208  * pages have been flushed, and will be respected by
3209  * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
3210  * of the whole object.
3211  *
3212  * This function returns when the move is complete, including waiting on
3213  * flushes to occur.
3214  */
3215 static int
3216 i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
3217                                           uint64_t offset, uint64_t size)
3218 {
3219         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3220         uint32_t old_read_domains;
3221         int i, ret;
3222
3223         if (offset == 0 && size == obj->size)
3224                 return i915_gem_object_set_to_cpu_domain(obj, 0);
3225
3226         i915_gem_object_flush_gpu_write_domain(obj);
3227         /* Wait on any GPU rendering and flushing to occur. */
3228         ret = i915_gem_object_wait_rendering(obj);
3229         if (ret != 0)
3230                 return ret;
3231         i915_gem_object_flush_gtt_write_domain(obj);
3232
3233         /* If we're already fully in the CPU read domain, we're done. */
3234         if (obj_priv->page_cpu_valid == NULL &&
3235             (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
3236                 return 0;
3237
3238         /* Otherwise, create/clear the per-page CPU read domain flag if we're
3239          * newly adding I915_GEM_DOMAIN_CPU
3240          */
3241         if (obj_priv->page_cpu_valid == NULL) {
3242                 obj_priv->page_cpu_valid = kzalloc(obj->size / PAGE_SIZE,
3243                                                    GFP_KERNEL);
3244                 if (obj_priv->page_cpu_valid == NULL)
3245                         return -ENOMEM;
3246         } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
3247                 memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
3248
3249         /* Flush the cache on any pages that are still invalid from the CPU's
3250          * perspective.
3251          */
3252         for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3253              i++) {
3254                 if (obj_priv->page_cpu_valid[i])
3255                         continue;
3256
3257                 drm_clflush_pages(obj_priv->pages + i, 1);
3258
3259                 obj_priv->page_cpu_valid[i] = 1;
3260         }
3261
3262         /* It should now be out of any other write domains, and we can update
3263          * the domain values for our changes.
3264          */
3265         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3266
3267         old_read_domains = obj->read_domains;
3268         obj->read_domains |= I915_GEM_DOMAIN_CPU;
3269
3270         trace_i915_gem_object_change_domain(obj,
3271                                             old_read_domains,
3272                                             obj->write_domain);
3273
3274         return 0;
3275 }
3276
3277 /**
3278  * Pin an object to the GTT and evaluate the relocations landing in it.
3279  */
3280 static int
3281 i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
3282                                  struct drm_file *file_priv,
3283                                  struct drm_i915_gem_exec_object2 *entry,
3284                                  struct drm_i915_gem_relocation_entry *relocs)
3285 {
3286         struct drm_device *dev = obj->dev;
3287         drm_i915_private_t *dev_priv = dev->dev_private;
3288         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3289         int i, ret;
3290         void __iomem *reloc_page;
3291         bool need_fence;
3292
3293         need_fence = entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
3294                      obj_priv->tiling_mode != I915_TILING_NONE;
3295
3296         /* Check fence reg constraints and rebind if necessary */
3297         if (need_fence &&
3298             !i915_gem_object_fence_offset_ok(obj,
3299                                              obj_priv->tiling_mode)) {
3300                 ret = i915_gem_object_unbind(obj);
3301                 if (ret)
3302                         return ret;
3303         }
3304
3305         /* Choose the GTT offset for our buffer and put it there. */
3306         ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment);
3307         if (ret)
3308                 return ret;
3309
3310         /*
3311          * Pre-965 chips need a fence register set up in order to
3312          * properly handle blits to/from tiled surfaces.
3313          */
3314         if (need_fence) {
3315                 ret = i915_gem_object_get_fence_reg(obj);
3316                 if (ret != 0) {
3317                         i915_gem_object_unpin(obj);
3318                         return ret;
3319                 }
3320         }
3321
3322         entry->offset = obj_priv->gtt_offset;
3323
3324         /* Apply the relocations, using the GTT aperture to avoid cache
3325          * flushing requirements.
3326          */
3327         for (i = 0; i < entry->relocation_count; i++) {
3328                 struct drm_i915_gem_relocation_entry *reloc= &relocs[i];
3329                 struct drm_gem_object *target_obj;
3330                 struct drm_i915_gem_object *target_obj_priv;
3331                 uint32_t reloc_val, reloc_offset;
3332                 uint32_t __iomem *reloc_entry;
3333
3334                 target_obj = drm_gem_object_lookup(obj->dev, file_priv,
3335                                                    reloc->target_handle);
3336                 if (target_obj == NULL) {
3337                         i915_gem_object_unpin(obj);
3338                         return -EBADF;
3339                 }
3340                 target_obj_priv = to_intel_bo(target_obj);
3341
3342 #if WATCH_RELOC
3343                 DRM_INFO("%s: obj %p offset %08x target %d "
3344                          "read %08x write %08x gtt %08x "
3345                          "presumed %08x delta %08x\n",
3346                          __func__,
3347                          obj,
3348                          (int) reloc->offset,
3349                          (int) reloc->target_handle,
3350                          (int) reloc->read_domains,
3351                          (int) reloc->write_domain,
3352                          (int) target_obj_priv->gtt_offset,
3353                          (int) reloc->presumed_offset,
3354                          reloc->delta);
3355 #endif
3356
3357                 /* The target buffer should have appeared before us in the
3358                  * exec_object list, so it should have a GTT space bound by now.
3359                  */
3360                 if (target_obj_priv->gtt_space == NULL) {
3361                         DRM_ERROR("No GTT space found for object %d\n",
3362                                   reloc->target_handle);
3363                         drm_gem_object_unreference(target_obj);
3364                         i915_gem_object_unpin(obj);
3365                         return -EINVAL;
3366                 }
3367
3368                 /* Validate that the target is in a valid r/w GPU domain */
3369                 if (reloc->write_domain & (reloc->write_domain - 1)) {
3370                         DRM_ERROR("reloc with multiple write domains: "
3371                                   "obj %p target %d offset %d "
3372                                   "read %08x write %08x",
3373                                   obj, reloc->target_handle,
3374                                   (int) reloc->offset,
3375                                   reloc->read_domains,
3376                                   reloc->write_domain);
3377                         return -EINVAL;
3378                 }
3379                 if (reloc->write_domain & I915_GEM_DOMAIN_CPU ||
3380                     reloc->read_domains & I915_GEM_DOMAIN_CPU) {
3381                         DRM_ERROR("reloc with read/write CPU domains: "
3382                                   "obj %p target %d offset %d "
3383                                   "read %08x write %08x",
3384                                   obj, reloc->target_handle,
3385                                   (int) reloc->offset,
3386                                   reloc->read_domains,
3387                                   reloc->write_domain);
3388                         drm_gem_object_unreference(target_obj);
3389                         i915_gem_object_unpin(obj);
3390                         return -EINVAL;
3391                 }
3392                 if (reloc->write_domain && target_obj->pending_write_domain &&
3393                     reloc->write_domain != target_obj->pending_write_domain) {
3394                         DRM_ERROR("Write domain conflict: "
3395                                   "obj %p target %d offset %d "
3396                                   "new %08x old %08x\n",
3397                                   obj, reloc->target_handle,
3398                                   (int) reloc->offset,
3399                                   reloc->write_domain,
3400                                   target_obj->pending_write_domain);
3401                         drm_gem_object_unreference(target_obj);
3402                         i915_gem_object_unpin(obj);
3403                         return -EINVAL;
3404                 }
3405
3406                 target_obj->pending_read_domains |= reloc->read_domains;
3407                 target_obj->pending_write_domain |= reloc->write_domain;
3408
3409                 /* If the relocation already has the right value in it, no
3410                  * more work needs to be done.
3411                  */
3412                 if (target_obj_priv->gtt_offset == reloc->presumed_offset) {
3413                         drm_gem_object_unreference(target_obj);
3414                         continue;
3415                 }
3416
3417                 /* Check that the relocation address is valid... */
3418                 if (reloc->offset > obj->size - 4) {
3419                         DRM_ERROR("Relocation beyond object bounds: "
3420                                   "obj %p target %d offset %d size %d.\n",
3421                                   obj, reloc->target_handle,
3422                                   (int) reloc->offset, (int) obj->size);
3423                         drm_gem_object_unreference(target_obj);
3424                         i915_gem_object_unpin(obj);
3425                         return -EINVAL;
3426                 }
3427                 if (reloc->offset & 3) {
3428                         DRM_ERROR("Relocation not 4-byte aligned: "
3429                                   "obj %p target %d offset %d.\n",
3430                                   obj, reloc->target_handle,
3431                                   (int) reloc->offset);
3432                         drm_gem_object_unreference(target_obj);
3433                         i915_gem_object_unpin(obj);
3434                         return -EINVAL;
3435                 }
3436
3437                 /* and points to somewhere within the target object. */
3438                 if (reloc->delta >= target_obj->size) {
3439                         DRM_ERROR("Relocation beyond target object bounds: "
3440                                   "obj %p target %d delta %d size %d.\n",
3441                                   obj, reloc->target_handle,
3442                                   (int) reloc->delta, (int) target_obj->size);
3443                         drm_gem_object_unreference(target_obj);
3444                         i915_gem_object_unpin(obj);
3445                         return -EINVAL;
3446                 }
3447
3448                 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
3449                 if (ret != 0) {
3450                         drm_gem_object_unreference(target_obj);
3451                         i915_gem_object_unpin(obj);
3452                         return -EINVAL;
3453                 }
3454
3455                 /* Map the page containing the relocation we're going to
3456                  * perform.
3457                  */
3458                 reloc_offset = obj_priv->gtt_offset + reloc->offset;
3459                 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
3460                                                       (reloc_offset &
3461                                                        ~(PAGE_SIZE - 1)));
3462                 reloc_entry = (uint32_t __iomem *)(reloc_page +
3463                                                    (reloc_offset & (PAGE_SIZE - 1)));
3464                 reloc_val = target_obj_priv->gtt_offset + reloc->delta;
3465
3466 #if WATCH_BUF
3467                 DRM_INFO("Applied relocation: %p@0x%08x %08x -> %08x\n",
3468                           obj, (unsigned int) reloc->offset,
3469                           readl(reloc_entry), reloc_val);
3470 #endif
3471                 writel(reloc_val, reloc_entry);
3472                 io_mapping_unmap_atomic(reloc_page);
3473
3474                 /* The updated presumed offset for this entry will be
3475                  * copied back out to the user.
3476                  */
3477                 reloc->presumed_offset = target_obj_priv->gtt_offset;
3478
3479                 drm_gem_object_unreference(target_obj);
3480         }
3481
3482 #if WATCH_BUF
3483         if (0)
3484                 i915_gem_dump_object(obj, 128, __func__, ~0);
3485 #endif
3486         return 0;
3487 }
3488
3489 /* Throttle our rendering by waiting until the ring has completed our requests
3490  * emitted over 20 msec ago.
3491  *
3492  * Note that if we were to use the current jiffies each time around the loop,
3493  * we wouldn't escape the function with any frames outstanding if the time to
3494  * render a frame was over 20ms.
3495  *
3496  * This should get us reasonable parallelism between CPU and GPU but also
3497  * relatively low latency when blocking on a particular request to finish.
3498  */
3499 static int
3500 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file_priv)
3501 {
3502         struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
3503         int ret = 0;
3504         unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
3505
3506         mutex_lock(&dev->struct_mutex);
3507         while (!list_empty(&i915_file_priv->mm.request_list)) {
3508                 struct drm_i915_gem_request *request;
3509
3510                 request = list_first_entry(&i915_file_priv->mm.request_list,
3511                                            struct drm_i915_gem_request,
3512                                            client_list);
3513
3514                 if (time_after_eq(request->emitted_jiffies, recent_enough))
3515                         break;
3516
3517                 ret = i915_wait_request(dev, request->seqno, request->ring);
3518                 if (ret != 0)
3519                         break;
3520         }
3521         mutex_unlock(&dev->struct_mutex);
3522
3523         return ret;
3524 }
3525
3526 static int
3527 i915_gem_get_relocs_from_user(struct drm_i915_gem_exec_object2 *exec_list,
3528                               uint32_t buffer_count,
3529                               struct drm_i915_gem_relocation_entry **relocs)
3530 {
3531         uint32_t reloc_count = 0, reloc_index = 0, i;
3532         int ret;
3533
3534         *relocs = NULL;
3535         for (i = 0; i < buffer_count; i++) {
3536                 if (reloc_count + exec_list[i].relocation_count < reloc_count)
3537                         return -EINVAL;
3538                 reloc_count += exec_list[i].relocation_count;
3539         }
3540
3541         *relocs = drm_calloc_large(reloc_count, sizeof(**relocs));
3542         if (*relocs == NULL) {
3543                 DRM_ERROR("failed to alloc relocs, count %d\n", reloc_count);
3544                 return -ENOMEM;
3545         }
3546
3547         for (i = 0; i < buffer_count; i++) {
3548                 struct drm_i915_gem_relocation_entry __user *user_relocs;
3549
3550                 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3551
3552                 ret = copy_from_user(&(*relocs)[reloc_index],
3553                                      user_relocs,
3554                                      exec_list[i].relocation_count *
3555                                      sizeof(**relocs));
3556                 if (ret != 0) {
3557                         drm_free_large(*relocs);
3558                         *relocs = NULL;
3559                         return -EFAULT;
3560                 }
3561
3562                 reloc_index += exec_list[i].relocation_count;
3563         }
3564
3565         return 0;
3566 }
3567
3568 static int
3569 i915_gem_put_relocs_to_user(struct drm_i915_gem_exec_object2 *exec_list,
3570                             uint32_t buffer_count,
3571                             struct drm_i915_gem_relocation_entry *relocs)
3572 {
3573         uint32_t reloc_count = 0, i;
3574         int ret = 0;
3575
3576         if (relocs == NULL)
3577             return 0;
3578
3579         for (i = 0; i < buffer_count; i++) {
3580                 struct drm_i915_gem_relocation_entry __user *user_relocs;
3581                 int unwritten;
3582
3583                 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3584
3585                 unwritten = copy_to_user(user_relocs,
3586                                          &relocs[reloc_count],
3587                                          exec_list[i].relocation_count *
3588                                          sizeof(*relocs));
3589
3590                 if (unwritten) {
3591                         ret = -EFAULT;
3592                         goto err;
3593                 }
3594
3595                 reloc_count += exec_list[i].relocation_count;
3596         }
3597
3598 err:
3599         drm_free_large(relocs);
3600
3601         return ret;
3602 }
3603
3604 static int
3605 i915_gem_check_execbuffer (struct drm_i915_gem_execbuffer2 *exec,
3606                            uint64_t exec_offset)
3607 {
3608         uint32_t exec_start, exec_len;
3609
3610         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3611         exec_len = (uint32_t) exec->batch_len;
3612
3613         if ((exec_start | exec_len) & 0x7)
3614                 return -EINVAL;
3615
3616         if (!exec_start)
3617                 return -EINVAL;
3618
3619         return 0;
3620 }
3621
3622 static int
3623 i915_gem_wait_for_pending_flip(struct drm_device *dev,
3624                                struct drm_gem_object **object_list,
3625                                int count)
3626 {
3627         drm_i915_private_t *dev_priv = dev->dev_private;
3628         struct drm_i915_gem_object *obj_priv;
3629         DEFINE_WAIT(wait);
3630         int i, ret = 0;
3631
3632         for (;;) {
3633                 prepare_to_wait(&dev_priv->pending_flip_queue,
3634                                 &wait, TASK_INTERRUPTIBLE);
3635                 for (i = 0; i < count; i++) {
3636                         obj_priv = to_intel_bo(object_list[i]);
3637                         if (atomic_read(&obj_priv->pending_flip) > 0)
3638                                 break;
3639                 }
3640                 if (i == count)
3641                         break;
3642
3643                 if (!signal_pending(current)) {
3644                         mutex_unlock(&dev->struct_mutex);
3645                         schedule();
3646                         mutex_lock(&dev->struct_mutex);
3647                         continue;
3648                 }
3649                 ret = -ERESTARTSYS;
3650                 break;
3651         }
3652         finish_wait(&dev_priv->pending_flip_queue, &wait);
3653
3654         return ret;
3655 }
3656
3657 int
3658 i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3659                        struct drm_file *file_priv,
3660                        struct drm_i915_gem_execbuffer2 *args,
3661                        struct drm_i915_gem_exec_object2 *exec_list)
3662 {
3663         drm_i915_private_t *dev_priv = dev->dev_private;
3664         struct drm_gem_object **object_list = NULL;
3665         struct drm_gem_object *batch_obj;
3666         struct drm_i915_gem_object *obj_priv;
3667         struct drm_clip_rect *cliprects = NULL;
3668         struct drm_i915_gem_relocation_entry *relocs = NULL;
3669         int ret = 0, ret2, i, pinned = 0;
3670         uint64_t exec_offset;
3671         uint32_t seqno, flush_domains, reloc_index;
3672         int pin_tries, flips;
3673
3674         struct intel_ring_buffer *ring = NULL;
3675
3676 #if WATCH_EXEC
3677         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3678                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3679 #endif
3680         if (args->flags & I915_EXEC_BSD) {
3681                 if (!HAS_BSD(dev)) {
3682                         DRM_ERROR("execbuf with wrong flag\n");
3683                         return -EINVAL;
3684                 }
3685                 ring = &dev_priv->bsd_ring;
3686         } else {
3687                 ring = &dev_priv->render_ring;
3688         }
3689
3690
3691         if (args->buffer_count < 1) {
3692                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3693                 return -EINVAL;
3694         }
3695         object_list = drm_malloc_ab(sizeof(*object_list), args->buffer_count);
3696         if (object_list == NULL) {
3697                 DRM_ERROR("Failed to allocate object list for %d buffers\n",
3698                           args->buffer_count);
3699                 ret = -ENOMEM;
3700                 goto pre_mutex_err;
3701         }
3702
3703         if (args->num_cliprects != 0) {
3704                 cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects),
3705                                     GFP_KERNEL);
3706                 if (cliprects == NULL) {
3707                         ret = -ENOMEM;
3708                         goto pre_mutex_err;
3709                 }
3710
3711                 ret = copy_from_user(cliprects,
3712                                      (struct drm_clip_rect __user *)
3713                                      (uintptr_t) args->cliprects_ptr,
3714                                      sizeof(*cliprects) * args->num_cliprects);
3715                 if (ret != 0) {
3716                         DRM_ERROR("copy %d cliprects failed: %d\n",
3717                                   args->num_cliprects, ret);
3718                         goto pre_mutex_err;
3719                 }
3720         }
3721
3722         ret = i915_gem_get_relocs_from_user(exec_list, args->buffer_count,
3723                                             &relocs);
3724         if (ret != 0)
3725                 goto pre_mutex_err;
3726
3727         mutex_lock(&dev->struct_mutex);
3728
3729         i915_verify_inactive(dev, __FILE__, __LINE__);
3730
3731         if (atomic_read(&dev_priv->mm.wedged)) {
3732                 mutex_unlock(&dev->struct_mutex);
3733                 ret = -EIO;
3734                 goto pre_mutex_err;
3735         }
3736
3737         if (dev_priv->mm.suspended) {
3738                 mutex_unlock(&dev->struct_mutex);
3739                 ret = -EBUSY;
3740                 goto pre_mutex_err;
3741         }
3742
3743         /* Look up object handles */
3744         flips = 0;
3745         for (i = 0; i < args->buffer_count; i++) {
3746                 object_list[i] = drm_gem_object_lookup(dev, file_priv,
3747                                                        exec_list[i].handle);
3748                 if (object_list[i] == NULL) {
3749                         DRM_ERROR("Invalid object handle %d at index %d\n",
3750                                    exec_list[i].handle, i);
3751                         /* prevent error path from reading uninitialized data */
3752                         args->buffer_count = i + 1;
3753                         ret = -EBADF;
3754                         goto err;
3755                 }
3756
3757                 obj_priv = to_intel_bo(object_list[i]);
3758                 if (obj_priv->in_execbuffer) {
3759                         DRM_ERROR("Object %p appears more than once in object list\n",
3760                                    object_list[i]);
3761                         /* prevent error path from reading uninitialized data */
3762                         args->buffer_count = i + 1;
3763                         ret = -EBADF;
3764                         goto err;
3765                 }
3766                 obj_priv->in_execbuffer = true;
3767                 flips += atomic_read(&obj_priv->pending_flip);
3768         }
3769
3770         if (flips > 0) {
3771                 ret = i915_gem_wait_for_pending_flip(dev, object_list,
3772                                                      args->buffer_count);
3773                 if (ret)
3774                         goto err;
3775         }
3776
3777         /* Pin and relocate */
3778         for (pin_tries = 0; ; pin_tries++) {
3779                 ret = 0;
3780                 reloc_index = 0;
3781
3782                 for (i = 0; i < args->buffer_count; i++) {
3783                         object_list[i]->pending_read_domains = 0;
3784                         object_list[i]->pending_write_domain = 0;
3785                         ret = i915_gem_object_pin_and_relocate(object_list[i],
3786                                                                file_priv,
3787                                                                &exec_list[i],
3788                                                                &relocs[reloc_index]);
3789                         if (ret)
3790                                 break;
3791                         pinned = i + 1;
3792                         reloc_index += exec_list[i].relocation_count;
3793                 }
3794                 /* success */
3795                 if (ret == 0)
3796                         break;
3797
3798                 /* error other than GTT full, or we've already tried again */
3799                 if (ret != -ENOSPC || pin_tries >= 1) {
3800                         if (ret != -ERESTARTSYS) {
3801                                 unsigned long long total_size = 0;
3802                                 int num_fences = 0;
3803                                 for (i = 0; i < args->buffer_count; i++) {
3804                                         obj_priv = object_list[i]->driver_private;
3805
3806                                         total_size += object_list[i]->size;
3807                                         num_fences +=
3808                                                 exec_list[i].flags & EXEC_OBJECT_NEEDS_FENCE &&
3809                                                 obj_priv->tiling_mode != I915_TILING_NONE;
3810                                 }
3811                                 DRM_ERROR("Failed to pin buffer %d of %d, total %llu bytes, %d fences: %d\n",
3812                                           pinned+1, args->buffer_count,
3813                                           total_size, num_fences,
3814                                           ret);
3815                                 DRM_ERROR("%d objects [%d pinned], "
3816                                           "%d object bytes [%d pinned], "
3817                                           "%d/%d gtt bytes\n",
3818                                           atomic_read(&dev->object_count),
3819                                           atomic_read(&dev->pin_count),
3820                                           atomic_read(&dev->object_memory),
3821                                           atomic_read(&dev->pin_memory),
3822                                           atomic_read(&dev->gtt_memory),
3823                                           dev->gtt_total);
3824                         }
3825                         goto err;
3826                 }
3827
3828                 /* unpin all of our buffers */
3829                 for (i = 0; i < pinned; i++)
3830                         i915_gem_object_unpin(object_list[i]);
3831                 pinned = 0;
3832
3833                 /* evict everyone we can from the aperture */
3834                 ret = i915_gem_evict_everything(dev);
3835                 if (ret && ret != -ENOSPC)
3836                         goto err;
3837         }
3838
3839         /* Set the pending read domains for the batch buffer to COMMAND */
3840         batch_obj = object_list[args->buffer_count-1];
3841         if (batch_obj->pending_write_domain) {
3842                 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
3843                 ret = -EINVAL;
3844                 goto err;
3845         }
3846         batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
3847
3848         /* Sanity check the batch buffer, prior to moving objects */
3849         exec_offset = exec_list[args->buffer_count - 1].offset;
3850         ret = i915_gem_check_execbuffer (args, exec_offset);
3851         if (ret != 0) {
3852                 DRM_ERROR("execbuf with invalid offset/length\n");
3853                 goto err;
3854         }
3855
3856         i915_verify_inactive(dev, __FILE__, __LINE__);
3857
3858         /* Zero the global flush/invalidate flags. These
3859          * will be modified as new domains are computed
3860          * for each object
3861          */
3862         dev->invalidate_domains = 0;
3863         dev->flush_domains = 0;
3864
3865         for (i = 0; i < args->buffer_count; i++) {
3866                 struct drm_gem_object *obj = object_list[i];
3867
3868                 /* Compute new gpu domains and update invalidate/flush */
3869                 i915_gem_object_set_to_gpu_domain(obj);
3870         }
3871
3872         i915_verify_inactive(dev, __FILE__, __LINE__);
3873
3874         if (dev->invalidate_domains | dev->flush_domains) {
3875 #if WATCH_EXEC
3876                 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
3877                           __func__,
3878                          dev->invalidate_domains,
3879                          dev->flush_domains);
3880 #endif
3881                 i915_gem_flush(dev,
3882                                dev->invalidate_domains,
3883                                dev->flush_domains);
3884                 if (dev->flush_domains & I915_GEM_GPU_DOMAINS) {
3885                         (void)i915_add_request(dev, file_priv,
3886                                         dev->flush_domains,
3887                                         &dev_priv->render_ring);
3888
3889                         if (HAS_BSD(dev))
3890                                 (void)i915_add_request(dev, file_priv,
3891                                                 dev->flush_domains,
3892                                                 &dev_priv->bsd_ring);
3893                 }
3894         }
3895
3896         for (i = 0; i < args->buffer_count; i++) {
3897                 struct drm_gem_object *obj = object_list[i];
3898                 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3899                 uint32_t old_write_domain = obj->write_domain;
3900
3901                 obj->write_domain = obj->pending_write_domain;
3902                 if (obj->write_domain)
3903                         list_move_tail(&obj_priv->gpu_write_list,
3904                                        &dev_priv->mm.gpu_write_list);
3905                 else
3906                         list_del_init(&obj_priv->gpu_write_list);
3907
3908                 trace_i915_gem_object_change_domain(obj,
3909                                                     obj->read_domains,
3910                                                     old_write_domain);
3911         }
3912
3913         i915_verify_inactive(dev, __FILE__, __LINE__);
3914
3915 #if WATCH_COHERENCY
3916         for (i = 0; i < args->buffer_count; i++) {
3917                 i915_gem_object_check_coherency(object_list[i],
3918                                                 exec_list[i].handle);
3919         }
3920 #endif
3921
3922 #if WATCH_EXEC
3923         i915_gem_dump_object(batch_obj,
3924                               args->batch_len,
3925                               __func__,
3926                               ~0);
3927 #endif
3928
3929         /* Exec the batchbuffer */
3930         ret = ring->dispatch_gem_execbuffer(dev, ring, args,
3931                         cliprects, exec_offset);
3932         if (ret) {
3933                 DRM_ERROR("dispatch failed %d\n", ret);
3934                 goto err;
3935         }
3936
3937         /*
3938          * Ensure that the commands in the batch buffer are
3939          * finished before the interrupt fires
3940          */
3941         flush_domains = i915_retire_commands(dev, ring);
3942
3943         i915_verify_inactive(dev, __FILE__, __LINE__);
3944
3945         /*
3946          * Get a seqno representing the execution of the current buffer,
3947          * which we can wait on.  We would like to mitigate these interrupts,
3948          * likely by only creating seqnos occasionally (so that we have
3949          * *some* interrupts representing completion of buffers that we can
3950          * wait on when trying to clear up gtt space).
3951          */
3952         seqno = i915_add_request(dev, file_priv, flush_domains, ring);
3953         BUG_ON(seqno == 0);
3954         for (i = 0; i < args->buffer_count; i++) {
3955                 struct drm_gem_object *obj = object_list[i];
3956                 obj_priv = to_intel_bo(obj);
3957
3958                 i915_gem_object_move_to_active(obj, seqno, ring);
3959 #if WATCH_LRU
3960                 DRM_INFO("%s: move to exec list %p\n", __func__, obj);
3961 #endif
3962         }
3963 #if WATCH_LRU
3964         i915_dump_lru(dev, __func__);
3965 #endif
3966
3967         i915_verify_inactive(dev, __FILE__, __LINE__);
3968
3969 err:
3970         for (i = 0; i < pinned; i++)
3971                 i915_gem_object_unpin(object_list[i]);
3972
3973         for (i = 0; i < args->buffer_count; i++) {
3974                 if (object_list[i]) {
3975                         obj_priv = to_intel_bo(object_list[i]);
3976                         obj_priv->in_execbuffer = false;
3977                 }
3978                 drm_gem_object_unreference(object_list[i]);
3979         }
3980
3981         mutex_unlock(&dev->struct_mutex);
3982
3983 pre_mutex_err:
3984         /* Copy the updated relocations out regardless of current error
3985          * state.  Failure to update the relocs would mean that the next
3986          * time userland calls execbuf, it would do so with presumed offset
3987          * state that didn't match the actual object state.
3988          */
3989         ret2 = i915_gem_put_relocs_to_user(exec_list, args->buffer_count,
3990                                            relocs);
3991         if (ret2 != 0) {
3992                 DRM_ERROR("Failed to copy relocations back out: %d\n", ret2);
3993
3994                 if (ret == 0)
3995                         ret = ret2;
3996         }
3997
3998         drm_free_large(object_list);
3999         kfree(cliprects);
4000
4001         return ret;
4002 }
4003
4004 /*
4005  * Legacy execbuffer just creates an exec2 list from the original exec object
4006  * list array and passes it to the real function.
4007  */
4008 int
4009 i915_gem_execbuffer(struct drm_device *dev, void *data,
4010                     struct drm_file *file_priv)
4011 {
4012         struct drm_i915_gem_execbuffer *args = data;
4013         struct drm_i915_gem_execbuffer2 exec2;
4014         struct drm_i915_gem_exec_object *exec_list = NULL;
4015         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
4016         int ret, i;
4017
4018 #if WATCH_EXEC
4019         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
4020                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
4021 #endif
4022
4023         if (args->buffer_count < 1) {
4024                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
4025                 return -EINVAL;
4026         }
4027
4028         /* Copy in the exec list from userland */
4029         exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
4030         exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4031         if (exec_list == NULL || exec2_list == NULL) {
4032                 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4033                           args->buffer_count);
4034                 drm_free_large(exec_list);
4035                 drm_free_large(exec2_list);
4036                 return -ENOMEM;
4037         }
4038         ret = copy_from_user(exec_list,
4039                              (struct drm_i915_relocation_entry __user *)
4040                              (uintptr_t) args->buffers_ptr,
4041                              sizeof(*exec_list) * args->buffer_count);
4042         if (ret != 0) {
4043                 DRM_ERROR("copy %d exec entries failed %d\n",
4044                           args->buffer_count, ret);
4045                 drm_free_large(exec_list);
4046                 drm_free_large(exec2_list);
4047                 return -EFAULT;
4048         }
4049
4050         for (i = 0; i < args->buffer_count; i++) {
4051                 exec2_list[i].handle = exec_list[i].handle;
4052                 exec2_list[i].relocation_count = exec_list[i].relocation_count;
4053                 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
4054                 exec2_list[i].alignment = exec_list[i].alignment;
4055                 exec2_list[i].offset = exec_list[i].offset;
4056                 if (!IS_I965G(dev))
4057                         exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
4058                 else
4059                         exec2_list[i].flags = 0;
4060         }
4061
4062         exec2.buffers_ptr = args->buffers_ptr;
4063         exec2.buffer_count = args->buffer_count;
4064         exec2.batch_start_offset = args->batch_start_offset;
4065         exec2.batch_len = args->batch_len;
4066         exec2.DR1 = args->DR1;
4067         exec2.DR4 = args->DR4;
4068         exec2.num_cliprects = args->num_cliprects;
4069         exec2.cliprects_ptr = args->cliprects_ptr;
4070         exec2.flags = I915_EXEC_RENDER;
4071
4072         ret = i915_gem_do_execbuffer(dev, data, file_priv, &exec2, exec2_list);
4073         if (!ret) {
4074                 /* Copy the new buffer offsets back to the user's exec list. */
4075                 for (i = 0; i < args->buffer_count; i++)
4076                         exec_list[i].offset = exec2_list[i].offset;
4077                 /* ... and back out to userspace */
4078                 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4079                                    (uintptr_t) args->buffers_ptr,
4080                                    exec_list,
4081                                    sizeof(*exec_list) * args->buffer_count);
4082                 if (ret) {
4083                         ret = -EFAULT;
4084                         DRM_ERROR("failed to copy %d exec entries "
4085                                   "back to user (%d)\n",
4086                                   args->buffer_count, ret);
4087                 }
4088         }
4089
4090         drm_free_large(exec_list);
4091         drm_free_large(exec2_list);
4092         return ret;
4093 }
4094
4095 int
4096 i915_gem_execbuffer2(struct drm_device *dev, void *data,
4097                      struct drm_file *file_priv)
4098 {
4099         struct drm_i915_gem_execbuffer2 *args = data;
4100         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
4101         int ret;
4102
4103 #if WATCH_EXEC
4104         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
4105                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
4106 #endif
4107
4108         if (args->buffer_count < 1) {
4109                 DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
4110                 return -EINVAL;
4111         }
4112
4113         exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4114         if (exec2_list == NULL) {
4115                 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4116                           args->buffer_count);
4117                 return -ENOMEM;
4118         }
4119         ret = copy_from_user(exec2_list,
4120                              (struct drm_i915_relocation_entry __user *)
4121                              (uintptr_t) args->buffers_ptr,
4122                              sizeof(*exec2_list) * args->buffer_count);
4123         if (ret != 0) {
4124                 DRM_ERROR("copy %d exec entries failed %d\n",
4125                           args->buffer_count, ret);
4126                 drm_free_large(exec2_list);
4127                 return -EFAULT;
4128         }
4129
4130         ret = i915_gem_do_execbuffer(dev, data, file_priv, args, exec2_list);
4131         if (!ret) {
4132                 /* Copy the new buffer offsets back to the user's exec list. */
4133                 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4134                                    (uintptr_t) args->buffers_ptr,
4135                                    exec2_list,
4136                                    sizeof(*exec2_list) * args->buffer_count);
4137                 if (ret) {
4138                         ret = -EFAULT;
4139                         DRM_ERROR("failed to copy %d exec entries "
4140                                   "back to user (%d)\n",
4141                                   args->buffer_count, ret);
4142                 }
4143         }
4144
4145         drm_free_large(exec2_list);
4146         return ret;
4147 }
4148
4149 int
4150 i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
4151 {
4152         struct drm_device *dev = obj->dev;
4153         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4154         int ret;
4155
4156         BUG_ON(obj_priv->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
4157
4158         i915_verify_inactive(dev, __FILE__, __LINE__);
4159
4160         if (obj_priv->gtt_space != NULL) {
4161                 if (alignment == 0)
4162                         alignment = i915_gem_get_gtt_alignment(obj);
4163                 if (obj_priv->gtt_offset & (alignment - 1)) {
4164                         ret = i915_gem_object_unbind(obj);
4165                         if (ret)
4166                                 return ret;
4167                 }
4168         }
4169
4170         if (obj_priv->gtt_space == NULL) {
4171                 ret = i915_gem_object_bind_to_gtt(obj, alignment);
4172                 if (ret)
4173                         return ret;
4174         }
4175
4176         obj_priv->pin_count++;
4177
4178         /* If the object is not active and not pending a flush,
4179          * remove it from the inactive list
4180          */
4181         if (obj_priv->pin_count == 1) {
4182                 atomic_inc(&dev->pin_count);
4183                 atomic_add(obj->size, &dev->pin_memory);
4184                 if (!obj_priv->active &&
4185                     (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0 &&
4186                     !list_empty(&obj_priv->list))
4187                         list_del_init(&obj_priv->list);
4188         }
4189         i915_verify_inactive(dev, __FILE__, __LINE__);
4190
4191         return 0;
4192 }
4193
4194 void
4195 i915_gem_object_unpin(struct drm_gem_object *obj)
4196 {
4197         struct drm_device *dev = obj->dev;
4198         drm_i915_private_t *dev_priv = dev->dev_private;
4199         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4200
4201         i915_verify_inactive(dev, __FILE__, __LINE__);
4202         obj_priv->pin_count--;
4203         BUG_ON(obj_priv->pin_count < 0);
4204         BUG_ON(obj_priv->gtt_space == NULL);
4205
4206         /* If the object is no longer pinned, and is
4207          * neither active nor being flushed, then stick it on
4208          * the inactive list
4209          */
4210         if (obj_priv->pin_count == 0) {
4211                 if (!obj_priv->active &&
4212                     (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
4213                         list_move_tail(&obj_priv->list,
4214                                        &dev_priv->mm.inactive_list);
4215                 atomic_dec(&dev->pin_count);
4216                 atomic_sub(obj->size, &dev->pin_memory);
4217         }
4218         i915_verify_inactive(dev, __FILE__, __LINE__);
4219 }
4220
4221 int
4222 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
4223                    struct drm_file *file_priv)
4224 {
4225         struct drm_i915_gem_pin *args = data;
4226         struct drm_gem_object *obj;
4227         struct drm_i915_gem_object *obj_priv;
4228         int ret;
4229
4230         mutex_lock(&dev->struct_mutex);
4231
4232         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4233         if (obj == NULL) {
4234                 DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n",
4235                           args->handle);
4236                 mutex_unlock(&dev->struct_mutex);
4237                 return -EBADF;
4238         }
4239         obj_priv = to_intel_bo(obj);
4240
4241         if (obj_priv->madv != I915_MADV_WILLNEED) {
4242                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
4243                 drm_gem_object_unreference(obj);
4244                 mutex_unlock(&dev->struct_mutex);
4245                 return -EINVAL;
4246         }
4247
4248         if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) {
4249                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
4250                           args->handle);
4251                 drm_gem_object_unreference(obj);
4252                 mutex_unlock(&dev->struct_mutex);
4253                 return -EINVAL;
4254         }
4255
4256         obj_priv->user_pin_count++;
4257         obj_priv->pin_filp = file_priv;
4258         if (obj_priv->user_pin_count == 1) {
4259                 ret = i915_gem_object_pin(obj, args->alignment);
4260                 if (ret != 0) {
4261                         drm_gem_object_unreference(obj);
4262                         mutex_unlock(&dev->struct_mutex);
4263                         return ret;
4264                 }
4265         }
4266
4267         /* XXX - flush the CPU caches for pinned objects
4268          * as the X server doesn't manage domains yet
4269          */
4270         i915_gem_object_flush_cpu_write_domain(obj);
4271         args->offset = obj_priv->gtt_offset;
4272         drm_gem_object_unreference(obj);
4273         mutex_unlock(&dev->struct_mutex);
4274
4275         return 0;
4276 }
4277
4278 int
4279 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
4280                      struct drm_file *file_priv)
4281 {
4282         struct drm_i915_gem_pin *args = data;
4283         struct drm_gem_object *obj;
4284         struct drm_i915_gem_object *obj_priv;
4285
4286         mutex_lock(&dev->struct_mutex);
4287
4288         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4289         if (obj == NULL) {
4290                 DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n",
4291                           args->handle);
4292                 mutex_unlock(&dev->struct_mutex);
4293                 return -EBADF;
4294         }
4295
4296         obj_priv = to_intel_bo(obj);
4297         if (obj_priv->pin_filp != file_priv) {
4298                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
4299                           args->handle);
4300                 drm_gem_object_unreference(obj);
4301                 mutex_unlock(&dev->struct_mutex);
4302                 return -EINVAL;
4303         }
4304         obj_priv->user_pin_count--;
4305         if (obj_priv->user_pin_count == 0) {
4306                 obj_priv->pin_filp = NULL;
4307                 i915_gem_object_unpin(obj);
4308         }
4309
4310         drm_gem_object_unreference(obj);
4311         mutex_unlock(&dev->struct_mutex);
4312         return 0;
4313 }
4314
4315 int
4316 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4317                     struct drm_file *file_priv)
4318 {
4319         struct drm_i915_gem_busy *args = data;
4320         struct drm_gem_object *obj;
4321         struct drm_i915_gem_object *obj_priv;
4322         drm_i915_private_t *dev_priv = dev->dev_private;
4323
4324         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4325         if (obj == NULL) {
4326                 DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n",
4327                           args->handle);
4328                 return -EBADF;
4329         }
4330
4331         mutex_lock(&dev->struct_mutex);
4332         /* Update the active list for the hardware's current position.
4333          * Otherwise this only updates on a delayed timer or when irqs are
4334          * actually unmasked, and our working set ends up being larger than
4335          * required.
4336          */
4337         i915_gem_retire_requests(dev, &dev_priv->render_ring);
4338
4339         if (HAS_BSD(dev))
4340                 i915_gem_retire_requests(dev, &dev_priv->bsd_ring);
4341
4342         obj_priv = to_intel_bo(obj);
4343         /* Don't count being on the flushing list against the object being
4344          * done.  Otherwise, a buffer left on the flushing list but not getting
4345          * flushed (because nobody's flushing that domain) won't ever return
4346          * unbusy and get reused by libdrm's bo cache.  The other expected
4347          * consumer of this interface, OpenGL's occlusion queries, also specs
4348          * that the objects get unbusy "eventually" without any interference.
4349          */
4350         args->busy = obj_priv->active && obj_priv->last_rendering_seqno != 0;
4351
4352         drm_gem_object_unreference(obj);
4353         mutex_unlock(&dev->struct_mutex);
4354         return 0;
4355 }
4356
4357 int
4358 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4359                         struct drm_file *file_priv)
4360 {
4361     return i915_gem_ring_throttle(dev, file_priv);
4362 }
4363
4364 int
4365 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4366                        struct drm_file *file_priv)
4367 {
4368         struct drm_i915_gem_madvise *args = data;
4369         struct drm_gem_object *obj;
4370         struct drm_i915_gem_object *obj_priv;
4371
4372         switch (args->madv) {
4373         case I915_MADV_DONTNEED:
4374         case I915_MADV_WILLNEED:
4375             break;
4376         default:
4377             return -EINVAL;
4378         }
4379
4380         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4381         if (obj == NULL) {
4382                 DRM_ERROR("Bad handle in i915_gem_madvise_ioctl(): %d\n",
4383                           args->handle);
4384                 return -EBADF;
4385         }
4386
4387         mutex_lock(&dev->struct_mutex);
4388         obj_priv = to_intel_bo(obj);
4389
4390         if (obj_priv->pin_count) {
4391                 drm_gem_object_unreference(obj);
4392                 mutex_unlock(&dev->struct_mutex);
4393
4394                 DRM_ERROR("Attempted i915_gem_madvise_ioctl() on a pinned object\n");
4395                 return -EINVAL;
4396         }
4397
4398         if (obj_priv->madv != __I915_MADV_PURGED)
4399                 obj_priv->madv = args->madv;
4400
4401         /* if the object is no longer bound, discard its backing storage */
4402         if (i915_gem_object_is_purgeable(obj_priv) &&
4403             obj_priv->gtt_space == NULL)
4404                 i915_gem_object_truncate(obj);
4405
4406         args->retained = obj_priv->madv != __I915_MADV_PURGED;
4407
4408         drm_gem_object_unreference(obj);
4409         mutex_unlock(&dev->struct_mutex);
4410
4411         return 0;
4412 }
4413
4414 struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev,
4415                                               size_t size)
4416 {
4417         struct drm_i915_gem_object *obj;
4418
4419         obj = kzalloc(sizeof(*obj), GFP_KERNEL);
4420         if (obj == NULL)
4421                 return NULL;
4422
4423         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4424                 kfree(obj);
4425                 return NULL;
4426         }
4427
4428         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4429         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4430
4431         obj->agp_type = AGP_USER_MEMORY;
4432         obj->base.driver_private = NULL;
4433         obj->fence_reg = I915_FENCE_REG_NONE;
4434         INIT_LIST_HEAD(&obj->list);
4435         INIT_LIST_HEAD(&obj->gpu_write_list);
4436         obj->madv = I915_MADV_WILLNEED;
4437
4438         trace_i915_gem_object_create(&obj->base);
4439
4440         return &obj->base;
4441 }
4442
4443 int i915_gem_init_object(struct drm_gem_object *obj)
4444 {
4445         BUG();
4446
4447         return 0;
4448 }
4449
4450 void i915_gem_free_object(struct drm_gem_object *obj)
4451 {
4452         struct drm_device *dev = obj->dev;
4453         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4454
4455         trace_i915_gem_object_destroy(obj);
4456
4457         while (obj_priv->pin_count > 0)
4458                 i915_gem_object_unpin(obj);
4459
4460         if (obj_priv->phys_obj)
4461                 i915_gem_detach_phys_object(dev, obj);
4462
4463         i915_gem_object_unbind(obj);
4464
4465         if (obj_priv->mmap_offset)
4466                 i915_gem_free_mmap_offset(obj);
4467
4468         drm_gem_object_release(obj);
4469
4470         kfree(obj_priv->page_cpu_valid);
4471         kfree(obj_priv->bit_17);
4472         kfree(obj_priv);
4473 }
4474
4475 /** Unbinds all inactive objects. */
4476 static int
4477 i915_gem_evict_from_inactive_list(struct drm_device *dev)
4478 {
4479         drm_i915_private_t *dev_priv = dev->dev_private;
4480
4481         while (!list_empty(&dev_priv->mm.inactive_list)) {
4482                 struct drm_gem_object *obj;
4483                 int ret;
4484
4485                 obj = &list_first_entry(&dev_priv->mm.inactive_list,
4486                                         struct drm_i915_gem_object,
4487                                         list)->base;
4488
4489                 ret = i915_gem_object_unbind(obj);
4490                 if (ret != 0) {
4491                         DRM_ERROR("Error unbinding object: %d\n", ret);
4492                         return ret;
4493                 }
4494         }
4495
4496         return 0;
4497 }
4498
4499 int
4500 i915_gem_idle(struct drm_device *dev)
4501 {
4502         drm_i915_private_t *dev_priv = dev->dev_private;
4503         int ret;
4504
4505         mutex_lock(&dev->struct_mutex);
4506
4507         if (dev_priv->mm.suspended ||
4508                         (dev_priv->render_ring.gem_object == NULL) ||
4509                         (HAS_BSD(dev) &&
4510                          dev_priv->bsd_ring.gem_object == NULL)) {
4511                 mutex_unlock(&dev->struct_mutex);
4512                 return 0;
4513         }
4514
4515         ret = i915_gpu_idle(dev);
4516         if (ret) {
4517                 mutex_unlock(&dev->struct_mutex);
4518                 return ret;
4519         }
4520
4521         /* Under UMS, be paranoid and evict. */
4522         if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
4523                 ret = i915_gem_evict_from_inactive_list(dev);
4524                 if (ret) {
4525                         mutex_unlock(&dev->struct_mutex);
4526                         return ret;
4527                 }
4528         }
4529
4530         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
4531          * We need to replace this with a semaphore, or something.
4532          * And not confound mm.suspended!
4533          */
4534         dev_priv->mm.suspended = 1;
4535         del_timer(&dev_priv->hangcheck_timer);
4536
4537         i915_kernel_lost_context(dev);
4538         i915_gem_cleanup_ringbuffer(dev);
4539
4540         mutex_unlock(&dev->struct_mutex);
4541
4542         /* Cancel the retire work handler, which should be idle now. */
4543         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4544
4545         return 0;
4546 }
4547
4548 /*
4549  * 965+ support PIPE_CONTROL commands, which provide finer grained control
4550  * over cache flushing.
4551  */
4552 static int
4553 i915_gem_init_pipe_control(struct drm_device *dev)
4554 {
4555         drm_i915_private_t *dev_priv = dev->dev_private;
4556         struct drm_gem_object *obj;
4557         struct drm_i915_gem_object *obj_priv;
4558         int ret;
4559
4560         obj = i915_gem_alloc_object(dev, 4096);
4561         if (obj == NULL) {
4562                 DRM_ERROR("Failed to allocate seqno page\n");
4563                 ret = -ENOMEM;
4564                 goto err;
4565         }
4566         obj_priv = to_intel_bo(obj);
4567         obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
4568
4569         ret = i915_gem_object_pin(obj, 4096);
4570         if (ret)
4571                 goto err_unref;
4572
4573         dev_priv->seqno_gfx_addr = obj_priv->gtt_offset;
4574         dev_priv->seqno_page =  kmap(obj_priv->pages[0]);
4575         if (dev_priv->seqno_page == NULL)
4576                 goto err_unpin;
4577
4578         dev_priv->seqno_obj = obj;
4579         memset(dev_priv->seqno_page, 0, PAGE_SIZE);
4580
4581         return 0;
4582
4583 err_unpin:
4584         i915_gem_object_unpin(obj);
4585 err_unref:
4586         drm_gem_object_unreference(obj);
4587 err:
4588         return ret;
4589 }
4590
4591
4592 static void
4593 i915_gem_cleanup_pipe_control(struct drm_device *dev)
4594 {
4595         drm_i915_private_t *dev_priv = dev->dev_private;
4596         struct drm_gem_object *obj;
4597         struct drm_i915_gem_object *obj_priv;
4598
4599         obj = dev_priv->seqno_obj;
4600         obj_priv = to_intel_bo(obj);
4601         kunmap(obj_priv->pages[0]);
4602         i915_gem_object_unpin(obj);
4603         drm_gem_object_unreference(obj);
4604         dev_priv->seqno_obj = NULL;
4605
4606         dev_priv->seqno_page = NULL;
4607 }
4608
4609 int
4610 i915_gem_init_ringbuffer(struct drm_device *dev)
4611 {
4612         drm_i915_private_t *dev_priv = dev->dev_private;
4613         int ret;
4614
4615         dev_priv->render_ring = render_ring;
4616
4617         if (!I915_NEED_GFX_HWS(dev)) {
4618                 dev_priv->render_ring.status_page.page_addr
4619                         = dev_priv->status_page_dmah->vaddr;
4620                 memset(dev_priv->render_ring.status_page.page_addr,
4621                                 0, PAGE_SIZE);
4622         }
4623
4624         if (HAS_PIPE_CONTROL(dev)) {
4625                 ret = i915_gem_init_pipe_control(dev);
4626                 if (ret)
4627                         return ret;
4628         }
4629
4630         ret = intel_init_ring_buffer(dev, &dev_priv->render_ring);
4631         if (ret)
4632                 goto cleanup_pipe_control;
4633
4634         if (HAS_BSD(dev)) {
4635                 dev_priv->bsd_ring = bsd_ring;
4636                 ret = intel_init_ring_buffer(dev, &dev_priv->bsd_ring);
4637                 if (ret)
4638                         goto cleanup_render_ring;
4639         }
4640
4641         return 0;
4642
4643 cleanup_render_ring:
4644         intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
4645 cleanup_pipe_control:
4646         if (HAS_PIPE_CONTROL(dev))
4647                 i915_gem_cleanup_pipe_control(dev);
4648         return ret;
4649 }
4650
4651 void
4652 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4653 {
4654         drm_i915_private_t *dev_priv = dev->dev_private;
4655
4656         intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
4657         if (HAS_BSD(dev))
4658                 intel_cleanup_ring_buffer(dev, &dev_priv->bsd_ring);
4659         if (HAS_PIPE_CONTROL(dev))
4660                 i915_gem_cleanup_pipe_control(dev);
4661 }
4662
4663 int
4664 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4665                        struct drm_file *file_priv)
4666 {
4667         drm_i915_private_t *dev_priv = dev->dev_private;
4668         int ret;
4669
4670         if (drm_core_check_feature(dev, DRIVER_MODESET))
4671                 return 0;
4672
4673         if (atomic_read(&dev_priv->mm.wedged)) {
4674                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
4675                 atomic_set(&dev_priv->mm.wedged, 0);
4676         }
4677
4678         mutex_lock(&dev->struct_mutex);
4679         dev_priv->mm.suspended = 0;
4680
4681         ret = i915_gem_init_ringbuffer(dev);
4682         if (ret != 0) {
4683                 mutex_unlock(&dev->struct_mutex);
4684                 return ret;
4685         }
4686
4687         spin_lock(&dev_priv->mm.active_list_lock);
4688         BUG_ON(!list_empty(&dev_priv->render_ring.active_list));
4689         BUG_ON(HAS_BSD(dev) && !list_empty(&dev_priv->bsd_ring.active_list));
4690         spin_unlock(&dev_priv->mm.active_list_lock);
4691
4692         BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
4693         BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
4694         BUG_ON(!list_empty(&dev_priv->render_ring.request_list));
4695         BUG_ON(HAS_BSD(dev) && !list_empty(&dev_priv->bsd_ring.request_list));
4696         mutex_unlock(&dev->struct_mutex);
4697
4698         drm_irq_install(dev);
4699
4700         return 0;
4701 }
4702
4703 int
4704 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
4705                        struct drm_file *file_priv)
4706 {
4707         if (drm_core_check_feature(dev, DRIVER_MODESET))
4708                 return 0;
4709
4710         drm_irq_uninstall(dev);
4711         return i915_gem_idle(dev);
4712 }
4713
4714 void
4715 i915_gem_lastclose(struct drm_device *dev)
4716 {
4717         int ret;
4718
4719         if (drm_core_check_feature(dev, DRIVER_MODESET))
4720                 return;
4721
4722         ret = i915_gem_idle(dev);
4723         if (ret)
4724                 DRM_ERROR("failed to idle hardware: %d\n", ret);
4725 }
4726
4727 void
4728 i915_gem_load(struct drm_device *dev)
4729 {
4730         int i;
4731         drm_i915_private_t *dev_priv = dev->dev_private;
4732
4733         spin_lock_init(&dev_priv->mm.active_list_lock);
4734         INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
4735         INIT_LIST_HEAD(&dev_priv->mm.gpu_write_list);
4736         INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
4737         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4738         INIT_LIST_HEAD(&dev_priv->render_ring.active_list);
4739         INIT_LIST_HEAD(&dev_priv->render_ring.request_list);
4740         if (HAS_BSD(dev)) {
4741                 INIT_LIST_HEAD(&dev_priv->bsd_ring.active_list);
4742                 INIT_LIST_HEAD(&dev_priv->bsd_ring.request_list);
4743         }
4744         for (i = 0; i < 16; i++)
4745                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
4746         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4747                           i915_gem_retire_work_handler);
4748         spin_lock(&shrink_list_lock);
4749         list_add(&dev_priv->mm.shrink_list, &shrink_list);
4750         spin_unlock(&shrink_list_lock);
4751
4752         /* Old X drivers will take 0-2 for front, back, depth buffers */
4753         if (!drm_core_check_feature(dev, DRIVER_MODESET))
4754                 dev_priv->fence_reg_start = 3;
4755
4756         if (IS_I965G(dev) || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4757                 dev_priv->num_fence_regs = 16;
4758         else
4759                 dev_priv->num_fence_regs = 8;
4760
4761         /* Initialize fence registers to zero */
4762         if (IS_I965G(dev)) {
4763                 for (i = 0; i < 16; i++)
4764                         I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0);
4765         } else {
4766                 for (i = 0; i < 8; i++)
4767                         I915_WRITE(FENCE_REG_830_0 + (i * 4), 0);
4768                 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4769                         for (i = 0; i < 8; i++)
4770                                 I915_WRITE(FENCE_REG_945_8 + (i * 4), 0);
4771         }
4772         i915_gem_detect_bit_6_swizzle(dev);
4773         init_waitqueue_head(&dev_priv->pending_flip_queue);
4774 }
4775
4776 /*
4777  * Create a physically contiguous memory object for this object
4778  * e.g. for cursor + overlay regs
4779  */
4780 int i915_gem_init_phys_object(struct drm_device *dev,
4781                               int id, int size)
4782 {
4783         drm_i915_private_t *dev_priv = dev->dev_private;
4784         struct drm_i915_gem_phys_object *phys_obj;
4785         int ret;
4786
4787         if (dev_priv->mm.phys_objs[id - 1] || !size)
4788                 return 0;
4789
4790         phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
4791         if (!phys_obj)
4792                 return -ENOMEM;
4793
4794         phys_obj->id = id;
4795
4796         phys_obj->handle = drm_pci_alloc(dev, size, 0);
4797         if (!phys_obj->handle) {
4798                 ret = -ENOMEM;
4799                 goto kfree_obj;
4800         }
4801 #ifdef CONFIG_X86
4802         set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4803 #endif
4804
4805         dev_priv->mm.phys_objs[id - 1] = phys_obj;
4806
4807         return 0;
4808 kfree_obj:
4809         kfree(phys_obj);
4810         return ret;
4811 }
4812
4813 void i915_gem_free_phys_object(struct drm_device *dev, int id)
4814 {
4815         drm_i915_private_t *dev_priv = dev->dev_private;
4816         struct drm_i915_gem_phys_object *phys_obj;
4817
4818         if (!dev_priv->mm.phys_objs[id - 1])
4819                 return;
4820
4821         phys_obj = dev_priv->mm.phys_objs[id - 1];
4822         if (phys_obj->cur_obj) {
4823                 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4824         }
4825
4826 #ifdef CONFIG_X86
4827         set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4828 #endif
4829         drm_pci_free(dev, phys_obj->handle);
4830         kfree(phys_obj);
4831         dev_priv->mm.phys_objs[id - 1] = NULL;
4832 }
4833
4834 void i915_gem_free_all_phys_object(struct drm_device *dev)
4835 {
4836         int i;
4837
4838         for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
4839                 i915_gem_free_phys_object(dev, i);
4840 }
4841
4842 void i915_gem_detach_phys_object(struct drm_device *dev,
4843                                  struct drm_gem_object *obj)
4844 {
4845         struct drm_i915_gem_object *obj_priv;
4846         int i;
4847         int ret;
4848         int page_count;
4849
4850         obj_priv = to_intel_bo(obj);
4851         if (!obj_priv->phys_obj)
4852                 return;
4853
4854         ret = i915_gem_object_get_pages(obj, 0);
4855         if (ret)
4856                 goto out;
4857
4858         page_count = obj->size / PAGE_SIZE;
4859
4860         for (i = 0; i < page_count; i++) {
4861                 char *dst = kmap_atomic(obj_priv->pages[i], KM_USER0);
4862                 char *src = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4863
4864                 memcpy(dst, src, PAGE_SIZE);
4865                 kunmap_atomic(dst, KM_USER0);
4866         }
4867         drm_clflush_pages(obj_priv->pages, page_count);
4868         drm_agp_chipset_flush(dev);
4869
4870         i915_gem_object_put_pages(obj);
4871 out:
4872         obj_priv->phys_obj->cur_obj = NULL;
4873         obj_priv->phys_obj = NULL;
4874 }
4875
4876 int
4877 i915_gem_attach_phys_object(struct drm_device *dev,
4878                             struct drm_gem_object *obj, int id)
4879 {
4880         drm_i915_private_t *dev_priv = dev->dev_private;
4881         struct drm_i915_gem_object *obj_priv;
4882         int ret = 0;
4883         int page_count;
4884         int i;
4885
4886         if (id > I915_MAX_PHYS_OBJECT)
4887                 return -EINVAL;
4888
4889         obj_priv = to_intel_bo(obj);
4890
4891         if (obj_priv->phys_obj) {
4892                 if (obj_priv->phys_obj->id == id)
4893                         return 0;
4894                 i915_gem_detach_phys_object(dev, obj);
4895         }
4896
4897
4898         /* create a new object */
4899         if (!dev_priv->mm.phys_objs[id - 1]) {
4900                 ret = i915_gem_init_phys_object(dev, id,
4901                                                 obj->size);
4902                 if (ret) {
4903                         DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size);
4904                         goto out;
4905                 }
4906         }
4907
4908         /* bind to the object */
4909         obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1];
4910         obj_priv->phys_obj->cur_obj = obj;
4911
4912         ret = i915_gem_object_get_pages(obj, 0);
4913         if (ret) {
4914                 DRM_ERROR("failed to get page list\n");
4915                 goto out;
4916         }
4917
4918         page_count = obj->size / PAGE_SIZE;
4919
4920         for (i = 0; i < page_count; i++) {
4921                 char *src = kmap_atomic(obj_priv->pages[i], KM_USER0);
4922                 char *dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4923
4924                 memcpy(dst, src, PAGE_SIZE);
4925                 kunmap_atomic(src, KM_USER0);
4926         }
4927
4928         i915_gem_object_put_pages(obj);
4929
4930         return 0;
4931 out:
4932         return ret;
4933 }
4934
4935 static int
4936 i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
4937                      struct drm_i915_gem_pwrite *args,
4938                      struct drm_file *file_priv)
4939 {
4940         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4941         void *obj_addr;
4942         int ret;
4943         char __user *user_data;
4944
4945         user_data = (char __user *) (uintptr_t) args->data_ptr;
4946         obj_addr = obj_priv->phys_obj->handle->vaddr + args->offset;
4947
4948         DRM_DEBUG_DRIVER("obj_addr %p, %lld\n", obj_addr, args->size);
4949         ret = copy_from_user(obj_addr, user_data, args->size);
4950         if (ret)
4951                 return -EFAULT;
4952
4953         drm_agp_chipset_flush(dev);
4954         return 0;
4955 }
4956
4957 void i915_gem_release(struct drm_device * dev, struct drm_file *file_priv)
4958 {
4959         struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
4960
4961         /* Clean up our request list when the client is going away, so that
4962          * later retire_requests won't dereference our soon-to-be-gone
4963          * file_priv.
4964          */
4965         mutex_lock(&dev->struct_mutex);
4966         while (!list_empty(&i915_file_priv->mm.request_list))
4967                 list_del_init(i915_file_priv->mm.request_list.next);
4968         mutex_unlock(&dev->struct_mutex);
4969 }
4970
4971 static int
4972 i915_gpu_is_active(struct drm_device *dev)
4973 {
4974         drm_i915_private_t *dev_priv = dev->dev_private;
4975         int lists_empty;
4976
4977         spin_lock(&dev_priv->mm.active_list_lock);
4978         lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
4979                       list_empty(&dev_priv->render_ring.active_list);
4980         if (HAS_BSD(dev))
4981                 lists_empty &= list_empty(&dev_priv->bsd_ring.active_list);
4982         spin_unlock(&dev_priv->mm.active_list_lock);
4983
4984         return !lists_empty;
4985 }
4986
4987 static int
4988 i915_gem_shrink(int nr_to_scan, gfp_t gfp_mask)
4989 {
4990         drm_i915_private_t *dev_priv, *next_dev;
4991         struct drm_i915_gem_object *obj_priv, *next_obj;
4992         int cnt = 0;
4993         int would_deadlock = 1;
4994
4995         /* "fast-path" to count number of available objects */
4996         if (nr_to_scan == 0) {
4997                 spin_lock(&shrink_list_lock);
4998                 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
4999                         struct drm_device *dev = dev_priv->dev;
5000
5001                         if (mutex_trylock(&dev->struct_mutex)) {
5002                                 list_for_each_entry(obj_priv,
5003                                                     &dev_priv->mm.inactive_list,
5004                                                     list)
5005                                         cnt++;
5006                                 mutex_unlock(&dev->struct_mutex);
5007                         }
5008                 }
5009                 spin_unlock(&shrink_list_lock);
5010
5011                 return (cnt / 100) * sysctl_vfs_cache_pressure;
5012         }
5013
5014         spin_lock(&shrink_list_lock);
5015
5016 rescan:
5017         /* first scan for clean buffers */
5018         list_for_each_entry_safe(dev_priv, next_dev,
5019                                  &shrink_list, mm.shrink_list) {
5020                 struct drm_device *dev = dev_priv->dev;
5021
5022                 if (! mutex_trylock(&dev->struct_mutex))
5023                         continue;
5024
5025                 spin_unlock(&shrink_list_lock);
5026                 i915_gem_retire_requests(dev, &dev_priv->render_ring);
5027
5028                 if (HAS_BSD(dev))
5029                         i915_gem_retire_requests(dev, &dev_priv->bsd_ring);
5030
5031                 list_for_each_entry_safe(obj_priv, next_obj,
5032                                          &dev_priv->mm.inactive_list,
5033                                          list) {
5034                         if (i915_gem_object_is_purgeable(obj_priv)) {
5035                                 i915_gem_object_unbind(&obj_priv->base);
5036                                 if (--nr_to_scan <= 0)
5037                                         break;
5038                         }
5039                 }
5040
5041                 spin_lock(&shrink_list_lock);
5042                 mutex_unlock(&dev->struct_mutex);
5043
5044                 would_deadlock = 0;
5045
5046                 if (nr_to_scan <= 0)
5047                         break;
5048         }
5049
5050         /* second pass, evict/count anything still on the inactive list */
5051         list_for_each_entry_safe(dev_priv, next_dev,
5052                                  &shrink_list, mm.shrink_list) {
5053                 struct drm_device *dev = dev_priv->dev;
5054
5055                 if (! mutex_trylock(&dev->struct_mutex))
5056                         continue;
5057
5058                 spin_unlock(&shrink_list_lock);
5059
5060                 list_for_each_entry_safe(obj_priv, next_obj,
5061                                          &dev_priv->mm.inactive_list,
5062                                          list) {
5063                         if (nr_to_scan > 0) {
5064                                 i915_gem_object_unbind(&obj_priv->base);
5065                                 nr_to_scan--;
5066                         } else
5067                                 cnt++;
5068                 }
5069
5070                 spin_lock(&shrink_list_lock);
5071                 mutex_unlock(&dev->struct_mutex);
5072
5073                 would_deadlock = 0;
5074         }
5075
5076         if (nr_to_scan) {
5077                 int active = 0;
5078
5079                 /*
5080                  * We are desperate for pages, so as a last resort, wait
5081                  * for the GPU to finish and discard whatever we can.
5082                  * This has a dramatic impact to reduce the number of
5083                  * OOM-killer events whilst running the GPU aggressively.
5084                  */
5085                 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
5086                         struct drm_device *dev = dev_priv->dev;
5087
5088                         if (!mutex_trylock(&dev->struct_mutex))
5089                                 continue;
5090
5091                         spin_unlock(&shrink_list_lock);
5092
5093                         if (i915_gpu_is_active(dev)) {
5094                                 i915_gpu_idle(dev);
5095                                 active++;
5096                         }
5097
5098                         spin_lock(&shrink_list_lock);
5099                         mutex_unlock(&dev->struct_mutex);
5100                 }
5101
5102                 if (active)
5103                         goto rescan;
5104         }
5105
5106         spin_unlock(&shrink_list_lock);
5107
5108         if (would_deadlock)
5109                 return -1;
5110         else if (cnt > 0)
5111                 return (cnt / 100) * sysctl_vfs_cache_pressure;
5112         else
5113                 return 0;
5114 }
5115
5116 static struct shrinker shrinker = {
5117         .shrink = i915_gem_shrink,
5118         .seeks = DEFAULT_SEEKS,
5119 };
5120
5121 __init void
5122 i915_gem_shrinker_init(void)
5123 {
5124     register_shrinker(&shrinker);
5125 }
5126
5127 __exit void
5128 i915_gem_shrinker_exit(void)
5129 {
5130     unregister_shrinker(&shrinker);
5131 }