drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  */
  27
  28 #include "drmP.h"
  29 #include "drm.h"
  30 #include "i915_drm.h"
  31 #include "i915_drv.h"
  32 #include "i915_trace.h"
  33 #include "intel_drv.h"
  34 #include <linux/slab.h>
  35 #include <linux/swap.h>
  36 #include <linux/pci.h>
  37 #include <linux/intel-gtt.h>
  38
  39 static uint32_t i915_gem_get_gtt_alignment(struct drm_gem_object *obj);
  40 static int i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj);
  41 static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
  42 static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
  43 static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
  44                                              int write);
  45 static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
  46                                                      uint64_t offset,
  47                                                      uint64_t size);
  48 static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
  49 static int i915_gem_object_wait_rendering(struct drm_gem_object *obj);
  50 static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
  51                                            unsigned alignment);
  52 static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
  53 static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
  54                                 struct drm_i915_gem_pwrite *args,
  55                                 struct drm_file *file_priv);
  56 static void i915_gem_free_object_tail(struct drm_gem_object *obj);
  57
  58 static LIST_HEAD(shrink_list);
  59 static DEFINE_SPINLOCK(shrink_list_lock);
  60
  61 static inline bool
  62 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj_priv)
  63 {
  64         return obj_priv->gtt_space &&
  65                 !obj_priv->active &&
  66                 obj_priv->pin_count == 0;
  67 }
  68
  69 int i915_gem_do_init(struct drm_device *dev, unsigned long start,
  70                      unsigned long end)
  71 {
  72         drm_i915_private_t *dev_priv = dev->dev_private;
  73
  74         if (start >= end ||
  75             (start & (PAGE_SIZE - 1)) != 0 ||
  76             (end & (PAGE_SIZE - 1)) != 0) {
  77                 return -EINVAL;
  78         }
  79
  80         drm_mm_init(&dev_priv->mm.gtt_space, start,
  81                     end - start);
  82
  83         dev->gtt_total = (uint32_t) (end - start);
  84
  85         return 0;
  86 }
  87
  88 int
  89 i915_gem_init_ioctl(struct drm_device *dev, void *data,
  90                     struct drm_file *file_priv)
  91 {
  92         struct drm_i915_gem_init *args = data;
  93         int ret;
  94
  95         mutex_lock(&dev->struct_mutex);
  96         ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end);
  97         mutex_unlock(&dev->struct_mutex);
  98
  99         return ret;
 100 }
 101
 102 int
 103 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 104                             struct drm_file *file_priv)
 105 {
 106         struct drm_i915_gem_get_aperture *args = data;
 107
 108         if (!(dev->driver->driver_features & DRIVER_GEM))
 109                 return -ENODEV;
 110
 111         args->aper_size = dev->gtt_total;
 112         args->aper_available_size = (args->aper_size -
 113                                      atomic_read(&dev->pin_memory));
 114
 115         return 0;
 116 }
 117
 118
 119 /**
 120  * Creates a new mm object and returns a handle to it.
 121  */
 122 int
 123 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 124                       struct drm_file *file_priv)
 125 {
 126         struct drm_i915_gem_create *args = data;
 127         struct drm_gem_object *obj;
 128         int ret;
 129         u32 handle;
 130
 131         args->size = roundup(args->size, PAGE_SIZE);
 132
 133         /* Allocate the new object */
 134         obj = i915_gem_alloc_object(dev, args->size);
 135         if (obj == NULL)
 136                 return -ENOMEM;
 137
 138         ret = drm_gem_handle_create(file_priv, obj, &handle);
 139         if (ret) {
 140                 drm_gem_object_unreference_unlocked(obj);
 141                 return ret;
 142         }
 143
 144         /* Sink the floating reference from kref_init(handlecount) */
 145         drm_gem_object_handle_unreference_unlocked(obj);
 146
 147         args->handle = handle;
 148         return 0;
 149 }
 150
 151 static inline int
 152 fast_shmem_read(struct page **pages,
 153                 loff_t page_base, int page_offset,
 154                 char __user *data,
 155                 int length)
 156 {
 157         char __iomem *vaddr;
 158         int unwritten;
 159
 160         vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
 161         if (vaddr == NULL)
 162                 return -ENOMEM;
 163         unwritten = __copy_to_user_inatomic(data, vaddr + page_offset, length);
 164         kunmap_atomic(vaddr, KM_USER0);
 165
 166         if (unwritten)
 167                 return -EFAULT;
 168
 169         return 0;
 170 }
 171
 172 static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
 173 {
 174         drm_i915_private_t *dev_priv = obj->dev->dev_private;
 175         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 176
 177         return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
 178                 obj_priv->tiling_mode != I915_TILING_NONE;
 179 }
 180
 181 static inline void
 182 slow_shmem_copy(struct page *dst_page,
 183                 int dst_offset,
 184                 struct page *src_page,
 185                 int src_offset,
 186                 int length)
 187 {
 188         char *dst_vaddr, *src_vaddr;
 189
 190         dst_vaddr = kmap(dst_page);
 191         src_vaddr = kmap(src_page);
 192
 193         memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
 194
 195         kunmap(src_page);
 196         kunmap(dst_page);
 197 }
 198
 199 static inline void
 200 slow_shmem_bit17_copy(struct page *gpu_page,
 201                       int gpu_offset,
 202                       struct page *cpu_page,
 203                       int cpu_offset,
 204                       int length,
 205                       int is_read)
 206 {
 207         char *gpu_vaddr, *cpu_vaddr;
 208
 209         /* Use the unswizzled path if this page isn't affected. */
 210         if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
 211                 if (is_read)
 212                         return slow_shmem_copy(cpu_page, cpu_offset,
 213                                                gpu_page, gpu_offset, length);
 214                 else
 215                         return slow_shmem_copy(gpu_page, gpu_offset,
 216                                                cpu_page, cpu_offset, length);
 217         }
 218
 219         gpu_vaddr = kmap(gpu_page);
 220         cpu_vaddr = kmap(cpu_page);
 221
 222         /* Copy the data, XORing A6 with A17 (1). The user already knows he's
 223          * XORing with the other bits (A9 for Y, A9 and A10 for X)
 224          */
 225         while (length > 0) {
 226                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 227                 int this_length = min(cacheline_end - gpu_offset, length);
 228                 int swizzled_gpu_offset = gpu_offset ^ 64;
 229
 230                 if (is_read) {
 231                         memcpy(cpu_vaddr + cpu_offset,
 232                                gpu_vaddr + swizzled_gpu_offset,
 233                                this_length);
 234                 } else {
 235                         memcpy(gpu_vaddr + swizzled_gpu_offset,
 236                                cpu_vaddr + cpu_offset,
 237                                this_length);
 238                 }
 239                 cpu_offset += this_length;
 240                 gpu_offset += this_length;
 241                 length -= this_length;
 242         }
 243
 244         kunmap(cpu_page);
 245         kunmap(gpu_page);
 246 }
 247
 248 /**
 249  * This is the fast shmem pread path, which attempts to copy_from_user directly
 250  * from the backing pages of the object to the user's address space.  On a
 251  * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
 252  */
 253 static int
 254 i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
 255                           struct drm_i915_gem_pread *args,
 256                           struct drm_file *file_priv)
 257 {
 258         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 259         ssize_t remain;
 260         loff_t offset, page_base;
 261         char __user *user_data;
 262         int page_offset, page_length;
 263         int ret;
 264
 265         user_data = (char __user *) (uintptr_t) args->data_ptr;
 266         remain = args->size;
 267
 268         mutex_lock(&dev->struct_mutex);
 269
 270         ret = i915_gem_object_get_pages(obj, 0);
 271         if (ret != 0)
 272                 goto fail_unlock;
 273
 274         ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
 275                                                         args->size);
 276         if (ret != 0)
 277                 goto fail_put_pages;
 278
 279         obj_priv = to_intel_bo(obj);
 280         offset = args->offset;
 281
 282         while (remain > 0) {
 283                 /* Operation in this page
 284                  *
 285                  * page_base = page offset within aperture
 286                  * page_offset = offset within page
 287                  * page_length = bytes to copy for this page
 288                  */
 289                 page_base = (offset & ~(PAGE_SIZE-1));
 290                 page_offset = offset & (PAGE_SIZE-1);
 291                 page_length = remain;
 292                 if ((page_offset + remain) > PAGE_SIZE)
 293                         page_length = PAGE_SIZE - page_offset;
 294
 295                 ret = fast_shmem_read(obj_priv->pages,
 296                                       page_base, page_offset,
 297                                       user_data, page_length);
 298                 if (ret)
 299                         goto fail_put_pages;
 300
 301                 remain -= page_length;
 302                 user_data += page_length;
 303                 offset += page_length;
 304         }
 305
 306 fail_put_pages:
 307         i915_gem_object_put_pages(obj);
 308 fail_unlock:
 309         mutex_unlock(&dev->struct_mutex);
 310
 311         return ret;
 312 }
 313
 314 static int
 315 i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
 316 {
 317         int ret;
 318
 319         ret = i915_gem_object_get_pages(obj, __GFP_NORETRY | __GFP_NOWARN);
 320
 321         /* If we've insufficient memory to map in the pages, attempt
 322          * to make some space by throwing out some old buffers.
 323          */
 324         if (ret == -ENOMEM) {
 325                 struct drm_device *dev = obj->dev;
 326
 327                 ret = i915_gem_evict_something(dev, obj->size,
 328                                                i915_gem_get_gtt_alignment(obj));
 329                 if (ret)
 330                         return ret;
 331
 332                 ret = i915_gem_object_get_pages(obj, 0);
 333         }
 334
 335         return ret;
 336 }
 337
 338 /**
 339  * This is the fallback shmem pread path, which allocates temporary storage
 340  * in kernel space to copy_to_user into outside of the struct_mutex, so we
 341  * can copy out of the object's backing pages while holding the struct mutex
 342  * and not take page faults.
 343  */
 344 static int
 345 i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
 346                           struct drm_i915_gem_pread *args,
 347                           struct drm_file *file_priv)
 348 {
 349         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 350         struct mm_struct *mm = current->mm;
 351         struct page **user_pages;
 352         ssize_t remain;
 353         loff_t offset, pinned_pages, i;
 354         loff_t first_data_page, last_data_page, num_pages;
 355         int shmem_page_index, shmem_page_offset;
 356         int data_page_index,  data_page_offset;
 357         int page_length;
 358         int ret;
 359         uint64_t data_ptr = args->data_ptr;
 360         int do_bit17_swizzling;
 361
 362         remain = args->size;
 363
 364         /* Pin the user pages containing the data.  We can't fault while
 365          * holding the struct mutex, yet we want to hold it while
 366          * dereferencing the user data.
 367          */
 368         first_data_page = data_ptr / PAGE_SIZE;
 369         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 370         num_pages = last_data_page - first_data_page + 1;
 371
 372         user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
 373         if (user_pages == NULL)
 374                 return -ENOMEM;
 375
 376         down_read(&mm->mmap_sem);
 377         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 378                                       num_pages, 1, 0, user_pages, NULL);
 379         up_read(&mm->mmap_sem);
 380         if (pinned_pages < num_pages) {
 381                 ret = -EFAULT;
 382                 goto fail_put_user_pages;
 383         }
 384
 385         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 386
 387         mutex_lock(&dev->struct_mutex);
 388
 389         ret = i915_gem_object_get_pages_or_evict(obj);
 390         if (ret)
 391                 goto fail_unlock;
 392
 393         ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
 394                                                         args->size);
 395         if (ret != 0)
 396                 goto fail_put_pages;
 397
 398         obj_priv = to_intel_bo(obj);
 399         offset = args->offset;
 400
 401         while (remain > 0) {
 402                 /* Operation in this page
 403                  *
 404                  * shmem_page_index = page number within shmem file
 405                  * shmem_page_offset = offset within page in shmem file
 406                  * data_page_index = page number in get_user_pages return
 407                  * data_page_offset = offset with data_page_index page.
 408                  * page_length = bytes to copy for this page
 409                  */
 410                 shmem_page_index = offset / PAGE_SIZE;
 411                 shmem_page_offset = offset & ~PAGE_MASK;
 412                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 413                 data_page_offset = data_ptr & ~PAGE_MASK;
 414
 415                 page_length = remain;
 416                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 417                         page_length = PAGE_SIZE - shmem_page_offset;
 418                 if ((data_page_offset + page_length) > PAGE_SIZE)
 419                         page_length = PAGE_SIZE - data_page_offset;
 420
 421                 if (do_bit17_swizzling) {
 422                         slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
 423                                               shmem_page_offset,
 424                                               user_pages[data_page_index],
 425                                               data_page_offset,
 426                                               page_length,
 427                                               1);
 428                 } else {
 429                         slow_shmem_copy(user_pages[data_page_index],
 430                                         data_page_offset,
 431                                         obj_priv->pages[shmem_page_index],
 432                                         shmem_page_offset,
 433                                         page_length);
 434                 }
 435
 436                 remain -= page_length;
 437                 data_ptr += page_length;
 438                 offset += page_length;
 439         }
 440
 441 fail_put_pages:
 442         i915_gem_object_put_pages(obj);
 443 fail_unlock:
 444         mutex_unlock(&dev->struct_mutex);
 445 fail_put_user_pages:
 446         for (i = 0; i < pinned_pages; i++) {
 447                 SetPageDirty(user_pages[i]);
 448                 page_cache_release(user_pages[i]);
 449         }
 450         drm_free_large(user_pages);
 451
 452         return ret;
 453 }
 454
 455 /**
 456  * Reads data from the object referenced by handle.
 457  *
 458  * On error, the contents of *data are undefined.
 459  */
 460 int
 461 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 462                      struct drm_file *file_priv)
 463 {
 464         struct drm_i915_gem_pread *args = data;
 465         struct drm_gem_object *obj;
 466         struct drm_i915_gem_object *obj_priv;
 467         int ret;
 468
 469         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 470         if (obj == NULL)
 471                 return -ENOENT;
 472         obj_priv = to_intel_bo(obj);
 473
 474         /* Bounds check source.
 475          *
 476          * XXX: This could use review for overflow issues...
 477          */
 478         if (args->offset > obj->size || args->size > obj->size ||
 479             args->offset + args->size > obj->size) {
 480                 drm_gem_object_unreference_unlocked(obj);
 481                 return -EINVAL;
 482         }
 483
 484         if (i915_gem_object_needs_bit17_swizzle(obj)) {
 485                 ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv);
 486         } else {
 487                 ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
 488                 if (ret != 0)
 489                         ret = i915_gem_shmem_pread_slow(dev, obj, args,
 490                                                         file_priv);
 491         }
 492
 493         drm_gem_object_unreference_unlocked(obj);
 494
 495         return ret;
 496 }
 497
 498 /* This is the fast write path which cannot handle
 499  * page faults in the source data
 500  */
 501
 502 static inline int
 503 fast_user_write(struct io_mapping *mapping,
 504                 loff_t page_base, int page_offset,
 505                 char __user *user_data,
 506                 int length)
 507 {
 508         char *vaddr_atomic;
 509         unsigned long unwritten;
 510
 511         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base, KM_USER0);
 512         unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
 513                                                       user_data, length);
 514         io_mapping_unmap_atomic(vaddr_atomic, KM_USER0);
 515         if (unwritten)
 516                 return -EFAULT;
 517         return 0;
 518 }
 519
 520 /* Here's the write path which can sleep for
 521  * page faults
 522  */
 523
 524 static inline void
 525 slow_kernel_write(struct io_mapping *mapping,
 526                   loff_t gtt_base, int gtt_offset,
 527                   struct page *user_page, int user_offset,
 528                   int length)
 529 {
 530         char __iomem *dst_vaddr;
 531         char *src_vaddr;
 532
 533         dst_vaddr = io_mapping_map_wc(mapping, gtt_base);
 534         src_vaddr = kmap(user_page);
 535
 536         memcpy_toio(dst_vaddr + gtt_offset,
 537                     src_vaddr + user_offset,
 538                     length);
 539
 540         kunmap(user_page);
 541         io_mapping_unmap(dst_vaddr);
 542 }
 543
 544 static inline int
 545 fast_shmem_write(struct page **pages,
 546                  loff_t page_base, int page_offset,
 547                  char __user *data,
 548                  int length)
 549 {
 550         char __iomem *vaddr;
 551         unsigned long unwritten;
 552
 553         vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
 554         if (vaddr == NULL)
 555                 return -ENOMEM;
 556         unwritten = __copy_from_user_inatomic(vaddr + page_offset, data, length);
 557         kunmap_atomic(vaddr, KM_USER0);
 558
 559         if (unwritten)
 560                 return -EFAULT;
 561         return 0;
 562 }
 563
 564 /**
 565  * This is the fast pwrite path, where we copy the data directly from the
 566  * user into the GTT, uncached.
 567  */
 568 static int
 569 i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
 570                          struct drm_i915_gem_pwrite *args,
 571                          struct drm_file *file_priv)
 572 {
 573         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 574         drm_i915_private_t *dev_priv = dev->dev_private;
 575         ssize_t remain;
 576         loff_t offset, page_base;
 577         char __user *user_data;
 578         int page_offset, page_length;
 579         int ret;
 580
 581         user_data = (char __user *) (uintptr_t) args->data_ptr;
 582         remain = args->size;
 583         if (!access_ok(VERIFY_READ, user_data, remain))
 584                 return -EFAULT;
 585
 586
 587         mutex_lock(&dev->struct_mutex);
 588         ret = i915_gem_object_pin(obj, 0);
 589         if (ret) {
 590                 mutex_unlock(&dev->struct_mutex);
 591                 return ret;
 592         }
 593         ret = i915_gem_object_set_to_gtt_domain(obj, 1);
 594         if (ret)
 595                 goto fail;
 596
 597         obj_priv = to_intel_bo(obj);
 598         offset = obj_priv->gtt_offset + args->offset;
 599
 600         while (remain > 0) {
 601                 /* Operation in this page
 602                  *
 603                  * page_base = page offset within aperture
 604                  * page_offset = offset within page
 605                  * page_length = bytes to copy for this page
 606                  */
 607                 page_base = (offset & ~(PAGE_SIZE-1));
 608                 page_offset = offset & (PAGE_SIZE-1);
 609                 page_length = remain;
 610                 if ((page_offset + remain) > PAGE_SIZE)
 611                         page_length = PAGE_SIZE - page_offset;
 612
 613                 ret = fast_user_write (dev_priv->mm.gtt_mapping, page_base,
 614                                        page_offset, user_data, page_length);
 615
 616                 /* If we get a fault while copying data, then (presumably) our
 617                  * source page isn't available.  Return the error and we'll
 618                  * retry in the slow path.
 619                  */
 620                 if (ret)
 621                         goto fail;
 622
 623                 remain -= page_length;
 624                 user_data += page_length;
 625                 offset += page_length;
 626         }
 627
 628 fail:
 629         i915_gem_object_unpin(obj);
 630         mutex_unlock(&dev->struct_mutex);
 631
 632         return ret;
 633 }
 634
 635 /**
 636  * This is the fallback GTT pwrite path, which uses get_user_pages to pin
 637  * the memory and maps it using kmap_atomic for copying.
 638  *
 639  * This code resulted in x11perf -rgb10text consuming about 10% more CPU
 640  * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
 641  */
 642 static int
 643 i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
 644                          struct drm_i915_gem_pwrite *args,
 645                          struct drm_file *file_priv)
 646 {
 647         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 648         drm_i915_private_t *dev_priv = dev->dev_private;
 649         ssize_t remain;
 650         loff_t gtt_page_base, offset;
 651         loff_t first_data_page, last_data_page, num_pages;
 652         loff_t pinned_pages, i;
 653         struct page **user_pages;
 654         struct mm_struct *mm = current->mm;
 655         int gtt_page_offset, data_page_offset, data_page_index, page_length;
 656         int ret;
 657         uint64_t data_ptr = args->data_ptr;
 658
 659         remain = args->size;
 660
 661         /* Pin the user pages containing the data.  We can't fault while
 662          * holding the struct mutex, and all of the pwrite implementations
 663          * want to hold it while dereferencing the user data.
 664          */
 665         first_data_page = data_ptr / PAGE_SIZE;
 666         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 667         num_pages = last_data_page - first_data_page + 1;
 668
 669         user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
 670         if (user_pages == NULL)
 671                 return -ENOMEM;
 672
 673         down_read(&mm->mmap_sem);
 674         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 675                                       num_pages, 0, 0, user_pages, NULL);
 676         up_read(&mm->mmap_sem);
 677         if (pinned_pages < num_pages) {
 678                 ret = -EFAULT;
 679                 goto out_unpin_pages;
 680         }
 681
 682         mutex_lock(&dev->struct_mutex);
 683         ret = i915_gem_object_pin(obj, 0);
 684         if (ret)
 685                 goto out_unlock;
 686
 687         ret = i915_gem_object_set_to_gtt_domain(obj, 1);
 688         if (ret)
 689                 goto out_unpin_object;
 690
 691         obj_priv = to_intel_bo(obj);
 692         offset = obj_priv->gtt_offset + args->offset;
 693
 694         while (remain > 0) {
 695                 /* Operation in this page
 696                  *
 697                  * gtt_page_base = page offset within aperture
 698                  * gtt_page_offset = offset within page in aperture
 699                  * data_page_index = page number in get_user_pages return
 700                  * data_page_offset = offset with data_page_index page.
 701                  * page_length = bytes to copy for this page
 702                  */
 703                 gtt_page_base = offset & PAGE_MASK;
 704                 gtt_page_offset = offset & ~PAGE_MASK;
 705                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 706                 data_page_offset = data_ptr & ~PAGE_MASK;
 707
 708                 page_length = remain;
 709                 if ((gtt_page_offset + page_length) > PAGE_SIZE)
 710                         page_length = PAGE_SIZE - gtt_page_offset;
 711                 if ((data_page_offset + page_length) > PAGE_SIZE)
 712                         page_length = PAGE_SIZE - data_page_offset;
 713
 714                 slow_kernel_write(dev_priv->mm.gtt_mapping,
 715                                   gtt_page_base, gtt_page_offset,
 716                                   user_pages[data_page_index],
 717                                   data_page_offset,
 718                                   page_length);
 719
 720                 remain -= page_length;
 721                 offset += page_length;
 722                 data_ptr += page_length;
 723         }
 724
 725 out_unpin_object:
 726         i915_gem_object_unpin(obj);
 727 out_unlock:
 728         mutex_unlock(&dev->struct_mutex);
 729 out_unpin_pages:
 730         for (i = 0; i < pinned_pages; i++)
 731                 page_cache_release(user_pages[i]);
 732         drm_free_large(user_pages);
 733
 734         return ret;
 735 }
 736
 737 /**
 738  * This is the fast shmem pwrite path, which attempts to directly
 739  * copy_from_user into the kmapped pages backing the object.
 740  */
 741 static int
 742 i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
 743                            struct drm_i915_gem_pwrite *args,
 744                            struct drm_file *file_priv)
 745 {
 746         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 747         ssize_t remain;
 748         loff_t offset, page_base;
 749         char __user *user_data;
 750         int page_offset, page_length;
 751         int ret;
 752
 753         user_data = (char __user *) (uintptr_t) args->data_ptr;
 754         remain = args->size;
 755
 756         mutex_lock(&dev->struct_mutex);
 757
 758         ret = i915_gem_object_get_pages(obj, 0);
 759         if (ret != 0)
 760                 goto fail_unlock;
 761
 762         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
 763         if (ret != 0)
 764                 goto fail_put_pages;
 765
 766         obj_priv = to_intel_bo(obj);
 767         offset = args->offset;
 768         obj_priv->dirty = 1;
 769
 770         while (remain > 0) {
 771                 /* Operation in this page
 772                  *
 773                  * page_base = page offset within aperture
 774                  * page_offset = offset within page
 775                  * page_length = bytes to copy for this page
 776                  */
 777                 page_base = (offset & ~(PAGE_SIZE-1));
 778                 page_offset = offset & (PAGE_SIZE-1);
 779                 page_length = remain;
 780                 if ((page_offset + remain) > PAGE_SIZE)
 781                         page_length = PAGE_SIZE - page_offset;
 782
 783                 ret = fast_shmem_write(obj_priv->pages,
 784                                        page_base, page_offset,
 785                                        user_data, page_length);
 786                 if (ret)
 787                         goto fail_put_pages;
 788
 789                 remain -= page_length;
 790                 user_data += page_length;
 791                 offset += page_length;
 792         }
 793
 794 fail_put_pages:
 795         i915_gem_object_put_pages(obj);
 796 fail_unlock:
 797         mutex_unlock(&dev->struct_mutex);
 798
 799         return ret;
 800 }
 801
 802 /**
 803  * This is the fallback shmem pwrite path, which uses get_user_pages to pin
 804  * the memory and maps it using kmap_atomic for copying.
 805  *
 806  * This avoids taking mmap_sem for faulting on the user's address while the
 807  * struct_mutex is held.
 808  */
 809 static int
 810 i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
 811                            struct drm_i915_gem_pwrite *args,
 812                            struct drm_file *file_priv)
 813 {
 814         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 815         struct mm_struct *mm = current->mm;
 816         struct page **user_pages;
 817         ssize_t remain;
 818         loff_t offset, pinned_pages, i;
 819         loff_t first_data_page, last_data_page, num_pages;
 820         int shmem_page_index, shmem_page_offset;
 821         int data_page_index,  data_page_offset;
 822         int page_length;
 823         int ret;
 824         uint64_t data_ptr = args->data_ptr;
 825         int do_bit17_swizzling;
 826
 827         remain = args->size;
 828
 829         /* Pin the user pages containing the data.  We can't fault while
 830          * holding the struct mutex, and all of the pwrite implementations
 831          * want to hold it while dereferencing the user data.
 832          */
 833         first_data_page = data_ptr / PAGE_SIZE;
 834         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 835         num_pages = last_data_page - first_data_page + 1;
 836
 837         user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
 838         if (user_pages == NULL)
 839                 return -ENOMEM;
 840
 841         down_read(&mm->mmap_sem);
 842         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 843                                       num_pages, 0, 0, user_pages, NULL);
 844         up_read(&mm->mmap_sem);
 845         if (pinned_pages < num_pages) {
 846                 ret = -EFAULT;
 847                 goto fail_put_user_pages;
 848         }
 849
 850         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 851
 852         mutex_lock(&dev->struct_mutex);
 853
 854         ret = i915_gem_object_get_pages_or_evict(obj);
 855         if (ret)
 856                 goto fail_unlock;
 857
 858         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
 859         if (ret != 0)
 860                 goto fail_put_pages;
 861
 862         obj_priv = to_intel_bo(obj);
 863         offset = args->offset;
 864         obj_priv->dirty = 1;
 865
 866         while (remain > 0) {
 867                 /* Operation in this page
 868                  *
 869                  * shmem_page_index = page number within shmem file
 870                  * shmem_page_offset = offset within page in shmem file
 871                  * data_page_index = page number in get_user_pages return
 872                  * data_page_offset = offset with data_page_index page.
 873                  * page_length = bytes to copy for this page
 874                  */
 875                 shmem_page_index = offset / PAGE_SIZE;
 876                 shmem_page_offset = offset & ~PAGE_MASK;
 877                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 878                 data_page_offset = data_ptr & ~PAGE_MASK;
 879
 880                 page_length = remain;
 881                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 882                         page_length = PAGE_SIZE - shmem_page_offset;
 883                 if ((data_page_offset + page_length) > PAGE_SIZE)
 884                         page_length = PAGE_SIZE - data_page_offset;
 885
 886                 if (do_bit17_swizzling) {
 887                         slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
 888                                               shmem_page_offset,
 889                                               user_pages[data_page_index],
 890                                               data_page_offset,
 891                                               page_length,
 892                                               0);
 893                 } else {
 894                         slow_shmem_copy(obj_priv->pages[shmem_page_index],
 895                                         shmem_page_offset,
 896                                         user_pages[data_page_index],
 897                                         data_page_offset,
 898                                         page_length);
 899                 }
 900
 901                 remain -= page_length;
 902                 data_ptr += page_length;
 903                 offset += page_length;
 904         }
 905
 906 fail_put_pages:
 907         i915_gem_object_put_pages(obj);
 908 fail_unlock:
 909         mutex_unlock(&dev->struct_mutex);
 910 fail_put_user_pages:
 911         for (i = 0; i < pinned_pages; i++)
 912                 page_cache_release(user_pages[i]);
 913         drm_free_large(user_pages);
 914
 915         return ret;
 916 }
 917
 918 /**
 919  * Writes data to the object referenced by handle.
 920  *
 921  * On error, the contents of the buffer that were to be modified are undefined.
 922  */
 923 int
 924 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 925                       struct drm_file *file_priv)
 926 {
 927         struct drm_i915_gem_pwrite *args = data;
 928         struct drm_gem_object *obj;
 929         struct drm_i915_gem_object *obj_priv;
 930         int ret = 0;
 931
 932         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 933         if (obj == NULL)
 934                 return -ENOENT;
 935         obj_priv = to_intel_bo(obj);
 936
 937         /* Bounds check destination.
 938          *
 939          * XXX: This could use review for overflow issues...
 940          */
 941         if (args->offset > obj->size || args->size > obj->size ||
 942             args->offset + args->size > obj->size) {
 943                 drm_gem_object_unreference_unlocked(obj);
 944                 return -EINVAL;
 945         }
 946
 947         /* We can only do the GTT pwrite on untiled buffers, as otherwise
 948          * it would end up going through the fenced access, and we'll get
 949          * different detiling behavior between reading and writing.
 950          * pread/pwrite currently are reading and writing from the CPU
 951          * perspective, requiring manual detiling by the client.
 952          */
 953         if (obj_priv->phys_obj)
 954                 ret = i915_gem_phys_pwrite(dev, obj, args, file_priv);
 955         else if (obj_priv->tiling_mode == I915_TILING_NONE &&
 956                  dev->gtt_total != 0 &&
 957                  obj->write_domain != I915_GEM_DOMAIN_CPU) {
 958                 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file_priv);
 959                 if (ret == -EFAULT) {
 960                         ret = i915_gem_gtt_pwrite_slow(dev, obj, args,
 961                                                        file_priv);
 962                 }
 963         } else if (i915_gem_object_needs_bit17_swizzle(obj)) {
 964                 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file_priv);
 965         } else {
 966                 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file_priv);
 967                 if (ret == -EFAULT) {
 968                         ret = i915_gem_shmem_pwrite_slow(dev, obj, args,
 969                                                          file_priv);
 970                 }
 971         }
 972
 973 #if WATCH_PWRITE
 974         if (ret)
 975                 DRM_INFO("pwrite failed %d\n", ret);
 976 #endif
 977
 978         drm_gem_object_unreference_unlocked(obj);
 979
 980         return ret;
 981 }
 982
 983 /**
 984  * Called when user space prepares to use an object with the CPU, either
 985  * through the mmap ioctl's mapping or a GTT mapping.
 986  */
 987 int
 988 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 989                           struct drm_file *file_priv)
 990 {
 991         struct drm_i915_private *dev_priv = dev->dev_private;
 992         struct drm_i915_gem_set_domain *args = data;
 993         struct drm_gem_object *obj;
 994         struct drm_i915_gem_object *obj_priv;
 995         uint32_t read_domains = args->read_domains;
 996         uint32_t write_domain = args->write_domain;
 997         int ret;
 998
 999         if (!(dev->driver->driver_features & DRIVER_GEM))
1000                 return -ENODEV;
1001
1002         /* Only handle setting domains to types used by the CPU. */
1003         if (write_domain & I915_GEM_GPU_DOMAINS)
1004                 return -EINVAL;
1005
1006         if (read_domains & I915_GEM_GPU_DOMAINS)
1007                 return -EINVAL;
1008
1009         /* Having something in the write domain implies it's in the read
1010          * domain, and only that read domain.  Enforce that in the request.
1011          */
1012         if (write_domain != 0 && read_domains != write_domain)
1013                 return -EINVAL;
1014
1015         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1016         if (obj == NULL)
1017                 return -ENOENT;
1018         obj_priv = to_intel_bo(obj);
1019
1020         mutex_lock(&dev->struct_mutex);
1021
1022         intel_mark_busy(dev, obj);
1023
1024 #if WATCH_BUF
1025         DRM_INFO("set_domain_ioctl %p(%zd), %08x %08x\n",
1026                  obj, obj->size, read_domains, write_domain);
1027 #endif
1028         if (read_domains & I915_GEM_DOMAIN_GTT) {
1029                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1030
1031                 /* Update the LRU on the fence for the CPU access that's
1032                  * about to occur.
1033                  */
1034                 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1035                         struct drm_i915_fence_reg *reg =
1036                                 &dev_priv->fence_regs[obj_priv->fence_reg];
1037                         list_move_tail(&reg->lru_list,
1038                                        &dev_priv->mm.fence_list);
1039                 }
1040
1041                 /* Silently promote "you're not bound, there was nothing to do"
1042                  * to success, since the client was just asking us to
1043                  * make sure everything was done.
1044                  */
1045                 if (ret == -EINVAL)
1046                         ret = 0;
1047         } else {
1048                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1049         }
1050
1051
1052         /* Maintain LRU order of "inactive" objects */
1053         if (ret == 0 && i915_gem_object_is_inactive(obj_priv))
1054                 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1055
1056         drm_gem_object_unreference(obj);
1057         mutex_unlock(&dev->struct_mutex);
1058         return ret;
1059 }
1060
1061 /**
1062  * Called when user space has done writes to this buffer
1063  */
1064 int
1065 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1066                       struct drm_file *file_priv)
1067 {
1068         struct drm_i915_gem_sw_finish *args = data;
1069         struct drm_gem_object *obj;
1070         struct drm_i915_gem_object *obj_priv;
1071         int ret = 0;
1072
1073         if (!(dev->driver->driver_features & DRIVER_GEM))
1074                 return -ENODEV;
1075
1076         mutex_lock(&dev->struct_mutex);
1077         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1078         if (obj == NULL) {
1079                 mutex_unlock(&dev->struct_mutex);
1080                 return -ENOENT;
1081         }
1082
1083 #if WATCH_BUF
1084         DRM_INFO("%s: sw_finish %d (%p %zd)\n",
1085                  __func__, args->handle, obj, obj->size);
1086 #endif
1087         obj_priv = to_intel_bo(obj);
1088
1089         /* Pinned buffers may be scanout, so flush the cache */
1090         if (obj_priv->pin_count)
1091                 i915_gem_object_flush_cpu_write_domain(obj);
1092
1093         drm_gem_object_unreference(obj);
1094         mutex_unlock(&dev->struct_mutex);
1095         return ret;
1096 }
1097
1098 /**
1099  * Maps the contents of an object, returning the address it is mapped
1100  * into.
1101  *
1102  * While the mapping holds a reference on the contents of the object, it doesn't
1103  * imply a ref on the object itself.
1104  */
1105 int
1106 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1107                    struct drm_file *file_priv)
1108 {
1109         struct drm_i915_gem_mmap *args = data;
1110         struct drm_gem_object *obj;
1111         loff_t offset;
1112         unsigned long addr;
1113
1114         if (!(dev->driver->driver_features & DRIVER_GEM))
1115                 return -ENODEV;
1116
1117         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1118         if (obj == NULL)
1119                 return -ENOENT;
1120
1121         offset = args->offset;
1122
1123         down_write(&current->mm->mmap_sem);
1124         addr = do_mmap(obj->filp, 0, args->size,
1125                        PROT_READ | PROT_WRITE, MAP_SHARED,
1126                        args->offset);
1127         up_write(&current->mm->mmap_sem);
1128         drm_gem_object_unreference_unlocked(obj);
1129         if (IS_ERR((void *)addr))
1130                 return addr;
1131
1132         args->addr_ptr = (uint64_t) addr;
1133
1134         return 0;
1135 }
1136
1137 /**
1138  * i915_gem_fault - fault a page into the GTT
1139  * vma: VMA in question
1140  * vmf: fault info
1141  *
1142  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1143  * from userspace.  The fault handler takes care of binding the object to
1144  * the GTT (if needed), allocating and programming a fence register (again,
1145  * only if needed based on whether the old reg is still valid or the object
1146  * is tiled) and inserting a new PTE into the faulting process.
1147  *
1148  * Note that the faulting process may involve evicting existing objects
1149  * from the GTT and/or fence registers to make room.  So performance may
1150  * suffer if the GTT working set is large or there are few fence registers
1151  * left.
1152  */
1153 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1154 {
1155         struct drm_gem_object *obj = vma->vm_private_data;
1156         struct drm_device *dev = obj->dev;
1157         drm_i915_private_t *dev_priv = dev->dev_private;
1158         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1159         pgoff_t page_offset;
1160         unsigned long pfn;
1161         int ret = 0;
1162         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1163
1164         /* We don't use vmf->pgoff since that has the fake offset */
1165         page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1166                 PAGE_SHIFT;
1167
1168         /* Now bind it into the GTT if needed */
1169         mutex_lock(&dev->struct_mutex);
1170         if (!obj_priv->gtt_space) {
1171                 ret = i915_gem_object_bind_to_gtt(obj, 0);
1172                 if (ret)
1173                         goto unlock;
1174
1175                 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1176                 if (ret)
1177                         goto unlock;
1178         }
1179
1180         /* Need a new fence register? */
1181         if (obj_priv->tiling_mode != I915_TILING_NONE) {
1182                 ret = i915_gem_object_get_fence_reg(obj);
1183                 if (ret)
1184                         goto unlock;
1185         }
1186
1187         if (i915_gem_object_is_inactive(obj_priv))
1188                 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1189
1190         pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
1191                 page_offset;
1192
1193         /* Finally, remap it using the new GTT offset */
1194         ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1195 unlock:
1196         mutex_unlock(&dev->struct_mutex);
1197
1198         switch (ret) {
1199         case 0:
1200         case -ERESTARTSYS:
1201                 return VM_FAULT_NOPAGE;
1202         case -ENOMEM:
1203         case -EAGAIN:
1204                 return VM_FAULT_OOM;
1205         default:
1206                 return VM_FAULT_SIGBUS;
1207         }
1208 }
1209
1210 /**
1211  * i915_gem_create_mmap_offset - create a fake mmap offset for an object
1212  * @obj: obj in question
1213  *
1214  * GEM memory mapping works by handing back to userspace a fake mmap offset
1215  * it can use in a subsequent mmap(2) call.  The DRM core code then looks
1216  * up the object based on the offset and sets up the various memory mapping
1217  * structures.
1218  *
1219  * This routine allocates and attaches a fake offset for @obj.
1220  */
1221 static int
1222 i915_gem_create_mmap_offset(struct drm_gem_object *obj)
1223 {
1224         struct drm_device *dev = obj->dev;
1225         struct drm_gem_mm *mm = dev->mm_private;
1226         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1227         struct drm_map_list *list;
1228         struct drm_local_map *map;
1229         int ret = 0;
1230
1231         /* Set the object up for mmap'ing */
1232         list = &obj->map_list;
1233         list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
1234         if (!list->map)
1235                 return -ENOMEM;
1236
1237         map = list->map;
1238         map->type = _DRM_GEM;
1239         map->size = obj->size;
1240         map->handle = obj;
1241
1242         /* Get a DRM GEM mmap offset allocated... */
1243         list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
1244                                                     obj->size / PAGE_SIZE, 0, 0);
1245         if (!list->file_offset_node) {
1246                 DRM_ERROR("failed to allocate offset for bo %d\n", obj->name);
1247                 ret = -ENOMEM;
1248                 goto out_free_list;
1249         }
1250
1251         list->file_offset_node = drm_mm_get_block(list->file_offset_node,
1252                                                   obj->size / PAGE_SIZE, 0);
1253         if (!list->file_offset_node) {
1254                 ret = -ENOMEM;
1255                 goto out_free_list;
1256         }
1257
1258         list->hash.key = list->file_offset_node->start;
1259         if (drm_ht_insert_item(&mm->offset_hash, &list->hash)) {
1260                 DRM_ERROR("failed to add to map hash\n");
1261                 ret = -ENOMEM;
1262                 goto out_free_mm;
1263         }
1264
1265         /* By now we should be all set, any drm_mmap request on the offset
1266          * below will get to our mmap & fault handler */
1267         obj_priv->mmap_offset = ((uint64_t) list->hash.key) << PAGE_SHIFT;
1268
1269         return 0;
1270
1271 out_free_mm:
1272         drm_mm_put_block(list->file_offset_node);
1273 out_free_list:
1274         kfree(list->map);
1275
1276         return ret;
1277 }
1278
1279 /**
1280  * i915_gem_release_mmap - remove physical page mappings
1281  * @obj: obj in question
1282  *
1283  * Preserve the reservation of the mmapping with the DRM core code, but
1284  * relinquish ownership of the pages back to the system.
1285  *
1286  * It is vital that we remove the page mapping if we have mapped a tiled
1287  * object through the GTT and then lose the fence register due to
1288  * resource pressure. Similarly if the object has been moved out of the
1289  * aperture, than pages mapped into userspace must be revoked. Removing the
1290  * mapping will then trigger a page fault on the next user access, allowing
1291  * fixup by i915_gem_fault().
1292  */
1293 void
1294 i915_gem_release_mmap(struct drm_gem_object *obj)
1295 {
1296         struct drm_device *dev = obj->dev;
1297         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1298
1299         if (dev->dev_mapping)
1300                 unmap_mapping_range(dev->dev_mapping,
1301                                     obj_priv->mmap_offset, obj->size, 1);
1302 }
1303
1304 static void
1305 i915_gem_free_mmap_offset(struct drm_gem_object *obj)
1306 {
1307         struct drm_device *dev = obj->dev;
1308         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1309         struct drm_gem_mm *mm = dev->mm_private;
1310         struct drm_map_list *list;
1311
1312         list = &obj->map_list;
1313         drm_ht_remove_item(&mm->offset_hash, &list->hash);
1314
1315         if (list->file_offset_node) {
1316                 drm_mm_put_block(list->file_offset_node);
1317                 list->file_offset_node = NULL;
1318         }
1319
1320         if (list->map) {
1321                 kfree(list->map);
1322                 list->map = NULL;
1323         }
1324
1325         obj_priv->mmap_offset = 0;
1326 }
1327
1328 /**
1329  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1330  * @obj: object to check
1331  *
1332  * Return the required GTT alignment for an object, taking into account
1333  * potential fence register mapping if needed.
1334  */
1335 static uint32_t
1336 i915_gem_get_gtt_alignment(struct drm_gem_object *obj)
1337 {
1338         struct drm_device *dev = obj->dev;
1339         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1340         int start, i;
1341
1342         /*
1343          * Minimum alignment is 4k (GTT page size), but might be greater
1344          * if a fence register is needed for the object.
1345          */
1346         if (IS_I965G(dev) || obj_priv->tiling_mode == I915_TILING_NONE)
1347                 return 4096;
1348
1349         /*
1350          * Previous chips need to be aligned to the size of the smallest
1351          * fence register that can contain the object.
1352          */
1353         if (IS_I9XX(dev))
1354                 start = 1024*1024;
1355         else
1356                 start = 512*1024;
1357
1358         for (i = start; i < obj->size; i <<= 1)
1359                 ;
1360
1361         return i;
1362 }
1363
1364 /**
1365  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1366  * @dev: DRM device
1367  * @data: GTT mapping ioctl data
1368  * @file_priv: GEM object info
1369  *
1370  * Simply returns the fake offset to userspace so it can mmap it.
1371  * The mmap call will end up in drm_gem_mmap(), which will set things
1372  * up so we can get faults in the handler above.
1373  *
1374  * The fault handler will take care of binding the object into the GTT
1375  * (since it may have been evicted to make room for something), allocating
1376  * a fence register, and mapping the appropriate aperture address into
1377  * userspace.
1378  */
1379 int
1380 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1381                         struct drm_file *file_priv)
1382 {
1383         struct drm_i915_gem_mmap_gtt *args = data;
1384         struct drm_gem_object *obj;
1385         struct drm_i915_gem_object *obj_priv;
1386         int ret;
1387
1388         if (!(dev->driver->driver_features & DRIVER_GEM))
1389                 return -ENODEV;
1390
1391         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1392         if (obj == NULL)
1393                 return -ENOENT;
1394
1395         mutex_lock(&dev->struct_mutex);
1396
1397         obj_priv = to_intel_bo(obj);
1398
1399         if (obj_priv->madv != I915_MADV_WILLNEED) {
1400                 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1401                 drm_gem_object_unreference(obj);
1402                 mutex_unlock(&dev->struct_mutex);
1403                 return -EINVAL;
1404         }
1405
1406
1407         if (!obj_priv->mmap_offset) {
1408                 ret = i915_gem_create_mmap_offset(obj);
1409                 if (ret) {
1410                         drm_gem_object_unreference(obj);
1411                         mutex_unlock(&dev->struct_mutex);
1412                         return ret;
1413                 }
1414         }
1415
1416         args->offset = obj_priv->mmap_offset;
1417
1418         /*
1419          * Pull it into the GTT so that we have a page list (makes the
1420          * initial fault faster and any subsequent flushing possible).
1421          */
1422         if (!obj_priv->agp_mem) {
1423                 ret = i915_gem_object_bind_to_gtt(obj, 0);
1424                 if (ret) {
1425                         drm_gem_object_unreference(obj);
1426                         mutex_unlock(&dev->struct_mutex);
1427                         return ret;
1428                 }
1429         }
1430
1431         drm_gem_object_unreference(obj);
1432         mutex_unlock(&dev->struct_mutex);
1433
1434         return 0;
1435 }
1436
1437 void
1438 i915_gem_object_put_pages(struct drm_gem_object *obj)
1439 {
1440         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1441         int page_count = obj->size / PAGE_SIZE;
1442         int i;
1443
1444         BUG_ON(obj_priv->pages_refcount == 0);
1445         BUG_ON(obj_priv->madv == __I915_MADV_PURGED);
1446
1447         if (--obj_priv->pages_refcount != 0)
1448                 return;
1449
1450         if (obj_priv->tiling_mode != I915_TILING_NONE)
1451                 i915_gem_object_save_bit_17_swizzle(obj);
1452
1453         if (obj_priv->madv == I915_MADV_DONTNEED)
1454                 obj_priv->dirty = 0;
1455
1456         for (i = 0; i < page_count; i++) {
1457                 if (obj_priv->dirty)
1458                         set_page_dirty(obj_priv->pages[i]);
1459
1460                 if (obj_priv->madv == I915_MADV_WILLNEED)
1461                         mark_page_accessed(obj_priv->pages[i]);
1462
1463                 page_cache_release(obj_priv->pages[i]);
1464         }
1465         obj_priv->dirty = 0;
1466
1467         drm_free_large(obj_priv->pages);
1468         obj_priv->pages = NULL;
1469 }
1470
1471 static void
1472 i915_gem_object_move_to_active(struct drm_gem_object *obj, uint32_t seqno,
1473                                struct intel_ring_buffer *ring)
1474 {
1475         struct drm_device *dev = obj->dev;
1476         drm_i915_private_t *dev_priv = dev->dev_private;
1477         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1478         BUG_ON(ring == NULL);
1479         obj_priv->ring = ring;
1480
1481         /* Add a reference if we're newly entering the active list. */
1482         if (!obj_priv->active) {
1483                 drm_gem_object_reference(obj);
1484                 obj_priv->active = 1;
1485         }
1486         /* Move from whatever list we were on to the tail of execution. */
1487         spin_lock(&dev_priv->mm.active_list_lock);
1488         list_move_tail(&obj_priv->list, &ring->active_list);
1489         spin_unlock(&dev_priv->mm.active_list_lock);
1490         obj_priv->last_rendering_seqno = seqno;
1491 }
1492
1493 static void
1494 i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
1495 {
1496         struct drm_device *dev = obj->dev;
1497         drm_i915_private_t *dev_priv = dev->dev_private;
1498         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1499
1500         BUG_ON(!obj_priv->active);
1501         list_move_tail(&obj_priv->list, &dev_priv->mm.flushing_list);
1502         obj_priv->last_rendering_seqno = 0;
1503 }
1504
1505 /* Immediately discard the backing storage */
1506 static void
1507 i915_gem_object_truncate(struct drm_gem_object *obj)
1508 {
1509         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1510         struct inode *inode;
1511
1512         /* Our goal here is to return as much of the memory as
1513          * is possible back to the system as we are called from OOM.
1514          * To do this we must instruct the shmfs to drop all of its
1515          * backing pages, *now*. Here we mirror the actions taken
1516          * when by shmem_delete_inode() to release the backing store.
1517          */
1518         inode = obj->filp->f_path.dentry->d_inode;
1519         truncate_inode_pages(inode->i_mapping, 0);
1520         if (inode->i_op->truncate_range)
1521                 inode->i_op->truncate_range(inode, 0, (loff_t)-1);
1522
1523         obj_priv->madv = __I915_MADV_PURGED;
1524 }
1525
1526 static inline int
1527 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj_priv)
1528 {
1529         return obj_priv->madv == I915_MADV_DONTNEED;
1530 }
1531
1532 static void
1533 i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
1534 {
1535         struct drm_device *dev = obj->dev;
1536         drm_i915_private_t *dev_priv = dev->dev_private;
1537         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1538
1539         i915_verify_inactive(dev, __FILE__, __LINE__);
1540         if (obj_priv->pin_count != 0)
1541                 list_del_init(&obj_priv->list);
1542         else
1543                 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1544
1545         BUG_ON(!list_empty(&obj_priv->gpu_write_list));
1546
1547         obj_priv->last_rendering_seqno = 0;
1548         obj_priv->ring = NULL;
1549         if (obj_priv->active) {
1550                 obj_priv->active = 0;
1551                 drm_gem_object_unreference(obj);
1552         }
1553         i915_verify_inactive(dev, __FILE__, __LINE__);
1554 }
1555
1556 static void
1557 i915_gem_process_flushing_list(struct drm_device *dev,
1558                                uint32_t flush_domains, uint32_t seqno,
1559                                struct intel_ring_buffer *ring)
1560 {
1561         drm_i915_private_t *dev_priv = dev->dev_private;
1562         struct drm_i915_gem_object *obj_priv, *next;
1563
1564         list_for_each_entry_safe(obj_priv, next,
1565                                  &dev_priv->mm.gpu_write_list,
1566                                  gpu_write_list) {
1567                 struct drm_gem_object *obj = &obj_priv->base;
1568
1569                 if ((obj->write_domain & flush_domains) ==
1570                     obj->write_domain &&
1571                     obj_priv->ring->ring_flag == ring->ring_flag) {
1572                         uint32_t old_write_domain = obj->write_domain;
1573
1574                         obj->write_domain = 0;
1575                         list_del_init(&obj_priv->gpu_write_list);
1576                         i915_gem_object_move_to_active(obj, seqno, ring);
1577
1578                         /* update the fence lru list */
1579                         if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1580                                 struct drm_i915_fence_reg *reg =
1581                                         &dev_priv->fence_regs[obj_priv->fence_reg];
1582                                 list_move_tail(&reg->lru_list,
1583                                                 &dev_priv->mm.fence_list);
1584                         }
1585
1586                         trace_i915_gem_object_change_domain(obj,
1587                                                             obj->read_domains,
1588                                                             old_write_domain);
1589                 }
1590         }
1591 }
1592
1593 uint32_t
1594 i915_add_request(struct drm_device *dev, struct drm_file *file_priv,
1595                  uint32_t flush_domains, struct intel_ring_buffer *ring)
1596 {
1597         drm_i915_private_t *dev_priv = dev->dev_private;
1598         struct drm_i915_file_private *i915_file_priv = NULL;
1599         struct drm_i915_gem_request *request;
1600         uint32_t seqno;
1601         int was_empty;
1602
1603         if (file_priv != NULL)
1604                 i915_file_priv = file_priv->driver_priv;
1605
1606         request = kzalloc(sizeof(*request), GFP_KERNEL);
1607         if (request == NULL)
1608                 return 0;
1609
1610         seqno = ring->add_request(dev, ring, file_priv, flush_domains);
1611
1612         request->seqno = seqno;
1613         request->ring = ring;
1614         request->emitted_jiffies = jiffies;
1615         was_empty = list_empty(&ring->request_list);
1616         list_add_tail(&request->list, &ring->request_list);
1617
1618         if (i915_file_priv) {
1619                 list_add_tail(&request->client_list,
1620                               &i915_file_priv->mm.request_list);
1621         } else {
1622                 INIT_LIST_HEAD(&request->client_list);
1623         }
1624
1625         /* Associate any objects on the flushing list matching the write
1626          * domain we're flushing with our flush.
1627          */
1628         if (flush_domains != 0)
1629                 i915_gem_process_flushing_list(dev, flush_domains, seqno, ring);
1630
1631         if (!dev_priv->mm.suspended) {
1632                 mod_timer(&dev_priv->hangcheck_timer, jiffies + DRM_I915_HANGCHECK_PERIOD);
1633                 if (was_empty)
1634                         queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1635         }
1636         return seqno;
1637 }
1638
1639 /**
1640  * Command execution barrier
1641  *
1642  * Ensures that all commands in the ring are finished
1643  * before signalling the CPU
1644  */
1645 static uint32_t
1646 i915_retire_commands(struct drm_device *dev, struct intel_ring_buffer *ring)
1647 {
1648         uint32_t flush_domains = 0;
1649
1650         /* The sampler always gets flushed on i965 (sigh) */
1651         if (IS_I965G(dev))
1652                 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
1653
1654         ring->flush(dev, ring,
1655                         I915_GEM_DOMAIN_COMMAND, flush_domains);
1656         return flush_domains;
1657 }
1658
1659 /**
1660  * Moves buffers associated only with the given active seqno from the active
1661  * to inactive list, potentially freeing them.
1662  */
1663 static void
1664 i915_gem_retire_request(struct drm_device *dev,
1665                         struct drm_i915_gem_request *request)
1666 {
1667         drm_i915_private_t *dev_priv = dev->dev_private;
1668
1669         trace_i915_gem_request_retire(dev, request->seqno);
1670
1671         /* Move any buffers on the active list that are no longer referenced
1672          * by the ringbuffer to the flushing/inactive lists as appropriate.
1673          */
1674         spin_lock(&dev_priv->mm.active_list_lock);
1675         while (!list_empty(&request->ring->active_list)) {
1676                 struct drm_gem_object *obj;
1677                 struct drm_i915_gem_object *obj_priv;
1678
1679                 obj_priv = list_first_entry(&request->ring->active_list,
1680                                             struct drm_i915_gem_object,
1681                                             list);
1682                 obj = &obj_priv->base;
1683
1684                 /* If the seqno being retired doesn't match the oldest in the
1685                  * list, then the oldest in the list must still be newer than
1686                  * this seqno.
1687                  */
1688                 if (obj_priv->last_rendering_seqno != request->seqno)
1689                         goto out;
1690
1691 #if WATCH_LRU
1692                 DRM_INFO("%s: retire %d moves to inactive list %p\n",
1693                          __func__, request->seqno, obj);
1694 #endif
1695
1696                 if (obj->write_domain != 0)
1697                         i915_gem_object_move_to_flushing(obj);
1698                 else {
1699                         /* Take a reference on the object so it won't be
1700                          * freed while the spinlock is held.  The list
1701                          * protection for this spinlock is safe when breaking
1702                          * the lock like this since the next thing we do
1703                          * is just get the head of the list again.
1704                          */
1705                         drm_gem_object_reference(obj);
1706                         i915_gem_object_move_to_inactive(obj);
1707                         spin_unlock(&dev_priv->mm.active_list_lock);
1708                         drm_gem_object_unreference(obj);
1709                         spin_lock(&dev_priv->mm.active_list_lock);
1710                 }
1711         }
1712 out:
1713         spin_unlock(&dev_priv->mm.active_list_lock);
1714 }
1715
1716 /**
1717  * Returns true if seq1 is later than seq2.
1718  */
1719 bool
1720 i915_seqno_passed(uint32_t seq1, uint32_t seq2)
1721 {
1722         return (int32_t)(seq1 - seq2) >= 0;
1723 }
1724
1725 uint32_t
1726 i915_get_gem_seqno(struct drm_device *dev,
1727                    struct intel_ring_buffer *ring)
1728 {
1729         return ring->get_gem_seqno(dev, ring);
1730 }
1731
1732 /**
1733  * This function clears the request list as sequence numbers are passed.
1734  */
1735 static void
1736 i915_gem_retire_requests_ring(struct drm_device *dev,
1737                               struct intel_ring_buffer *ring)
1738 {
1739         drm_i915_private_t *dev_priv = dev->dev_private;
1740         uint32_t seqno;
1741
1742         if (!ring->status_page.page_addr
1743                         || list_empty(&ring->request_list))
1744                 return;
1745
1746         seqno = i915_get_gem_seqno(dev, ring);
1747
1748         while (!list_empty(&ring->request_list)) {
1749                 struct drm_i915_gem_request *request;
1750                 uint32_t retiring_seqno;
1751
1752                 request = list_first_entry(&ring->request_list,
1753                                            struct drm_i915_gem_request,
1754                                            list);
1755                 retiring_seqno = request->seqno;
1756
1757                 if (i915_seqno_passed(seqno, retiring_seqno) ||
1758                     atomic_read(&dev_priv->mm.wedged)) {
1759                         i915_gem_retire_request(dev, request);
1760
1761                         list_del(&request->list);
1762                         list_del(&request->client_list);
1763                         kfree(request);
1764                 } else
1765                         break;
1766         }
1767
1768         if (unlikely (dev_priv->trace_irq_seqno &&
1769                       i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) {
1770
1771                 ring->user_irq_put(dev, ring);
1772                 dev_priv->trace_irq_seqno = 0;
1773         }
1774 }
1775
1776 void
1777 i915_gem_retire_requests(struct drm_device *dev)
1778 {
1779         drm_i915_private_t *dev_priv = dev->dev_private;
1780
1781         if (!list_empty(&dev_priv->mm.deferred_free_list)) {
1782             struct drm_i915_gem_object *obj_priv, *tmp;
1783
1784             /* We must be careful that during unbind() we do not
1785              * accidentally infinitely recurse into retire requests.
1786              * Currently:
1787              *   retire -> free -> unbind -> wait -> retire_ring
1788              */
1789             list_for_each_entry_safe(obj_priv, tmp,
1790                                      &dev_priv->mm.deferred_free_list,
1791                                      list)
1792                     i915_gem_free_object_tail(&obj_priv->base);
1793         }
1794
1795         i915_gem_retire_requests_ring(dev, &dev_priv->render_ring);
1796         if (HAS_BSD(dev))
1797                 i915_gem_retire_requests_ring(dev, &dev_priv->bsd_ring);
1798 }
1799
1800 void
1801 i915_gem_retire_work_handler(struct work_struct *work)
1802 {
1803         drm_i915_private_t *dev_priv;
1804         struct drm_device *dev;
1805
1806         dev_priv = container_of(work, drm_i915_private_t,
1807                                 mm.retire_work.work);
1808         dev = dev_priv->dev;
1809
1810         mutex_lock(&dev->struct_mutex);
1811         i915_gem_retire_requests(dev);
1812
1813         if (!dev_priv->mm.suspended &&
1814                 (!list_empty(&dev_priv->render_ring.request_list) ||
1815                         (HAS_BSD(dev) &&
1816                          !list_empty(&dev_priv->bsd_ring.request_list))))
1817                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1818         mutex_unlock(&dev->struct_mutex);
1819 }
1820
1821 int
1822 i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
1823                 int interruptible, struct intel_ring_buffer *ring)
1824 {
1825         drm_i915_private_t *dev_priv = dev->dev_private;
1826         u32 ier;
1827         int ret = 0;
1828
1829         BUG_ON(seqno == 0);
1830
1831         if (atomic_read(&dev_priv->mm.wedged))
1832                 return -EIO;
1833
1834         if (!i915_seqno_passed(ring->get_gem_seqno(dev, ring), seqno)) {
1835                 if (HAS_PCH_SPLIT(dev))
1836                         ier = I915_READ(DEIER) | I915_READ(GTIER);
1837                 else
1838                         ier = I915_READ(IER);
1839                 if (!ier) {
1840                         DRM_ERROR("something (likely vbetool) disabled "
1841                                   "interrupts, re-enabling\n");
1842                         i915_driver_irq_preinstall(dev);
1843                         i915_driver_irq_postinstall(dev);
1844                 }
1845
1846                 trace_i915_gem_request_wait_begin(dev, seqno);
1847
1848                 ring->waiting_gem_seqno = seqno;
1849                 ring->user_irq_get(dev, ring);
1850                 if (interruptible)
1851                         ret = wait_event_interruptible(ring->irq_queue,
1852                                 i915_seqno_passed(
1853                                         ring->get_gem_seqno(dev, ring), seqno)
1854                                 || atomic_read(&dev_priv->mm.wedged));
1855                 else
1856                         wait_event(ring->irq_queue,
1857                                 i915_seqno_passed(
1858                                         ring->get_gem_seqno(dev, ring), seqno)
1859                                 || atomic_read(&dev_priv->mm.wedged));
1860
1861                 ring->user_irq_put(dev, ring);
1862                 ring->waiting_gem_seqno = 0;
1863
1864                 trace_i915_gem_request_wait_end(dev, seqno);
1865         }
1866         if (atomic_read(&dev_priv->mm.wedged))
1867                 ret = -EIO;
1868
1869         if (ret && ret != -ERESTARTSYS)
1870                 DRM_ERROR("%s returns %d (awaiting %d at %d)\n",
1871                           __func__, ret, seqno, ring->get_gem_seqno(dev, ring));
1872
1873         /* Directly dispatch request retiring.  While we have the work queue
1874          * to handle this, the waiter on a request often wants an associated
1875          * buffer to have made it to the inactive list, and we would need
1876          * a separate wait queue to handle that.
1877          */
1878         if (ret == 0)
1879                 i915_gem_retire_requests_ring(dev, ring);
1880
1881         return ret;
1882 }
1883
1884 /**
1885  * Waits for a sequence number to be signaled, and cleans up the
1886  * request and object lists appropriately for that event.
1887  */
1888 static int
1889 i915_wait_request(struct drm_device *dev, uint32_t seqno,
1890                 struct intel_ring_buffer *ring)
1891 {
1892         return i915_do_wait_request(dev, seqno, 1, ring);
1893 }
1894
1895 static void
1896 i915_gem_flush(struct drm_device *dev,
1897                uint32_t invalidate_domains,
1898                uint32_t flush_domains)
1899 {
1900         drm_i915_private_t *dev_priv = dev->dev_private;
1901         if (flush_domains & I915_GEM_DOMAIN_CPU)
1902                 drm_agp_chipset_flush(dev);
1903         dev_priv->render_ring.flush(dev, &dev_priv->render_ring,
1904                         invalidate_domains,
1905                         flush_domains);
1906
1907         if (HAS_BSD(dev))
1908                 dev_priv->bsd_ring.flush(dev, &dev_priv->bsd_ring,
1909                                 invalidate_domains,
1910                                 flush_domains);
1911 }
1912
1913 /**
1914  * Ensures that all rendering to the object has completed and the object is
1915  * safe to unbind from the GTT or access from the CPU.
1916  */
1917 static int
1918 i915_gem_object_wait_rendering(struct drm_gem_object *obj)
1919 {
1920         struct drm_device *dev = obj->dev;
1921         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1922         int ret;
1923
1924         /* This function only exists to support waiting for existing rendering,
1925          * not for emitting required flushes.
1926          */
1927         BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0);
1928
1929         /* If there is rendering queued on the buffer being evicted, wait for
1930          * it.
1931          */
1932         if (obj_priv->active) {
1933 #if WATCH_BUF
1934                 DRM_INFO("%s: object %p wait for seqno %08x\n",
1935                           __func__, obj, obj_priv->last_rendering_seqno);
1936 #endif
1937                 ret = i915_wait_request(dev,
1938                                 obj_priv->last_rendering_seqno, obj_priv->ring);
1939                 if (ret != 0)
1940                         return ret;
1941         }
1942
1943         return 0;
1944 }
1945
1946 /**
1947  * Unbinds an object from the GTT aperture.
1948  */
1949 int
1950 i915_gem_object_unbind(struct drm_gem_object *obj)
1951 {
1952         struct drm_device *dev = obj->dev;
1953         drm_i915_private_t *dev_priv = dev->dev_private;
1954         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1955         int ret = 0;
1956
1957 #if WATCH_BUF
1958         DRM_INFO("%s:%d %p\n", __func__, __LINE__, obj);
1959         DRM_INFO("gtt_space %p\n", obj_priv->gtt_space);
1960 #endif
1961         if (obj_priv->gtt_space == NULL)
1962                 return 0;
1963
1964         if (obj_priv->pin_count != 0) {
1965                 DRM_ERROR("Attempting to unbind pinned buffer\n");
1966                 return -EINVAL;
1967         }
1968
1969         /* blow away mappings if mapped through GTT */
1970         i915_gem_release_mmap(obj);
1971
1972         /* Move the object to the CPU domain to ensure that
1973          * any possible CPU writes while it's not in the GTT
1974          * are flushed when we go to remap it. This will
1975          * also ensure that all pending GPU writes are finished
1976          * before we unbind.
1977          */
1978         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
1979         if (ret == -ERESTARTSYS)
1980                 return ret;
1981         /* Continue on if we fail due to EIO, the GPU is hung so we
1982          * should be safe and we need to cleanup or else we might
1983          * cause memory corruption through use-after-free.
1984          */
1985
1986         /* release the fence reg _after_ flushing */
1987         if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
1988                 i915_gem_clear_fence_reg(obj);
1989
1990         if (obj_priv->agp_mem != NULL) {
1991                 drm_unbind_agp(obj_priv->agp_mem);
1992                 drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
1993                 obj_priv->agp_mem = NULL;
1994         }
1995
1996         i915_gem_object_put_pages(obj);
1997         BUG_ON(obj_priv->pages_refcount);
1998
1999         if (obj_priv->gtt_space) {
2000                 atomic_dec(&dev->gtt_count);
2001                 atomic_sub(obj->size, &dev->gtt_memory);
2002
2003                 drm_mm_put_block(obj_priv->gtt_space);
2004                 obj_priv->gtt_space = NULL;
2005         }
2006
2007         /* Remove ourselves from the LRU list if present. */
2008         spin_lock(&dev_priv->mm.active_list_lock);
2009         if (!list_empty(&obj_priv->list))
2010                 list_del_init(&obj_priv->list);
2011         spin_unlock(&dev_priv->mm.active_list_lock);
2012
2013         if (i915_gem_object_is_purgeable(obj_priv))
2014                 i915_gem_object_truncate(obj);
2015
2016         trace_i915_gem_object_unbind(obj);
2017
2018         return ret;
2019 }
2020
2021 int
2022 i915_gpu_idle(struct drm_device *dev)
2023 {
2024         drm_i915_private_t *dev_priv = dev->dev_private;
2025         bool lists_empty;
2026         uint32_t seqno1, seqno2;
2027         int ret;
2028
2029         spin_lock(&dev_priv->mm.active_list_lock);
2030         lists_empty = (list_empty(&dev_priv->mm.flushing_list) &&
2031                        list_empty(&dev_priv->render_ring.active_list) &&
2032                        (!HAS_BSD(dev) ||
2033                         list_empty(&dev_priv->bsd_ring.active_list)));
2034         spin_unlock(&dev_priv->mm.active_list_lock);
2035
2036         if (lists_empty)
2037                 return 0;
2038
2039         /* Flush everything onto the inactive list. */
2040         i915_gem_flush(dev, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2041         seqno1 = i915_add_request(dev, NULL, I915_GEM_GPU_DOMAINS,
2042                         &dev_priv->render_ring);
2043         if (seqno1 == 0)
2044                 return -ENOMEM;
2045         ret = i915_wait_request(dev, seqno1, &dev_priv->render_ring);
2046
2047         if (HAS_BSD(dev)) {
2048                 seqno2 = i915_add_request(dev, NULL, I915_GEM_GPU_DOMAINS,
2049                                 &dev_priv->bsd_ring);
2050                 if (seqno2 == 0)
2051                         return -ENOMEM;
2052
2053                 ret = i915_wait_request(dev, seqno2, &dev_priv->bsd_ring);
2054                 if (ret)
2055                         return ret;
2056         }
2057
2058
2059         return ret;
2060 }
2061
2062 int
2063 i915_gem_object_get_pages(struct drm_gem_object *obj,
2064                           gfp_t gfpmask)
2065 {
2066         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2067         int page_count, i;
2068         struct address_space *mapping;
2069         struct inode *inode;
2070         struct page *page;
2071
2072         BUG_ON(obj_priv->pages_refcount
2073                         == DRM_I915_GEM_OBJECT_MAX_PAGES_REFCOUNT);
2074
2075         if (obj_priv->pages_refcount++ != 0)
2076                 return 0;
2077
2078         /* Get the list of pages out of our struct file.  They'll be pinned
2079          * at this point until we release them.
2080          */
2081         page_count = obj->size / PAGE_SIZE;
2082         BUG_ON(obj_priv->pages != NULL);
2083         obj_priv->pages = drm_calloc_large(page_count, sizeof(struct page *));
2084         if (obj_priv->pages == NULL) {
2085                 obj_priv->pages_refcount--;
2086                 return -ENOMEM;
2087         }
2088
2089         inode = obj->filp->f_path.dentry->d_inode;
2090         mapping = inode->i_mapping;
2091         for (i = 0; i < page_count; i++) {
2092                 page = read_cache_page_gfp(mapping, i,
2093                                            GFP_HIGHUSER |
2094                                            __GFP_COLD |
2095                                            __GFP_RECLAIMABLE |
2096                                            gfpmask);
2097                 if (IS_ERR(page))
2098                         goto err_pages;
2099
2100                 obj_priv->pages[i] = page;
2101         }
2102
2103         if (obj_priv->tiling_mode != I915_TILING_NONE)
2104                 i915_gem_object_do_bit_17_swizzle(obj);
2105
2106         return 0;
2107
2108 err_pages:
2109         while (i--)
2110                 page_cache_release(obj_priv->pages[i]);
2111
2112         drm_free_large(obj_priv->pages);
2113         obj_priv->pages = NULL;
2114         obj_priv->pages_refcount--;
2115         return PTR_ERR(page);
2116 }
2117
2118 static void sandybridge_write_fence_reg(struct drm_i915_fence_reg *reg)
2119 {
2120         struct drm_gem_object *obj = reg->obj;
2121         struct drm_device *dev = obj->dev;
2122         drm_i915_private_t *dev_priv = dev->dev_private;
2123         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2124         int regnum = obj_priv->fence_reg;
2125         uint64_t val;
2126
2127         val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2128                     0xfffff000) << 32;
2129         val |= obj_priv->gtt_offset & 0xfffff000;
2130         val |= (uint64_t)((obj_priv->stride / 128) - 1) <<
2131                 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2132
2133         if (obj_priv->tiling_mode == I915_TILING_Y)
2134                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2135         val |= I965_FENCE_REG_VALID;
2136
2137         I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (regnum * 8), val);
2138 }
2139
2140 static void i965_write_fence_reg(struct drm_i915_fence_reg *reg)
2141 {
2142         struct drm_gem_object *obj = reg->obj;
2143         struct drm_device *dev = obj->dev;
2144         drm_i915_private_t *dev_priv = dev->dev_private;
2145         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2146         int regnum = obj_priv->fence_reg;
2147         uint64_t val;
2148
2149         val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2150                     0xfffff000) << 32;
2151         val |= obj_priv->gtt_offset & 0xfffff000;
2152         val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2153         if (obj_priv->tiling_mode == I915_TILING_Y)
2154                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2155         val |= I965_FENCE_REG_VALID;
2156
2157         I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val);
2158 }
2159
2160 static void i915_write_fence_reg(struct drm_i915_fence_reg *reg)
2161 {
2162         struct drm_gem_object *obj = reg->obj;
2163         struct drm_device *dev = obj->dev;
2164         drm_i915_private_t *dev_priv = dev->dev_private;
2165         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2166         int regnum = obj_priv->fence_reg;
2167         int tile_width;
2168         uint32_t fence_reg, val;
2169         uint32_t pitch_val;
2170
2171         if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) ||
2172             (obj_priv->gtt_offset & (obj->size - 1))) {
2173                 WARN(1, "%s: object 0x%08x not 1M or size (0x%zx) aligned\n",
2174                      __func__, obj_priv->gtt_offset, obj->size);
2175                 return;
2176         }
2177
2178         if (obj_priv->tiling_mode == I915_TILING_Y &&
2179             HAS_128_BYTE_Y_TILING(dev))
2180                 tile_width = 128;
2181         else
2182                 tile_width = 512;
2183
2184         /* Note: pitch better be a power of two tile widths */
2185         pitch_val = obj_priv->stride / tile_width;
2186         pitch_val = ffs(pitch_val) - 1;
2187
2188         if (obj_priv->tiling_mode == I915_TILING_Y &&
2189             HAS_128_BYTE_Y_TILING(dev))
2190                 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2191         else
2192                 WARN_ON(pitch_val > I915_FENCE_MAX_PITCH_VAL);
2193
2194         val = obj_priv->gtt_offset;
2195         if (obj_priv->tiling_mode == I915_TILING_Y)
2196                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2197         val |= I915_FENCE_SIZE_BITS(obj->size);
2198         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2199         val |= I830_FENCE_REG_VALID;
2200
2201         if (regnum < 8)
2202                 fence_reg = FENCE_REG_830_0 + (regnum * 4);
2203         else
2204                 fence_reg = FENCE_REG_945_8 + ((regnum - 8) * 4);
2205         I915_WRITE(fence_reg, val);
2206 }
2207
2208 static void i830_write_fence_reg(struct drm_i915_fence_reg *reg)
2209 {
2210         struct drm_gem_object *obj = reg->obj;
2211         struct drm_device *dev = obj->dev;
2212         drm_i915_private_t *dev_priv = dev->dev_private;
2213         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2214         int regnum = obj_priv->fence_reg;
2215         uint32_t val;
2216         uint32_t pitch_val;
2217         uint32_t fence_size_bits;
2218
2219         if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) ||
2220             (obj_priv->gtt_offset & (obj->size - 1))) {
2221                 WARN(1, "%s: object 0x%08x not 512K or size aligned\n",
2222                      __func__, obj_priv->gtt_offset);
2223                 return;
2224         }
2225
2226         pitch_val = obj_priv->stride / 128;
2227         pitch_val = ffs(pitch_val) - 1;
2228         WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2229
2230         val = obj_priv->gtt_offset;
2231         if (obj_priv->tiling_mode == I915_TILING_Y)
2232                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2233         fence_size_bits = I830_FENCE_SIZE_BITS(obj->size);
2234         WARN_ON(fence_size_bits & ~0x00000f00);
2235         val |= fence_size_bits;
2236         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2237         val |= I830_FENCE_REG_VALID;
2238
2239         I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val);
2240 }
2241
2242 static int i915_find_fence_reg(struct drm_device *dev)
2243 {
2244         struct drm_i915_fence_reg *reg = NULL;
2245         struct drm_i915_gem_object *obj_priv = NULL;
2246         struct drm_i915_private *dev_priv = dev->dev_private;
2247         struct drm_gem_object *obj = NULL;
2248         int i, avail, ret;
2249
2250         /* First try to find a free reg */
2251         avail = 0;
2252         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2253                 reg = &dev_priv->fence_regs[i];
2254                 if (!reg->obj)
2255                         return i;
2256
2257                 obj_priv = to_intel_bo(reg->obj);
2258                 if (!obj_priv->pin_count)
2259                     avail++;
2260         }
2261
2262         if (avail == 0)
2263                 return -ENOSPC;
2264
2265         /* None available, try to steal one or wait for a user to finish */
2266         i = I915_FENCE_REG_NONE;
2267         list_for_each_entry(reg, &dev_priv->mm.fence_list,
2268                             lru_list) {
2269                 obj = reg->obj;
2270                 obj_priv = to_intel_bo(obj);
2271
2272                 if (obj_priv->pin_count)
2273                         continue;
2274
2275                 /* found one! */
2276                 i = obj_priv->fence_reg;
2277                 break;
2278         }
2279
2280         BUG_ON(i == I915_FENCE_REG_NONE);
2281
2282         /* We only have a reference on obj from the active list. put_fence_reg
2283          * might drop that one, causing a use-after-free in it. So hold a
2284          * private reference to obj like the other callers of put_fence_reg
2285          * (set_tiling ioctl) do. */
2286         drm_gem_object_reference(obj);
2287         ret = i915_gem_object_put_fence_reg(obj);
2288         drm_gem_object_unreference(obj);
2289         if (ret != 0)
2290                 return ret;
2291
2292         return i;
2293 }
2294
2295 /**
2296  * i915_gem_object_get_fence_reg - set up a fence reg for an object
2297  * @obj: object to map through a fence reg
2298  *
2299  * When mapping objects through the GTT, userspace wants to be able to write
2300  * to them without having to worry about swizzling if the object is tiled.
2301  *
2302  * This function walks the fence regs looking for a free one for @obj,
2303  * stealing one if it can't find any.
2304  *
2305  * It then sets up the reg based on the object's properties: address, pitch
2306  * and tiling format.
2307  */
2308 int
2309 i915_gem_object_get_fence_reg(struct drm_gem_object *obj)
2310 {
2311         struct drm_device *dev = obj->dev;
2312         struct drm_i915_private *dev_priv = dev->dev_private;
2313         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2314         struct drm_i915_fence_reg *reg = NULL;
2315         int ret;
2316
2317         /* Just update our place in the LRU if our fence is getting used. */
2318         if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
2319                 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2320                 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2321                 return 0;
2322         }
2323
2324         switch (obj_priv->tiling_mode) {
2325         case I915_TILING_NONE:
2326                 WARN(1, "allocating a fence for non-tiled object?\n");
2327                 break;
2328         case I915_TILING_X:
2329                 if (!obj_priv->stride)
2330                         return -EINVAL;
2331                 WARN((obj_priv->stride & (512 - 1)),
2332                      "object 0x%08x is X tiled but has non-512B pitch\n",
2333                      obj_priv->gtt_offset);
2334                 break;
2335         case I915_TILING_Y:
2336                 if (!obj_priv->stride)
2337                         return -EINVAL;
2338                 WARN((obj_priv->stride & (128 - 1)),
2339                      "object 0x%08x is Y tiled but has non-128B pitch\n",
2340                      obj_priv->gtt_offset);
2341                 break;
2342         }
2343
2344         ret = i915_find_fence_reg(dev);
2345         if (ret < 0)
2346                 return ret;
2347
2348         obj_priv->fence_reg = ret;
2349         reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2350         list_add_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2351
2352         reg->obj = obj;
2353
2354         if (IS_GEN6(dev))
2355                 sandybridge_write_fence_reg(reg);
2356         else if (IS_I965G(dev))
2357                 i965_write_fence_reg(reg);
2358         else if (IS_I9XX(dev))
2359                 i915_write_fence_reg(reg);
2360         else
2361                 i830_write_fence_reg(reg);
2362
2363         trace_i915_gem_object_get_fence(obj, obj_priv->fence_reg,
2364                         obj_priv->tiling_mode);
2365
2366         return 0;
2367 }
2368
2369 /**
2370  * i915_gem_clear_fence_reg - clear out fence register info
2371  * @obj: object to clear
2372  *
2373  * Zeroes out the fence register itself and clears out the associated
2374  * data structures in dev_priv and obj_priv.
2375  */
2376 static void
2377 i915_gem_clear_fence_reg(struct drm_gem_object *obj)
2378 {
2379         struct drm_device *dev = obj->dev;
2380         drm_i915_private_t *dev_priv = dev->dev_private;
2381         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2382         struct drm_i915_fence_reg *reg =
2383                 &dev_priv->fence_regs[obj_priv->fence_reg];
2384
2385         if (IS_GEN6(dev)) {
2386                 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 +
2387                              (obj_priv->fence_reg * 8), 0);
2388         } else if (IS_I965G(dev)) {
2389                 I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0);
2390         } else {
2391                 uint32_t fence_reg;
2392
2393                 if (obj_priv->fence_reg < 8)
2394                         fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4;
2395                 else
2396                         fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg -
2397                                                        8) * 4;
2398
2399                 I915_WRITE(fence_reg, 0);
2400         }
2401
2402         reg->obj = NULL;
2403         obj_priv->fence_reg = I915_FENCE_REG_NONE;
2404         list_del_init(&reg->lru_list);
2405 }
2406
2407 /**
2408  * i915_gem_object_put_fence_reg - waits on outstanding fenced access
2409  * to the buffer to finish, and then resets the fence register.
2410  * @obj: tiled object holding a fence register.
2411  *
2412  * Zeroes out the fence register itself and clears out the associated
2413  * data structures in dev_priv and obj_priv.
2414  */
2415 int
2416 i915_gem_object_put_fence_reg(struct drm_gem_object *obj)
2417 {
2418         struct drm_device *dev = obj->dev;
2419         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2420
2421         if (obj_priv->fence_reg == I915_FENCE_REG_NONE)
2422                 return 0;
2423
2424         /* If we've changed tiling, GTT-mappings of the object
2425          * need to re-fault to ensure that the correct fence register
2426          * setup is in place.
2427          */
2428         i915_gem_release_mmap(obj);
2429
2430         /* On the i915, GPU access to tiled buffers is via a fence,
2431          * therefore we must wait for any outstanding access to complete
2432          * before clearing the fence.
2433          */
2434         if (!IS_I965G(dev)) {
2435                 int ret;
2436
2437                 ret = i915_gem_object_flush_gpu_write_domain(obj);
2438                 if (ret != 0)
2439                         return ret;
2440
2441                 ret = i915_gem_object_wait_rendering(obj);
2442                 if (ret != 0)
2443                         return ret;
2444         }
2445
2446         i915_gem_object_flush_gtt_write_domain(obj);
2447         i915_gem_clear_fence_reg (obj);
2448
2449         return 0;
2450 }
2451
2452 /**
2453  * Finds free space in the GTT aperture and binds the object there.
2454  */
2455 static int
2456 i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
2457 {
2458         struct drm_device *dev = obj->dev;
2459         drm_i915_private_t *dev_priv = dev->dev_private;
2460         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2461         struct drm_mm_node *free_space;
2462         gfp_t gfpmask =  __GFP_NORETRY | __GFP_NOWARN;
2463         int ret;
2464
2465         if (obj_priv->madv != I915_MADV_WILLNEED) {
2466                 DRM_ERROR("Attempting to bind a purgeable object\n");
2467                 return -EINVAL;
2468         }
2469
2470         if (alignment == 0)
2471                 alignment = i915_gem_get_gtt_alignment(obj);
2472         if (alignment & (i915_gem_get_gtt_alignment(obj) - 1)) {
2473                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2474                 return -EINVAL;
2475         }
2476
2477         /* If the object is bigger than the entire aperture, reject it early
2478          * before evicting everything in a vain attempt to find space.
2479          */
2480         if (obj->size > dev->gtt_total) {
2481                 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
2482                 return -E2BIG;
2483         }
2484
2485  search_free:
2486         free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2487                                         obj->size, alignment, 0);
2488         if (free_space != NULL) {
2489                 obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size,
2490                                                        alignment);
2491                 if (obj_priv->gtt_space != NULL)
2492                         obj_priv->gtt_offset = obj_priv->gtt_space->start;
2493         }
2494         if (obj_priv->gtt_space == NULL) {
2495                 /* If the gtt is empty and we're still having trouble
2496                  * fitting our object in, we're out of memory.
2497                  */
2498 #if WATCH_LRU
2499                 DRM_INFO("%s: GTT full, evicting something\n", __func__);
2500 #endif
2501                 ret = i915_gem_evict_something(dev, obj->size, alignment);
2502                 if (ret)
2503                         return ret;
2504
2505                 goto search_free;
2506         }
2507
2508 #if WATCH_BUF
2509         DRM_INFO("Binding object of size %zd at 0x%08x\n",
2510                  obj->size, obj_priv->gtt_offset);
2511 #endif
2512         ret = i915_gem_object_get_pages(obj, gfpmask);
2513         if (ret) {
2514                 drm_mm_put_block(obj_priv->gtt_space);
2515                 obj_priv->gtt_space = NULL;
2516
2517                 if (ret == -ENOMEM) {
2518                         /* first try to clear up some space from the GTT */
2519                         ret = i915_gem_evict_something(dev, obj->size,
2520                                                        alignment);
2521                         if (ret) {
2522                                 /* now try to shrink everyone else */
2523                                 if (gfpmask) {
2524                                         gfpmask = 0;
2525                                         goto search_free;
2526                                 }
2527
2528                                 return ret;
2529                         }
2530
2531                         goto search_free;
2532                 }
2533
2534                 return ret;
2535         }
2536
2537         /* Create an AGP memory structure pointing at our pages, and bind it
2538          * into the GTT.
2539          */
2540         obj_priv->agp_mem = drm_agp_bind_pages(dev,
2541                                                obj_priv->pages,
2542                                                obj->size >> PAGE_SHIFT,
2543                                                obj_priv->gtt_offset,
2544                                                obj_priv->agp_type);
2545         if (obj_priv->agp_mem == NULL) {
2546                 i915_gem_object_put_pages(obj);
2547                 drm_mm_put_block(obj_priv->gtt_space);
2548                 obj_priv->gtt_space = NULL;
2549
2550                 ret = i915_gem_evict_something(dev, obj->size, alignment);
2551                 if (ret)
2552                         return ret;
2553
2554                 goto search_free;
2555         }
2556         atomic_inc(&dev->gtt_count);
2557         atomic_add(obj->size, &dev->gtt_memory);
2558
2559         /* keep track of bounds object by adding it to the inactive list */
2560         list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
2561
2562         /* Assert that the object is not currently in any GPU domain. As it
2563          * wasn't in the GTT, there shouldn't be any way it could have been in
2564          * a GPU cache
2565          */
2566         BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
2567         BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
2568
2569         trace_i915_gem_object_bind(obj, obj_priv->gtt_offset);
2570
2571         return 0;
2572 }
2573
2574 void
2575 i915_gem_clflush_object(struct drm_gem_object *obj)
2576 {
2577         struct drm_i915_gem_object      *obj_priv = to_intel_bo(obj);
2578
2579         /* If we don't have a page list set up, then we're not pinned
2580          * to GPU, and we can ignore the cache flush because it'll happen
2581          * again at bind time.
2582          */
2583         if (obj_priv->pages == NULL)
2584                 return;
2585
2586         trace_i915_gem_object_clflush(obj);
2587
2588         drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE);
2589 }
2590
2591 /** Flushes any GPU write domain for the object if it's dirty. */
2592 static int
2593 i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj)
2594 {
2595         struct drm_device *dev = obj->dev;
2596         uint32_t old_write_domain;
2597         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2598
2599         if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
2600                 return 0;
2601
2602         /* Queue the GPU write cache flushing we need. */
2603         old_write_domain = obj->write_domain;
2604         i915_gem_flush(dev, 0, obj->write_domain);
2605         if (i915_add_request(dev, NULL, obj->write_domain, obj_priv->ring) == 0)
2606                 return -ENOMEM;
2607
2608         trace_i915_gem_object_change_domain(obj,
2609                                             obj->read_domains,
2610                                             old_write_domain);
2611         return 0;
2612 }
2613
2614 /** Flushes the GTT write domain for the object if it's dirty. */
2615 static void
2616 i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
2617 {
2618         uint32_t old_write_domain;
2619
2620         if (obj->write_domain != I915_GEM_DOMAIN_GTT)
2621                 return;
2622
2623         /* No actual flushing is required for the GTT write domain.   Writes
2624          * to it immediately go to main memory as far as we know, so there's
2625          * no chipset flush.  It also doesn't land in render cache.
2626          */
2627         old_write_domain = obj->write_domain;
2628         obj->write_domain = 0;
2629
2630         trace_i915_gem_object_change_domain(obj,
2631                                             obj->read_domains,
2632                                             old_write_domain);
2633 }
2634
2635 /** Flushes the CPU write domain for the object if it's dirty. */
2636 static void
2637 i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
2638 {
2639         struct drm_device *dev = obj->dev;
2640         uint32_t old_write_domain;
2641
2642         if (obj->write_domain != I915_GEM_DOMAIN_CPU)
2643                 return;
2644
2645         i915_gem_clflush_object(obj);
2646         drm_agp_chipset_flush(dev);
2647         old_write_domain = obj->write_domain;
2648         obj->write_domain = 0;
2649
2650         trace_i915_gem_object_change_domain(obj,
2651                                             obj->read_domains,
2652                                             old_write_domain);
2653 }
2654
2655 int
2656 i915_gem_object_flush_write_domain(struct drm_gem_object *obj)
2657 {
2658         int ret = 0;
2659
2660         switch (obj->write_domain) {
2661         case I915_GEM_DOMAIN_GTT:
2662                 i915_gem_object_flush_gtt_write_domain(obj);
2663                 break;
2664         case I915_GEM_DOMAIN_CPU:
2665                 i915_gem_object_flush_cpu_write_domain(obj);
2666                 break;
2667         default:
2668                 ret = i915_gem_object_flush_gpu_write_domain(obj);
2669                 break;
2670         }
2671
2672         return ret;
2673 }
2674
2675 /**
2676  * Moves a single object to the GTT read, and possibly write domain.
2677  *
2678  * This function returns when the move is complete, including waiting on
2679  * flushes to occur.
2680  */
2681 int
2682 i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
2683 {
2684         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2685         uint32_t old_write_domain, old_read_domains;
2686         int ret;
2687
2688         /* Not valid to be called on unbound objects. */
2689         if (obj_priv->gtt_space == NULL)
2690                 return -EINVAL;
2691
2692         ret = i915_gem_object_flush_gpu_write_domain(obj);
2693         if (ret != 0)
2694                 return ret;
2695
2696         /* Wait on any GPU rendering and flushing to occur. */
2697         ret = i915_gem_object_wait_rendering(obj);
2698         if (ret != 0)
2699                 return ret;
2700
2701         old_write_domain = obj->write_domain;
2702         old_read_domains = obj->read_domains;
2703
2704         /* If we're writing through the GTT domain, then CPU and GPU caches
2705          * will need to be invalidated at next use.
2706          */
2707         if (write)
2708                 obj->read_domains &= I915_GEM_DOMAIN_GTT;
2709
2710         i915_gem_object_flush_cpu_write_domain(obj);
2711
2712         /* It should now be out of any other write domains, and we can update
2713          * the domain values for our changes.
2714          */
2715         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2716         obj->read_domains |= I915_GEM_DOMAIN_GTT;
2717         if (write) {
2718                 obj->write_domain = I915_GEM_DOMAIN_GTT;
2719                 obj_priv->dirty = 1;
2720         }
2721
2722         trace_i915_gem_object_change_domain(obj,
2723                                             old_read_domains,
2724                                             old_write_domain);
2725
2726         return 0;
2727 }
2728
2729 /*
2730  * Prepare buffer for display plane. Use uninterruptible for possible flush
2731  * wait, as in modesetting process we're not supposed to be interrupted.
2732  */
2733 int
2734 i915_gem_object_set_to_display_plane(struct drm_gem_object *obj)
2735 {
2736         struct drm_device *dev = obj->dev;
2737         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2738         uint32_t old_write_domain, old_read_domains;
2739         int ret;
2740
2741         /* Not valid to be called on unbound objects. */
2742         if (obj_priv->gtt_space == NULL)
2743                 return -EINVAL;
2744
2745         ret = i915_gem_object_flush_gpu_write_domain(obj);
2746         if (ret)
2747                 return ret;
2748
2749         /* Wait on any GPU rendering and flushing to occur. */
2750         if (obj_priv->active) {
2751 #if WATCH_BUF
2752                 DRM_INFO("%s: object %p wait for seqno %08x\n",
2753                           __func__, obj, obj_priv->last_rendering_seqno);
2754 #endif
2755                 ret = i915_do_wait_request(dev,
2756                                 obj_priv->last_rendering_seqno,
2757                                 0,
2758                                 obj_priv->ring);
2759                 if (ret != 0)
2760                         return ret;
2761         }
2762
2763         i915_gem_object_flush_cpu_write_domain(obj);
2764
2765         old_write_domain = obj->write_domain;
2766         old_read_domains = obj->read_domains;
2767
2768         /* It should now be out of any other write domains, and we can update
2769          * the domain values for our changes.
2770          */
2771         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2772         obj->read_domains = I915_GEM_DOMAIN_GTT;
2773         obj->write_domain = I915_GEM_DOMAIN_GTT;
2774         obj_priv->dirty = 1;
2775
2776         trace_i915_gem_object_change_domain(obj,
2777                                             old_read_domains,
2778                                             old_write_domain);
2779
2780         return 0;
2781 }
2782
2783 /**
2784  * Moves a single object to the CPU read, and possibly write domain.
2785  *
2786  * This function returns when the move is complete, including waiting on
2787  * flushes to occur.
2788  */
2789 static int
2790 i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
2791 {
2792         uint32_t old_write_domain, old_read_domains;
2793         int ret;
2794
2795         ret = i915_gem_object_flush_gpu_write_domain(obj);
2796         if (ret)
2797                 return ret;
2798
2799         /* Wait on any GPU rendering and flushing to occur. */
2800         ret = i915_gem_object_wait_rendering(obj);
2801         if (ret != 0)
2802                 return ret;
2803
2804         i915_gem_object_flush_gtt_write_domain(obj);
2805
2806         /* If we have a partially-valid cache of the object in the CPU,
2807          * finish invalidating it and free the per-page flags.
2808          */
2809         i915_gem_object_set_to_full_cpu_read_domain(obj);
2810
2811         old_write_domain = obj->write_domain;
2812         old_read_domains = obj->read_domains;
2813
2814         /* Flush the CPU cache if it's still invalid. */
2815         if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2816                 i915_gem_clflush_object(obj);
2817
2818                 obj->read_domains |= I915_GEM_DOMAIN_CPU;
2819         }
2820
2821         /* It should now be out of any other write domains, and we can update
2822          * the domain values for our changes.
2823          */
2824         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2825
2826         /* If we're writing through the CPU, then the GPU read domains will
2827          * need to be invalidated at next use.
2828          */
2829         if (write) {
2830                 obj->read_domains &= I915_GEM_DOMAIN_CPU;
2831                 obj->write_domain = I915_GEM_DOMAIN_CPU;
2832         }
2833
2834         trace_i915_gem_object_change_domain(obj,
2835                                             old_read_domains,
2836                                             old_write_domain);
2837
2838         return 0;
2839 }
2840
2841 /*
2842  * Set the next domain for the specified object. This
2843  * may not actually perform the necessary flushing/invaliding though,
2844  * as that may want to be batched with other set_domain operations
2845  *
2846  * This is (we hope) the only really tricky part of gem. The goal
2847  * is fairly simple -- track which caches hold bits of the object
2848  * and make sure they remain coherent. A few concrete examples may
2849  * help to explain how it works. For shorthand, we use the notation
2850  * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
2851  * a pair of read and write domain masks.
2852  *
2853  * Case 1: the batch buffer
2854  *
2855  *      1. Allocated
2856  *      2. Written by CPU
2857  *      3. Mapped to GTT
2858  *      4. Read by GPU
2859  *      5. Unmapped from GTT
2860  *      6. Freed
2861  *
2862  *      Let's take these a step at a time
2863  *
2864  *      1. Allocated
2865  *              Pages allocated from the kernel may still have
2866  *              cache contents, so we set them to (CPU, CPU) always.
2867  *      2. Written by CPU (using pwrite)
2868  *              The pwrite function calls set_domain (CPU, CPU) and
2869  *              this function does nothing (as nothing changes)
2870  *      3. Mapped by GTT
2871  *              This function asserts that the object is not
2872  *              currently in any GPU-based read or write domains
2873  *      4. Read by GPU
2874  *              i915_gem_execbuffer calls set_domain (COMMAND, 0).
2875  *              As write_domain is zero, this function adds in the
2876  *              current read domains (CPU+COMMAND, 0).
2877  *              flush_domains is set to CPU.
2878  *              invalidate_domains is set to COMMAND
2879  *              clflush is run to get data out of the CPU caches
2880  *              then i915_dev_set_domain calls i915_gem_flush to
2881  *              emit an MI_FLUSH and drm_agp_chipset_flush
2882  *      5. Unmapped from GTT
2883  *              i915_gem_object_unbind calls set_domain (CPU, CPU)
2884  *              flush_domains and invalidate_domains end up both zero
2885  *              so no flushing/invalidating happens
2886  *      6. Freed
2887  *              yay, done
2888  *
2889  * Case 2: The shared render buffer
2890  *
2891  *      1. Allocated
2892  *      2. Mapped to GTT
2893  *      3. Read/written by GPU
2894  *      4. set_domain to (CPU,CPU)
2895  *      5. Read/written by CPU
2896  *      6. Read/written by GPU
2897  *
2898  *      1. Allocated
2899  *              Same as last example, (CPU, CPU)
2900  *      2. Mapped to GTT
2901  *              Nothing changes (assertions find that it is not in the GPU)
2902  *      3. Read/written by GPU
2903  *              execbuffer calls set_domain (RENDER, RENDER)
2904  *              flush_domains gets CPU
2905  *              invalidate_domains gets GPU
2906  *              clflush (obj)
2907  *              MI_FLUSH and drm_agp_chipset_flush
2908  *      4. set_domain (CPU, CPU)
2909  *              flush_domains gets GPU
2910  *              invalidate_domains gets CPU
2911  *              wait_rendering (obj) to make sure all drawing is complete.
2912  *              This will include an MI_FLUSH to get the data from GPU
2913  *              to memory
2914  *              clflush (obj) to invalidate the CPU cache
2915  *              Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
2916  *      5. Read/written by CPU
2917  *              cache lines are loaded and dirtied
2918  *      6. Read written by GPU
2919  *              Same as last GPU access
2920  *
2921  * Case 3: The constant buffer
2922  *
2923  *      1. Allocated
2924  *      2. Written by CPU
2925  *      3. Read by GPU
2926  *      4. Updated (written) by CPU again
2927  *      5. Read by GPU
2928  *
2929  *      1. Allocated
2930  *              (CPU, CPU)
2931  *      2. Written by CPU
2932  *              (CPU, CPU)
2933  *      3. Read by GPU
2934  *              (CPU+RENDER, 0)
2935  *              flush_domains = CPU
2936  *              invalidate_domains = RENDER
2937  *              clflush (obj)
2938  *              MI_FLUSH
2939  *              drm_agp_chipset_flush
2940  *      4. Updated (written) by CPU again
2941  *              (CPU, CPU)
2942  *              flush_domains = 0 (no previous write domain)
2943  *              invalidate_domains = 0 (no new read domains)
2944  *      5. Read by GPU
2945  *              (CPU+RENDER, 0)
2946  *              flush_domains = CPU
2947  *              invalidate_domains = RENDER
2948  *              clflush (obj)
2949  *              MI_FLUSH
2950  *              drm_agp_chipset_flush
2951  */
2952 static void
2953 i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
2954 {
2955         struct drm_device               *dev = obj->dev;
2956         drm_i915_private_t              *dev_priv = dev->dev_private;
2957         struct drm_i915_gem_object      *obj_priv = to_intel_bo(obj);
2958         uint32_t                        invalidate_domains = 0;
2959         uint32_t                        flush_domains = 0;
2960         uint32_t                        old_read_domains;
2961
2962         BUG_ON(obj->pending_read_domains & I915_GEM_DOMAIN_CPU);
2963         BUG_ON(obj->pending_write_domain == I915_GEM_DOMAIN_CPU);
2964
2965         intel_mark_busy(dev, obj);
2966
2967 #if WATCH_BUF
2968         DRM_INFO("%s: object %p read %08x -> %08x write %08x -> %08x\n",
2969                  __func__, obj,
2970                  obj->read_domains, obj->pending_read_domains,
2971                  obj->write_domain, obj->pending_write_domain);
2972 #endif
2973         /*
2974          * If the object isn't moving to a new write domain,
2975          * let the object stay in multiple read domains
2976          */
2977         if (obj->pending_write_domain == 0)
2978                 obj->pending_read_domains |= obj->read_domains;
2979         else
2980                 obj_priv->dirty = 1;
2981
2982         /*
2983          * Flush the current write domain if
2984          * the new read domains don't match. Invalidate
2985          * any read domains which differ from the old
2986          * write domain
2987          */
2988         if (obj->write_domain &&
2989             obj->write_domain != obj->pending_read_domains) {
2990                 flush_domains |= obj->write_domain;
2991                 invalidate_domains |=
2992                         obj->pending_read_domains & ~obj->write_domain;
2993         }
2994         /*
2995          * Invalidate any read caches which may have
2996          * stale data. That is, any new read domains.
2997          */
2998         invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
2999         if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) {
3000 #if WATCH_BUF
3001                 DRM_INFO("%s: CPU domain flush %08x invalidate %08x\n",
3002                          __func__, flush_domains, invalidate_domains);
3003 #endif
3004                 i915_gem_clflush_object(obj);
3005         }
3006
3007         old_read_domains = obj->read_domains;
3008
3009         /* The actual obj->write_domain will be updated with
3010          * pending_write_domain after we emit the accumulated flush for all
3011          * of our domain changes in execbuffers (which clears objects'
3012          * write_domains).  So if we have a current write domain that we
3013          * aren't changing, set pending_write_domain to that.
3014          */
3015         if (flush_domains == 0 && obj->pending_write_domain == 0)
3016                 obj->pending_write_domain = obj->write_domain;
3017         obj->read_domains = obj->pending_read_domains;
3018
3019         if (flush_domains & I915_GEM_GPU_DOMAINS) {
3020                 if (obj_priv->ring == &dev_priv->render_ring)
3021                         dev_priv->flush_rings |= FLUSH_RENDER_RING;
3022                 else if (obj_priv->ring == &dev_priv->bsd_ring)
3023                         dev_priv->flush_rings |= FLUSH_BSD_RING;
3024         }
3025
3026         dev->invalidate_domains |= invalidate_domains;
3027         dev->flush_domains |= flush_domains;
3028 #if WATCH_BUF
3029         DRM_INFO("%s: read %08x write %08x invalidate %08x flush %08x\n",
3030                  __func__,
3031                  obj->read_domains, obj->write_domain,
3032                  dev->invalidate_domains, dev->flush_domains);
3033 #endif
3034
3035         trace_i915_gem_object_change_domain(obj,
3036                                             old_read_domains,
3037                                             obj->write_domain);
3038 }
3039
3040 /**
3041  * Moves the object from a partially CPU read to a full one.
3042  *
3043  * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
3044  * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
3045  */
3046 static void
3047 i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
3048 {
3049         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3050
3051         if (!obj_priv->page_cpu_valid)
3052                 return;
3053
3054         /* If we're partially in the CPU read domain, finish moving it in.
3055          */
3056         if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
3057                 int i;
3058
3059                 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
3060                         if (obj_priv->page_cpu_valid[i])
3061                                 continue;
3062                         drm_clflush_pages(obj_priv->pages + i, 1);
3063                 }
3064         }
3065
3066         /* Free the page_cpu_valid mappings which are now stale, whether
3067          * or not we've got I915_GEM_DOMAIN_CPU.
3068          */
3069         kfree(obj_priv->page_cpu_valid);
3070         obj_priv->page_cpu_valid = NULL;
3071 }
3072
3073 /**
3074  * Set the CPU read domain on a range of the object.
3075  *
3076  * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
3077  * not entirely valid.  The page_cpu_valid member of the object flags which
3078  * pages have been flushed, and will be respected by
3079  * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
3080  * of the whole object.
3081  *
3082  * This function returns when the move is complete, including waiting on
3083  * flushes to occur.
3084  */
3085 static int
3086 i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
3087                                           uint64_t offset, uint64_t size)
3088 {
3089         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3090         uint32_t old_read_domains;
3091         int i, ret;
3092
3093         if (offset == 0 && size == obj->size)
3094                 return i915_gem_object_set_to_cpu_domain(obj, 0);
3095
3096         ret = i915_gem_object_flush_gpu_write_domain(obj);
3097         if (ret)
3098                 return ret;
3099
3100         /* Wait on any GPU rendering and flushing to occur. */
3101         ret = i915_gem_object_wait_rendering(obj);
3102         if (ret != 0)
3103                 return ret;
3104         i915_gem_object_flush_gtt_write_domain(obj);
3105
3106         /* If we're already fully in the CPU read domain, we're done. */
3107         if (obj_priv->page_cpu_valid == NULL &&
3108             (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
3109                 return 0;
3110
3111         /* Otherwise, create/clear the per-page CPU read domain flag if we're
3112          * newly adding I915_GEM_DOMAIN_CPU
3113          */
3114         if (obj_priv->page_cpu_valid == NULL) {
3115                 obj_priv->page_cpu_valid = kzalloc(obj->size / PAGE_SIZE,
3116                                                    GFP_KERNEL);
3117                 if (obj_priv->page_cpu_valid == NULL)
3118                         return -ENOMEM;
3119         } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
3120                 memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
3121
3122         /* Flush the cache on any pages that are still invalid from the CPU's
3123          * perspective.
3124          */
3125         for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3126              i++) {
3127                 if (obj_priv->page_cpu_valid[i])
3128                         continue;
3129
3130                 drm_clflush_pages(obj_priv->pages + i, 1);
3131
3132                 obj_priv->page_cpu_valid[i] = 1;
3133         }
3134
3135         /* It should now be out of any other write domains, and we can update
3136          * the domain values for our changes.
3137          */
3138         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3139
3140         old_read_domains = obj->read_domains;
3141         obj->read_domains |= I915_GEM_DOMAIN_CPU;
3142
3143         trace_i915_gem_object_change_domain(obj,
3144                                             old_read_domains,
3145                                             obj->write_domain);
3146
3147         return 0;
3148 }
3149
3150 /**
3151  * Pin an object to the GTT and evaluate the relocations landing in it.
3152  */
3153 static int
3154 i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
3155                                  struct drm_file *file_priv,
3156                                  struct drm_i915_gem_exec_object2 *entry,
3157                                  struct drm_i915_gem_relocation_entry *relocs)
3158 {
3159         struct drm_device *dev = obj->dev;
3160         drm_i915_private_t *dev_priv = dev->dev_private;
3161         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3162         int i, ret;
3163         void __iomem *reloc_page;
3164         bool need_fence;
3165
3166         need_fence = entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
3167                      obj_priv->tiling_mode != I915_TILING_NONE;
3168
3169         /* Check fence reg constraints and rebind if necessary */
3170         if (need_fence &&
3171             !i915_gem_object_fence_offset_ok(obj,
3172                                              obj_priv->tiling_mode)) {
3173                 ret = i915_gem_object_unbind(obj);
3174                 if (ret)
3175                         return ret;
3176         }
3177
3178         /* Choose the GTT offset for our buffer and put it there. */
3179         ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment);
3180         if (ret)
3181                 return ret;
3182
3183         /*
3184          * Pre-965 chips need a fence register set up in order to
3185          * properly handle blits to/from tiled surfaces.
3186          */
3187         if (need_fence) {
3188                 ret = i915_gem_object_get_fence_reg(obj);
3189                 if (ret != 0) {
3190                         i915_gem_object_unpin(obj);
3191                         return ret;
3192                 }
3193         }
3194
3195         entry->offset = obj_priv->gtt_offset;
3196
3197         /* Apply the relocations, using the GTT aperture to avoid cache
3198          * flushing requirements.
3199          */
3200         for (i = 0; i < entry->relocation_count; i++) {
3201                 struct drm_i915_gem_relocation_entry *reloc= &relocs[i];
3202                 struct drm_gem_object *target_obj;
3203                 struct drm_i915_gem_object *target_obj_priv;
3204                 uint32_t reloc_val, reloc_offset;
3205                 uint32_t __iomem *reloc_entry;
3206
3207                 target_obj = drm_gem_object_lookup(obj->dev, file_priv,
3208                                                    reloc->target_handle);
3209                 if (target_obj == NULL) {
3210                         i915_gem_object_unpin(obj);
3211                         return -ENOENT;
3212                 }
3213                 target_obj_priv = to_intel_bo(target_obj);
3214
3215 #if WATCH_RELOC
3216                 DRM_INFO("%s: obj %p offset %08x target %d "
3217                          "read %08x write %08x gtt %08x "
3218                          "presumed %08x delta %08x\n",
3219                          __func__,
3220                          obj,
3221                          (int) reloc->offset,
3222                          (int) reloc->target_handle,
3223                          (int) reloc->read_domains,
3224                          (int) reloc->write_domain,
3225                          (int) target_obj_priv->gtt_offset,
3226                          (int) reloc->presumed_offset,
3227                          reloc->delta);
3228 #endif
3229
3230                 /* The target buffer should have appeared before us in the
3231                  * exec_object list, so it should have a GTT space bound by now.
3232                  */
3233                 if (target_obj_priv->gtt_space == NULL) {
3234                         DRM_ERROR("No GTT space found for object %d\n",
3235                                   reloc->target_handle);
3236                         drm_gem_object_unreference(target_obj);
3237                         i915_gem_object_unpin(obj);
3238                         return -EINVAL;
3239                 }
3240
3241                 /* Validate that the target is in a valid r/w GPU domain */
3242                 if (reloc->write_domain & (reloc->write_domain - 1)) {
3243                         DRM_ERROR("reloc with multiple write domains: "
3244                                   "obj %p target %d offset %d "
3245                                   "read %08x write %08x",
3246                                   obj, reloc->target_handle,
3247                                   (int) reloc->offset,
3248                                   reloc->read_domains,
3249                                   reloc->write_domain);
3250                         return -EINVAL;
3251                 }
3252                 if (reloc->write_domain & I915_GEM_DOMAIN_CPU ||
3253                     reloc->read_domains & I915_GEM_DOMAIN_CPU) {
3254                         DRM_ERROR("reloc with read/write CPU domains: "
3255                                   "obj %p target %d offset %d "
3256                                   "read %08x write %08x",
3257                                   obj, reloc->target_handle,
3258                                   (int) reloc->offset,
3259                                   reloc->read_domains,
3260                                   reloc->write_domain);
3261                         drm_gem_object_unreference(target_obj);
3262                         i915_gem_object_unpin(obj);
3263                         return -EINVAL;
3264                 }
3265                 if (reloc->write_domain && target_obj->pending_write_domain &&
3266                     reloc->write_domain != target_obj->pending_write_domain) {
3267                         DRM_ERROR("Write domain conflict: "
3268                                   "obj %p target %d offset %d "
3269                                   "new %08x old %08x\n",
3270                                   obj, reloc->target_handle,
3271                                   (int) reloc->offset,
3272                                   reloc->write_domain,
3273                                   target_obj->pending_write_domain);
3274                         drm_gem_object_unreference(target_obj);
3275                         i915_gem_object_unpin(obj);
3276                         return -EINVAL;
3277                 }
3278
3279                 target_obj->pending_read_domains |= reloc->read_domains;
3280                 target_obj->pending_write_domain |= reloc->write_domain;
3281
3282                 /* If the relocation already has the right value in it, no
3283                  * more work needs to be done.
3284                  */
3285                 if (target_obj_priv->gtt_offset == reloc->presumed_offset) {
3286                         drm_gem_object_unreference(target_obj);
3287                         continue;
3288                 }
3289
3290                 /* Check that the relocation address is valid... */
3291                 if (reloc->offset > obj->size - 4) {
3292                         DRM_ERROR("Relocation beyond object bounds: "
3293                                   "obj %p target %d offset %d size %d.\n",
3294                                   obj, reloc->target_handle,
3295                                   (int) reloc->offset, (int) obj->size);
3296                         drm_gem_object_unreference(target_obj);
3297                         i915_gem_object_unpin(obj);
3298                         return -EINVAL;
3299                 }
3300                 if (reloc->offset & 3) {
3301                         DRM_ERROR("Relocation not 4-byte aligned: "
3302                                   "obj %p target %d offset %d.\n",
3303                                   obj, reloc->target_handle,
3304                                   (int) reloc->offset);
3305                         drm_gem_object_unreference(target_obj);
3306                         i915_gem_object_unpin(obj);
3307                         return -EINVAL;
3308                 }
3309
3310                 /* and points to somewhere within the target object. */
3311                 if (reloc->delta >= target_obj->size) {
3312                         DRM_ERROR("Relocation beyond target object bounds: "
3313                                   "obj %p target %d delta %d size %d.\n",
3314                                   obj, reloc->target_handle,
3315                                   (int) reloc->delta, (int) target_obj->size);
3316                         drm_gem_object_unreference(target_obj);
3317                         i915_gem_object_unpin(obj);
3318                         return -EINVAL;
3319                 }
3320
3321                 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
3322                 if (ret != 0) {
3323                         drm_gem_object_unreference(target_obj);
3324                         i915_gem_object_unpin(obj);
3325                         return -EINVAL;
3326                 }
3327
3328                 /* Map the page containing the relocation we're going to
3329                  * perform.
3330                  */
3331                 reloc_offset = obj_priv->gtt_offset + reloc->offset;
3332                 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
3333                                                       (reloc_offset &
3334                                                        ~(PAGE_SIZE - 1)),
3335                                                       KM_USER0);
3336                 reloc_entry = (uint32_t __iomem *)(reloc_page +
3337                                                    (reloc_offset & (PAGE_SIZE - 1)));
3338                 reloc_val = target_obj_priv->gtt_offset + reloc->delta;
3339
3340 #if WATCH_BUF
3341                 DRM_INFO("Applied relocation: %p@0x%08x %08x -> %08x\n",
3342                           obj, (unsigned int) reloc->offset,
3343                           readl(reloc_entry), reloc_val);
3344 #endif
3345                 writel(reloc_val, reloc_entry);
3346                 io_mapping_unmap_atomic(reloc_page, KM_USER0);
3347
3348                 /* The updated presumed offset for this entry will be
3349                  * copied back out to the user.
3350                  */
3351                 reloc->presumed_offset = target_obj_priv->gtt_offset;
3352
3353                 drm_gem_object_unreference(target_obj);
3354         }
3355
3356 #if WATCH_BUF
3357         if (0)
3358                 i915_gem_dump_object(obj, 128, __func__, ~0);
3359 #endif
3360         return 0;
3361 }
3362
3363 /* Throttle our rendering by waiting until the ring has completed our requests
3364  * emitted over 20 msec ago.
3365  *
3366  * Note that if we were to use the current jiffies each time around the loop,
3367  * we wouldn't escape the function with any frames outstanding if the time to
3368  * render a frame was over 20ms.
3369  *
3370  * This should get us reasonable parallelism between CPU and GPU but also
3371  * relatively low latency when blocking on a particular request to finish.
3372  */
3373 static int
3374 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file_priv)
3375 {
3376         struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
3377         int ret = 0;
3378         unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
3379
3380         mutex_lock(&dev->struct_mutex);
3381         while (!list_empty(&i915_file_priv->mm.request_list)) {
3382                 struct drm_i915_gem_request *request;
3383
3384                 request = list_first_entry(&i915_file_priv->mm.request_list,
3385                                            struct drm_i915_gem_request,
3386                                            client_list);
3387
3388                 if (time_after_eq(request->emitted_jiffies, recent_enough))
3389                         break;
3390
3391                 ret = i915_wait_request(dev, request->seqno, request->ring);
3392                 if (ret != 0)
3393                         break;
3394         }
3395         mutex_unlock(&dev->struct_mutex);
3396
3397         return ret;
3398 }
3399
3400 static int
3401 i915_gem_get_relocs_from_user(struct drm_i915_gem_exec_object2 *exec_list,
3402                               uint32_t buffer_count,
3403                               struct drm_i915_gem_relocation_entry **relocs)
3404 {
3405         uint32_t reloc_count = 0, reloc_index = 0, i;
3406         int ret;
3407
3408         *relocs = NULL;
3409         for (i = 0; i < buffer_count; i++) {
3410                 if (reloc_count + exec_list[i].relocation_count < reloc_count)
3411                         return -EINVAL;
3412                 reloc_count += exec_list[i].relocation_count;
3413         }
3414
3415         *relocs = drm_calloc_large(reloc_count, sizeof(**relocs));
3416         if (*relocs == NULL) {
3417                 DRM_ERROR("failed to alloc relocs, count %d\n", reloc_count);
3418                 return -ENOMEM;
3419         }
3420
3421         for (i = 0; i < buffer_count; i++) {
3422                 struct drm_i915_gem_relocation_entry __user *user_relocs;
3423
3424                 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3425
3426                 ret = copy_from_user(&(*relocs)[reloc_index],
3427                                      user_relocs,
3428                                      exec_list[i].relocation_count *
3429                                      sizeof(**relocs));
3430                 if (ret != 0) {
3431                         drm_free_large(*relocs);
3432                         *relocs = NULL;
3433                         return -EFAULT;
3434                 }
3435
3436                 reloc_index += exec_list[i].relocation_count;
3437         }
3438
3439         return 0;
3440 }
3441
3442 static int
3443 i915_gem_put_relocs_to_user(struct drm_i915_gem_exec_object2 *exec_list,
3444                             uint32_t buffer_count,
3445                             struct drm_i915_gem_relocation_entry *relocs)
3446 {
3447         uint32_t reloc_count = 0, i;
3448         int ret = 0;
3449
3450         if (relocs == NULL)
3451             return 0;
3452
3453         for (i = 0; i < buffer_count; i++) {
3454                 struct drm_i915_gem_relocation_entry __user *user_relocs;
3455                 int unwritten;
3456
3457                 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3458
3459                 unwritten = copy_to_user(user_relocs,
3460                                          &relocs[reloc_count],
3461                                          exec_list[i].relocation_count *
3462                                          sizeof(*relocs));
3463
3464                 if (unwritten) {
3465                         ret = -EFAULT;
3466                         goto err;
3467                 }
3468
3469                 reloc_count += exec_list[i].relocation_count;
3470         }
3471
3472 err:
3473         drm_free_large(relocs);
3474
3475         return ret;
3476 }
3477
3478 static int
3479 i915_gem_check_execbuffer (struct drm_i915_gem_execbuffer2 *exec,
3480                            uint64_t exec_offset)
3481 {
3482         uint32_t exec_start, exec_len;
3483
3484         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3485         exec_len = (uint32_t) exec->batch_len;
3486
3487         if ((exec_start | exec_len) & 0x7)
3488                 return -EINVAL;
3489
3490         if (!exec_start)
3491                 return -EINVAL;
3492
3493         return 0;
3494 }
3495
3496 static int
3497 i915_gem_wait_for_pending_flip(struct drm_device *dev,
3498                                struct drm_gem_object **object_list,
3499                                int count)
3500 {
3501         drm_i915_private_t *dev_priv = dev->dev_private;
3502         struct drm_i915_gem_object *obj_priv;
3503         DEFINE_WAIT(wait);
3504         int i, ret = 0;
3505
3506         for (;;) {
3507                 prepare_to_wait(&dev_priv->pending_flip_queue,
3508                                 &wait, TASK_INTERRUPTIBLE);
3509                 for (i = 0; i < count; i++) {
3510                         obj_priv = to_intel_bo(object_list[i]);
3511                         if (atomic_read(&obj_priv->pending_flip) > 0)
3512                                 break;
3513                 }
3514                 if (i == count)
3515                         break;
3516
3517                 if (!signal_pending(current)) {
3518                         mutex_unlock(&dev->struct_mutex);
3519                         schedule();
3520                         mutex_lock(&dev->struct_mutex);
3521                         continue;
3522                 }
3523                 ret = -ERESTARTSYS;
3524                 break;
3525         }
3526         finish_wait(&dev_priv->pending_flip_queue, &wait);
3527
3528         return ret;
3529 }
3530
3531
3532 int
3533 i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3534                        struct drm_file *file_priv,
3535                        struct drm_i915_gem_execbuffer2 *args,
3536                        struct drm_i915_gem_exec_object2 *exec_list)
3537 {
3538         drm_i915_private_t *dev_priv = dev->dev_private;
3539         struct drm_gem_object **object_list = NULL;
3540         struct drm_gem_object *batch_obj;
3541         struct drm_i915_gem_object *obj_priv;
3542         struct drm_clip_rect *cliprects = NULL;
3543         struct drm_i915_gem_relocation_entry *relocs = NULL;
3544         int ret = 0, ret2, i, pinned = 0;
3545         uint64_t exec_offset;
3546         uint32_t seqno, flush_domains, reloc_index;
3547         int pin_tries, flips;
3548
3549         struct intel_ring_buffer *ring = NULL;
3550
3551 #if WATCH_EXEC
3552         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3553                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3554 #endif
3555         if (args->flags & I915_EXEC_BSD) {
3556                 if (!HAS_BSD(dev)) {
3557                         DRM_ERROR("execbuf with wrong flag\n");
3558                         return -EINVAL;
3559                 }
3560                 ring = &dev_priv->bsd_ring;
3561         } else {
3562                 ring = &dev_priv->render_ring;
3563         }
3564
3565         if (args->buffer_count < 1) {
3566                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3567                 return -EINVAL;
3568         }
3569         object_list = drm_malloc_ab(sizeof(*object_list), args->buffer_count);
3570         if (object_list == NULL) {
3571                 DRM_ERROR("Failed to allocate object list for %d buffers\n",
3572                           args->buffer_count);
3573                 ret = -ENOMEM;
3574                 goto pre_mutex_err;
3575         }
3576
3577         if (args->num_cliprects != 0) {
3578                 cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects),
3579                                     GFP_KERNEL);
3580                 if (cliprects == NULL) {
3581                         ret = -ENOMEM;
3582                         goto pre_mutex_err;
3583                 }
3584
3585                 ret = copy_from_user(cliprects,
3586                                      (struct drm_clip_rect __user *)
3587                                      (uintptr_t) args->cliprects_ptr,
3588                                      sizeof(*cliprects) * args->num_cliprects);
3589                 if (ret != 0) {
3590                         DRM_ERROR("copy %d cliprects failed: %d\n",
3591                                   args->num_cliprects, ret);
3592                         ret = -EFAULT;
3593                         goto pre_mutex_err;
3594                 }
3595         }
3596
3597         ret = i915_gem_get_relocs_from_user(exec_list, args->buffer_count,
3598                                             &relocs);
3599         if (ret != 0)
3600                 goto pre_mutex_err;
3601
3602         mutex_lock(&dev->struct_mutex);
3603
3604         i915_verify_inactive(dev, __FILE__, __LINE__);
3605
3606         if (atomic_read(&dev_priv->mm.wedged)) {
3607                 mutex_unlock(&dev->struct_mutex);
3608                 ret = -EIO;
3609                 goto pre_mutex_err;
3610         }
3611
3612         if (dev_priv->mm.suspended) {
3613                 mutex_unlock(&dev->struct_mutex);
3614                 ret = -EBUSY;
3615                 goto pre_mutex_err;
3616         }
3617
3618         /* Look up object handles */
3619         flips = 0;
3620         for (i = 0; i < args->buffer_count; i++) {
3621                 object_list[i] = drm_gem_object_lookup(dev, file_priv,
3622                                                        exec_list[i].handle);
3623                 if (object_list[i] == NULL) {
3624                         DRM_ERROR("Invalid object handle %d at index %d\n",
3625                                    exec_list[i].handle, i);
3626                         /* prevent error path from reading uninitialized data */
3627                         args->buffer_count = i + 1;
3628                         ret = -ENOENT;
3629                         goto err;
3630                 }
3631
3632                 obj_priv = to_intel_bo(object_list[i]);
3633                 if (obj_priv->in_execbuffer) {
3634                         DRM_ERROR("Object %p appears more than once in object list\n",
3635                                    object_list[i]);
3636                         /* prevent error path from reading uninitialized data */
3637                         args->buffer_count = i + 1;
3638                         ret = -EINVAL;
3639                         goto err;
3640                 }
3641                 obj_priv->in_execbuffer = true;
3642                 flips += atomic_read(&obj_priv->pending_flip);
3643         }
3644
3645         if (flips > 0) {
3646                 ret = i915_gem_wait_for_pending_flip(dev, object_list,
3647                                                      args->buffer_count);
3648                 if (ret)
3649                         goto err;
3650         }
3651
3652         /* Pin and relocate */
3653         for (pin_tries = 0; ; pin_tries++) {
3654                 ret = 0;
3655                 reloc_index = 0;
3656
3657                 for (i = 0; i < args->buffer_count; i++) {
3658                         object_list[i]->pending_read_domains = 0;
3659                         object_list[i]->pending_write_domain = 0;
3660                         ret = i915_gem_object_pin_and_relocate(object_list[i],
3661                                                                file_priv,
3662                                                                &exec_list[i],
3663                                                                &relocs[reloc_index]);
3664                         if (ret)
3665                                 break;
3666                         pinned = i + 1;
3667                         reloc_index += exec_list[i].relocation_count;
3668                 }
3669                 /* success */
3670                 if (ret == 0)
3671                         break;
3672
3673                 /* error other than GTT full, or we've already tried again */
3674                 if (ret != -ENOSPC || pin_tries >= 1) {
3675                         if (ret != -ERESTARTSYS) {
3676                                 unsigned long long total_size = 0;
3677                                 int num_fences = 0;
3678                                 for (i = 0; i < args->buffer_count; i++) {
3679                                         obj_priv = to_intel_bo(object_list[i]);
3680
3681                                         total_size += object_list[i]->size;
3682                                         num_fences +=
3683                                                 exec_list[i].flags & EXEC_OBJECT_NEEDS_FENCE &&
3684                                                 obj_priv->tiling_mode != I915_TILING_NONE;
3685                                 }
3686                                 DRM_ERROR("Failed to pin buffer %d of %d, total %llu bytes, %d fences: %d\n",
3687                                           pinned+1, args->buffer_count,
3688                                           total_size, num_fences,
3689                                           ret);
3690                                 DRM_ERROR("%d objects [%d pinned], "
3691                                           "%d object bytes [%d pinned], "
3692                                           "%d/%d gtt bytes\n",
3693                                           atomic_read(&dev->object_count),
3694                                           atomic_read(&dev->pin_count),
3695                                           atomic_read(&dev->object_memory),
3696                                           atomic_read(&dev->pin_memory),
3697                                           atomic_read(&dev->gtt_memory),
3698                                           dev->gtt_total);
3699                         }
3700                         goto err;
3701                 }
3702
3703                 /* unpin all of our buffers */
3704                 for (i = 0; i < pinned; i++)
3705                         i915_gem_object_unpin(object_list[i]);
3706                 pinned = 0;
3707
3708                 /* evict everyone we can from the aperture */
3709                 ret = i915_gem_evict_everything(dev);
3710                 if (ret && ret != -ENOSPC)
3711                         goto err;
3712         }
3713
3714         /* Set the pending read domains for the batch buffer to COMMAND */
3715         batch_obj = object_list[args->buffer_count-1];
3716         if (batch_obj->pending_write_domain) {
3717                 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
3718                 ret = -EINVAL;
3719                 goto err;
3720         }
3721         batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
3722
3723         /* Sanity check the batch buffer, prior to moving objects */
3724         exec_offset = exec_list[args->buffer_count - 1].offset;
3725         ret = i915_gem_check_execbuffer (args, exec_offset);
3726         if (ret != 0) {
3727                 DRM_ERROR("execbuf with invalid offset/length\n");
3728                 goto err;
3729         }
3730
3731         i915_verify_inactive(dev, __FILE__, __LINE__);
3732
3733         /* Zero the global flush/invalidate flags. These
3734          * will be modified as new domains are computed
3735          * for each object
3736          */
3737         dev->invalidate_domains = 0;
3738         dev->flush_domains = 0;
3739         dev_priv->flush_rings = 0;
3740
3741         for (i = 0; i < args->buffer_count; i++) {
3742                 struct drm_gem_object *obj = object_list[i];
3743
3744                 /* Compute new gpu domains and update invalidate/flush */
3745                 i915_gem_object_set_to_gpu_domain(obj);
3746         }
3747
3748         i915_verify_inactive(dev, __FILE__, __LINE__);
3749
3750         if (dev->invalidate_domains | dev->flush_domains) {
3751 #if WATCH_EXEC
3752                 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
3753                           __func__,
3754                          dev->invalidate_domains,
3755                          dev->flush_domains);
3756 #endif
3757                 i915_gem_flush(dev,
3758                                dev->invalidate_domains,
3759                                dev->flush_domains);
3760                 if (dev_priv->flush_rings & FLUSH_RENDER_RING)
3761                         (void)i915_add_request(dev, file_priv,
3762                                                dev->flush_domains,
3763                                                &dev_priv->render_ring);
3764                 if (dev_priv->flush_rings & FLUSH_BSD_RING)
3765                         (void)i915_add_request(dev, file_priv,
3766                                                dev->flush_domains,
3767                                                &dev_priv->bsd_ring);
3768         }
3769
3770         for (i = 0; i < args->buffer_count; i++) {
3771                 struct drm_gem_object *obj = object_list[i];
3772                 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3773                 uint32_t old_write_domain = obj->write_domain;
3774
3775                 obj->write_domain = obj->pending_write_domain;
3776                 if (obj->write_domain)
3777                         list_move_tail(&obj_priv->gpu_write_list,
3778                                        &dev_priv->mm.gpu_write_list);
3779                 else
3780                         list_del_init(&obj_priv->gpu_write_list);
3781
3782                 trace_i915_gem_object_change_domain(obj,
3783                                                     obj->read_domains,
3784                                                     old_write_domain);
3785         }
3786
3787         i915_verify_inactive(dev, __FILE__, __LINE__);
3788
3789 #if WATCH_COHERENCY
3790         for (i = 0; i < args->buffer_count; i++) {
3791                 i915_gem_object_check_coherency(object_list[i],
3792                                                 exec_list[i].handle);
3793         }
3794 #endif
3795
3796 #if WATCH_EXEC
3797         i915_gem_dump_object(batch_obj,
3798                               args->batch_len,
3799                               __func__,
3800                               ~0);
3801 #endif
3802
3803         /* Exec the batchbuffer */
3804         ret = ring->dispatch_gem_execbuffer(dev, ring, args,
3805                         cliprects, exec_offset);
3806         if (ret) {
3807                 DRM_ERROR("dispatch failed %d\n", ret);
3808                 goto err;
3809         }
3810
3811         /*
3812          * Ensure that the commands in the batch buffer are
3813          * finished before the interrupt fires
3814          */
3815         flush_domains = i915_retire_commands(dev, ring);
3816
3817         i915_verify_inactive(dev, __FILE__, __LINE__);
3818
3819         /*
3820          * Get a seqno representing the execution of the current buffer,
3821          * which we can wait on.  We would like to mitigate these interrupts,
3822          * likely by only creating seqnos occasionally (so that we have
3823          * *some* interrupts representing completion of buffers that we can
3824          * wait on when trying to clear up gtt space).
3825          */
3826         seqno = i915_add_request(dev, file_priv, flush_domains, ring);
3827         BUG_ON(seqno == 0);
3828         for (i = 0; i < args->buffer_count; i++) {
3829                 struct drm_gem_object *obj = object_list[i];
3830                 obj_priv = to_intel_bo(obj);
3831
3832                 i915_gem_object_move_to_active(obj, seqno, ring);
3833 #if WATCH_LRU
3834                 DRM_INFO("%s: move to exec list %p\n", __func__, obj);
3835 #endif
3836         }
3837 #if WATCH_LRU
3838         i915_dump_lru(dev, __func__);
3839 #endif
3840
3841         i915_verify_inactive(dev, __FILE__, __LINE__);
3842
3843 err:
3844         for (i = 0; i < pinned; i++)
3845                 i915_gem_object_unpin(object_list[i]);
3846
3847         for (i = 0; i < args->buffer_count; i++) {
3848                 if (object_list[i]) {
3849                         obj_priv = to_intel_bo(object_list[i]);
3850                         obj_priv->in_execbuffer = false;
3851                 }
3852                 drm_gem_object_unreference(object_list[i]);
3853         }
3854
3855         mutex_unlock(&dev->struct_mutex);
3856
3857 pre_mutex_err:
3858         /* Copy the updated relocations out regardless of current error
3859          * state.  Failure to update the relocs would mean that the next
3860          * time userland calls execbuf, it would do so with presumed offset
3861          * state that didn't match the actual object state.
3862          */
3863         ret2 = i915_gem_put_relocs_to_user(exec_list, args->buffer_count,
3864                                            relocs);
3865         if (ret2 != 0) {
3866                 DRM_ERROR("Failed to copy relocations back out: %d\n", ret2);
3867
3868                 if (ret == 0)
3869                         ret = ret2;
3870         }
3871
3872         drm_free_large(object_list);
3873         kfree(cliprects);
3874
3875         return ret;
3876 }
3877
3878 /*
3879  * Legacy execbuffer just creates an exec2 list from the original exec object
3880  * list array and passes it to the real function.
3881  */
3882 int
3883 i915_gem_execbuffer(struct drm_device *dev, void *data,
3884                     struct drm_file *file_priv)
3885 {
3886         struct drm_i915_gem_execbuffer *args = data;
3887         struct drm_i915_gem_execbuffer2 exec2;
3888         struct drm_i915_gem_exec_object *exec_list = NULL;
3889         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
3890         int ret, i;
3891
3892 #if WATCH_EXEC
3893         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3894                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3895 #endif
3896
3897         if (args->buffer_count < 1) {
3898                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3899                 return -EINVAL;
3900         }
3901
3902         /* Copy in the exec list from userland */
3903         exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
3904         exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
3905         if (exec_list == NULL || exec2_list == NULL) {
3906                 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
3907                           args->buffer_count);
3908                 drm_free_large(exec_list);
3909                 drm_free_large(exec2_list);
3910                 return -ENOMEM;
3911         }
3912         ret = copy_from_user(exec_list,
3913                              (struct drm_i915_relocation_entry __user *)
3914                              (uintptr_t) args->buffers_ptr,
3915                              sizeof(*exec_list) * args->buffer_count);
3916         if (ret != 0) {
3917                 DRM_ERROR("copy %d exec entries failed %d\n",
3918                           args->buffer_count, ret);
3919                 drm_free_large(exec_list);
3920                 drm_free_large(exec2_list);
3921                 return -EFAULT;
3922         }
3923
3924         for (i = 0; i < args->buffer_count; i++) {
3925                 exec2_list[i].handle = exec_list[i].handle;
3926                 exec2_list[i].relocation_count = exec_list[i].relocation_count;
3927                 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
3928                 exec2_list[i].alignment = exec_list[i].alignment;
3929                 exec2_list[i].offset = exec_list[i].offset;
3930                 if (!IS_I965G(dev))
3931                         exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
3932                 else
3933                         exec2_list[i].flags = 0;
3934         }
3935
3936         exec2.buffers_ptr = args->buffers_ptr;
3937         exec2.buffer_count = args->buffer_count;
3938         exec2.batch_start_offset = args->batch_start_offset;
3939         exec2.batch_len = args->batch_len;
3940         exec2.DR1 = args->DR1;
3941         exec2.DR4 = args->DR4;
3942         exec2.num_cliprects = args->num_cliprects;
3943         exec2.cliprects_ptr = args->cliprects_ptr;
3944         exec2.flags = I915_EXEC_RENDER;
3945
3946         ret = i915_gem_do_execbuffer(dev, data, file_priv, &exec2, exec2_list);
3947         if (!ret) {
3948                 /* Copy the new buffer offsets back to the user's exec list. */
3949                 for (i = 0; i < args->buffer_count; i++)
3950                         exec_list[i].offset = exec2_list[i].offset;
3951                 /* ... and back out to userspace */
3952                 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
3953                                    (uintptr_t) args->buffers_ptr,
3954                                    exec_list,
3955                                    sizeof(*exec_list) * args->buffer_count);
3956                 if (ret) {
3957                         ret = -EFAULT;
3958                         DRM_ERROR("failed to copy %d exec entries "
3959                                   "back to user (%d)\n",
3960                                   args->buffer_count, ret);
3961                 }
3962         }
3963
3964         drm_free_large(exec_list);
3965         drm_free_large(exec2_list);
3966         return ret;
3967 }
3968
3969 int
3970 i915_gem_execbuffer2(struct drm_device *dev, void *data,
3971                      struct drm_file *file_priv)
3972 {
3973         struct drm_i915_gem_execbuffer2 *args = data;
3974         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
3975         int ret;
3976
3977 #if WATCH_EXEC
3978         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3979                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3980 #endif
3981
3982         if (args->buffer_count < 1) {
3983                 DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
3984                 return -EINVAL;
3985         }
3986
3987         exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
3988         if (exec2_list == NULL) {
3989                 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
3990                           args->buffer_count);
3991                 return -ENOMEM;
3992         }
3993         ret = copy_from_user(exec2_list,
3994                              (struct drm_i915_relocation_entry __user *)
3995                              (uintptr_t) args->buffers_ptr,
3996                              sizeof(*exec2_list) * args->buffer_count);
3997         if (ret != 0) {
3998                 DRM_ERROR("copy %d exec entries failed %d\n",
3999                           args->buffer_count, ret);
4000                 drm_free_large(exec2_list);
4001                 return -EFAULT;
4002         }
4003
4004         ret = i915_gem_do_execbuffer(dev, data, file_priv, args, exec2_list);
4005         if (!ret) {
4006                 /* Copy the new buffer offsets back to the user's exec list. */
4007                 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4008                                    (uintptr_t) args->buffers_ptr,
4009                                    exec2_list,
4010                                    sizeof(*exec2_list) * args->buffer_count);
4011                 if (ret) {
4012                         ret = -EFAULT;
4013                         DRM_ERROR("failed to copy %d exec entries "
4014                                   "back to user (%d)\n",
4015                                   args->buffer_count, ret);
4016                 }
4017         }
4018
4019         drm_free_large(exec2_list);
4020         return ret;
4021 }
4022
4023 int
4024 i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
4025 {
4026         struct drm_device *dev = obj->dev;
4027         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4028         int ret;
4029
4030         BUG_ON(obj_priv->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
4031
4032         i915_verify_inactive(dev, __FILE__, __LINE__);
4033
4034         if (obj_priv->gtt_space != NULL) {
4035                 if (alignment == 0)
4036                         alignment = i915_gem_get_gtt_alignment(obj);
4037                 if (obj_priv->gtt_offset & (alignment - 1)) {
4038                         WARN(obj_priv->pin_count,
4039                              "bo is already pinned with incorrect alignment:"
4040                              " offset=%x, req.alignment=%x\n",
4041                              obj_priv->gtt_offset, alignment);
4042                         ret = i915_gem_object_unbind(obj);
4043                         if (ret)
4044                                 return ret;
4045                 }
4046         }
4047
4048         if (obj_priv->gtt_space == NULL) {
4049                 ret = i915_gem_object_bind_to_gtt(obj, alignment);
4050                 if (ret)
4051                         return ret;
4052         }
4053
4054         obj_priv->pin_count++;
4055
4056         /* If the object is not active and not pending a flush,
4057          * remove it from the inactive list
4058          */
4059         if (obj_priv->pin_count == 1) {
4060                 atomic_inc(&dev->pin_count);
4061                 atomic_add(obj->size, &dev->pin_memory);
4062                 if (!obj_priv->active &&
4063                     (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
4064                         list_del_init(&obj_priv->list);
4065         }
4066         i915_verify_inactive(dev, __FILE__, __LINE__);
4067
4068         return 0;
4069 }
4070
4071 void
4072 i915_gem_object_unpin(struct drm_gem_object *obj)
4073 {
4074         struct drm_device *dev = obj->dev;
4075         drm_i915_private_t *dev_priv = dev->dev_private;
4076         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4077
4078         i915_verify_inactive(dev, __FILE__, __LINE__);
4079         obj_priv->pin_count--;
4080         BUG_ON(obj_priv->pin_count < 0);
4081         BUG_ON(obj_priv->gtt_space == NULL);
4082
4083         /* If the object is no longer pinned, and is
4084          * neither active nor being flushed, then stick it on
4085          * the inactive list
4086          */
4087         if (obj_priv->pin_count == 0) {
4088                 if (!obj_priv->active &&
4089                     (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
4090                         list_move_tail(&obj_priv->list,
4091                                        &dev_priv->mm.inactive_list);
4092                 atomic_dec(&dev->pin_count);
4093                 atomic_sub(obj->size, &dev->pin_memory);
4094         }
4095         i915_verify_inactive(dev, __FILE__, __LINE__);
4096 }
4097
4098 int
4099 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
4100                    struct drm_file *file_priv)
4101 {
4102         struct drm_i915_gem_pin *args = data;
4103         struct drm_gem_object *obj;
4104         struct drm_i915_gem_object *obj_priv;
4105         int ret;
4106
4107         mutex_lock(&dev->struct_mutex);
4108
4109         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4110         if (obj == NULL) {
4111                 DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n",
4112                           args->handle);
4113                 mutex_unlock(&dev->struct_mutex);
4114                 return -ENOENT;
4115         }
4116         obj_priv = to_intel_bo(obj);
4117
4118         if (obj_priv->madv != I915_MADV_WILLNEED) {
4119                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
4120                 drm_gem_object_unreference(obj);
4121                 mutex_unlock(&dev->struct_mutex);
4122                 return -EINVAL;
4123         }
4124
4125         if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) {
4126                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
4127                           args->handle);
4128                 drm_gem_object_unreference(obj);
4129                 mutex_unlock(&dev->struct_mutex);
4130                 return -EINVAL;
4131         }
4132
4133         obj_priv->user_pin_count++;
4134         obj_priv->pin_filp = file_priv;
4135         if (obj_priv->user_pin_count == 1) {
4136                 ret = i915_gem_object_pin(obj, args->alignment);
4137                 if (ret != 0) {
4138                         drm_gem_object_unreference(obj);
4139                         mutex_unlock(&dev->struct_mutex);
4140                         return ret;
4141                 }
4142         }
4143
4144         /* XXX - flush the CPU caches for pinned objects
4145          * as the X server doesn't manage domains yet
4146          */
4147         i915_gem_object_flush_cpu_write_domain(obj);
4148         args->offset = obj_priv->gtt_offset;
4149         drm_gem_object_unreference(obj);
4150         mutex_unlock(&dev->struct_mutex);
4151
4152         return 0;
4153 }
4154
4155 int
4156 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
4157                      struct drm_file *file_priv)
4158 {
4159         struct drm_i915_gem_pin *args = data;
4160         struct drm_gem_object *obj;
4161         struct drm_i915_gem_object *obj_priv;
4162
4163         mutex_lock(&dev->struct_mutex);
4164
4165         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4166         if (obj == NULL) {
4167                 DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n",
4168                           args->handle);
4169                 mutex_unlock(&dev->struct_mutex);
4170                 return -ENOENT;
4171         }
4172
4173         obj_priv = to_intel_bo(obj);
4174         if (obj_priv->pin_filp != file_priv) {
4175                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
4176                           args->handle);
4177                 drm_gem_object_unreference(obj);
4178                 mutex_unlock(&dev->struct_mutex);
4179                 return -EINVAL;
4180         }
4181         obj_priv->user_pin_count--;
4182         if (obj_priv->user_pin_count == 0) {
4183                 obj_priv->pin_filp = NULL;
4184                 i915_gem_object_unpin(obj);
4185         }
4186
4187         drm_gem_object_unreference(obj);
4188         mutex_unlock(&dev->struct_mutex);
4189         return 0;
4190 }
4191
4192 int
4193 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4194                     struct drm_file *file_priv)
4195 {
4196         struct drm_i915_gem_busy *args = data;
4197         struct drm_gem_object *obj;
4198         struct drm_i915_gem_object *obj_priv;
4199
4200         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4201         if (obj == NULL) {
4202                 DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n",
4203                           args->handle);
4204                 return -ENOENT;
4205         }
4206
4207         mutex_lock(&dev->struct_mutex);
4208
4209         /* Count all active objects as busy, even if they are currently not used
4210          * by the gpu. Users of this interface expect objects to eventually
4211          * become non-busy without any further actions, therefore emit any
4212          * necessary flushes here.
4213          */
4214         obj_priv = to_intel_bo(obj);
4215         args->busy = obj_priv->active;
4216         if (args->busy) {
4217                 /* Unconditionally flush objects, even when the gpu still uses this
4218                  * object. Userspace calling this function indicates that it wants to
4219                  * use this buffer rather sooner than later, so issuing the required
4220                  * flush earlier is beneficial.
4221                  */
4222                 if (obj->write_domain) {
4223                         i915_gem_flush(dev, 0, obj->write_domain);
4224                         (void)i915_add_request(dev, file_priv, obj->write_domain, obj_priv->ring);
4225                 }
4226
4227                 /* Update the active list for the hardware's current position.
4228                  * Otherwise this only updates on a delayed timer or when irqs
4229                  * are actually unmasked, and our working set ends up being
4230                  * larger than required.
4231                  */
4232                 i915_gem_retire_requests_ring(dev, obj_priv->ring);
4233
4234                 args->busy = obj_priv->active;
4235         }
4236
4237         drm_gem_object_unreference(obj);
4238         mutex_unlock(&dev->struct_mutex);
4239         return 0;
4240 }
4241
4242 int
4243 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4244                         struct drm_file *file_priv)
4245 {
4246     return i915_gem_ring_throttle(dev, file_priv);
4247 }
4248
4249 int
4250 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4251                        struct drm_file *file_priv)
4252 {
4253         struct drm_i915_gem_madvise *args = data;
4254         struct drm_gem_object *obj;
4255         struct drm_i915_gem_object *obj_priv;
4256
4257         switch (args->madv) {
4258         case I915_MADV_DONTNEED:
4259         case I915_MADV_WILLNEED:
4260             break;
4261         default:
4262             return -EINVAL;
4263         }
4264
4265         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4266         if (obj == NULL) {
4267                 DRM_ERROR("Bad handle in i915_gem_madvise_ioctl(): %d\n",
4268                           args->handle);
4269                 return -ENOENT;
4270         }
4271
4272         mutex_lock(&dev->struct_mutex);
4273         obj_priv = to_intel_bo(obj);
4274
4275         if (obj_priv->pin_count) {
4276                 drm_gem_object_unreference(obj);
4277                 mutex_unlock(&dev->struct_mutex);
4278
4279                 DRM_ERROR("Attempted i915_gem_madvise_ioctl() on a pinned object\n");
4280                 return -EINVAL;
4281         }
4282
4283         if (obj_priv->madv != __I915_MADV_PURGED)
4284                 obj_priv->madv = args->madv;
4285
4286         /* if the object is no longer bound, discard its backing storage */
4287         if (i915_gem_object_is_purgeable(obj_priv) &&
4288             obj_priv->gtt_space == NULL)
4289                 i915_gem_object_truncate(obj);
4290
4291         args->retained = obj_priv->madv != __I915_MADV_PURGED;
4292
4293         drm_gem_object_unreference(obj);
4294         mutex_unlock(&dev->struct_mutex);
4295
4296         return 0;
4297 }
4298
4299 struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev,
4300                                               size_t size)
4301 {
4302         struct drm_i915_gem_object *obj;
4303
4304         obj = kzalloc(sizeof(*obj), GFP_KERNEL);
4305         if (obj == NULL)
4306                 return NULL;
4307
4308         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4309                 kfree(obj);
4310                 return NULL;
4311         }
4312
4313         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4314         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4315
4316         obj->agp_type = AGP_USER_MEMORY;
4317         obj->base.driver_private = NULL;
4318         obj->fence_reg = I915_FENCE_REG_NONE;
4319         INIT_LIST_HEAD(&obj->list);
4320         INIT_LIST_HEAD(&obj->gpu_write_list);
4321         obj->madv = I915_MADV_WILLNEED;
4322
4323         trace_i915_gem_object_create(&obj->base);
4324
4325         return &obj->base;
4326 }
4327
4328 int i915_gem_init_object(struct drm_gem_object *obj)
4329 {
4330         BUG();
4331
4332         return 0;
4333 }
4334
4335 static void i915_gem_free_object_tail(struct drm_gem_object *obj)
4336 {
4337         struct drm_device *dev = obj->dev;
4338         drm_i915_private_t *dev_priv = dev->dev_private;
4339         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4340         int ret;
4341
4342         ret = i915_gem_object_unbind(obj);
4343         if (ret == -ERESTARTSYS) {
4344                 list_move(&obj_priv->list,
4345                           &dev_priv->mm.deferred_free_list);
4346                 return;
4347         }
4348
4349         if (obj_priv->mmap_offset)
4350                 i915_gem_free_mmap_offset(obj);
4351
4352         drm_gem_object_release(obj);
4353
4354         kfree(obj_priv->page_cpu_valid);
4355         kfree(obj_priv->bit_17);
4356         kfree(obj_priv);
4357 }
4358
4359 void i915_gem_free_object(struct drm_gem_object *obj)
4360 {
4361         struct drm_device *dev = obj->dev;
4362         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4363
4364         trace_i915_gem_object_destroy(obj);
4365
4366         while (obj_priv->pin_count > 0)
4367                 i915_gem_object_unpin(obj);
4368
4369         if (obj_priv->phys_obj)
4370                 i915_gem_detach_phys_object(dev, obj);
4371
4372         i915_gem_free_object_tail(obj);
4373 }
4374
4375 int
4376 i915_gem_idle(struct drm_device *dev)
4377 {
4378         drm_i915_private_t *dev_priv = dev->dev_private;
4379         int ret;
4380
4381         mutex_lock(&dev->struct_mutex);
4382
4383         if (dev_priv->mm.suspended ||
4384                         (dev_priv->render_ring.gem_object == NULL) ||
4385                         (HAS_BSD(dev) &&
4386                          dev_priv->bsd_ring.gem_object == NULL)) {
4387                 mutex_unlock(&dev->struct_mutex);
4388                 return 0;
4389         }
4390
4391         ret = i915_gpu_idle(dev);
4392         if (ret) {
4393                 mutex_unlock(&dev->struct_mutex);
4394                 return ret;
4395         }
4396
4397         /* Under UMS, be paranoid and evict. */
4398         if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
4399                 ret = i915_gem_evict_inactive(dev);
4400                 if (ret) {
4401                         mutex_unlock(&dev->struct_mutex);
4402                         return ret;
4403                 }
4404         }
4405
4406         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
4407          * We need to replace this with a semaphore, or something.
4408          * And not confound mm.suspended!
4409          */
4410         dev_priv->mm.suspended = 1;
4411         del_timer(&dev_priv->hangcheck_timer);
4412
4413         i915_kernel_lost_context(dev);
4414         i915_gem_cleanup_ringbuffer(dev);
4415
4416         mutex_unlock(&dev->struct_mutex);
4417
4418         /* Cancel the retire work handler, which should be idle now. */
4419         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4420
4421         return 0;
4422 }
4423
4424 /*
4425  * 965+ support PIPE_CONTROL commands, which provide finer grained control
4426  * over cache flushing.
4427  */
4428 static int
4429 i915_gem_init_pipe_control(struct drm_device *dev)
4430 {
4431         drm_i915_private_t *dev_priv = dev->dev_private;
4432         struct drm_gem_object *obj;
4433         struct drm_i915_gem_object *obj_priv;
4434         int ret;
4435
4436         obj = i915_gem_alloc_object(dev, 4096);
4437         if (obj == NULL) {
4438                 DRM_ERROR("Failed to allocate seqno page\n");
4439                 ret = -ENOMEM;
4440                 goto err;
4441         }
4442         obj_priv = to_intel_bo(obj);
4443         obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
4444
4445         ret = i915_gem_object_pin(obj, 4096);
4446         if (ret)
4447                 goto err_unref;
4448
4449         dev_priv->seqno_gfx_addr = obj_priv->gtt_offset;
4450         dev_priv->seqno_page =  kmap(obj_priv->pages[0]);
4451         if (dev_priv->seqno_page == NULL)
4452                 goto err_unpin;
4453
4454         dev_priv->seqno_obj = obj;
4455         memset(dev_priv->seqno_page, 0, PAGE_SIZE);
4456
4457         return 0;
4458
4459 err_unpin:
4460         i915_gem_object_unpin(obj);
4461 err_unref:
4462         drm_gem_object_unreference(obj);
4463 err:
4464         return ret;
4465 }
4466
4467
4468 static void
4469 i915_gem_cleanup_pipe_control(struct drm_device *dev)
4470 {
4471         drm_i915_private_t *dev_priv = dev->dev_private;
4472         struct drm_gem_object *obj;
4473         struct drm_i915_gem_object *obj_priv;
4474
4475         obj = dev_priv->seqno_obj;
4476         obj_priv = to_intel_bo(obj);
4477         kunmap(obj_priv->pages[0]);
4478         i915_gem_object_unpin(obj);
4479         drm_gem_object_unreference(obj);
4480         dev_priv->seqno_obj = NULL;
4481
4482         dev_priv->seqno_page = NULL;
4483 }
4484
4485 int
4486 i915_gem_init_ringbuffer(struct drm_device *dev)
4487 {
4488         drm_i915_private_t *dev_priv = dev->dev_private;
4489         int ret;
4490
4491         dev_priv->render_ring = render_ring;
4492
4493         if (!I915_NEED_GFX_HWS(dev)) {
4494                 dev_priv->render_ring.status_page.page_addr
4495                         = dev_priv->status_page_dmah->vaddr;
4496                 memset(dev_priv->render_ring.status_page.page_addr,
4497                                 0, PAGE_SIZE);
4498         }
4499
4500         if (HAS_PIPE_CONTROL(dev)) {
4501                 ret = i915_gem_init_pipe_control(dev);
4502                 if (ret)
4503                         return ret;
4504         }
4505
4506         ret = intel_init_ring_buffer(dev, &dev_priv->render_ring);
4507         if (ret)
4508                 goto cleanup_pipe_control;
4509
4510         if (HAS_BSD(dev)) {
4511                 dev_priv->bsd_ring = bsd_ring;
4512                 ret = intel_init_ring_buffer(dev, &dev_priv->bsd_ring);
4513                 if (ret)
4514                         goto cleanup_render_ring;
4515         }
4516
4517         dev_priv->next_seqno = 1;
4518
4519         return 0;
4520
4521 cleanup_render_ring:
4522         intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
4523 cleanup_pipe_control:
4524         if (HAS_PIPE_CONTROL(dev))
4525                 i915_gem_cleanup_pipe_control(dev);
4526         return ret;
4527 }
4528
4529 void
4530 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4531 {
4532         drm_i915_private_t *dev_priv = dev->dev_private;
4533
4534         intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
4535         if (HAS_BSD(dev))
4536                 intel_cleanup_ring_buffer(dev, &dev_priv->bsd_ring);
4537         if (HAS_PIPE_CONTROL(dev))
4538                 i915_gem_cleanup_pipe_control(dev);
4539 }
4540
4541 int
4542 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4543                        struct drm_file *file_priv)
4544 {
4545         drm_i915_private_t *dev_priv = dev->dev_private;
4546         int ret;
4547
4548         if (drm_core_check_feature(dev, DRIVER_MODESET))
4549                 return 0;
4550
4551         if (atomic_read(&dev_priv->mm.wedged)) {
4552                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
4553                 atomic_set(&dev_priv->mm.wedged, 0);
4554         }
4555
4556         mutex_lock(&dev->struct_mutex);
4557         dev_priv->mm.suspended = 0;
4558
4559         ret = i915_gem_init_ringbuffer(dev);
4560         if (ret != 0) {
4561                 mutex_unlock(&dev->struct_mutex);
4562                 return ret;
4563         }
4564
4565         spin_lock(&dev_priv->mm.active_list_lock);
4566         BUG_ON(!list_empty(&dev_priv->render_ring.active_list));
4567         BUG_ON(HAS_BSD(dev) && !list_empty(&dev_priv->bsd_ring.active_list));
4568         spin_unlock(&dev_priv->mm.active_list_lock);
4569
4570         BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
4571         BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
4572         BUG_ON(!list_empty(&dev_priv->render_ring.request_list));
4573         BUG_ON(HAS_BSD(dev) && !list_empty(&dev_priv->bsd_ring.request_list));
4574         mutex_unlock(&dev->struct_mutex);
4575
4576         ret = drm_irq_install(dev);
4577         if (ret)
4578                 goto cleanup_ringbuffer;
4579
4580         return 0;
4581
4582 cleanup_ringbuffer:
4583         mutex_lock(&dev->struct_mutex);
4584         i915_gem_cleanup_ringbuffer(dev);
4585         dev_priv->mm.suspended = 1;
4586         mutex_unlock(&dev->struct_mutex);
4587
4588         return ret;
4589 }
4590
4591 int
4592 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
4593                        struct drm_file *file_priv)
4594 {
4595         if (drm_core_check_feature(dev, DRIVER_MODESET))
4596                 return 0;
4597
4598         drm_irq_uninstall(dev);
4599         return i915_gem_idle(dev);
4600 }
4601
4602 void
4603 i915_gem_lastclose(struct drm_device *dev)
4604 {
4605         int ret;
4606
4607         if (drm_core_check_feature(dev, DRIVER_MODESET))
4608                 return;
4609
4610         ret = i915_gem_idle(dev);
4611         if (ret)
4612                 DRM_ERROR("failed to idle hardware: %d\n", ret);
4613 }
4614
4615 void
4616 i915_gem_load(struct drm_device *dev)
4617 {
4618         int i;
4619         drm_i915_private_t *dev_priv = dev->dev_private;
4620
4621         spin_lock_init(&dev_priv->mm.active_list_lock);
4622         INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
4623         INIT_LIST_HEAD(&dev_priv->mm.gpu_write_list);
4624         INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
4625         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4626         INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
4627         INIT_LIST_HEAD(&dev_priv->render_ring.active_list);
4628         INIT_LIST_HEAD(&dev_priv->render_ring.request_list);
4629         if (HAS_BSD(dev)) {
4630                 INIT_LIST_HEAD(&dev_priv->bsd_ring.active_list);
4631                 INIT_LIST_HEAD(&dev_priv->bsd_ring.request_list);
4632         }
4633         for (i = 0; i < 16; i++)
4634                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
4635         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4636                           i915_gem_retire_work_handler);
4637         spin_lock(&shrink_list_lock);
4638         list_add(&dev_priv->mm.shrink_list, &shrink_list);
4639         spin_unlock(&shrink_list_lock);
4640
4641         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
4642         if (IS_GEN3(dev)) {
4643                 u32 tmp = I915_READ(MI_ARB_STATE);
4644                 if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) {
4645                         /* arb state is a masked write, so set bit + bit in mask */
4646                         tmp = MI_ARB_C3_LP_WRITE_ENABLE | (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT);
4647                         I915_WRITE(MI_ARB_STATE, tmp);
4648                 }
4649         }
4650
4651         /* Old X drivers will take 0-2 for front, back, depth buffers */
4652         if (!drm_core_check_feature(dev, DRIVER_MODESET))
4653                 dev_priv->fence_reg_start = 3;
4654
4655         if (IS_I965G(dev) || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4656                 dev_priv->num_fence_regs = 16;
4657         else
4658                 dev_priv->num_fence_regs = 8;
4659
4660         /* Initialize fence registers to zero */
4661         if (IS_I965G(dev)) {
4662                 for (i = 0; i < 16; i++)
4663                         I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0);
4664         } else {
4665                 for (i = 0; i < 8; i++)
4666                         I915_WRITE(FENCE_REG_830_0 + (i * 4), 0);
4667                 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4668                         for (i = 0; i < 8; i++)
4669                                 I915_WRITE(FENCE_REG_945_8 + (i * 4), 0);
4670         }
4671         i915_gem_detect_bit_6_swizzle(dev);
4672         init_waitqueue_head(&dev_priv->pending_flip_queue);
4673 }
4674
4675 /*
4676  * Create a physically contiguous memory object for this object
4677  * e.g. for cursor + overlay regs
4678  */
4679 int i915_gem_init_phys_object(struct drm_device *dev,
4680                               int id, int size, int align)
4681 {
4682         drm_i915_private_t *dev_priv = dev->dev_private;
4683         struct drm_i915_gem_phys_object *phys_obj;
4684         int ret;
4685
4686         if (dev_priv->mm.phys_objs[id - 1] || !size)
4687                 return 0;
4688
4689         phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
4690         if (!phys_obj)
4691                 return -ENOMEM;
4692
4693         phys_obj->id = id;
4694
4695         phys_obj->handle = drm_pci_alloc(dev, size, align);
4696         if (!phys_obj->handle) {
4697                 ret = -ENOMEM;
4698                 goto kfree_obj;
4699         }
4700 #ifdef CONFIG_X86
4701         set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4702 #endif
4703
4704         dev_priv->mm.phys_objs[id - 1] = phys_obj;
4705
4706         return 0;
4707 kfree_obj:
4708         kfree(phys_obj);
4709         return ret;
4710 }
4711
4712 void i915_gem_free_phys_object(struct drm_device *dev, int id)
4713 {
4714         drm_i915_private_t *dev_priv = dev->dev_private;
4715         struct drm_i915_gem_phys_object *phys_obj;
4716
4717         if (!dev_priv->mm.phys_objs[id - 1])
4718                 return;
4719
4720         phys_obj = dev_priv->mm.phys_objs[id - 1];
4721         if (phys_obj->cur_obj) {
4722                 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4723         }
4724
4725 #ifdef CONFIG_X86
4726         set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4727 #endif
4728         drm_pci_free(dev, phys_obj->handle);
4729         kfree(phys_obj);
4730         dev_priv->mm.phys_objs[id - 1] = NULL;
4731 }
4732
4733 void i915_gem_free_all_phys_object(struct drm_device *dev)
4734 {
4735         int i;
4736
4737         for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
4738                 i915_gem_free_phys_object(dev, i);
4739 }
4740
4741 void i915_gem_detach_phys_object(struct drm_device *dev,
4742                                  struct drm_gem_object *obj)
4743 {
4744         struct drm_i915_gem_object *obj_priv;
4745         int i;
4746         int ret;
4747         int page_count;
4748
4749         obj_priv = to_intel_bo(obj);
4750         if (!obj_priv->phys_obj)
4751                 return;
4752
4753         ret = i915_gem_object_get_pages(obj, 0);
4754         if (ret)
4755                 goto out;
4756
4757         page_count = obj->size / PAGE_SIZE;
4758
4759         for (i = 0; i < page_count; i++) {
4760                 char *dst = kmap_atomic(obj_priv->pages[i], KM_USER0);
4761                 char *src = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4762
4763                 memcpy(dst, src, PAGE_SIZE);
4764                 kunmap_atomic(dst, KM_USER0);
4765         }
4766         drm_clflush_pages(obj_priv->pages, page_count);
4767         drm_agp_chipset_flush(dev);
4768
4769         i915_gem_object_put_pages(obj);
4770 out:
4771         obj_priv->phys_obj->cur_obj = NULL;
4772         obj_priv->phys_obj = NULL;
4773 }
4774
4775 int
4776 i915_gem_attach_phys_object(struct drm_device *dev,
4777                             struct drm_gem_object *obj,
4778                             int id,
4779                             int align)
4780 {
4781         drm_i915_private_t *dev_priv = dev->dev_private;
4782         struct drm_i915_gem_object *obj_priv;
4783         int ret = 0;
4784         int page_count;
4785         int i;
4786
4787         if (id > I915_MAX_PHYS_OBJECT)
4788                 return -EINVAL;
4789
4790         obj_priv = to_intel_bo(obj);
4791
4792         if (obj_priv->phys_obj) {
4793                 if (obj_priv->phys_obj->id == id)
4794                         return 0;
4795                 i915_gem_detach_phys_object(dev, obj);
4796         }
4797
4798         /* create a new object */
4799         if (!dev_priv->mm.phys_objs[id - 1]) {
4800                 ret = i915_gem_init_phys_object(dev, id,
4801                                                 obj->size, align);
4802                 if (ret) {
4803                         DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size);
4804                         goto out;
4805                 }
4806         }
4807
4808         /* bind to the object */
4809         obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1];
4810         obj_priv->phys_obj->cur_obj = obj;
4811
4812         ret = i915_gem_object_get_pages(obj, 0);
4813         if (ret) {
4814                 DRM_ERROR("failed to get page list\n");
4815                 goto out;
4816         }
4817
4818         page_count = obj->size / PAGE_SIZE;
4819
4820         for (i = 0; i < page_count; i++) {
4821                 char *src = kmap_atomic(obj_priv->pages[i], KM_USER0);
4822                 char *dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4823
4824                 memcpy(dst, src, PAGE_SIZE);
4825                 kunmap_atomic(src, KM_USER0);
4826         }
4827
4828         i915_gem_object_put_pages(obj);
4829
4830         return 0;
4831 out:
4832         return ret;
4833 }
4834
4835 static int
4836 i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
4837                      struct drm_i915_gem_pwrite *args,
4838                      struct drm_file *file_priv)
4839 {
4840         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4841         void *obj_addr;
4842         int ret;
4843         char __user *user_data;
4844
4845         user_data = (char __user *) (uintptr_t) args->data_ptr;
4846         obj_addr = obj_priv->phys_obj->handle->vaddr + args->offset;
4847
4848         DRM_DEBUG_DRIVER("obj_addr %p, %lld\n", obj_addr, args->size);
4849         ret = copy_from_user(obj_addr, user_data, args->size);
4850         if (ret)
4851                 return -EFAULT;
4852
4853         drm_agp_chipset_flush(dev);
4854         return 0;
4855 }
4856
4857 void i915_gem_release(struct drm_device * dev, struct drm_file *file_priv)
4858 {
4859         struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
4860
4861         /* Clean up our request list when the client is going away, so that
4862          * later retire_requests won't dereference our soon-to-be-gone
4863          * file_priv.
4864          */
4865         mutex_lock(&dev->struct_mutex);
4866         while (!list_empty(&i915_file_priv->mm.request_list))
4867                 list_del_init(i915_file_priv->mm.request_list.next);
4868         mutex_unlock(&dev->struct_mutex);
4869 }
4870
4871 static int
4872 i915_gpu_is_active(struct drm_device *dev)
4873 {
4874         drm_i915_private_t *dev_priv = dev->dev_private;
4875         int lists_empty;
4876
4877         spin_lock(&dev_priv->mm.active_list_lock);
4878         lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
4879                       list_empty(&dev_priv->render_ring.active_list);
4880         if (HAS_BSD(dev))
4881                 lists_empty &= list_empty(&dev_priv->bsd_ring.active_list);
4882         spin_unlock(&dev_priv->mm.active_list_lock);
4883
4884         return !lists_empty;
4885 }
4886
4887 static int
4888 i915_gem_shrink(struct shrinker *shrink, int nr_to_scan, gfp_t gfp_mask)
4889 {
4890         drm_i915_private_t *dev_priv, *next_dev;
4891         struct drm_i915_gem_object *obj_priv, *next_obj;
4892         int cnt = 0;
4893         int would_deadlock = 1;
4894
4895         /* "fast-path" to count number of available objects */
4896         if (nr_to_scan == 0) {
4897                 spin_lock(&shrink_list_lock);
4898                 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
4899                         struct drm_device *dev = dev_priv->dev;
4900
4901                         if (mutex_trylock(&dev->struct_mutex)) {
4902                                 list_for_each_entry(obj_priv,
4903                                                     &dev_priv->mm.inactive_list,
4904                                                     list)
4905                                         cnt++;
4906                                 mutex_unlock(&dev->struct_mutex);
4907                         }
4908                 }
4909                 spin_unlock(&shrink_list_lock);
4910
4911                 return (cnt / 100) * sysctl_vfs_cache_pressure;
4912         }
4913
4914         spin_lock(&shrink_list_lock);
4915
4916 rescan:
4917         /* first scan for clean buffers */
4918         list_for_each_entry_safe(dev_priv, next_dev,
4919                                  &shrink_list, mm.shrink_list) {
4920                 struct drm_device *dev = dev_priv->dev;
4921
4922                 if (! mutex_trylock(&dev->struct_mutex))
4923                         continue;
4924
4925                 spin_unlock(&shrink_list_lock);
4926                 i915_gem_retire_requests(dev);
4927
4928                 list_for_each_entry_safe(obj_priv, next_obj,
4929                                          &dev_priv->mm.inactive_list,
4930                                          list) {
4931                         if (i915_gem_object_is_purgeable(obj_priv)) {
4932                                 i915_gem_object_unbind(&obj_priv->base);
4933                                 if (--nr_to_scan <= 0)
4934                                         break;
4935                         }
4936                 }
4937
4938                 spin_lock(&shrink_list_lock);
4939                 mutex_unlock(&dev->struct_mutex);
4940
4941                 would_deadlock = 0;
4942
4943                 if (nr_to_scan <= 0)
4944                         break;
4945         }
4946
4947         /* second pass, evict/count anything still on the inactive list */
4948         list_for_each_entry_safe(dev_priv, next_dev,
4949                                  &shrink_list, mm.shrink_list) {
4950                 struct drm_device *dev = dev_priv->dev;
4951
4952                 if (! mutex_trylock(&dev->struct_mutex))
4953                         continue;
4954
4955                 spin_unlock(&shrink_list_lock);
4956
4957                 list_for_each_entry_safe(obj_priv, next_obj,
4958                                          &dev_priv->mm.inactive_list,
4959                                          list) {
4960                         if (nr_to_scan > 0) {
4961                                 i915_gem_object_unbind(&obj_priv->base);
4962                                 nr_to_scan--;
4963                         } else
4964                                 cnt++;
4965                 }
4966
4967                 spin_lock(&shrink_list_lock);
4968                 mutex_unlock(&dev->struct_mutex);
4969
4970                 would_deadlock = 0;
4971         }
4972
4973         if (nr_to_scan) {
4974                 int active = 0;
4975
4976                 /*
4977                  * We are desperate for pages, so as a last resort, wait
4978                  * for the GPU to finish and discard whatever we can.
4979                  * This has a dramatic impact to reduce the number of
4980                  * OOM-killer events whilst running the GPU aggressively.
4981                  */
4982                 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
4983                         struct drm_device *dev = dev_priv->dev;
4984
4985                         if (!mutex_trylock(&dev->struct_mutex))
4986                                 continue;
4987
4988                         spin_unlock(&shrink_list_lock);
4989
4990                         if (i915_gpu_is_active(dev)) {
4991                                 i915_gpu_idle(dev);
4992                                 active++;
4993                         }
4994
4995                         spin_lock(&shrink_list_lock);
4996                         mutex_unlock(&dev->struct_mutex);
4997                 }
4998
4999                 if (active)
5000                         goto rescan;
5001         }
5002
5003         spin_unlock(&shrink_list_lock);
5004
5005         if (would_deadlock)
5006                 return -1;
5007         else if (cnt > 0)
5008                 return (cnt / 100) * sysctl_vfs_cache_pressure;
5009         else
5010                 return 0;
5011 }
5012
5013 static struct shrinker shrinker = {
5014         .shrink = i915_gem_shrink,
5015         .seeks = DEFAULT_SEEKS,
5016 };
5017
5018 __init void
5019 i915_gem_shrinker_init(void)
5020 {
5021     register_shrinker(&shrinker);
5022 }
5023
5024 __exit void
5025 i915_gem_shrinker_exit(void)
5026 {
5027     unregister_shrinker(&shrinker);
5028 }