drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  */
  27
  28 #include "drmP.h"
  29 #include "drm.h"
  30 #include "i915_drm.h"
  31 #include "i915_drv.h"
  32 #include "i915_trace.h"
  33 #include "intel_drv.h"
  34 #include <linux/slab.h>
  35 #include <linux/swap.h>
  36 #include <linux/pci.h>
  37 #include <linux/intel-gtt.h>
  38
  39 static uint32_t i915_gem_get_gtt_alignment(struct drm_gem_object *obj);
  40 static int i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj);
  41 static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
  42 static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
  43 static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
  44                                              int write);
  45 static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
  46                                                      uint64_t offset,
  47                                                      uint64_t size);
  48 static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
  49 static int i915_gem_object_wait_rendering(struct drm_gem_object *obj);
  50 static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
  51                                            unsigned alignment);
  52 static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
  53 static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
  54                                 struct drm_i915_gem_pwrite *args,
  55                                 struct drm_file *file_priv);
  56 static void i915_gem_free_object_tail(struct drm_gem_object *obj);
  57
  58 static LIST_HEAD(shrink_list);
  59 static DEFINE_SPINLOCK(shrink_list_lock);
  60
  61 static inline bool
  62 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj_priv)
  63 {
  64         return obj_priv->gtt_space &&
  65                 !obj_priv->active &&
  66                 obj_priv->pin_count == 0;
  67 }
  68
  69 int i915_gem_do_init(struct drm_device *dev, unsigned long start,
  70                      unsigned long end)
  71 {
  72         drm_i915_private_t *dev_priv = dev->dev_private;
  73
  74         if (start >= end ||
  75             (start & (PAGE_SIZE - 1)) != 0 ||
  76             (end & (PAGE_SIZE - 1)) != 0) {
  77                 return -EINVAL;
  78         }
  79
  80         drm_mm_init(&dev_priv->mm.gtt_space, start,
  81                     end - start);
  82
  83         dev->gtt_total = (uint32_t) (end - start);
  84
  85         return 0;
  86 }
  87
  88 int
  89 i915_gem_init_ioctl(struct drm_device *dev, void *data,
  90                     struct drm_file *file_priv)
  91 {
  92         struct drm_i915_gem_init *args = data;
  93         int ret;
  94
  95         mutex_lock(&dev->struct_mutex);
  96         ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end);
  97         mutex_unlock(&dev->struct_mutex);
  98
  99         return ret;
 100 }
 101
 102 int
 103 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 104                             struct drm_file *file_priv)
 105 {
 106         struct drm_i915_gem_get_aperture *args = data;
 107
 108         if (!(dev->driver->driver_features & DRIVER_GEM))
 109                 return -ENODEV;
 110
 111         args->aper_size = dev->gtt_total;
 112         args->aper_available_size = (args->aper_size -
 113                                      atomic_read(&dev->pin_memory));
 114
 115         return 0;
 116 }
 117
 118
 119 /**
 120  * Creates a new mm object and returns a handle to it.
 121  */
 122 int
 123 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 124                       struct drm_file *file_priv)
 125 {
 126         struct drm_i915_gem_create *args = data;
 127         struct drm_gem_object *obj;
 128         int ret;
 129         u32 handle;
 130
 131         args->size = roundup(args->size, PAGE_SIZE);
 132
 133         /* Allocate the new object */
 134         obj = i915_gem_alloc_object(dev, args->size);
 135         if (obj == NULL)
 136                 return -ENOMEM;
 137
 138         ret = drm_gem_handle_create(file_priv, obj, &handle);
 139         if (ret) {
 140                 drm_gem_object_unreference_unlocked(obj);
 141                 return ret;
 142         }
 143
 144         /* Sink the floating reference from kref_init(handlecount) */
 145         drm_gem_object_handle_unreference_unlocked(obj);
 146
 147         args->handle = handle;
 148         return 0;
 149 }
 150
 151 static inline int
 152 fast_shmem_read(struct page **pages,
 153                 loff_t page_base, int page_offset,
 154                 char __user *data,
 155                 int length)
 156 {
 157         char __iomem *vaddr;
 158         int unwritten;
 159
 160         vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
 161         if (vaddr == NULL)
 162                 return -ENOMEM;
 163         unwritten = __copy_to_user_inatomic(data, vaddr + page_offset, length);
 164         kunmap_atomic(vaddr, KM_USER0);
 165
 166         if (unwritten)
 167                 return -EFAULT;
 168
 169         return 0;
 170 }
 171
 172 static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
 173 {
 174         drm_i915_private_t *dev_priv = obj->dev->dev_private;
 175         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 176
 177         return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
 178                 obj_priv->tiling_mode != I915_TILING_NONE;
 179 }
 180
 181 static inline void
 182 slow_shmem_copy(struct page *dst_page,
 183                 int dst_offset,
 184                 struct page *src_page,
 185                 int src_offset,
 186                 int length)
 187 {
 188         char *dst_vaddr, *src_vaddr;
 189
 190         dst_vaddr = kmap(dst_page);
 191         src_vaddr = kmap(src_page);
 192
 193         memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
 194
 195         kunmap(src_page);
 196         kunmap(dst_page);
 197 }
 198
 199 static inline void
 200 slow_shmem_bit17_copy(struct page *gpu_page,
 201                       int gpu_offset,
 202                       struct page *cpu_page,
 203                       int cpu_offset,
 204                       int length,
 205                       int is_read)
 206 {
 207         char *gpu_vaddr, *cpu_vaddr;
 208
 209         /* Use the unswizzled path if this page isn't affected. */
 210         if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
 211                 if (is_read)
 212                         return slow_shmem_copy(cpu_page, cpu_offset,
 213                                                gpu_page, gpu_offset, length);
 214                 else
 215                         return slow_shmem_copy(gpu_page, gpu_offset,
 216                                                cpu_page, cpu_offset, length);
 217         }
 218
 219         gpu_vaddr = kmap(gpu_page);
 220         cpu_vaddr = kmap(cpu_page);
 221
 222         /* Copy the data, XORing A6 with A17 (1). The user already knows he's
 223          * XORing with the other bits (A9 for Y, A9 and A10 for X)
 224          */
 225         while (length > 0) {
 226                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 227                 int this_length = min(cacheline_end - gpu_offset, length);
 228                 int swizzled_gpu_offset = gpu_offset ^ 64;
 229
 230                 if (is_read) {
 231                         memcpy(cpu_vaddr + cpu_offset,
 232                                gpu_vaddr + swizzled_gpu_offset,
 233                                this_length);
 234                 } else {
 235                         memcpy(gpu_vaddr + swizzled_gpu_offset,
 236                                cpu_vaddr + cpu_offset,
 237                                this_length);
 238                 }
 239                 cpu_offset += this_length;
 240                 gpu_offset += this_length;
 241                 length -= this_length;
 242         }
 243
 244         kunmap(cpu_page);
 245         kunmap(gpu_page);
 246 }
 247
 248 /**
 249  * This is the fast shmem pread path, which attempts to copy_from_user directly
 250  * from the backing pages of the object to the user's address space.  On a
 251  * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
 252  */
 253 static int
 254 i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
 255                           struct drm_i915_gem_pread *args,
 256                           struct drm_file *file_priv)
 257 {
 258         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 259         ssize_t remain;
 260         loff_t offset, page_base;
 261         char __user *user_data;
 262         int page_offset, page_length;
 263         int ret;
 264
 265         user_data = (char __user *) (uintptr_t) args->data_ptr;
 266         remain = args->size;
 267
 268         mutex_lock(&dev->struct_mutex);
 269
 270         ret = i915_gem_object_get_pages(obj, 0);
 271         if (ret != 0)
 272                 goto fail_unlock;
 273
 274         ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
 275                                                         args->size);
 276         if (ret != 0)
 277                 goto fail_put_pages;
 278
 279         obj_priv = to_intel_bo(obj);
 280         offset = args->offset;
 281
 282         while (remain > 0) {
 283                 /* Operation in this page
 284                  *
 285                  * page_base = page offset within aperture
 286                  * page_offset = offset within page
 287                  * page_length = bytes to copy for this page
 288                  */
 289                 page_base = (offset & ~(PAGE_SIZE-1));
 290                 page_offset = offset & (PAGE_SIZE-1);
 291                 page_length = remain;
 292                 if ((page_offset + remain) > PAGE_SIZE)
 293                         page_length = PAGE_SIZE - page_offset;
 294
 295                 ret = fast_shmem_read(obj_priv->pages,
 296                                       page_base, page_offset,
 297                                       user_data, page_length);
 298                 if (ret)
 299                         goto fail_put_pages;
 300
 301                 remain -= page_length;
 302                 user_data += page_length;
 303                 offset += page_length;
 304         }
 305
 306 fail_put_pages:
 307         i915_gem_object_put_pages(obj);
 308 fail_unlock:
 309         mutex_unlock(&dev->struct_mutex);
 310
 311         return ret;
 312 }
 313
 314 static int
 315 i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
 316 {
 317         int ret;
 318
 319         ret = i915_gem_object_get_pages(obj, __GFP_NORETRY | __GFP_NOWARN);
 320
 321         /* If we've insufficient memory to map in the pages, attempt
 322          * to make some space by throwing out some old buffers.
 323          */
 324         if (ret == -ENOMEM) {
 325                 struct drm_device *dev = obj->dev;
 326
 327                 ret = i915_gem_evict_something(dev, obj->size,
 328                                                i915_gem_get_gtt_alignment(obj));
 329                 if (ret)
 330                         return ret;
 331
 332                 ret = i915_gem_object_get_pages(obj, 0);
 333         }
 334
 335         return ret;
 336 }
 337
 338 /**
 339  * This is the fallback shmem pread path, which allocates temporary storage
 340  * in kernel space to copy_to_user into outside of the struct_mutex, so we
 341  * can copy out of the object's backing pages while holding the struct mutex
 342  * and not take page faults.
 343  */
 344 static int
 345 i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
 346                           struct drm_i915_gem_pread *args,
 347                           struct drm_file *file_priv)
 348 {
 349         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 350         struct mm_struct *mm = current->mm;
 351         struct page **user_pages;
 352         ssize_t remain;
 353         loff_t offset, pinned_pages, i;
 354         loff_t first_data_page, last_data_page, num_pages;
 355         int shmem_page_index, shmem_page_offset;
 356         int data_page_index,  data_page_offset;
 357         int page_length;
 358         int ret;
 359         uint64_t data_ptr = args->data_ptr;
 360         int do_bit17_swizzling;
 361
 362         remain = args->size;
 363
 364         /* Pin the user pages containing the data.  We can't fault while
 365          * holding the struct mutex, yet we want to hold it while
 366          * dereferencing the user data.
 367          */
 368         first_data_page = data_ptr / PAGE_SIZE;
 369         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 370         num_pages = last_data_page - first_data_page + 1;
 371
 372         user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
 373         if (user_pages == NULL)
 374                 return -ENOMEM;
 375
 376         down_read(&mm->mmap_sem);
 377         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 378                                       num_pages, 1, 0, user_pages, NULL);
 379         up_read(&mm->mmap_sem);
 380         if (pinned_pages < num_pages) {
 381                 ret = -EFAULT;
 382                 goto fail_put_user_pages;
 383         }
 384
 385         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 386
 387         mutex_lock(&dev->struct_mutex);
 388
 389         ret = i915_gem_object_get_pages_or_evict(obj);
 390         if (ret)
 391                 goto fail_unlock;
 392
 393         ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
 394                                                         args->size);
 395         if (ret != 0)
 396                 goto fail_put_pages;
 397
 398         obj_priv = to_intel_bo(obj);
 399         offset = args->offset;
 400
 401         while (remain > 0) {
 402                 /* Operation in this page
 403                  *
 404                  * shmem_page_index = page number within shmem file
 405                  * shmem_page_offset = offset within page in shmem file
 406                  * data_page_index = page number in get_user_pages return
 407                  * data_page_offset = offset with data_page_index page.
 408                  * page_length = bytes to copy for this page
 409                  */
 410                 shmem_page_index = offset / PAGE_SIZE;
 411                 shmem_page_offset = offset & ~PAGE_MASK;
 412                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 413                 data_page_offset = data_ptr & ~PAGE_MASK;
 414
 415                 page_length = remain;
 416                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 417                         page_length = PAGE_SIZE - shmem_page_offset;
 418                 if ((data_page_offset + page_length) > PAGE_SIZE)
 419                         page_length = PAGE_SIZE - data_page_offset;
 420
 421                 if (do_bit17_swizzling) {
 422                         slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
 423                                               shmem_page_offset,
 424                                               user_pages[data_page_index],
 425                                               data_page_offset,
 426                                               page_length,
 427                                               1);
 428                 } else {
 429                         slow_shmem_copy(user_pages[data_page_index],
 430                                         data_page_offset,
 431                                         obj_priv->pages[shmem_page_index],
 432                                         shmem_page_offset,
 433                                         page_length);
 434                 }
 435
 436                 remain -= page_length;
 437                 data_ptr += page_length;
 438                 offset += page_length;
 439         }
 440
 441 fail_put_pages:
 442         i915_gem_object_put_pages(obj);
 443 fail_unlock:
 444         mutex_unlock(&dev->struct_mutex);
 445 fail_put_user_pages:
 446         for (i = 0; i < pinned_pages; i++) {
 447                 SetPageDirty(user_pages[i]);
 448                 page_cache_release(user_pages[i]);
 449         }
 450         drm_free_large(user_pages);
 451
 452         return ret;
 453 }
 454
 455 /**
 456  * Reads data from the object referenced by handle.
 457  *
 458  * On error, the contents of *data are undefined.
 459  */
 460 int
 461 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 462                      struct drm_file *file_priv)
 463 {
 464         struct drm_i915_gem_pread *args = data;
 465         struct drm_gem_object *obj;
 466         struct drm_i915_gem_object *obj_priv;
 467         int ret;
 468
 469         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 470         if (obj == NULL)
 471                 return -ENOENT;
 472         obj_priv = to_intel_bo(obj);
 473
 474         /* Bounds check source.
 475          *
 476          * XXX: This could use review for overflow issues...
 477          */
 478         if (args->offset > obj->size || args->size > obj->size ||
 479             args->offset + args->size > obj->size) {
 480                 drm_gem_object_unreference_unlocked(obj);
 481                 return -EINVAL;
 482         }
 483
 484         if (i915_gem_object_needs_bit17_swizzle(obj)) {
 485                 ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv);
 486         } else {
 487                 ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
 488                 if (ret != 0)
 489                         ret = i915_gem_shmem_pread_slow(dev, obj, args,
 490                                                         file_priv);
 491         }
 492
 493         drm_gem_object_unreference_unlocked(obj);
 494
 495         return ret;
 496 }
 497
 498 /* This is the fast write path which cannot handle
 499  * page faults in the source data
 500  */
 501
 502 static inline int
 503 fast_user_write(struct io_mapping *mapping,
 504                 loff_t page_base, int page_offset,
 505                 char __user *user_data,
 506                 int length)
 507 {
 508         char *vaddr_atomic;
 509         unsigned long unwritten;
 510
 511         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base, KM_USER0);
 512         unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
 513                                                       user_data, length);
 514         io_mapping_unmap_atomic(vaddr_atomic, KM_USER0);
 515         if (unwritten)
 516                 return -EFAULT;
 517         return 0;
 518 }
 519
 520 /* Here's the write path which can sleep for
 521  * page faults
 522  */
 523
 524 static inline void
 525 slow_kernel_write(struct io_mapping *mapping,
 526                   loff_t gtt_base, int gtt_offset,
 527                   struct page *user_page, int user_offset,
 528                   int length)
 529 {
 530         char __iomem *dst_vaddr;
 531         char *src_vaddr;
 532
 533         dst_vaddr = io_mapping_map_wc(mapping, gtt_base);
 534         src_vaddr = kmap(user_page);
 535
 536         memcpy_toio(dst_vaddr + gtt_offset,
 537                     src_vaddr + user_offset,
 538                     length);
 539
 540         kunmap(user_page);
 541         io_mapping_unmap(dst_vaddr);
 542 }
 543
 544 static inline int
 545 fast_shmem_write(struct page **pages,
 546                  loff_t page_base, int page_offset,
 547                  char __user *data,
 548                  int length)
 549 {
 550         char __iomem *vaddr;
 551         unsigned long unwritten;
 552
 553         vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
 554         if (vaddr == NULL)
 555                 return -ENOMEM;
 556         unwritten = __copy_from_user_inatomic(vaddr + page_offset, data, length);
 557         kunmap_atomic(vaddr, KM_USER0);
 558
 559         if (unwritten)
 560                 return -EFAULT;
 561         return 0;
 562 }
 563
 564 /**
 565  * This is the fast pwrite path, where we copy the data directly from the
 566  * user into the GTT, uncached.
 567  */
 568 static int
 569 i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
 570                          struct drm_i915_gem_pwrite *args,
 571                          struct drm_file *file_priv)
 572 {
 573         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 574         drm_i915_private_t *dev_priv = dev->dev_private;
 575         ssize_t remain;
 576         loff_t offset, page_base;
 577         char __user *user_data;
 578         int page_offset, page_length;
 579         int ret;
 580
 581         user_data = (char __user *) (uintptr_t) args->data_ptr;
 582         remain = args->size;
 583         if (!access_ok(VERIFY_READ, user_data, remain))
 584                 return -EFAULT;
 585
 586
 587         mutex_lock(&dev->struct_mutex);
 588         ret = i915_gem_object_pin(obj, 0);
 589         if (ret) {
 590                 mutex_unlock(&dev->struct_mutex);
 591                 return ret;
 592         }
 593         ret = i915_gem_object_set_to_gtt_domain(obj, 1);
 594         if (ret)
 595                 goto fail;
 596
 597         obj_priv = to_intel_bo(obj);
 598         offset = obj_priv->gtt_offset + args->offset;
 599
 600         while (remain > 0) {
 601                 /* Operation in this page
 602                  *
 603                  * page_base = page offset within aperture
 604                  * page_offset = offset within page
 605                  * page_length = bytes to copy for this page
 606                  */
 607                 page_base = (offset & ~(PAGE_SIZE-1));
 608                 page_offset = offset & (PAGE_SIZE-1);
 609                 page_length = remain;
 610                 if ((page_offset + remain) > PAGE_SIZE)
 611                         page_length = PAGE_SIZE - page_offset;
 612
 613                 ret = fast_user_write (dev_priv->mm.gtt_mapping, page_base,
 614                                        page_offset, user_data, page_length);
 615
 616                 /* If we get a fault while copying data, then (presumably) our
 617                  * source page isn't available.  Return the error and we'll
 618                  * retry in the slow path.
 619                  */
 620                 if (ret)
 621                         goto fail;
 622
 623                 remain -= page_length;
 624                 user_data += page_length;
 625                 offset += page_length;
 626         }
 627
 628 fail:
 629         i915_gem_object_unpin(obj);
 630         mutex_unlock(&dev->struct_mutex);
 631
 632         return ret;
 633 }
 634
 635 /**
 636  * This is the fallback GTT pwrite path, which uses get_user_pages to pin
 637  * the memory and maps it using kmap_atomic for copying.
 638  *
 639  * This code resulted in x11perf -rgb10text consuming about 10% more CPU
 640  * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
 641  */
 642 static int
 643 i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
 644                          struct drm_i915_gem_pwrite *args,
 645                          struct drm_file *file_priv)
 646 {
 647         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 648         drm_i915_private_t *dev_priv = dev->dev_private;
 649         ssize_t remain;
 650         loff_t gtt_page_base, offset;
 651         loff_t first_data_page, last_data_page, num_pages;
 652         loff_t pinned_pages, i;
 653         struct page **user_pages;
 654         struct mm_struct *mm = current->mm;
 655         int gtt_page_offset, data_page_offset, data_page_index, page_length;
 656         int ret;
 657         uint64_t data_ptr = args->data_ptr;
 658
 659         remain = args->size;
 660
 661         /* Pin the user pages containing the data.  We can't fault while
 662          * holding the struct mutex, and all of the pwrite implementations
 663          * want to hold it while dereferencing the user data.
 664          */
 665         first_data_page = data_ptr / PAGE_SIZE;
 666         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 667         num_pages = last_data_page - first_data_page + 1;
 668
 669         user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
 670         if (user_pages == NULL)
 671                 return -ENOMEM;
 672
 673         down_read(&mm->mmap_sem);
 674         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 675                                       num_pages, 0, 0, user_pages, NULL);
 676         up_read(&mm->mmap_sem);
 677         if (pinned_pages < num_pages) {
 678                 ret = -EFAULT;
 679                 goto out_unpin_pages;
 680         }
 681
 682         mutex_lock(&dev->struct_mutex);
 683         ret = i915_gem_object_pin(obj, 0);
 684         if (ret)
 685                 goto out_unlock;
 686
 687         ret = i915_gem_object_set_to_gtt_domain(obj, 1);
 688         if (ret)
 689                 goto out_unpin_object;
 690
 691         obj_priv = to_intel_bo(obj);
 692         offset = obj_priv->gtt_offset + args->offset;
 693
 694         while (remain > 0) {
 695                 /* Operation in this page
 696                  *
 697                  * gtt_page_base = page offset within aperture
 698                  * gtt_page_offset = offset within page in aperture
 699                  * data_page_index = page number in get_user_pages return
 700                  * data_page_offset = offset with data_page_index page.
 701                  * page_length = bytes to copy for this page
 702                  */
 703                 gtt_page_base = offset & PAGE_MASK;
 704                 gtt_page_offset = offset & ~PAGE_MASK;
 705                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 706                 data_page_offset = data_ptr & ~PAGE_MASK;
 707
 708                 page_length = remain;
 709                 if ((gtt_page_offset + page_length) > PAGE_SIZE)
 710                         page_length = PAGE_SIZE - gtt_page_offset;
 711                 if ((data_page_offset + page_length) > PAGE_SIZE)
 712                         page_length = PAGE_SIZE - data_page_offset;
 713
 714                 slow_kernel_write(dev_priv->mm.gtt_mapping,
 715                                   gtt_page_base, gtt_page_offset,
 716                                   user_pages[data_page_index],
 717                                   data_page_offset,
 718                                   page_length);
 719
 720                 remain -= page_length;
 721                 offset += page_length;
 722                 data_ptr += page_length;
 723         }
 724
 725 out_unpin_object:
 726         i915_gem_object_unpin(obj);
 727 out_unlock:
 728         mutex_unlock(&dev->struct_mutex);
 729 out_unpin_pages:
 730         for (i = 0; i < pinned_pages; i++)
 731                 page_cache_release(user_pages[i]);
 732         drm_free_large(user_pages);
 733
 734         return ret;
 735 }
 736
 737 /**
 738  * This is the fast shmem pwrite path, which attempts to directly
 739  * copy_from_user into the kmapped pages backing the object.
 740  */
 741 static int
 742 i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
 743                            struct drm_i915_gem_pwrite *args,
 744                            struct drm_file *file_priv)
 745 {
 746         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 747         ssize_t remain;
 748         loff_t offset, page_base;
 749         char __user *user_data;
 750         int page_offset, page_length;
 751         int ret;
 752
 753         user_data = (char __user *) (uintptr_t) args->data_ptr;
 754         remain = args->size;
 755
 756         mutex_lock(&dev->struct_mutex);
 757
 758         ret = i915_gem_object_get_pages(obj, 0);
 759         if (ret != 0)
 760                 goto fail_unlock;
 761
 762         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
 763         if (ret != 0)
 764                 goto fail_put_pages;
 765
 766         obj_priv = to_intel_bo(obj);
 767         offset = args->offset;
 768         obj_priv->dirty = 1;
 769
 770         while (remain > 0) {
 771                 /* Operation in this page
 772                  *
 773                  * page_base = page offset within aperture
 774                  * page_offset = offset within page
 775                  * page_length = bytes to copy for this page
 776                  */
 777                 page_base = (offset & ~(PAGE_SIZE-1));
 778                 page_offset = offset & (PAGE_SIZE-1);
 779                 page_length = remain;
 780                 if ((page_offset + remain) > PAGE_SIZE)
 781                         page_length = PAGE_SIZE - page_offset;
 782
 783                 ret = fast_shmem_write(obj_priv->pages,
 784                                        page_base, page_offset,
 785                                        user_data, page_length);
 786                 if (ret)
 787                         goto fail_put_pages;
 788
 789                 remain -= page_length;
 790                 user_data += page_length;
 791                 offset += page_length;
 792         }
 793
 794 fail_put_pages:
 795         i915_gem_object_put_pages(obj);
 796 fail_unlock:
 797         mutex_unlock(&dev->struct_mutex);
 798
 799         return ret;
 800 }
 801
 802 /**
 803  * This is the fallback shmem pwrite path, which uses get_user_pages to pin
 804  * the memory and maps it using kmap_atomic for copying.
 805  *
 806  * This avoids taking mmap_sem for faulting on the user's address while the
 807  * struct_mutex is held.
 808  */
 809 static int
 810 i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
 811                            struct drm_i915_gem_pwrite *args,
 812                            struct drm_file *file_priv)
 813 {
 814         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 815         struct mm_struct *mm = current->mm;
 816         struct page **user_pages;
 817         ssize_t remain;
 818         loff_t offset, pinned_pages, i;
 819         loff_t first_data_page, last_data_page, num_pages;
 820         int shmem_page_index, shmem_page_offset;
 821         int data_page_index,  data_page_offset;
 822         int page_length;
 823         int ret;
 824         uint64_t data_ptr = args->data_ptr;
 825         int do_bit17_swizzling;
 826
 827         remain = args->size;
 828
 829         /* Pin the user pages containing the data.  We can't fault while
 830          * holding the struct mutex, and all of the pwrite implementations
 831          * want to hold it while dereferencing the user data.
 832          */
 833         first_data_page = data_ptr / PAGE_SIZE;
 834         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 835         num_pages = last_data_page - first_data_page + 1;
 836
 837         user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
 838         if (user_pages == NULL)
 839                 return -ENOMEM;
 840
 841         down_read(&mm->mmap_sem);
 842         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 843                                       num_pages, 0, 0, user_pages, NULL);
 844         up_read(&mm->mmap_sem);
 845         if (pinned_pages < num_pages) {
 846                 ret = -EFAULT;
 847                 goto fail_put_user_pages;
 848         }
 849
 850         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 851
 852         mutex_lock(&dev->struct_mutex);
 853
 854         ret = i915_gem_object_get_pages_or_evict(obj);
 855         if (ret)
 856                 goto fail_unlock;
 857
 858         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
 859         if (ret != 0)
 860                 goto fail_put_pages;
 861
 862         obj_priv = to_intel_bo(obj);
 863         offset = args->offset;
 864         obj_priv->dirty = 1;
 865
 866         while (remain > 0) {
 867                 /* Operation in this page
 868                  *
 869                  * shmem_page_index = page number within shmem file
 870                  * shmem_page_offset = offset within page in shmem file
 871                  * data_page_index = page number in get_user_pages return
 872                  * data_page_offset = offset with data_page_index page.
 873                  * page_length = bytes to copy for this page
 874                  */
 875                 shmem_page_index = offset / PAGE_SIZE;
 876                 shmem_page_offset = offset & ~PAGE_MASK;
 877                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 878                 data_page_offset = data_ptr & ~PAGE_MASK;
 879
 880                 page_length = remain;
 881                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 882                         page_length = PAGE_SIZE - shmem_page_offset;
 883                 if ((data_page_offset + page_length) > PAGE_SIZE)
 884                         page_length = PAGE_SIZE - data_page_offset;
 885
 886                 if (do_bit17_swizzling) {
 887                         slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
 888                                               shmem_page_offset,
 889                                               user_pages[data_page_index],
 890                                               data_page_offset,
 891                                               page_length,
 892                                               0);
 893                 } else {
 894                         slow_shmem_copy(obj_priv->pages[shmem_page_index],
 895                                         shmem_page_offset,
 896                                         user_pages[data_page_index],
 897                                         data_page_offset,
 898                                         page_length);
 899                 }
 900
 901                 remain -= page_length;
 902                 data_ptr += page_length;
 903                 offset += page_length;
 904         }
 905
 906 fail_put_pages:
 907         i915_gem_object_put_pages(obj);
 908 fail_unlock:
 909         mutex_unlock(&dev->struct_mutex);
 910 fail_put_user_pages:
 911         for (i = 0; i < pinned_pages; i++)
 912                 page_cache_release(user_pages[i]);
 913         drm_free_large(user_pages);
 914
 915         return ret;
 916 }
 917
 918 /**
 919  * Writes data to the object referenced by handle.
 920  *
 921  * On error, the contents of the buffer that were to be modified are undefined.
 922  */
 923 int
 924 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 925                       struct drm_file *file_priv)
 926 {
 927         struct drm_i915_gem_pwrite *args = data;
 928         struct drm_gem_object *obj;
 929         struct drm_i915_gem_object *obj_priv;
 930         int ret = 0;
 931
 932         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 933         if (obj == NULL)
 934                 return -ENOENT;
 935         obj_priv = to_intel_bo(obj);
 936
 937         /* Bounds check destination.
 938          *
 939          * XXX: This could use review for overflow issues...
 940          */
 941         if (args->offset > obj->size || args->size > obj->size ||
 942             args->offset + args->size > obj->size) {
 943                 drm_gem_object_unreference_unlocked(obj);
 944                 return -EINVAL;
 945         }
 946
 947         /* We can only do the GTT pwrite on untiled buffers, as otherwise
 948          * it would end up going through the fenced access, and we'll get
 949          * different detiling behavior between reading and writing.
 950          * pread/pwrite currently are reading and writing from the CPU
 951          * perspective, requiring manual detiling by the client.
 952          */
 953         if (obj_priv->phys_obj)
 954                 ret = i915_gem_phys_pwrite(dev, obj, args, file_priv);
 955         else if (obj_priv->tiling_mode == I915_TILING_NONE &&
 956                  dev->gtt_total != 0 &&
 957                  obj->write_domain != I915_GEM_DOMAIN_CPU) {
 958                 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file_priv);
 959                 if (ret == -EFAULT) {
 960                         ret = i915_gem_gtt_pwrite_slow(dev, obj, args,
 961                                                        file_priv);
 962                 }
 963         } else if (i915_gem_object_needs_bit17_swizzle(obj)) {
 964                 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file_priv);
 965         } else {
 966                 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file_priv);
 967                 if (ret == -EFAULT) {
 968                         ret = i915_gem_shmem_pwrite_slow(dev, obj, args,
 969                                                          file_priv);
 970                 }
 971         }
 972
 973 #if WATCH_PWRITE
 974         if (ret)
 975                 DRM_INFO("pwrite failed %d\n", ret);
 976 #endif
 977
 978         drm_gem_object_unreference_unlocked(obj);
 979
 980         return ret;
 981 }
 982
 983 /**
 984  * Called when user space prepares to use an object with the CPU, either
 985  * through the mmap ioctl's mapping or a GTT mapping.
 986  */
 987 int
 988 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 989                           struct drm_file *file_priv)
 990 {
 991         struct drm_i915_private *dev_priv = dev->dev_private;
 992         struct drm_i915_gem_set_domain *args = data;
 993         struct drm_gem_object *obj;
 994         struct drm_i915_gem_object *obj_priv;
 995         uint32_t read_domains = args->read_domains;
 996         uint32_t write_domain = args->write_domain;
 997         int ret;
 998
 999         if (!(dev->driver->driver_features & DRIVER_GEM))
1000                 return -ENODEV;
1001
1002         /* Only handle setting domains to types used by the CPU. */
1003         if (write_domain & I915_GEM_GPU_DOMAINS)
1004                 return -EINVAL;
1005
1006         if (read_domains & I915_GEM_GPU_DOMAINS)
1007                 return -EINVAL;
1008
1009         /* Having something in the write domain implies it's in the read
1010          * domain, and only that read domain.  Enforce that in the request.
1011          */
1012         if (write_domain != 0 && read_domains != write_domain)
1013                 return -EINVAL;
1014
1015         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1016         if (obj == NULL)
1017                 return -ENOENT;
1018         obj_priv = to_intel_bo(obj);
1019
1020         mutex_lock(&dev->struct_mutex);
1021
1022         intel_mark_busy(dev, obj);
1023
1024 #if WATCH_BUF
1025         DRM_INFO("set_domain_ioctl %p(%zd), %08x %08x\n",
1026                  obj, obj->size, read_domains, write_domain);
1027 #endif
1028         if (read_domains & I915_GEM_DOMAIN_GTT) {
1029                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1030
1031                 /* Update the LRU on the fence for the CPU access that's
1032                  * about to occur.
1033                  */
1034                 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1035                         struct drm_i915_fence_reg *reg =
1036                                 &dev_priv->fence_regs[obj_priv->fence_reg];
1037                         list_move_tail(&reg->lru_list,
1038                                        &dev_priv->mm.fence_list);
1039                 }
1040
1041                 /* Silently promote "you're not bound, there was nothing to do"
1042                  * to success, since the client was just asking us to
1043                  * make sure everything was done.
1044                  */
1045                 if (ret == -EINVAL)
1046                         ret = 0;
1047         } else {
1048                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1049         }
1050
1051
1052         /* Maintain LRU order of "inactive" objects */
1053         if (ret == 0 && i915_gem_object_is_inactive(obj_priv))
1054                 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1055
1056         drm_gem_object_unreference(obj);
1057         mutex_unlock(&dev->struct_mutex);
1058         return ret;
1059 }
1060
1061 /**
1062  * Called when user space has done writes to this buffer
1063  */
1064 int
1065 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1066                       struct drm_file *file_priv)
1067 {
1068         struct drm_i915_gem_sw_finish *args = data;
1069         struct drm_gem_object *obj;
1070         struct drm_i915_gem_object *obj_priv;
1071         int ret = 0;
1072
1073         if (!(dev->driver->driver_features & DRIVER_GEM))
1074                 return -ENODEV;
1075
1076         mutex_lock(&dev->struct_mutex);
1077         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1078         if (obj == NULL) {
1079                 mutex_unlock(&dev->struct_mutex);
1080                 return -ENOENT;
1081         }
1082
1083 #if WATCH_BUF
1084         DRM_INFO("%s: sw_finish %d (%p %zd)\n",
1085                  __func__, args->handle, obj, obj->size);
1086 #endif
1087         obj_priv = to_intel_bo(obj);
1088
1089         /* Pinned buffers may be scanout, so flush the cache */
1090         if (obj_priv->pin_count)
1091                 i915_gem_object_flush_cpu_write_domain(obj);
1092
1093         drm_gem_object_unreference(obj);
1094         mutex_unlock(&dev->struct_mutex);
1095         return ret;
1096 }
1097
1098 /**
1099  * Maps the contents of an object, returning the address it is mapped
1100  * into.
1101  *
1102  * While the mapping holds a reference on the contents of the object, it doesn't
1103  * imply a ref on the object itself.
1104  */
1105 int
1106 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1107                    struct drm_file *file_priv)
1108 {
1109         struct drm_i915_gem_mmap *args = data;
1110         struct drm_gem_object *obj;
1111         loff_t offset;
1112         unsigned long addr;
1113
1114         if (!(dev->driver->driver_features & DRIVER_GEM))
1115                 return -ENODEV;
1116
1117         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1118         if (obj == NULL)
1119                 return -ENOENT;
1120
1121         offset = args->offset;
1122
1123         down_write(&current->mm->mmap_sem);
1124         addr = do_mmap(obj->filp, 0, args->size,
1125                        PROT_READ | PROT_WRITE, MAP_SHARED,
1126                        args->offset);
1127         up_write(&current->mm->mmap_sem);
1128         drm_gem_object_unreference_unlocked(obj);
1129         if (IS_ERR((void *)addr))
1130                 return addr;
1131
1132         args->addr_ptr = (uint64_t) addr;
1133
1134         return 0;
1135 }
1136
1137 /**
1138  * i915_gem_fault - fault a page into the GTT
1139  * vma: VMA in question
1140  * vmf: fault info
1141  *
1142  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1143  * from userspace.  The fault handler takes care of binding the object to
1144  * the GTT (if needed), allocating and programming a fence register (again,
1145  * only if needed based on whether the old reg is still valid or the object
1146  * is tiled) and inserting a new PTE into the faulting process.
1147  *
1148  * Note that the faulting process may involve evicting existing objects
1149  * from the GTT and/or fence registers to make room.  So performance may
1150  * suffer if the GTT working set is large or there are few fence registers
1151  * left.
1152  */
1153 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1154 {
1155         struct drm_gem_object *obj = vma->vm_private_data;
1156         struct drm_device *dev = obj->dev;
1157         drm_i915_private_t *dev_priv = dev->dev_private;
1158         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1159         pgoff_t page_offset;
1160         unsigned long pfn;
1161         int ret = 0;
1162         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1163
1164         /* We don't use vmf->pgoff since that has the fake offset */
1165         page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1166                 PAGE_SHIFT;
1167
1168         /* Now bind it into the GTT if needed */
1169         mutex_lock(&dev->struct_mutex);
1170         if (!obj_priv->gtt_space) {
1171                 ret = i915_gem_object_bind_to_gtt(obj, 0);
1172                 if (ret)
1173                         goto unlock;
1174
1175                 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1176                 if (ret)
1177                         goto unlock;
1178         }
1179
1180         /* Need a new fence register? */
1181         if (obj_priv->tiling_mode != I915_TILING_NONE) {
1182                 ret = i915_gem_object_get_fence_reg(obj);
1183                 if (ret)
1184                         goto unlock;
1185         }
1186
1187         if (i915_gem_object_is_inactive(obj_priv))
1188                 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1189
1190         pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
1191                 page_offset;
1192
1193         /* Finally, remap it using the new GTT offset */
1194         ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1195 unlock:
1196         mutex_unlock(&dev->struct_mutex);
1197
1198         switch (ret) {
1199         case 0:
1200         case -ERESTARTSYS:
1201                 return VM_FAULT_NOPAGE;
1202         case -ENOMEM:
1203         case -EAGAIN:
1204                 return VM_FAULT_OOM;
1205         default:
1206                 return VM_FAULT_SIGBUS;
1207         }
1208 }
1209
1210 /**
1211  * i915_gem_create_mmap_offset - create a fake mmap offset for an object
1212  * @obj: obj in question
1213  *
1214  * GEM memory mapping works by handing back to userspace a fake mmap offset
1215  * it can use in a subsequent mmap(2) call.  The DRM core code then looks
1216  * up the object based on the offset and sets up the various memory mapping
1217  * structures.
1218  *
1219  * This routine allocates and attaches a fake offset for @obj.
1220  */
1221 static int
1222 i915_gem_create_mmap_offset(struct drm_gem_object *obj)
1223 {
1224         struct drm_device *dev = obj->dev;
1225         struct drm_gem_mm *mm = dev->mm_private;
1226         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1227         struct drm_map_list *list;
1228         struct drm_local_map *map;
1229         int ret = 0;
1230
1231         /* Set the object up for mmap'ing */
1232         list = &obj->map_list;
1233         list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
1234         if (!list->map)
1235                 return -ENOMEM;
1236
1237         map = list->map;
1238         map->type = _DRM_GEM;
1239         map->size = obj->size;
1240         map->handle = obj;
1241
1242         /* Get a DRM GEM mmap offset allocated... */
1243         list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
1244                                                     obj->size / PAGE_SIZE, 0, 0);
1245         if (!list->file_offset_node) {
1246                 DRM_ERROR("failed to allocate offset for bo %d\n", obj->name);
1247                 ret = -ENOMEM;
1248                 goto out_free_list;
1249         }
1250
1251         list->file_offset_node = drm_mm_get_block(list->file_offset_node,
1252                                                   obj->size / PAGE_SIZE, 0);
1253         if (!list->file_offset_node) {
1254                 ret = -ENOMEM;
1255                 goto out_free_list;
1256         }
1257
1258         list->hash.key = list->file_offset_node->start;
1259         if (drm_ht_insert_item(&mm->offset_hash, &list->hash)) {
1260                 DRM_ERROR("failed to add to map hash\n");
1261                 ret = -ENOMEM;
1262                 goto out_free_mm;
1263         }
1264
1265         /* By now we should be all set, any drm_mmap request on the offset
1266          * below will get to our mmap & fault handler */
1267         obj_priv->mmap_offset = ((uint64_t) list->hash.key) << PAGE_SHIFT;
1268
1269         return 0;
1270
1271 out_free_mm:
1272         drm_mm_put_block(list->file_offset_node);
1273 out_free_list:
1274         kfree(list->map);
1275
1276         return ret;
1277 }
1278
1279 /**
1280  * i915_gem_release_mmap - remove physical page mappings
1281  * @obj: obj in question
1282  *
1283  * Preserve the reservation of the mmapping with the DRM core code, but
1284  * relinquish ownership of the pages back to the system.
1285  *
1286  * It is vital that we remove the page mapping if we have mapped a tiled
1287  * object through the GTT and then lose the fence register due to
1288  * resource pressure. Similarly if the object has been moved out of the
1289  * aperture, than pages mapped into userspace must be revoked. Removing the
1290  * mapping will then trigger a page fault on the next user access, allowing
1291  * fixup by i915_gem_fault().
1292  */
1293 void
1294 i915_gem_release_mmap(struct drm_gem_object *obj)
1295 {
1296         struct drm_device *dev = obj->dev;
1297         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1298
1299         if (dev->dev_mapping)
1300                 unmap_mapping_range(dev->dev_mapping,
1301                                     obj_priv->mmap_offset, obj->size, 1);
1302 }
1303
1304 static void
1305 i915_gem_free_mmap_offset(struct drm_gem_object *obj)
1306 {
1307         struct drm_device *dev = obj->dev;
1308         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1309         struct drm_gem_mm *mm = dev->mm_private;
1310         struct drm_map_list *list;
1311
1312         list = &obj->map_list;
1313         drm_ht_remove_item(&mm->offset_hash, &list->hash);
1314
1315         if (list->file_offset_node) {
1316                 drm_mm_put_block(list->file_offset_node);
1317                 list->file_offset_node = NULL;
1318         }
1319
1320         if (list->map) {
1321                 kfree(list->map);
1322                 list->map = NULL;
1323         }
1324
1325         obj_priv->mmap_offset = 0;
1326 }
1327
1328 /**
1329  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1330  * @obj: object to check
1331  *
1332  * Return the required GTT alignment for an object, taking into account
1333  * potential fence register mapping if needed.
1334  */
1335 static uint32_t
1336 i915_gem_get_gtt_alignment(struct drm_gem_object *obj)
1337 {
1338         struct drm_device *dev = obj->dev;
1339         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1340         int start, i;
1341
1342         /*
1343          * Minimum alignment is 4k (GTT page size), but might be greater
1344          * if a fence register is needed for the object.
1345          */
1346         if (IS_I965G(dev) || obj_priv->tiling_mode == I915_TILING_NONE)
1347                 return 4096;
1348
1349         /*
1350          * Previous chips need to be aligned to the size of the smallest
1351          * fence register that can contain the object.
1352          */
1353         if (IS_I9XX(dev))
1354                 start = 1024*1024;
1355         else
1356                 start = 512*1024;
1357
1358         for (i = start; i < obj->size; i <<= 1)
1359                 ;
1360
1361         return i;
1362 }
1363
1364 /**
1365  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1366  * @dev: DRM device
1367  * @data: GTT mapping ioctl data
1368  * @file_priv: GEM object info
1369  *
1370  * Simply returns the fake offset to userspace so it can mmap it.
1371  * The mmap call will end up in drm_gem_mmap(), which will set things
1372  * up so we can get faults in the handler above.
1373  *
1374  * The fault handler will take care of binding the object into the GTT
1375  * (since it may have been evicted to make room for something), allocating
1376  * a fence register, and mapping the appropriate aperture address into
1377  * userspace.
1378  */
1379 int
1380 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1381                         struct drm_file *file_priv)
1382 {
1383         struct drm_i915_gem_mmap_gtt *args = data;
1384         struct drm_gem_object *obj;
1385         struct drm_i915_gem_object *obj_priv;
1386         int ret;
1387
1388         if (!(dev->driver->driver_features & DRIVER_GEM))
1389                 return -ENODEV;
1390
1391         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1392         if (obj == NULL)
1393                 return -ENOENT;
1394
1395         mutex_lock(&dev->struct_mutex);
1396
1397         obj_priv = to_intel_bo(obj);
1398
1399         if (obj_priv->madv != I915_MADV_WILLNEED) {
1400                 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1401                 drm_gem_object_unreference(obj);
1402                 mutex_unlock(&dev->struct_mutex);
1403                 return -EINVAL;
1404         }
1405
1406
1407         if (!obj_priv->mmap_offset) {
1408                 ret = i915_gem_create_mmap_offset(obj);
1409                 if (ret) {
1410                         drm_gem_object_unreference(obj);
1411                         mutex_unlock(&dev->struct_mutex);
1412                         return ret;
1413                 }
1414         }
1415
1416         args->offset = obj_priv->mmap_offset;
1417
1418         /*
1419          * Pull it into the GTT so that we have a page list (makes the
1420          * initial fault faster and any subsequent flushing possible).
1421          */
1422         if (!obj_priv->agp_mem) {
1423                 ret = i915_gem_object_bind_to_gtt(obj, 0);
1424                 if (ret) {
1425                         drm_gem_object_unreference(obj);
1426                         mutex_unlock(&dev->struct_mutex);
1427                         return ret;
1428                 }
1429         }
1430
1431         drm_gem_object_unreference(obj);
1432         mutex_unlock(&dev->struct_mutex);
1433
1434         return 0;
1435 }
1436
1437 void
1438 i915_gem_object_put_pages(struct drm_gem_object *obj)
1439 {
1440         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1441         int page_count = obj->size / PAGE_SIZE;
1442         int i;
1443
1444         BUG_ON(obj_priv->pages_refcount == 0);
1445         BUG_ON(obj_priv->madv == __I915_MADV_PURGED);
1446
1447         if (--obj_priv->pages_refcount != 0)
1448                 return;
1449
1450         if (obj_priv->tiling_mode != I915_TILING_NONE)
1451                 i915_gem_object_save_bit_17_swizzle(obj);
1452
1453         if (obj_priv->madv == I915_MADV_DONTNEED)
1454                 obj_priv->dirty = 0;
1455
1456         for (i = 0; i < page_count; i++) {
1457                 if (obj_priv->dirty)
1458                         set_page_dirty(obj_priv->pages[i]);
1459
1460                 if (obj_priv->madv == I915_MADV_WILLNEED)
1461                         mark_page_accessed(obj_priv->pages[i]);
1462
1463                 page_cache_release(obj_priv->pages[i]);
1464         }
1465         obj_priv->dirty = 0;
1466
1467         drm_free_large(obj_priv->pages);
1468         obj_priv->pages = NULL;
1469 }
1470
1471 static void
1472 i915_gem_object_move_to_active(struct drm_gem_object *obj, uint32_t seqno,
1473                                struct intel_ring_buffer *ring)
1474 {
1475         struct drm_device *dev = obj->dev;
1476         drm_i915_private_t *dev_priv = dev->dev_private;
1477         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1478         BUG_ON(ring == NULL);
1479         obj_priv->ring = ring;
1480
1481         /* Add a reference if we're newly entering the active list. */
1482         if (!obj_priv->active) {
1483                 drm_gem_object_reference(obj);
1484                 obj_priv->active = 1;
1485         }
1486         /* Move from whatever list we were on to the tail of execution. */
1487         spin_lock(&dev_priv->mm.active_list_lock);
1488         list_move_tail(&obj_priv->list, &ring->active_list);
1489         spin_unlock(&dev_priv->mm.active_list_lock);
1490         obj_priv->last_rendering_seqno = seqno;
1491 }
1492
1493 static void
1494 i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
1495 {
1496         struct drm_device *dev = obj->dev;
1497         drm_i915_private_t *dev_priv = dev->dev_private;
1498         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1499
1500         BUG_ON(!obj_priv->active);
1501         list_move_tail(&obj_priv->list, &dev_priv->mm.flushing_list);
1502         obj_priv->last_rendering_seqno = 0;
1503 }
1504
1505 /* Immediately discard the backing storage */
1506 static void
1507 i915_gem_object_truncate(struct drm_gem_object *obj)
1508 {
1509         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1510         struct inode *inode;
1511
1512         /* Our goal here is to return as much of the memory as
1513          * is possible back to the system as we are called from OOM.
1514          * To do this we must instruct the shmfs to drop all of its
1515          * backing pages, *now*. Here we mirror the actions taken
1516          * when by shmem_delete_inode() to release the backing store.
1517          */
1518         inode = obj->filp->f_path.dentry->d_inode;
1519         truncate_inode_pages(inode->i_mapping, 0);
1520         if (inode->i_op->truncate_range)
1521                 inode->i_op->truncate_range(inode, 0, (loff_t)-1);
1522
1523         obj_priv->madv = __I915_MADV_PURGED;
1524 }
1525
1526 static inline int
1527 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj_priv)
1528 {
1529         return obj_priv->madv == I915_MADV_DONTNEED;
1530 }
1531
1532 static void
1533 i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
1534 {
1535         struct drm_device *dev = obj->dev;
1536         drm_i915_private_t *dev_priv = dev->dev_private;
1537         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1538
1539         i915_verify_inactive(dev, __FILE__, __LINE__);
1540         if (obj_priv->pin_count != 0)
1541                 list_del_init(&obj_priv->list);
1542         else
1543                 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1544
1545         BUG_ON(!list_empty(&obj_priv->gpu_write_list));
1546
1547         obj_priv->last_rendering_seqno = 0;
1548         obj_priv->ring = NULL;
1549         if (obj_priv->active) {
1550                 obj_priv->active = 0;
1551                 drm_gem_object_unreference(obj);
1552         }
1553         i915_verify_inactive(dev, __FILE__, __LINE__);
1554 }
1555
1556 static void
1557 i915_gem_process_flushing_list(struct drm_device *dev,
1558                                uint32_t flush_domains, uint32_t seqno,
1559                                struct intel_ring_buffer *ring)
1560 {
1561         drm_i915_private_t *dev_priv = dev->dev_private;
1562         struct drm_i915_gem_object *obj_priv, *next;
1563
1564         list_for_each_entry_safe(obj_priv, next,
1565                                  &dev_priv->mm.gpu_write_list,
1566                                  gpu_write_list) {
1567                 struct drm_gem_object *obj = &obj_priv->base;
1568
1569                 if ((obj->write_domain & flush_domains) ==
1570                     obj->write_domain &&
1571                     obj_priv->ring->ring_flag == ring->ring_flag) {
1572                         uint32_t old_write_domain = obj->write_domain;
1573
1574                         obj->write_domain = 0;
1575                         list_del_init(&obj_priv->gpu_write_list);
1576                         i915_gem_object_move_to_active(obj, seqno, ring);
1577
1578                         /* update the fence lru list */
1579                         if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1580                                 struct drm_i915_fence_reg *reg =
1581                                         &dev_priv->fence_regs[obj_priv->fence_reg];
1582                                 list_move_tail(&reg->lru_list,
1583                                                 &dev_priv->mm.fence_list);
1584                         }
1585
1586                         trace_i915_gem_object_change_domain(obj,
1587                                                             obj->read_domains,
1588                                                             old_write_domain);
1589                 }
1590         }
1591 }
1592
1593 uint32_t
1594 i915_add_request(struct drm_device *dev, struct drm_file *file_priv,
1595                  uint32_t flush_domains, struct intel_ring_buffer *ring)
1596 {
1597         drm_i915_private_t *dev_priv = dev->dev_private;
1598         struct drm_i915_file_private *i915_file_priv = NULL;
1599         struct drm_i915_gem_request *request;
1600         uint32_t seqno;
1601         int was_empty;
1602
1603         if (file_priv != NULL)
1604                 i915_file_priv = file_priv->driver_priv;
1605
1606         request = kzalloc(sizeof(*request), GFP_KERNEL);
1607         if (request == NULL)
1608                 return 0;
1609
1610         seqno = ring->add_request(dev, ring, file_priv, flush_domains);
1611
1612         request->seqno = seqno;
1613         request->ring = ring;
1614         request->emitted_jiffies = jiffies;
1615         was_empty = list_empty(&ring->request_list);
1616         list_add_tail(&request->list, &ring->request_list);
1617
1618         if (i915_file_priv) {
1619                 list_add_tail(&request->client_list,
1620                               &i915_file_priv->mm.request_list);
1621         } else {
1622                 INIT_LIST_HEAD(&request->client_list);
1623         }
1624
1625         /* Associate any objects on the flushing list matching the write
1626          * domain we're flushing with our flush.
1627          */
1628         if (flush_domains != 0)
1629                 i915_gem_process_flushing_list(dev, flush_domains, seqno, ring);
1630
1631         if (!dev_priv->mm.suspended) {
1632                 mod_timer(&dev_priv->hangcheck_timer, jiffies + DRM_I915_HANGCHECK_PERIOD);
1633                 if (was_empty)
1634                         queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1635         }
1636         return seqno;
1637 }
1638
1639 /**
1640  * Command execution barrier
1641  *
1642  * Ensures that all commands in the ring are finished
1643  * before signalling the CPU
1644  */
1645 static uint32_t
1646 i915_retire_commands(struct drm_device *dev, struct intel_ring_buffer *ring)
1647 {
1648         uint32_t flush_domains = 0;
1649
1650         /* The sampler always gets flushed on i965 (sigh) */
1651         if (IS_I965G(dev))
1652                 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
1653
1654         ring->flush(dev, ring,
1655                         I915_GEM_DOMAIN_COMMAND, flush_domains);
1656         return flush_domains;
1657 }
1658
1659 /**
1660  * Moves buffers associated only with the given active seqno from the active
1661  * to inactive list, potentially freeing them.
1662  */
1663 static void
1664 i915_gem_retire_request(struct drm_device *dev,
1665                         struct drm_i915_gem_request *request)
1666 {
1667         drm_i915_private_t *dev_priv = dev->dev_private;
1668
1669         trace_i915_gem_request_retire(dev, request->seqno);
1670
1671         /* Move any buffers on the active list that are no longer referenced
1672          * by the ringbuffer to the flushing/inactive lists as appropriate.
1673          */
1674         spin_lock(&dev_priv->mm.active_list_lock);
1675         while (!list_empty(&request->ring->active_list)) {
1676                 struct drm_gem_object *obj;
1677                 struct drm_i915_gem_object *obj_priv;
1678
1679                 obj_priv = list_first_entry(&request->ring->active_list,
1680                                             struct drm_i915_gem_object,
1681                                             list);
1682                 obj = &obj_priv->base;
1683
1684                 /* If the seqno being retired doesn't match the oldest in the
1685                  * list, then the oldest in the list must still be newer than
1686                  * this seqno.
1687                  */
1688                 if (obj_priv->last_rendering_seqno != request->seqno)
1689                         goto out;
1690
1691 #if WATCH_LRU
1692                 DRM_INFO("%s: retire %d moves to inactive list %p\n",
1693                          __func__, request->seqno, obj);
1694 #endif
1695
1696                 if (obj->write_domain != 0)
1697                         i915_gem_object_move_to_flushing(obj);
1698                 else {
1699                         /* Take a reference on the object so it won't be
1700                          * freed while the spinlock is held.  The list
1701                          * protection for this spinlock is safe when breaking
1702                          * the lock like this since the next thing we do
1703                          * is just get the head of the list again.
1704                          */
1705                         drm_gem_object_reference(obj);
1706                         i915_gem_object_move_to_inactive(obj);
1707                         spin_unlock(&dev_priv->mm.active_list_lock);
1708                         drm_gem_object_unreference(obj);
1709                         spin_lock(&dev_priv->mm.active_list_lock);
1710                 }
1711         }
1712 out:
1713         spin_unlock(&dev_priv->mm.active_list_lock);
1714 }
1715
1716 /**
1717  * Returns true if seq1 is later than seq2.
1718  */
1719 bool
1720 i915_seqno_passed(uint32_t seq1, uint32_t seq2)
1721 {
1722         return (int32_t)(seq1 - seq2) >= 0;
1723 }
1724
1725 uint32_t
1726 i915_get_gem_seqno(struct drm_device *dev,
1727                    struct intel_ring_buffer *ring)
1728 {
1729         return ring->get_gem_seqno(dev, ring);
1730 }
1731
1732 /**
1733  * This function clears the request list as sequence numbers are passed.
1734  */
1735 static void
1736 i915_gem_retire_requests_ring(struct drm_device *dev,
1737                               struct intel_ring_buffer *ring)
1738 {
1739         drm_i915_private_t *dev_priv = dev->dev_private;
1740         uint32_t seqno;
1741
1742         if (!ring->status_page.page_addr
1743                         || list_empty(&ring->request_list))
1744                 return;
1745
1746         seqno = i915_get_gem_seqno(dev, ring);
1747
1748         while (!list_empty(&ring->request_list)) {
1749                 struct drm_i915_gem_request *request;
1750                 uint32_t retiring_seqno;
1751
1752                 request = list_first_entry(&ring->request_list,
1753                                            struct drm_i915_gem_request,
1754                                            list);
1755                 retiring_seqno = request->seqno;
1756
1757                 if (i915_seqno_passed(seqno, retiring_seqno) ||
1758                     atomic_read(&dev_priv->mm.wedged)) {
1759                         i915_gem_retire_request(dev, request);
1760
1761                         list_del(&request->list);
1762                         list_del(&request->client_list);
1763                         kfree(request);
1764                 } else
1765                         break;
1766         }
1767
1768         if (unlikely (dev_priv->trace_irq_seqno &&
1769                       i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) {
1770
1771                 ring->user_irq_put(dev, ring);
1772                 dev_priv->trace_irq_seqno = 0;
1773         }
1774 }
1775
1776 void
1777 i915_gem_retire_requests(struct drm_device *dev)
1778 {
1779         drm_i915_private_t *dev_priv = dev->dev_private;
1780
1781         if (!list_empty(&dev_priv->mm.deferred_free_list)) {
1782             struct drm_i915_gem_object *obj_priv, *tmp;
1783
1784             /* We must be careful that during unbind() we do not
1785              * accidentally infinitely recurse into retire requests.
1786              * Currently:
1787              *   retire -> free -> unbind -> wait -> retire_ring
1788              */
1789             list_for_each_entry_safe(obj_priv, tmp,
1790                                      &dev_priv->mm.deferred_free_list,
1791                                      list)
1792                     i915_gem_free_object_tail(&obj_priv->base);
1793         }
1794
1795         i915_gem_retire_requests_ring(dev, &dev_priv->render_ring);
1796         if (HAS_BSD(dev))
1797                 i915_gem_retire_requests_ring(dev, &dev_priv->bsd_ring);
1798 }
1799
1800 void
1801 i915_gem_retire_work_handler(struct work_struct *work)
1802 {
1803         drm_i915_private_t *dev_priv;
1804         struct drm_device *dev;
1805
1806         dev_priv = container_of(work, drm_i915_private_t,
1807                                 mm.retire_work.work);
1808         dev = dev_priv->dev;
1809
1810         mutex_lock(&dev->struct_mutex);
1811         i915_gem_retire_requests(dev);
1812
1813         if (!dev_priv->mm.suspended &&
1814                 (!list_empty(&dev_priv->render_ring.request_list) ||
1815                         (HAS_BSD(dev) &&
1816                          !list_empty(&dev_priv->bsd_ring.request_list))))
1817                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1818         mutex_unlock(&dev->struct_mutex);
1819 }
1820
1821 int
1822 i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
1823                 int interruptible, struct intel_ring_buffer *ring)
1824 {
1825         drm_i915_private_t *dev_priv = dev->dev_private;
1826         u32 ier;
1827         int ret = 0;
1828
1829         BUG_ON(seqno == 0);
1830
1831         if (atomic_read(&dev_priv->mm.wedged))
1832                 return -EIO;
1833
1834         if (!i915_seqno_passed(ring->get_gem_seqno(dev, ring), seqno)) {
1835                 if (HAS_PCH_SPLIT(dev))
1836                         ier = I915_READ(DEIER) | I915_READ(GTIER);
1837                 else
1838                         ier = I915_READ(IER);
1839                 if (!ier) {
1840                         DRM_ERROR("something (likely vbetool) disabled "
1841                                   "interrupts, re-enabling\n");
1842                         i915_driver_irq_preinstall(dev);
1843                         i915_driver_irq_postinstall(dev);
1844                 }
1845
1846                 trace_i915_gem_request_wait_begin(dev, seqno);
1847
1848                 ring->waiting_gem_seqno = seqno;
1849                 ring->user_irq_get(dev, ring);
1850                 if (interruptible)
1851                         ret = wait_event_interruptible(ring->irq_queue,
1852                                 i915_seqno_passed(
1853                                         ring->get_gem_seqno(dev, ring), seqno)
1854                                 || atomic_read(&dev_priv->mm.wedged));
1855                 else
1856                         wait_event(ring->irq_queue,
1857                                 i915_seqno_passed(
1858                                         ring->get_gem_seqno(dev, ring), seqno)
1859                                 || atomic_read(&dev_priv->mm.wedged));
1860
1861                 ring->user_irq_put(dev, ring);
1862                 ring->waiting_gem_seqno = 0;
1863
1864                 trace_i915_gem_request_wait_end(dev, seqno);
1865         }
1866         if (atomic_read(&dev_priv->mm.wedged))
1867                 ret = -EIO;
1868
1869         if (ret && ret != -ERESTARTSYS)
1870                 DRM_ERROR("%s returns %d (awaiting %d at %d)\n",
1871                           __func__, ret, seqno, ring->get_gem_seqno(dev, ring));
1872
1873         /* Directly dispatch request retiring.  While we have the work queue
1874          * to handle this, the waiter on a request often wants an associated
1875          * buffer to have made it to the inactive list, and we would need
1876          * a separate wait queue to handle that.
1877          */
1878         if (ret == 0)
1879                 i915_gem_retire_requests_ring(dev, ring);
1880
1881         return ret;
1882 }
1883
1884 /**
1885  * Waits for a sequence number to be signaled, and cleans up the
1886  * request and object lists appropriately for that event.
1887  */
1888 static int
1889 i915_wait_request(struct drm_device *dev, uint32_t seqno,
1890                 struct intel_ring_buffer *ring)
1891 {
1892         return i915_do_wait_request(dev, seqno, 1, ring);
1893 }
1894
1895 static void
1896 i915_gem_flush(struct drm_device *dev,
1897                uint32_t invalidate_domains,
1898                uint32_t flush_domains)
1899 {
1900         drm_i915_private_t *dev_priv = dev->dev_private;
1901         if (flush_domains & I915_GEM_DOMAIN_CPU)
1902                 drm_agp_chipset_flush(dev);
1903         dev_priv->render_ring.flush(dev, &dev_priv->render_ring,
1904                         invalidate_domains,
1905                         flush_domains);
1906
1907         if (HAS_BSD(dev))
1908                 dev_priv->bsd_ring.flush(dev, &dev_priv->bsd_ring,
1909                                 invalidate_domains,
1910                                 flush_domains);
1911 }
1912
1913 /**
1914  * Ensures that all rendering to the object has completed and the object is
1915  * safe to unbind from the GTT or access from the CPU.
1916  */
1917 static int
1918 i915_gem_object_wait_rendering(struct drm_gem_object *obj)
1919 {
1920         struct drm_device *dev = obj->dev;
1921         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1922         int ret;
1923
1924         /* This function only exists to support waiting for existing rendering,
1925          * not for emitting required flushes.
1926          */
1927         BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0);
1928
1929         /* If there is rendering queued on the buffer being evicted, wait for
1930          * it.
1931          */
1932         if (obj_priv->active) {
1933 #if WATCH_BUF
1934                 DRM_INFO("%s: object %p wait for seqno %08x\n",
1935                           __func__, obj, obj_priv->last_rendering_seqno);
1936 #endif
1937                 ret = i915_wait_request(dev,
1938                                 obj_priv->last_rendering_seqno, obj_priv->ring);
1939                 if (ret != 0)
1940                         return ret;
1941         }
1942
1943         return 0;
1944 }
1945
1946 /**
1947  * Unbinds an object from the GTT aperture.
1948  */
1949 int
1950 i915_gem_object_unbind(struct drm_gem_object *obj)
1951 {
1952         struct drm_device *dev = obj->dev;
1953         drm_i915_private_t *dev_priv = dev->dev_private;
1954         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1955         int ret = 0;
1956
1957 #if WATCH_BUF
1958         DRM_INFO("%s:%d %p\n", __func__, __LINE__, obj);
1959         DRM_INFO("gtt_space %p\n", obj_priv->gtt_space);
1960 #endif
1961         if (obj_priv->gtt_space == NULL)
1962                 return 0;
1963
1964         if (obj_priv->pin_count != 0) {
1965                 DRM_ERROR("Attempting to unbind pinned buffer\n");
1966                 return -EINVAL;
1967         }
1968
1969         /* blow away mappings if mapped through GTT */
1970         i915_gem_release_mmap(obj);
1971
1972         /* Move the object to the CPU domain to ensure that
1973          * any possible CPU writes while it's not in the GTT
1974          * are flushed when we go to remap it. This will
1975          * also ensure that all pending GPU writes are finished
1976          * before we unbind.
1977          */
1978         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
1979         if (ret == -ERESTARTSYS)
1980                 return ret;
1981         /* Continue on if we fail due to EIO, the GPU is hung so we
1982          * should be safe and we need to cleanup or else we might
1983          * cause memory corruption through use-after-free.
1984          */
1985
1986         /* release the fence reg _after_ flushing */
1987         if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
1988                 i915_gem_clear_fence_reg(obj);
1989
1990         if (obj_priv->agp_mem != NULL) {
1991                 drm_unbind_agp(obj_priv->agp_mem);
1992                 drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
1993                 obj_priv->agp_mem = NULL;
1994         }
1995
1996         i915_gem_object_put_pages(obj);
1997         BUG_ON(obj_priv->pages_refcount);
1998
1999         if (obj_priv->gtt_space) {
2000                 atomic_dec(&dev->gtt_count);
2001                 atomic_sub(obj->size, &dev->gtt_memory);
2002
2003                 drm_mm_put_block(obj_priv->gtt_space);
2004                 obj_priv->gtt_space = NULL;
2005         }
2006
2007         /* Remove ourselves from the LRU list if present. */
2008         spin_lock(&dev_priv->mm.active_list_lock);
2009         if (!list_empty(&obj_priv->list))
2010                 list_del_init(&obj_priv->list);
2011         spin_unlock(&dev_priv->mm.active_list_lock);
2012
2013         if (i915_gem_object_is_purgeable(obj_priv))
2014                 i915_gem_object_truncate(obj);
2015
2016         trace_i915_gem_object_unbind(obj);
2017
2018         return ret;
2019 }
2020
2021 int
2022 i915_gpu_idle(struct drm_device *dev)
2023 {
2024         drm_i915_private_t *dev_priv = dev->dev_private;
2025         bool lists_empty;
2026         uint32_t seqno1, seqno2;
2027         int ret;
2028
2029         spin_lock(&dev_priv->mm.active_list_lock);
2030         lists_empty = (list_empty(&dev_priv->mm.flushing_list) &&
2031                        list_empty(&dev_priv->render_ring.active_list) &&
2032                        (!HAS_BSD(dev) ||
2033                         list_empty(&dev_priv->bsd_ring.active_list)));
2034         spin_unlock(&dev_priv->mm.active_list_lock);
2035
2036         if (lists_empty)
2037                 return 0;
2038
2039         /* Flush everything onto the inactive list. */
2040         i915_gem_flush(dev, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2041         seqno1 = i915_add_request(dev, NULL, I915_GEM_GPU_DOMAINS,
2042                         &dev_priv->render_ring);
2043         if (seqno1 == 0)
2044                 return -ENOMEM;
2045         ret = i915_wait_request(dev, seqno1, &dev_priv->render_ring);
2046
2047         if (HAS_BSD(dev)) {
2048                 seqno2 = i915_add_request(dev, NULL, I915_GEM_GPU_DOMAINS,
2049                                 &dev_priv->bsd_ring);
2050                 if (seqno2 == 0)
2051                         return -ENOMEM;
2052
2053                 ret = i915_wait_request(dev, seqno2, &dev_priv->bsd_ring);
2054                 if (ret)
2055                         return ret;
2056         }
2057
2058
2059         return ret;
2060 }
2061
2062 int
2063 i915_gem_object_get_pages(struct drm_gem_object *obj,
2064                           gfp_t gfpmask)
2065 {
2066         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2067         int page_count, i;
2068         struct address_space *mapping;
2069         struct inode *inode;
2070         struct page *page;
2071
2072         BUG_ON(obj_priv->pages_refcount
2073                         == DRM_I915_GEM_OBJECT_MAX_PAGES_REFCOUNT);
2074
2075         if (obj_priv->pages_refcount++ != 0)
2076                 return 0;
2077
2078         /* Get the list of pages out of our struct file.  They'll be pinned
2079          * at this point until we release them.
2080          */
2081         page_count = obj->size / PAGE_SIZE;
2082         BUG_ON(obj_priv->pages != NULL);
2083         obj_priv->pages = drm_calloc_large(page_count, sizeof(struct page *));
2084         if (obj_priv->pages == NULL) {
2085                 obj_priv->pages_refcount--;
2086                 return -ENOMEM;
2087         }
2088
2089         inode = obj->filp->f_path.dentry->d_inode;
2090         mapping = inode->i_mapping;
2091         for (i = 0; i < page_count; i++) {
2092                 page = read_cache_page_gfp(mapping, i,
2093                                            GFP_HIGHUSER |
2094                                            __GFP_COLD |
2095                                            __GFP_RECLAIMABLE |
2096                                            gfpmask);
2097                 if (IS_ERR(page))
2098                         goto err_pages;
2099
2100                 obj_priv->pages[i] = page;
2101         }
2102
2103         if (obj_priv->tiling_mode != I915_TILING_NONE)
2104                 i915_gem_object_do_bit_17_swizzle(obj);
2105
2106         return 0;
2107
2108 err_pages:
2109         while (i--)
2110                 page_cache_release(obj_priv->pages[i]);
2111
2112         drm_free_large(obj_priv->pages);
2113         obj_priv->pages = NULL;
2114         obj_priv->pages_refcount--;
2115         return PTR_ERR(page);
2116 }
2117
2118 static void sandybridge_write_fence_reg(struct drm_i915_fence_reg *reg)
2119 {
2120         struct drm_gem_object *obj = reg->obj;
2121         struct drm_device *dev = obj->dev;
2122         drm_i915_private_t *dev_priv = dev->dev_private;
2123         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2124         int regnum = obj_priv->fence_reg;
2125         uint64_t val;
2126
2127         val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2128                     0xfffff000) << 32;
2129         val |= obj_priv->gtt_offset & 0xfffff000;
2130         val |= (uint64_t)((obj_priv->stride / 128) - 1) <<
2131                 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2132
2133         if (obj_priv->tiling_mode == I915_TILING_Y)
2134                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2135         val |= I965_FENCE_REG_VALID;
2136
2137         I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (regnum * 8), val);
2138 }
2139
2140 static void i965_write_fence_reg(struct drm_i915_fence_reg *reg)
2141 {
2142         struct drm_gem_object *obj = reg->obj;
2143         struct drm_device *dev = obj->dev;
2144         drm_i915_private_t *dev_priv = dev->dev_private;
2145         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2146         int regnum = obj_priv->fence_reg;
2147         uint64_t val;
2148
2149         val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2150                     0xfffff000) << 32;
2151         val |= obj_priv->gtt_offset & 0xfffff000;
2152         val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2153         if (obj_priv->tiling_mode == I915_TILING_Y)
2154                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2155         val |= I965_FENCE_REG_VALID;
2156
2157         I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val);
2158 }
2159
2160 static void i915_write_fence_reg(struct drm_i915_fence_reg *reg)
2161 {
2162         struct drm_gem_object *obj = reg->obj;
2163         struct drm_device *dev = obj->dev;
2164         drm_i915_private_t *dev_priv = dev->dev_private;
2165         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2166         int regnum = obj_priv->fence_reg;
2167         int tile_width;
2168         uint32_t fence_reg, val;
2169         uint32_t pitch_val;
2170
2171         if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) ||
2172             (obj_priv->gtt_offset & (obj->size - 1))) {
2173                 WARN(1, "%s: object 0x%08x not 1M or size (0x%zx) aligned\n",
2174                      __func__, obj_priv->gtt_offset, obj->size);
2175                 return;
2176         }
2177
2178         if (obj_priv->tiling_mode == I915_TILING_Y &&
2179             HAS_128_BYTE_Y_TILING(dev))
2180                 tile_width = 128;
2181         else
2182                 tile_width = 512;
2183
2184         /* Note: pitch better be a power of two tile widths */
2185         pitch_val = obj_priv->stride / tile_width;
2186         pitch_val = ffs(pitch_val) - 1;
2187
2188         if (obj_priv->tiling_mode == I915_TILING_Y &&
2189             HAS_128_BYTE_Y_TILING(dev))
2190                 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2191         else
2192                 WARN_ON(pitch_val > I915_FENCE_MAX_PITCH_VAL);
2193
2194         val = obj_priv->gtt_offset;
2195         if (obj_priv->tiling_mode == I915_TILING_Y)
2196                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2197         val |= I915_FENCE_SIZE_BITS(obj->size);
2198         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2199         val |= I830_FENCE_REG_VALID;
2200
2201         if (regnum < 8)
2202                 fence_reg = FENCE_REG_830_0 + (regnum * 4);
2203         else
2204                 fence_reg = FENCE_REG_945_8 + ((regnum - 8) * 4);
2205         I915_WRITE(fence_reg, val);
2206 }
2207
2208 static void i830_write_fence_reg(struct drm_i915_fence_reg *reg)
2209 {
2210         struct drm_gem_object *obj = reg->obj;
2211         struct drm_device *dev = obj->dev;
2212         drm_i915_private_t *dev_priv = dev->dev_private;
2213         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2214         int regnum = obj_priv->fence_reg;
2215         uint32_t val;
2216         uint32_t pitch_val;
2217         uint32_t fence_size_bits;
2218
2219         if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) ||
2220             (obj_priv->gtt_offset & (obj->size - 1))) {
2221                 WARN(1, "%s: object 0x%08x not 512K or size aligned\n",
2222                      __func__, obj_priv->gtt_offset);
2223                 return;
2224         }
2225
2226         pitch_val = obj_priv->stride / 128;
2227         pitch_val = ffs(pitch_val) - 1;
2228         WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2229
2230         val = obj_priv->gtt_offset;
2231         if (obj_priv->tiling_mode == I915_TILING_Y)
2232                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2233         fence_size_bits = I830_FENCE_SIZE_BITS(obj->size);
2234         WARN_ON(fence_size_bits & ~0x00000f00);
2235         val |= fence_size_bits;
2236         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2237         val |= I830_FENCE_REG_VALID;
2238
2239         I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val);
2240 }
2241
2242 static int i915_find_fence_reg(struct drm_device *dev)
2243 {
2244         struct drm_i915_fence_reg *reg = NULL;
2245         struct drm_i915_gem_object *obj_priv = NULL;
2246         struct drm_i915_private *dev_priv = dev->dev_private;
2247         struct drm_gem_object *obj = NULL;
2248         int i, avail, ret;
2249
2250         /* First try to find a free reg */
2251         avail = 0;
2252         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2253                 reg = &dev_priv->fence_regs[i];
2254                 if (!reg->obj)
2255                         return i;
2256
2257                 obj_priv = to_intel_bo(reg->obj);
2258                 if (!obj_priv->pin_count)
2259                     avail++;
2260         }
2261
2262         if (avail == 0)
2263                 return -ENOSPC;
2264
2265         /* None available, try to steal one or wait for a user to finish */
2266         i = I915_FENCE_REG_NONE;
2267         list_for_each_entry(reg, &dev_priv->mm.fence_list,
2268                             lru_list) {
2269                 obj = reg->obj;
2270                 obj_priv = to_intel_bo(obj);
2271
2272                 if (obj_priv->pin_count)
2273                         continue;
2274
2275                 /* found one! */
2276                 i = obj_priv->fence_reg;
2277                 break;
2278         }
2279
2280         BUG_ON(i == I915_FENCE_REG_NONE);
2281
2282         /* We only have a reference on obj from the active list. put_fence_reg
2283          * might drop that one, causing a use-after-free in it. So hold a
2284          * private reference to obj like the other callers of put_fence_reg
2285          * (set_tiling ioctl) do. */
2286         drm_gem_object_reference(obj);
2287         ret = i915_gem_object_put_fence_reg(obj);
2288         drm_gem_object_unreference(obj);
2289         if (ret != 0)
2290                 return ret;
2291
2292         return i;
2293 }
2294
2295 /**
2296  * i915_gem_object_get_fence_reg - set up a fence reg for an object
2297  * @obj: object to map through a fence reg
2298  *
2299  * When mapping objects through the GTT, userspace wants to be able to write
2300  * to them without having to worry about swizzling if the object is tiled.
2301  *
2302  * This function walks the fence regs looking for a free one for @obj,
2303  * stealing one if it can't find any.
2304  *
2305  * It then sets up the reg based on the object's properties: address, pitch
2306  * and tiling format.
2307  */
2308 int
2309 i915_gem_object_get_fence_reg(struct drm_gem_object *obj)
2310 {
2311         struct drm_device *dev = obj->dev;
2312         struct drm_i915_private *dev_priv = dev->dev_private;
2313         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2314         struct drm_i915_fence_reg *reg = NULL;
2315         int ret;
2316
2317         /* Just update our place in the LRU if our fence is getting used. */
2318         if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
2319                 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2320                 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2321                 return 0;
2322         }
2323
2324         switch (obj_priv->tiling_mode) {
2325         case I915_TILING_NONE:
2326                 WARN(1, "allocating a fence for non-tiled object?\n");
2327                 break;
2328         case I915_TILING_X:
2329                 if (!obj_priv->stride)
2330                         return -EINVAL;
2331                 WARN((obj_priv->stride & (512 - 1)),
2332                      "object 0x%08x is X tiled but has non-512B pitch\n",
2333                      obj_priv->gtt_offset);
2334                 break;
2335         case I915_TILING_Y:
2336                 if (!obj_priv->stride)
2337                         return -EINVAL;
2338                 WARN((obj_priv->stride & (128 - 1)),
2339                      "object 0x%08x is Y tiled but has non-128B pitch\n",
2340                      obj_priv->gtt_offset);
2341                 break;
2342         }
2343
2344         ret = i915_find_fence_reg(dev);
2345         if (ret < 0)
2346                 return ret;
2347
2348         obj_priv->fence_reg = ret;
2349         reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2350         list_add_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2351
2352         reg->obj = obj;
2353
2354         switch (INTEL_INFO(dev)->gen) {
2355         case 6:
2356                 sandybridge_write_fence_reg(reg);
2357                 break;
2358         case 5:
2359         case 4:
2360                 i965_write_fence_reg(reg);
2361                 break;
2362         case 3:
2363                 i915_write_fence_reg(reg);
2364                 break;
2365         case 2:
2366                 i830_write_fence_reg(reg);
2367                 break;
2368         }
2369
2370         trace_i915_gem_object_get_fence(obj, obj_priv->fence_reg,
2371                         obj_priv->tiling_mode);
2372
2373         return 0;
2374 }
2375
2376 /**
2377  * i915_gem_clear_fence_reg - clear out fence register info
2378  * @obj: object to clear
2379  *
2380  * Zeroes out the fence register itself and clears out the associated
2381  * data structures in dev_priv and obj_priv.
2382  */
2383 static void
2384 i915_gem_clear_fence_reg(struct drm_gem_object *obj)
2385 {
2386         struct drm_device *dev = obj->dev;
2387         drm_i915_private_t *dev_priv = dev->dev_private;
2388         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2389         struct drm_i915_fence_reg *reg =
2390                 &dev_priv->fence_regs[obj_priv->fence_reg];
2391         uint32_t fence_reg;
2392
2393         switch (INTEL_INFO(dev)->gen) {
2394         case 6:
2395                 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 +
2396                              (obj_priv->fence_reg * 8), 0);
2397                 break;
2398         case 5:
2399         case 4:
2400                 I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0);
2401                 break;
2402         case 3:
2403                 if (obj_priv->fence_reg > 8)
2404                         fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg - 8) * 4;
2405                 else
2406         case 2:
2407                         fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4;
2408
2409                 I915_WRITE(fence_reg, 0);
2410                 break;
2411         }
2412
2413         reg->obj = NULL;
2414         obj_priv->fence_reg = I915_FENCE_REG_NONE;
2415         list_del_init(&reg->lru_list);
2416 }
2417
2418 /**
2419  * i915_gem_object_put_fence_reg - waits on outstanding fenced access
2420  * to the buffer to finish, and then resets the fence register.
2421  * @obj: tiled object holding a fence register.
2422  *
2423  * Zeroes out the fence register itself and clears out the associated
2424  * data structures in dev_priv and obj_priv.
2425  */
2426 int
2427 i915_gem_object_put_fence_reg(struct drm_gem_object *obj)
2428 {
2429         struct drm_device *dev = obj->dev;
2430         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2431
2432         if (obj_priv->fence_reg == I915_FENCE_REG_NONE)
2433                 return 0;
2434
2435         /* If we've changed tiling, GTT-mappings of the object
2436          * need to re-fault to ensure that the correct fence register
2437          * setup is in place.
2438          */
2439         i915_gem_release_mmap(obj);
2440
2441         /* On the i915, GPU access to tiled buffers is via a fence,
2442          * therefore we must wait for any outstanding access to complete
2443          * before clearing the fence.
2444          */
2445         if (!IS_I965G(dev)) {
2446                 int ret;
2447
2448                 ret = i915_gem_object_flush_gpu_write_domain(obj);
2449                 if (ret != 0)
2450                         return ret;
2451
2452                 ret = i915_gem_object_wait_rendering(obj);
2453                 if (ret != 0)
2454                         return ret;
2455         }
2456
2457         i915_gem_object_flush_gtt_write_domain(obj);
2458         i915_gem_clear_fence_reg (obj);
2459
2460         return 0;
2461 }
2462
2463 /**
2464  * Finds free space in the GTT aperture and binds the object there.
2465  */
2466 static int
2467 i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
2468 {
2469         struct drm_device *dev = obj->dev;
2470         drm_i915_private_t *dev_priv = dev->dev_private;
2471         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2472         struct drm_mm_node *free_space;
2473         gfp_t gfpmask =  __GFP_NORETRY | __GFP_NOWARN;
2474         int ret;
2475
2476         if (obj_priv->madv != I915_MADV_WILLNEED) {
2477                 DRM_ERROR("Attempting to bind a purgeable object\n");
2478                 return -EINVAL;
2479         }
2480
2481         if (alignment == 0)
2482                 alignment = i915_gem_get_gtt_alignment(obj);
2483         if (alignment & (i915_gem_get_gtt_alignment(obj) - 1)) {
2484                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2485                 return -EINVAL;
2486         }
2487
2488         /* If the object is bigger than the entire aperture, reject it early
2489          * before evicting everything in a vain attempt to find space.
2490          */
2491         if (obj->size > dev->gtt_total) {
2492                 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
2493                 return -E2BIG;
2494         }
2495
2496  search_free:
2497         free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2498                                         obj->size, alignment, 0);
2499         if (free_space != NULL) {
2500                 obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size,
2501                                                        alignment);
2502                 if (obj_priv->gtt_space != NULL)
2503                         obj_priv->gtt_offset = obj_priv->gtt_space->start;
2504         }
2505         if (obj_priv->gtt_space == NULL) {
2506                 /* If the gtt is empty and we're still having trouble
2507                  * fitting our object in, we're out of memory.
2508                  */
2509 #if WATCH_LRU
2510                 DRM_INFO("%s: GTT full, evicting something\n", __func__);
2511 #endif
2512                 ret = i915_gem_evict_something(dev, obj->size, alignment);
2513                 if (ret)
2514                         return ret;
2515
2516                 goto search_free;
2517         }
2518
2519 #if WATCH_BUF
2520         DRM_INFO("Binding object of size %zd at 0x%08x\n",
2521                  obj->size, obj_priv->gtt_offset);
2522 #endif
2523         ret = i915_gem_object_get_pages(obj, gfpmask);
2524         if (ret) {
2525                 drm_mm_put_block(obj_priv->gtt_space);
2526                 obj_priv->gtt_space = NULL;
2527
2528                 if (ret == -ENOMEM) {
2529                         /* first try to clear up some space from the GTT */
2530                         ret = i915_gem_evict_something(dev, obj->size,
2531                                                        alignment);
2532                         if (ret) {
2533                                 /* now try to shrink everyone else */
2534                                 if (gfpmask) {
2535                                         gfpmask = 0;
2536                                         goto search_free;
2537                                 }
2538
2539                                 return ret;
2540                         }
2541
2542                         goto search_free;
2543                 }
2544
2545                 return ret;
2546         }
2547
2548         /* Create an AGP memory structure pointing at our pages, and bind it
2549          * into the GTT.
2550          */
2551         obj_priv->agp_mem = drm_agp_bind_pages(dev,
2552                                                obj_priv->pages,
2553                                                obj->size >> PAGE_SHIFT,
2554                                                obj_priv->gtt_offset,
2555                                                obj_priv->agp_type);
2556         if (obj_priv->agp_mem == NULL) {
2557                 i915_gem_object_put_pages(obj);
2558                 drm_mm_put_block(obj_priv->gtt_space);
2559                 obj_priv->gtt_space = NULL;
2560
2561                 ret = i915_gem_evict_something(dev, obj->size, alignment);
2562                 if (ret)
2563                         return ret;
2564
2565                 goto search_free;
2566         }
2567         atomic_inc(&dev->gtt_count);
2568         atomic_add(obj->size, &dev->gtt_memory);
2569
2570         /* keep track of bounds object by adding it to the inactive list */
2571         list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
2572
2573         /* Assert that the object is not currently in any GPU domain. As it
2574          * wasn't in the GTT, there shouldn't be any way it could have been in
2575          * a GPU cache
2576          */
2577         BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
2578         BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
2579
2580         trace_i915_gem_object_bind(obj, obj_priv->gtt_offset);
2581
2582         return 0;
2583 }
2584
2585 void
2586 i915_gem_clflush_object(struct drm_gem_object *obj)
2587 {
2588         struct drm_i915_gem_object      *obj_priv = to_intel_bo(obj);
2589
2590         /* If we don't have a page list set up, then we're not pinned
2591          * to GPU, and we can ignore the cache flush because it'll happen
2592          * again at bind time.
2593          */
2594         if (obj_priv->pages == NULL)
2595                 return;
2596
2597         trace_i915_gem_object_clflush(obj);
2598
2599         drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE);
2600 }
2601
2602 /** Flushes any GPU write domain for the object if it's dirty. */
2603 static int
2604 i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj)
2605 {
2606         struct drm_device *dev = obj->dev;
2607         uint32_t old_write_domain;
2608         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2609
2610         if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
2611                 return 0;
2612
2613         /* Queue the GPU write cache flushing we need. */
2614         old_write_domain = obj->write_domain;
2615         i915_gem_flush(dev, 0, obj->write_domain);
2616         if (i915_add_request(dev, NULL, obj->write_domain, obj_priv->ring) == 0)
2617                 return -ENOMEM;
2618
2619         trace_i915_gem_object_change_domain(obj,
2620                                             obj->read_domains,
2621                                             old_write_domain);
2622         return 0;
2623 }
2624
2625 /** Flushes the GTT write domain for the object if it's dirty. */
2626 static void
2627 i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
2628 {
2629         uint32_t old_write_domain;
2630
2631         if (obj->write_domain != I915_GEM_DOMAIN_GTT)
2632                 return;
2633
2634         /* No actual flushing is required for the GTT write domain.   Writes
2635          * to it immediately go to main memory as far as we know, so there's
2636          * no chipset flush.  It also doesn't land in render cache.
2637          */
2638         old_write_domain = obj->write_domain;
2639         obj->write_domain = 0;
2640
2641         trace_i915_gem_object_change_domain(obj,
2642                                             obj->read_domains,
2643                                             old_write_domain);
2644 }
2645
2646 /** Flushes the CPU write domain for the object if it's dirty. */
2647 static void
2648 i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
2649 {
2650         struct drm_device *dev = obj->dev;
2651         uint32_t old_write_domain;
2652
2653         if (obj->write_domain != I915_GEM_DOMAIN_CPU)
2654                 return;
2655
2656         i915_gem_clflush_object(obj);
2657         drm_agp_chipset_flush(dev);
2658         old_write_domain = obj->write_domain;
2659         obj->write_domain = 0;
2660
2661         trace_i915_gem_object_change_domain(obj,
2662                                             obj->read_domains,
2663                                             old_write_domain);
2664 }
2665
2666 int
2667 i915_gem_object_flush_write_domain(struct drm_gem_object *obj)
2668 {
2669         int ret = 0;
2670
2671         switch (obj->write_domain) {
2672         case I915_GEM_DOMAIN_GTT:
2673                 i915_gem_object_flush_gtt_write_domain(obj);
2674                 break;
2675         case I915_GEM_DOMAIN_CPU:
2676                 i915_gem_object_flush_cpu_write_domain(obj);
2677                 break;
2678         default:
2679                 ret = i915_gem_object_flush_gpu_write_domain(obj);
2680                 break;
2681         }
2682
2683         return ret;
2684 }
2685
2686 /**
2687  * Moves a single object to the GTT read, and possibly write domain.
2688  *
2689  * This function returns when the move is complete, including waiting on
2690  * flushes to occur.
2691  */
2692 int
2693 i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
2694 {
2695         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2696         uint32_t old_write_domain, old_read_domains;
2697         int ret;
2698
2699         /* Not valid to be called on unbound objects. */
2700         if (obj_priv->gtt_space == NULL)
2701                 return -EINVAL;
2702
2703         ret = i915_gem_object_flush_gpu_write_domain(obj);
2704         if (ret != 0)
2705                 return ret;
2706
2707         /* Wait on any GPU rendering and flushing to occur. */
2708         ret = i915_gem_object_wait_rendering(obj);
2709         if (ret != 0)
2710                 return ret;
2711
2712         old_write_domain = obj->write_domain;
2713         old_read_domains = obj->read_domains;
2714
2715         /* If we're writing through the GTT domain, then CPU and GPU caches
2716          * will need to be invalidated at next use.
2717          */
2718         if (write)
2719                 obj->read_domains &= I915_GEM_DOMAIN_GTT;
2720
2721         i915_gem_object_flush_cpu_write_domain(obj);
2722
2723         /* It should now be out of any other write domains, and we can update
2724          * the domain values for our changes.
2725          */
2726         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2727         obj->read_domains |= I915_GEM_DOMAIN_GTT;
2728         if (write) {
2729                 obj->write_domain = I915_GEM_DOMAIN_GTT;
2730                 obj_priv->dirty = 1;
2731         }
2732
2733         trace_i915_gem_object_change_domain(obj,
2734                                             old_read_domains,
2735                                             old_write_domain);
2736
2737         return 0;
2738 }
2739
2740 /*
2741  * Prepare buffer for display plane. Use uninterruptible for possible flush
2742  * wait, as in modesetting process we're not supposed to be interrupted.
2743  */
2744 int
2745 i915_gem_object_set_to_display_plane(struct drm_gem_object *obj)
2746 {
2747         struct drm_device *dev = obj->dev;
2748         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2749         uint32_t old_write_domain, old_read_domains;
2750         int ret;
2751
2752         /* Not valid to be called on unbound objects. */
2753         if (obj_priv->gtt_space == NULL)
2754                 return -EINVAL;
2755
2756         ret = i915_gem_object_flush_gpu_write_domain(obj);
2757         if (ret)
2758                 return ret;
2759
2760         /* Wait on any GPU rendering and flushing to occur. */
2761         if (obj_priv->active) {
2762 #if WATCH_BUF
2763                 DRM_INFO("%s: object %p wait for seqno %08x\n",
2764                           __func__, obj, obj_priv->last_rendering_seqno);
2765 #endif
2766                 ret = i915_do_wait_request(dev,
2767                                 obj_priv->last_rendering_seqno,
2768                                 0,
2769                                 obj_priv->ring);
2770                 if (ret != 0)
2771                         return ret;
2772         }
2773
2774         i915_gem_object_flush_cpu_write_domain(obj);
2775
2776         old_write_domain = obj->write_domain;
2777         old_read_domains = obj->read_domains;
2778
2779         /* It should now be out of any other write domains, and we can update
2780          * the domain values for our changes.
2781          */
2782         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2783         obj->read_domains = I915_GEM_DOMAIN_GTT;
2784         obj->write_domain = I915_GEM_DOMAIN_GTT;
2785         obj_priv->dirty = 1;
2786
2787         trace_i915_gem_object_change_domain(obj,
2788                                             old_read_domains,
2789                                             old_write_domain);
2790
2791         return 0;
2792 }
2793
2794 /**
2795  * Moves a single object to the CPU read, and possibly write domain.
2796  *
2797  * This function returns when the move is complete, including waiting on
2798  * flushes to occur.
2799  */
2800 static int
2801 i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
2802 {
2803         uint32_t old_write_domain, old_read_domains;
2804         int ret;
2805
2806         ret = i915_gem_object_flush_gpu_write_domain(obj);
2807         if (ret)
2808                 return ret;
2809
2810         /* Wait on any GPU rendering and flushing to occur. */
2811         ret = i915_gem_object_wait_rendering(obj);
2812         if (ret != 0)
2813                 return ret;
2814
2815         i915_gem_object_flush_gtt_write_domain(obj);
2816
2817         /* If we have a partially-valid cache of the object in the CPU,
2818          * finish invalidating it and free the per-page flags.
2819          */
2820         i915_gem_object_set_to_full_cpu_read_domain(obj);
2821
2822         old_write_domain = obj->write_domain;
2823         old_read_domains = obj->read_domains;
2824
2825         /* Flush the CPU cache if it's still invalid. */
2826         if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2827                 i915_gem_clflush_object(obj);
2828
2829                 obj->read_domains |= I915_GEM_DOMAIN_CPU;
2830         }
2831
2832         /* It should now be out of any other write domains, and we can update
2833          * the domain values for our changes.
2834          */
2835         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2836
2837         /* If we're writing through the CPU, then the GPU read domains will
2838          * need to be invalidated at next use.
2839          */
2840         if (write) {
2841                 obj->read_domains &= I915_GEM_DOMAIN_CPU;
2842                 obj->write_domain = I915_GEM_DOMAIN_CPU;
2843         }
2844
2845         trace_i915_gem_object_change_domain(obj,
2846                                             old_read_domains,
2847                                             old_write_domain);
2848
2849         return 0;
2850 }
2851
2852 /*
2853  * Set the next domain for the specified object. This
2854  * may not actually perform the necessary flushing/invaliding though,
2855  * as that may want to be batched with other set_domain operations
2856  *
2857  * This is (we hope) the only really tricky part of gem. The goal
2858  * is fairly simple -- track which caches hold bits of the object
2859  * and make sure they remain coherent. A few concrete examples may
2860  * help to explain how it works. For shorthand, we use the notation
2861  * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
2862  * a pair of read and write domain masks.
2863  *
2864  * Case 1: the batch buffer
2865  *
2866  *      1. Allocated
2867  *      2. Written by CPU
2868  *      3. Mapped to GTT
2869  *      4. Read by GPU
2870  *      5. Unmapped from GTT
2871  *      6. Freed
2872  *
2873  *      Let's take these a step at a time
2874  *
2875  *      1. Allocated
2876  *              Pages allocated from the kernel may still have
2877  *              cache contents, so we set them to (CPU, CPU) always.
2878  *      2. Written by CPU (using pwrite)
2879  *              The pwrite function calls set_domain (CPU, CPU) and
2880  *              this function does nothing (as nothing changes)
2881  *      3. Mapped by GTT
2882  *              This function asserts that the object is not
2883  *              currently in any GPU-based read or write domains
2884  *      4. Read by GPU
2885  *              i915_gem_execbuffer calls set_domain (COMMAND, 0).
2886  *              As write_domain is zero, this function adds in the
2887  *              current read domains (CPU+COMMAND, 0).
2888  *              flush_domains is set to CPU.
2889  *              invalidate_domains is set to COMMAND
2890  *              clflush is run to get data out of the CPU caches
2891  *              then i915_dev_set_domain calls i915_gem_flush to
2892  *              emit an MI_FLUSH and drm_agp_chipset_flush
2893  *      5. Unmapped from GTT
2894  *              i915_gem_object_unbind calls set_domain (CPU, CPU)
2895  *              flush_domains and invalidate_domains end up both zero
2896  *              so no flushing/invalidating happens
2897  *      6. Freed
2898  *              yay, done
2899  *
2900  * Case 2: The shared render buffer
2901  *
2902  *      1. Allocated
2903  *      2. Mapped to GTT
2904  *      3. Read/written by GPU
2905  *      4. set_domain to (CPU,CPU)
2906  *      5. Read/written by CPU
2907  *      6. Read/written by GPU
2908  *
2909  *      1. Allocated
2910  *              Same as last example, (CPU, CPU)
2911  *      2. Mapped to GTT
2912  *              Nothing changes (assertions find that it is not in the GPU)
2913  *      3. Read/written by GPU
2914  *              execbuffer calls set_domain (RENDER, RENDER)
2915  *              flush_domains gets CPU
2916  *              invalidate_domains gets GPU
2917  *              clflush (obj)
2918  *              MI_FLUSH and drm_agp_chipset_flush
2919  *      4. set_domain (CPU, CPU)
2920  *              flush_domains gets GPU
2921  *              invalidate_domains gets CPU
2922  *              wait_rendering (obj) to make sure all drawing is complete.
2923  *              This will include an MI_FLUSH to get the data from GPU
2924  *              to memory
2925  *              clflush (obj) to invalidate the CPU cache
2926  *              Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
2927  *      5. Read/written by CPU
2928  *              cache lines are loaded and dirtied
2929  *      6. Read written by GPU
2930  *              Same as last GPU access
2931  *
2932  * Case 3: The constant buffer
2933  *
2934  *      1. Allocated
2935  *      2. Written by CPU
2936  *      3. Read by GPU
2937  *      4. Updated (written) by CPU again
2938  *      5. Read by GPU
2939  *
2940  *      1. Allocated
2941  *              (CPU, CPU)
2942  *      2. Written by CPU
2943  *              (CPU, CPU)
2944  *      3. Read by GPU
2945  *              (CPU+RENDER, 0)
2946  *              flush_domains = CPU
2947  *              invalidate_domains = RENDER
2948  *              clflush (obj)
2949  *              MI_FLUSH
2950  *              drm_agp_chipset_flush
2951  *      4. Updated (written) by CPU again
2952  *              (CPU, CPU)
2953  *              flush_domains = 0 (no previous write domain)
2954  *              invalidate_domains = 0 (no new read domains)
2955  *      5. Read by GPU
2956  *              (CPU+RENDER, 0)
2957  *              flush_domains = CPU
2958  *              invalidate_domains = RENDER
2959  *              clflush (obj)
2960  *              MI_FLUSH
2961  *              drm_agp_chipset_flush
2962  */
2963 static void
2964 i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
2965 {
2966         struct drm_device               *dev = obj->dev;
2967         drm_i915_private_t              *dev_priv = dev->dev_private;
2968         struct drm_i915_gem_object      *obj_priv = to_intel_bo(obj);
2969         uint32_t                        invalidate_domains = 0;
2970         uint32_t                        flush_domains = 0;
2971         uint32_t                        old_read_domains;
2972
2973         BUG_ON(obj->pending_read_domains & I915_GEM_DOMAIN_CPU);
2974         BUG_ON(obj->pending_write_domain == I915_GEM_DOMAIN_CPU);
2975
2976         intel_mark_busy(dev, obj);
2977
2978 #if WATCH_BUF
2979         DRM_INFO("%s: object %p read %08x -> %08x write %08x -> %08x\n",
2980                  __func__, obj,
2981                  obj->read_domains, obj->pending_read_domains,
2982                  obj->write_domain, obj->pending_write_domain);
2983 #endif
2984         /*
2985          * If the object isn't moving to a new write domain,
2986          * let the object stay in multiple read domains
2987          */
2988         if (obj->pending_write_domain == 0)
2989                 obj->pending_read_domains |= obj->read_domains;
2990         else
2991                 obj_priv->dirty = 1;
2992
2993         /*
2994          * Flush the current write domain if
2995          * the new read domains don't match. Invalidate
2996          * any read domains which differ from the old
2997          * write domain
2998          */
2999         if (obj->write_domain &&
3000             obj->write_domain != obj->pending_read_domains) {
3001                 flush_domains |= obj->write_domain;
3002                 invalidate_domains |=
3003                         obj->pending_read_domains & ~obj->write_domain;
3004         }
3005         /*
3006          * Invalidate any read caches which may have
3007          * stale data. That is, any new read domains.
3008          */
3009         invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
3010         if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) {
3011 #if WATCH_BUF
3012                 DRM_INFO("%s: CPU domain flush %08x invalidate %08x\n",
3013                          __func__, flush_domains, invalidate_domains);
3014 #endif
3015                 i915_gem_clflush_object(obj);
3016         }
3017
3018         old_read_domains = obj->read_domains;
3019
3020         /* The actual obj->write_domain will be updated with
3021          * pending_write_domain after we emit the accumulated flush for all
3022          * of our domain changes in execbuffers (which clears objects'
3023          * write_domains).  So if we have a current write domain that we
3024          * aren't changing, set pending_write_domain to that.
3025          */
3026         if (flush_domains == 0 && obj->pending_write_domain == 0)
3027                 obj->pending_write_domain = obj->write_domain;
3028         obj->read_domains = obj->pending_read_domains;
3029
3030         if (flush_domains & I915_GEM_GPU_DOMAINS) {
3031                 if (obj_priv->ring == &dev_priv->render_ring)
3032                         dev_priv->flush_rings |= FLUSH_RENDER_RING;
3033                 else if (obj_priv->ring == &dev_priv->bsd_ring)
3034                         dev_priv->flush_rings |= FLUSH_BSD_RING;
3035         }
3036
3037         dev->invalidate_domains |= invalidate_domains;
3038         dev->flush_domains |= flush_domains;
3039 #if WATCH_BUF
3040         DRM_INFO("%s: read %08x write %08x invalidate %08x flush %08x\n",
3041                  __func__,
3042                  obj->read_domains, obj->write_domain,
3043                  dev->invalidate_domains, dev->flush_domains);
3044 #endif
3045
3046         trace_i915_gem_object_change_domain(obj,
3047                                             old_read_domains,
3048                                             obj->write_domain);
3049 }
3050
3051 /**
3052  * Moves the object from a partially CPU read to a full one.
3053  *
3054  * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
3055  * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
3056  */
3057 static void
3058 i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
3059 {
3060         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3061
3062         if (!obj_priv->page_cpu_valid)
3063                 return;
3064
3065         /* If we're partially in the CPU read domain, finish moving it in.
3066          */
3067         if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
3068                 int i;
3069
3070                 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
3071                         if (obj_priv->page_cpu_valid[i])
3072                                 continue;
3073                         drm_clflush_pages(obj_priv->pages + i, 1);
3074                 }
3075         }
3076
3077         /* Free the page_cpu_valid mappings which are now stale, whether
3078          * or not we've got I915_GEM_DOMAIN_CPU.
3079          */
3080         kfree(obj_priv->page_cpu_valid);
3081         obj_priv->page_cpu_valid = NULL;
3082 }
3083
3084 /**
3085  * Set the CPU read domain on a range of the object.
3086  *
3087  * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
3088  * not entirely valid.  The page_cpu_valid member of the object flags which
3089  * pages have been flushed, and will be respected by
3090  * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
3091  * of the whole object.
3092  *
3093  * This function returns when the move is complete, including waiting on
3094  * flushes to occur.
3095  */
3096 static int
3097 i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
3098                                           uint64_t offset, uint64_t size)
3099 {
3100         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3101         uint32_t old_read_domains;
3102         int i, ret;
3103
3104         if (offset == 0 && size == obj->size)
3105                 return i915_gem_object_set_to_cpu_domain(obj, 0);
3106
3107         ret = i915_gem_object_flush_gpu_write_domain(obj);
3108         if (ret)
3109                 return ret;
3110
3111         /* Wait on any GPU rendering and flushing to occur. */
3112         ret = i915_gem_object_wait_rendering(obj);
3113         if (ret != 0)
3114                 return ret;
3115         i915_gem_object_flush_gtt_write_domain(obj);
3116
3117         /* If we're already fully in the CPU read domain, we're done. */
3118         if (obj_priv->page_cpu_valid == NULL &&
3119             (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
3120                 return 0;
3121
3122         /* Otherwise, create/clear the per-page CPU read domain flag if we're
3123          * newly adding I915_GEM_DOMAIN_CPU
3124          */
3125         if (obj_priv->page_cpu_valid == NULL) {
3126                 obj_priv->page_cpu_valid = kzalloc(obj->size / PAGE_SIZE,
3127                                                    GFP_KERNEL);
3128                 if (obj_priv->page_cpu_valid == NULL)
3129                         return -ENOMEM;
3130         } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
3131                 memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
3132
3133         /* Flush the cache on any pages that are still invalid from the CPU's
3134          * perspective.
3135          */
3136         for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3137              i++) {
3138                 if (obj_priv->page_cpu_valid[i])
3139                         continue;
3140
3141                 drm_clflush_pages(obj_priv->pages + i, 1);
3142
3143                 obj_priv->page_cpu_valid[i] = 1;
3144         }
3145
3146         /* It should now be out of any other write domains, and we can update
3147          * the domain values for our changes.
3148          */
3149         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3150
3151         old_read_domains = obj->read_domains;
3152         obj->read_domains |= I915_GEM_DOMAIN_CPU;
3153
3154         trace_i915_gem_object_change_domain(obj,
3155                                             old_read_domains,
3156                                             obj->write_domain);
3157
3158         return 0;
3159 }
3160
3161 /**
3162  * Pin an object to the GTT and evaluate the relocations landing in it.
3163  */
3164 static int
3165 i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
3166                                  struct drm_file *file_priv,
3167                                  struct drm_i915_gem_exec_object2 *entry,
3168                                  struct drm_i915_gem_relocation_entry *relocs)
3169 {
3170         struct drm_device *dev = obj->dev;
3171         drm_i915_private_t *dev_priv = dev->dev_private;
3172         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3173         int i, ret;
3174         void __iomem *reloc_page;
3175         bool need_fence;
3176
3177         need_fence = entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
3178                      obj_priv->tiling_mode != I915_TILING_NONE;
3179
3180         /* Check fence reg constraints and rebind if necessary */
3181         if (need_fence &&
3182             !i915_gem_object_fence_offset_ok(obj,
3183                                              obj_priv->tiling_mode)) {
3184                 ret = i915_gem_object_unbind(obj);
3185                 if (ret)
3186                         return ret;
3187         }
3188
3189         /* Choose the GTT offset for our buffer and put it there. */
3190         ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment);
3191         if (ret)
3192                 return ret;
3193
3194         /*
3195          * Pre-965 chips need a fence register set up in order to
3196          * properly handle blits to/from tiled surfaces.
3197          */
3198         if (need_fence) {
3199                 ret = i915_gem_object_get_fence_reg(obj);
3200                 if (ret != 0) {
3201                         i915_gem_object_unpin(obj);
3202                         return ret;
3203                 }
3204         }
3205
3206         entry->offset = obj_priv->gtt_offset;
3207
3208         /* Apply the relocations, using the GTT aperture to avoid cache
3209          * flushing requirements.
3210          */
3211         for (i = 0; i < entry->relocation_count; i++) {
3212                 struct drm_i915_gem_relocation_entry *reloc= &relocs[i];
3213                 struct drm_gem_object *target_obj;
3214                 struct drm_i915_gem_object *target_obj_priv;
3215                 uint32_t reloc_val, reloc_offset;
3216                 uint32_t __iomem *reloc_entry;
3217
3218                 target_obj = drm_gem_object_lookup(obj->dev, file_priv,
3219                                                    reloc->target_handle);
3220                 if (target_obj == NULL) {
3221                         i915_gem_object_unpin(obj);
3222                         return -ENOENT;
3223                 }
3224                 target_obj_priv = to_intel_bo(target_obj);
3225
3226 #if WATCH_RELOC
3227                 DRM_INFO("%s: obj %p offset %08x target %d "
3228                          "read %08x write %08x gtt %08x "
3229                          "presumed %08x delta %08x\n",
3230                          __func__,
3231                          obj,
3232                          (int) reloc->offset,
3233                          (int) reloc->target_handle,
3234                          (int) reloc->read_domains,
3235                          (int) reloc->write_domain,
3236                          (int) target_obj_priv->gtt_offset,
3237                          (int) reloc->presumed_offset,
3238                          reloc->delta);
3239 #endif
3240
3241                 /* The target buffer should have appeared before us in the
3242                  * exec_object list, so it should have a GTT space bound by now.
3243                  */
3244                 if (target_obj_priv->gtt_space == NULL) {
3245                         DRM_ERROR("No GTT space found for object %d\n",
3246                                   reloc->target_handle);
3247                         drm_gem_object_unreference(target_obj);
3248                         i915_gem_object_unpin(obj);
3249                         return -EINVAL;
3250                 }
3251
3252                 /* Validate that the target is in a valid r/w GPU domain */
3253                 if (reloc->write_domain & (reloc->write_domain - 1)) {
3254                         DRM_ERROR("reloc with multiple write domains: "
3255                                   "obj %p target %d offset %d "
3256                                   "read %08x write %08x",
3257                                   obj, reloc->target_handle,
3258                                   (int) reloc->offset,
3259                                   reloc->read_domains,
3260                                   reloc->write_domain);
3261                         return -EINVAL;
3262                 }
3263                 if (reloc->write_domain & I915_GEM_DOMAIN_CPU ||
3264                     reloc->read_domains & I915_GEM_DOMAIN_CPU) {
3265                         DRM_ERROR("reloc with read/write CPU domains: "
3266                                   "obj %p target %d offset %d "
3267                                   "read %08x write %08x",
3268                                   obj, reloc->target_handle,
3269                                   (int) reloc->offset,
3270                                   reloc->read_domains,
3271                                   reloc->write_domain);
3272                         drm_gem_object_unreference(target_obj);
3273                         i915_gem_object_unpin(obj);
3274                         return -EINVAL;
3275                 }
3276                 if (reloc->write_domain && target_obj->pending_write_domain &&
3277                     reloc->write_domain != target_obj->pending_write_domain) {
3278                         DRM_ERROR("Write domain conflict: "
3279                                   "obj %p target %d offset %d "
3280                                   "new %08x old %08x\n",
3281                                   obj, reloc->target_handle,
3282                                   (int) reloc->offset,
3283                                   reloc->write_domain,
3284                                   target_obj->pending_write_domain);
3285                         drm_gem_object_unreference(target_obj);
3286                         i915_gem_object_unpin(obj);
3287                         return -EINVAL;
3288                 }
3289
3290                 target_obj->pending_read_domains |= reloc->read_domains;
3291                 target_obj->pending_write_domain |= reloc->write_domain;
3292
3293                 /* If the relocation already has the right value in it, no
3294                  * more work needs to be done.
3295                  */
3296                 if (target_obj_priv->gtt_offset == reloc->presumed_offset) {
3297                         drm_gem_object_unreference(target_obj);
3298                         continue;
3299                 }
3300
3301                 /* Check that the relocation address is valid... */
3302                 if (reloc->offset > obj->size - 4) {
3303                         DRM_ERROR("Relocation beyond object bounds: "
3304                                   "obj %p target %d offset %d size %d.\n",
3305                                   obj, reloc->target_handle,
3306                                   (int) reloc->offset, (int) obj->size);
3307                         drm_gem_object_unreference(target_obj);
3308                         i915_gem_object_unpin(obj);
3309                         return -EINVAL;
3310                 }
3311                 if (reloc->offset & 3) {
3312                         DRM_ERROR("Relocation not 4-byte aligned: "
3313                                   "obj %p target %d offset %d.\n",
3314                                   obj, reloc->target_handle,
3315                                   (int) reloc->offset);
3316                         drm_gem_object_unreference(target_obj);
3317                         i915_gem_object_unpin(obj);
3318                         return -EINVAL;
3319                 }
3320
3321                 /* and points to somewhere within the target object. */
3322                 if (reloc->delta >= target_obj->size) {
3323                         DRM_ERROR("Relocation beyond target object bounds: "
3324                                   "obj %p target %d delta %d size %d.\n",
3325                                   obj, reloc->target_handle,
3326                                   (int) reloc->delta, (int) target_obj->size);
3327                         drm_gem_object_unreference(target_obj);
3328                         i915_gem_object_unpin(obj);
3329                         return -EINVAL;
3330                 }
3331
3332                 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
3333                 if (ret != 0) {
3334                         drm_gem_object_unreference(target_obj);
3335                         i915_gem_object_unpin(obj);
3336                         return -EINVAL;
3337                 }
3338
3339                 /* Map the page containing the relocation we're going to
3340                  * perform.
3341                  */
3342                 reloc_offset = obj_priv->gtt_offset + reloc->offset;
3343                 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
3344                                                       (reloc_offset &
3345                                                        ~(PAGE_SIZE - 1)),
3346                                                       KM_USER0);
3347                 reloc_entry = (uint32_t __iomem *)(reloc_page +
3348                                                    (reloc_offset & (PAGE_SIZE - 1)));
3349                 reloc_val = target_obj_priv->gtt_offset + reloc->delta;
3350
3351 #if WATCH_BUF
3352                 DRM_INFO("Applied relocation: %p@0x%08x %08x -> %08x\n",
3353                           obj, (unsigned int) reloc->offset,
3354                           readl(reloc_entry), reloc_val);
3355 #endif
3356                 writel(reloc_val, reloc_entry);
3357                 io_mapping_unmap_atomic(reloc_page, KM_USER0);
3358
3359                 /* The updated presumed offset for this entry will be
3360                  * copied back out to the user.
3361                  */
3362                 reloc->presumed_offset = target_obj_priv->gtt_offset;
3363
3364                 drm_gem_object_unreference(target_obj);
3365         }
3366
3367 #if WATCH_BUF
3368         if (0)
3369                 i915_gem_dump_object(obj, 128, __func__, ~0);
3370 #endif
3371         return 0;
3372 }
3373
3374 /* Throttle our rendering by waiting until the ring has completed our requests
3375  * emitted over 20 msec ago.
3376  *
3377  * Note that if we were to use the current jiffies each time around the loop,
3378  * we wouldn't escape the function with any frames outstanding if the time to
3379  * render a frame was over 20ms.
3380  *
3381  * This should get us reasonable parallelism between CPU and GPU but also
3382  * relatively low latency when blocking on a particular request to finish.
3383  */
3384 static int
3385 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file_priv)
3386 {
3387         struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
3388         int ret = 0;
3389         unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
3390
3391         mutex_lock(&dev->struct_mutex);
3392         while (!list_empty(&i915_file_priv->mm.request_list)) {
3393                 struct drm_i915_gem_request *request;
3394
3395                 request = list_first_entry(&i915_file_priv->mm.request_list,
3396                                            struct drm_i915_gem_request,
3397                                            client_list);
3398
3399                 if (time_after_eq(request->emitted_jiffies, recent_enough))
3400                         break;
3401
3402                 ret = i915_wait_request(dev, request->seqno, request->ring);
3403                 if (ret != 0)
3404                         break;
3405         }
3406         mutex_unlock(&dev->struct_mutex);
3407
3408         return ret;
3409 }
3410
3411 static int
3412 i915_gem_get_relocs_from_user(struct drm_i915_gem_exec_object2 *exec_list,
3413                               uint32_t buffer_count,
3414                               struct drm_i915_gem_relocation_entry **relocs)
3415 {
3416         uint32_t reloc_count = 0, reloc_index = 0, i;
3417         int ret;
3418
3419         *relocs = NULL;
3420         for (i = 0; i < buffer_count; i++) {
3421                 if (reloc_count + exec_list[i].relocation_count < reloc_count)
3422                         return -EINVAL;
3423                 reloc_count += exec_list[i].relocation_count;
3424         }
3425
3426         *relocs = drm_calloc_large(reloc_count, sizeof(**relocs));
3427         if (*relocs == NULL) {
3428                 DRM_ERROR("failed to alloc relocs, count %d\n", reloc_count);
3429                 return -ENOMEM;
3430         }
3431
3432         for (i = 0; i < buffer_count; i++) {
3433                 struct drm_i915_gem_relocation_entry __user *user_relocs;
3434
3435                 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3436
3437                 ret = copy_from_user(&(*relocs)[reloc_index],
3438                                      user_relocs,
3439                                      exec_list[i].relocation_count *
3440                                      sizeof(**relocs));
3441                 if (ret != 0) {
3442                         drm_free_large(*relocs);
3443                         *relocs = NULL;
3444                         return -EFAULT;
3445                 }
3446
3447                 reloc_index += exec_list[i].relocation_count;
3448         }
3449
3450         return 0;
3451 }
3452
3453 static int
3454 i915_gem_put_relocs_to_user(struct drm_i915_gem_exec_object2 *exec_list,
3455                             uint32_t buffer_count,
3456                             struct drm_i915_gem_relocation_entry *relocs)
3457 {
3458         uint32_t reloc_count = 0, i;
3459         int ret = 0;
3460
3461         if (relocs == NULL)
3462             return 0;
3463
3464         for (i = 0; i < buffer_count; i++) {
3465                 struct drm_i915_gem_relocation_entry __user *user_relocs;
3466                 int unwritten;
3467
3468                 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3469
3470                 unwritten = copy_to_user(user_relocs,
3471                                          &relocs[reloc_count],
3472                                          exec_list[i].relocation_count *
3473                                          sizeof(*relocs));
3474
3475                 if (unwritten) {
3476                         ret = -EFAULT;
3477                         goto err;
3478                 }
3479
3480                 reloc_count += exec_list[i].relocation_count;
3481         }
3482
3483 err:
3484         drm_free_large(relocs);
3485
3486         return ret;
3487 }
3488
3489 static int
3490 i915_gem_check_execbuffer (struct drm_i915_gem_execbuffer2 *exec,
3491                            uint64_t exec_offset)
3492 {
3493         uint32_t exec_start, exec_len;
3494
3495         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3496         exec_len = (uint32_t) exec->batch_len;
3497
3498         if ((exec_start | exec_len) & 0x7)
3499                 return -EINVAL;
3500
3501         if (!exec_start)
3502                 return -EINVAL;
3503
3504         return 0;
3505 }
3506
3507 static int
3508 i915_gem_wait_for_pending_flip(struct drm_device *dev,
3509                                struct drm_gem_object **object_list,
3510                                int count)
3511 {
3512         drm_i915_private_t *dev_priv = dev->dev_private;
3513         struct drm_i915_gem_object *obj_priv;
3514         DEFINE_WAIT(wait);
3515         int i, ret = 0;
3516
3517         for (;;) {
3518                 prepare_to_wait(&dev_priv->pending_flip_queue,
3519                                 &wait, TASK_INTERRUPTIBLE);
3520                 for (i = 0; i < count; i++) {
3521                         obj_priv = to_intel_bo(object_list[i]);
3522                         if (atomic_read(&obj_priv->pending_flip) > 0)
3523                                 break;
3524                 }
3525                 if (i == count)
3526                         break;
3527
3528                 if (!signal_pending(current)) {
3529                         mutex_unlock(&dev->struct_mutex);
3530                         schedule();
3531                         mutex_lock(&dev->struct_mutex);
3532                         continue;
3533                 }
3534                 ret = -ERESTARTSYS;
3535                 break;
3536         }
3537         finish_wait(&dev_priv->pending_flip_queue, &wait);
3538
3539         return ret;
3540 }
3541
3542
3543 int
3544 i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3545                        struct drm_file *file_priv,
3546                        struct drm_i915_gem_execbuffer2 *args,
3547                        struct drm_i915_gem_exec_object2 *exec_list)
3548 {
3549         drm_i915_private_t *dev_priv = dev->dev_private;
3550         struct drm_gem_object **object_list = NULL;
3551         struct drm_gem_object *batch_obj;
3552         struct drm_i915_gem_object *obj_priv;
3553         struct drm_clip_rect *cliprects = NULL;
3554         struct drm_i915_gem_relocation_entry *relocs = NULL;
3555         int ret = 0, ret2, i, pinned = 0;
3556         uint64_t exec_offset;
3557         uint32_t seqno, flush_domains, reloc_index;
3558         int pin_tries, flips;
3559
3560         struct intel_ring_buffer *ring = NULL;
3561
3562 #if WATCH_EXEC
3563         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3564                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3565 #endif
3566         if (args->flags & I915_EXEC_BSD) {
3567                 if (!HAS_BSD(dev)) {
3568                         DRM_ERROR("execbuf with wrong flag\n");
3569                         return -EINVAL;
3570                 }
3571                 ring = &dev_priv->bsd_ring;
3572         } else {
3573                 ring = &dev_priv->render_ring;
3574         }
3575
3576         if (args->buffer_count < 1) {
3577                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3578                 return -EINVAL;
3579         }
3580         object_list = drm_malloc_ab(sizeof(*object_list), args->buffer_count);
3581         if (object_list == NULL) {
3582                 DRM_ERROR("Failed to allocate object list for %d buffers\n",
3583                           args->buffer_count);
3584                 ret = -ENOMEM;
3585                 goto pre_mutex_err;
3586         }
3587
3588         if (args->num_cliprects != 0) {
3589                 cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects),
3590                                     GFP_KERNEL);
3591                 if (cliprects == NULL) {
3592                         ret = -ENOMEM;
3593                         goto pre_mutex_err;
3594                 }
3595
3596                 ret = copy_from_user(cliprects,
3597                                      (struct drm_clip_rect __user *)
3598                                      (uintptr_t) args->cliprects_ptr,
3599                                      sizeof(*cliprects) * args->num_cliprects);
3600                 if (ret != 0) {
3601                         DRM_ERROR("copy %d cliprects failed: %d\n",
3602                                   args->num_cliprects, ret);
3603                         ret = -EFAULT;
3604                         goto pre_mutex_err;
3605                 }
3606         }
3607
3608         ret = i915_gem_get_relocs_from_user(exec_list, args->buffer_count,
3609                                             &relocs);
3610         if (ret != 0)
3611                 goto pre_mutex_err;
3612
3613         mutex_lock(&dev->struct_mutex);
3614
3615         i915_verify_inactive(dev, __FILE__, __LINE__);
3616
3617         if (atomic_read(&dev_priv->mm.wedged)) {
3618                 mutex_unlock(&dev->struct_mutex);
3619                 ret = -EIO;
3620                 goto pre_mutex_err;
3621         }
3622
3623         if (dev_priv->mm.suspended) {
3624                 mutex_unlock(&dev->struct_mutex);
3625                 ret = -EBUSY;
3626                 goto pre_mutex_err;
3627         }
3628
3629         /* Look up object handles */
3630         flips = 0;
3631         for (i = 0; i < args->buffer_count; i++) {
3632                 object_list[i] = drm_gem_object_lookup(dev, file_priv,
3633                                                        exec_list[i].handle);
3634                 if (object_list[i] == NULL) {
3635                         DRM_ERROR("Invalid object handle %d at index %d\n",
3636                                    exec_list[i].handle, i);
3637                         /* prevent error path from reading uninitialized data */
3638                         args->buffer_count = i + 1;
3639                         ret = -ENOENT;
3640                         goto err;
3641                 }
3642
3643                 obj_priv = to_intel_bo(object_list[i]);
3644                 if (obj_priv->in_execbuffer) {
3645                         DRM_ERROR("Object %p appears more than once in object list\n",
3646                                    object_list[i]);
3647                         /* prevent error path from reading uninitialized data */
3648                         args->buffer_count = i + 1;
3649                         ret = -EINVAL;
3650                         goto err;
3651                 }
3652                 obj_priv->in_execbuffer = true;
3653                 flips += atomic_read(&obj_priv->pending_flip);
3654         }
3655
3656         if (flips > 0) {
3657                 ret = i915_gem_wait_for_pending_flip(dev, object_list,
3658                                                      args->buffer_count);
3659                 if (ret)
3660                         goto err;
3661         }
3662
3663         /* Pin and relocate */
3664         for (pin_tries = 0; ; pin_tries++) {
3665                 ret = 0;
3666                 reloc_index = 0;
3667
3668                 for (i = 0; i < args->buffer_count; i++) {
3669                         object_list[i]->pending_read_domains = 0;
3670                         object_list[i]->pending_write_domain = 0;
3671                         ret = i915_gem_object_pin_and_relocate(object_list[i],
3672                                                                file_priv,
3673                                                                &exec_list[i],
3674                                                                &relocs[reloc_index]);
3675                         if (ret)
3676                                 break;
3677                         pinned = i + 1;
3678                         reloc_index += exec_list[i].relocation_count;
3679                 }
3680                 /* success */
3681                 if (ret == 0)
3682                         break;
3683
3684                 /* error other than GTT full, or we've already tried again */
3685                 if (ret != -ENOSPC || pin_tries >= 1) {
3686                         if (ret != -ERESTARTSYS) {
3687                                 unsigned long long total_size = 0;
3688                                 int num_fences = 0;
3689                                 for (i = 0; i < args->buffer_count; i++) {
3690                                         obj_priv = to_intel_bo(object_list[i]);
3691
3692                                         total_size += object_list[i]->size;
3693                                         num_fences +=
3694                                                 exec_list[i].flags & EXEC_OBJECT_NEEDS_FENCE &&
3695                                                 obj_priv->tiling_mode != I915_TILING_NONE;
3696                                 }
3697                                 DRM_ERROR("Failed to pin buffer %d of %d, total %llu bytes, %d fences: %d\n",
3698                                           pinned+1, args->buffer_count,
3699                                           total_size, num_fences,
3700                                           ret);
3701                                 DRM_ERROR("%d objects [%d pinned], "
3702                                           "%d object bytes [%d pinned], "
3703                                           "%d/%d gtt bytes\n",
3704                                           atomic_read(&dev->object_count),
3705                                           atomic_read(&dev->pin_count),
3706                                           atomic_read(&dev->object_memory),
3707                                           atomic_read(&dev->pin_memory),
3708                                           atomic_read(&dev->gtt_memory),
3709                                           dev->gtt_total);
3710                         }
3711                         goto err;
3712                 }
3713
3714                 /* unpin all of our buffers */
3715                 for (i = 0; i < pinned; i++)
3716                         i915_gem_object_unpin(object_list[i]);
3717                 pinned = 0;
3718
3719                 /* evict everyone we can from the aperture */
3720                 ret = i915_gem_evict_everything(dev);
3721                 if (ret && ret != -ENOSPC)
3722                         goto err;
3723         }
3724
3725         /* Set the pending read domains for the batch buffer to COMMAND */
3726         batch_obj = object_list[args->buffer_count-1];
3727         if (batch_obj->pending_write_domain) {
3728                 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
3729                 ret = -EINVAL;
3730                 goto err;
3731         }
3732         batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
3733
3734         /* Sanity check the batch buffer, prior to moving objects */
3735         exec_offset = exec_list[args->buffer_count - 1].offset;
3736         ret = i915_gem_check_execbuffer (args, exec_offset);
3737         if (ret != 0) {
3738                 DRM_ERROR("execbuf with invalid offset/length\n");
3739                 goto err;
3740         }
3741
3742         i915_verify_inactive(dev, __FILE__, __LINE__);
3743
3744         /* Zero the global flush/invalidate flags. These
3745          * will be modified as new domains are computed
3746          * for each object
3747          */
3748         dev->invalidate_domains = 0;
3749         dev->flush_domains = 0;
3750         dev_priv->flush_rings = 0;
3751
3752         for (i = 0; i < args->buffer_count; i++) {
3753                 struct drm_gem_object *obj = object_list[i];
3754
3755                 /* Compute new gpu domains and update invalidate/flush */
3756                 i915_gem_object_set_to_gpu_domain(obj);
3757         }
3758
3759         i915_verify_inactive(dev, __FILE__, __LINE__);
3760
3761         if (dev->invalidate_domains | dev->flush_domains) {
3762 #if WATCH_EXEC
3763                 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
3764                           __func__,
3765                          dev->invalidate_domains,
3766                          dev->flush_domains);
3767 #endif
3768                 i915_gem_flush(dev,
3769                                dev->invalidate_domains,
3770                                dev->flush_domains);
3771                 if (dev_priv->flush_rings & FLUSH_RENDER_RING)
3772                         (void)i915_add_request(dev, file_priv,
3773                                                dev->flush_domains,
3774                                                &dev_priv->render_ring);
3775                 if (dev_priv->flush_rings & FLUSH_BSD_RING)
3776                         (void)i915_add_request(dev, file_priv,
3777                                                dev->flush_domains,
3778                                                &dev_priv->bsd_ring);
3779         }
3780
3781         for (i = 0; i < args->buffer_count; i++) {
3782                 struct drm_gem_object *obj = object_list[i];
3783                 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3784                 uint32_t old_write_domain = obj->write_domain;
3785
3786                 obj->write_domain = obj->pending_write_domain;
3787                 if (obj->write_domain)
3788                         list_move_tail(&obj_priv->gpu_write_list,
3789                                        &dev_priv->mm.gpu_write_list);
3790                 else
3791                         list_del_init(&obj_priv->gpu_write_list);
3792
3793                 trace_i915_gem_object_change_domain(obj,
3794                                                     obj->read_domains,
3795                                                     old_write_domain);
3796         }
3797
3798         i915_verify_inactive(dev, __FILE__, __LINE__);
3799
3800 #if WATCH_COHERENCY
3801         for (i = 0; i < args->buffer_count; i++) {
3802                 i915_gem_object_check_coherency(object_list[i],
3803                                                 exec_list[i].handle);
3804         }
3805 #endif
3806
3807 #if WATCH_EXEC
3808         i915_gem_dump_object(batch_obj,
3809                               args->batch_len,
3810                               __func__,
3811                               ~0);
3812 #endif
3813
3814         /* Exec the batchbuffer */
3815         ret = ring->dispatch_gem_execbuffer(dev, ring, args,
3816                         cliprects, exec_offset);
3817         if (ret) {
3818                 DRM_ERROR("dispatch failed %d\n", ret);
3819                 goto err;
3820         }
3821
3822         /*
3823          * Ensure that the commands in the batch buffer are
3824          * finished before the interrupt fires
3825          */
3826         flush_domains = i915_retire_commands(dev, ring);
3827
3828         i915_verify_inactive(dev, __FILE__, __LINE__);
3829
3830         /*
3831          * Get a seqno representing the execution of the current buffer,
3832          * which we can wait on.  We would like to mitigate these interrupts,
3833          * likely by only creating seqnos occasionally (so that we have
3834          * *some* interrupts representing completion of buffers that we can
3835          * wait on when trying to clear up gtt space).
3836          */
3837         seqno = i915_add_request(dev, file_priv, flush_domains, ring);
3838         BUG_ON(seqno == 0);
3839         for (i = 0; i < args->buffer_count; i++) {
3840                 struct drm_gem_object *obj = object_list[i];
3841                 obj_priv = to_intel_bo(obj);
3842
3843                 i915_gem_object_move_to_active(obj, seqno, ring);
3844 #if WATCH_LRU
3845                 DRM_INFO("%s: move to exec list %p\n", __func__, obj);
3846 #endif
3847         }
3848 #if WATCH_LRU
3849         i915_dump_lru(dev, __func__);
3850 #endif
3851
3852         i915_verify_inactive(dev, __FILE__, __LINE__);
3853
3854 err:
3855         for (i = 0; i < pinned; i++)
3856                 i915_gem_object_unpin(object_list[i]);
3857
3858         for (i = 0; i < args->buffer_count; i++) {
3859                 if (object_list[i]) {
3860                         obj_priv = to_intel_bo(object_list[i]);
3861                         obj_priv->in_execbuffer = false;
3862                 }
3863                 drm_gem_object_unreference(object_list[i]);
3864         }
3865
3866         mutex_unlock(&dev->struct_mutex);
3867
3868 pre_mutex_err:
3869         /* Copy the updated relocations out regardless of current error
3870          * state.  Failure to update the relocs would mean that the next
3871          * time userland calls execbuf, it would do so with presumed offset
3872          * state that didn't match the actual object state.
3873          */
3874         ret2 = i915_gem_put_relocs_to_user(exec_list, args->buffer_count,
3875                                            relocs);
3876         if (ret2 != 0) {
3877                 DRM_ERROR("Failed to copy relocations back out: %d\n", ret2);
3878
3879                 if (ret == 0)
3880                         ret = ret2;
3881         }
3882
3883         drm_free_large(object_list);
3884         kfree(cliprects);
3885
3886         return ret;
3887 }
3888
3889 /*
3890  * Legacy execbuffer just creates an exec2 list from the original exec object
3891  * list array and passes it to the real function.
3892  */
3893 int
3894 i915_gem_execbuffer(struct drm_device *dev, void *data,
3895                     struct drm_file *file_priv)
3896 {
3897         struct drm_i915_gem_execbuffer *args = data;
3898         struct drm_i915_gem_execbuffer2 exec2;
3899         struct drm_i915_gem_exec_object *exec_list = NULL;
3900         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
3901         int ret, i;
3902
3903 #if WATCH_EXEC
3904         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3905                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3906 #endif
3907
3908         if (args->buffer_count < 1) {
3909                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3910                 return -EINVAL;
3911         }
3912
3913         /* Copy in the exec list from userland */
3914         exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
3915         exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
3916         if (exec_list == NULL || exec2_list == NULL) {
3917                 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
3918                           args->buffer_count);
3919                 drm_free_large(exec_list);
3920                 drm_free_large(exec2_list);
3921                 return -ENOMEM;
3922         }
3923         ret = copy_from_user(exec_list,
3924                              (struct drm_i915_relocation_entry __user *)
3925                              (uintptr_t) args->buffers_ptr,
3926                              sizeof(*exec_list) * args->buffer_count);
3927         if (ret != 0) {
3928                 DRM_ERROR("copy %d exec entries failed %d\n",
3929                           args->buffer_count, ret);
3930                 drm_free_large(exec_list);
3931                 drm_free_large(exec2_list);
3932                 return -EFAULT;
3933         }
3934
3935         for (i = 0; i < args->buffer_count; i++) {
3936                 exec2_list[i].handle = exec_list[i].handle;
3937                 exec2_list[i].relocation_count = exec_list[i].relocation_count;
3938                 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
3939                 exec2_list[i].alignment = exec_list[i].alignment;
3940                 exec2_list[i].offset = exec_list[i].offset;
3941                 if (!IS_I965G(dev))
3942                         exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
3943                 else
3944                         exec2_list[i].flags = 0;
3945         }
3946
3947         exec2.buffers_ptr = args->buffers_ptr;
3948         exec2.buffer_count = args->buffer_count;
3949         exec2.batch_start_offset = args->batch_start_offset;
3950         exec2.batch_len = args->batch_len;
3951         exec2.DR1 = args->DR1;
3952         exec2.DR4 = args->DR4;
3953         exec2.num_cliprects = args->num_cliprects;
3954         exec2.cliprects_ptr = args->cliprects_ptr;
3955         exec2.flags = I915_EXEC_RENDER;
3956
3957         ret = i915_gem_do_execbuffer(dev, data, file_priv, &exec2, exec2_list);
3958         if (!ret) {
3959                 /* Copy the new buffer offsets back to the user's exec list. */
3960                 for (i = 0; i < args->buffer_count; i++)
3961                         exec_list[i].offset = exec2_list[i].offset;
3962                 /* ... and back out to userspace */
3963                 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
3964                                    (uintptr_t) args->buffers_ptr,
3965                                    exec_list,
3966                                    sizeof(*exec_list) * args->buffer_count);
3967                 if (ret) {
3968                         ret = -EFAULT;
3969                         DRM_ERROR("failed to copy %d exec entries "
3970                                   "back to user (%d)\n",
3971                                   args->buffer_count, ret);
3972                 }
3973         }
3974
3975         drm_free_large(exec_list);
3976         drm_free_large(exec2_list);
3977         return ret;
3978 }
3979
3980 int
3981 i915_gem_execbuffer2(struct drm_device *dev, void *data,
3982                      struct drm_file *file_priv)
3983 {
3984         struct drm_i915_gem_execbuffer2 *args = data;
3985         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
3986         int ret;
3987
3988 #if WATCH_EXEC
3989         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3990                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3991 #endif
3992
3993         if (args->buffer_count < 1) {
3994                 DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
3995                 return -EINVAL;
3996         }
3997
3998         exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
3999         if (exec2_list == NULL) {
4000                 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4001                           args->buffer_count);
4002                 return -ENOMEM;
4003         }
4004         ret = copy_from_user(exec2_list,
4005                              (struct drm_i915_relocation_entry __user *)
4006                              (uintptr_t) args->buffers_ptr,
4007                              sizeof(*exec2_list) * args->buffer_count);
4008         if (ret != 0) {
4009                 DRM_ERROR("copy %d exec entries failed %d\n",
4010                           args->buffer_count, ret);
4011                 drm_free_large(exec2_list);
4012                 return -EFAULT;
4013         }
4014
4015         ret = i915_gem_do_execbuffer(dev, data, file_priv, args, exec2_list);
4016         if (!ret) {
4017                 /* Copy the new buffer offsets back to the user's exec list. */
4018                 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4019                                    (uintptr_t) args->buffers_ptr,
4020                                    exec2_list,
4021                                    sizeof(*exec2_list) * args->buffer_count);
4022                 if (ret) {
4023                         ret = -EFAULT;
4024                         DRM_ERROR("failed to copy %d exec entries "
4025                                   "back to user (%d)\n",
4026                                   args->buffer_count, ret);
4027                 }
4028         }
4029
4030         drm_free_large(exec2_list);
4031         return ret;
4032 }
4033
4034 int
4035 i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
4036 {
4037         struct drm_device *dev = obj->dev;
4038         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4039         int ret;
4040
4041         BUG_ON(obj_priv->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
4042
4043         i915_verify_inactive(dev, __FILE__, __LINE__);
4044
4045         if (obj_priv->gtt_space != NULL) {
4046                 if (alignment == 0)
4047                         alignment = i915_gem_get_gtt_alignment(obj);
4048                 if (obj_priv->gtt_offset & (alignment - 1)) {
4049                         WARN(obj_priv->pin_count,
4050                              "bo is already pinned with incorrect alignment:"
4051                              " offset=%x, req.alignment=%x\n",
4052                              obj_priv->gtt_offset, alignment);
4053                         ret = i915_gem_object_unbind(obj);
4054                         if (ret)
4055                                 return ret;
4056                 }
4057         }
4058
4059         if (obj_priv->gtt_space == NULL) {
4060                 ret = i915_gem_object_bind_to_gtt(obj, alignment);
4061                 if (ret)
4062                         return ret;
4063         }
4064
4065         obj_priv->pin_count++;
4066
4067         /* If the object is not active and not pending a flush,
4068          * remove it from the inactive list
4069          */
4070         if (obj_priv->pin_count == 1) {
4071                 atomic_inc(&dev->pin_count);
4072                 atomic_add(obj->size, &dev->pin_memory);
4073                 if (!obj_priv->active &&
4074                     (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
4075                         list_del_init(&obj_priv->list);
4076         }
4077         i915_verify_inactive(dev, __FILE__, __LINE__);
4078
4079         return 0;
4080 }
4081
4082 void
4083 i915_gem_object_unpin(struct drm_gem_object *obj)
4084 {
4085         struct drm_device *dev = obj->dev;
4086         drm_i915_private_t *dev_priv = dev->dev_private;
4087         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4088
4089         i915_verify_inactive(dev, __FILE__, __LINE__);
4090         obj_priv->pin_count--;
4091         BUG_ON(obj_priv->pin_count < 0);
4092         BUG_ON(obj_priv->gtt_space == NULL);
4093
4094         /* If the object is no longer pinned, and is
4095          * neither active nor being flushed, then stick it on
4096          * the inactive list
4097          */
4098         if (obj_priv->pin_count == 0) {
4099                 if (!obj_priv->active &&
4100                     (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
4101                         list_move_tail(&obj_priv->list,
4102                                        &dev_priv->mm.inactive_list);
4103                 atomic_dec(&dev->pin_count);
4104                 atomic_sub(obj->size, &dev->pin_memory);
4105         }
4106         i915_verify_inactive(dev, __FILE__, __LINE__);
4107 }
4108
4109 int
4110 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
4111                    struct drm_file *file_priv)
4112 {
4113         struct drm_i915_gem_pin *args = data;
4114         struct drm_gem_object *obj;
4115         struct drm_i915_gem_object *obj_priv;
4116         int ret;
4117
4118         mutex_lock(&dev->struct_mutex);
4119
4120         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4121         if (obj == NULL) {
4122                 DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n",
4123                           args->handle);
4124                 mutex_unlock(&dev->struct_mutex);
4125                 return -ENOENT;
4126         }
4127         obj_priv = to_intel_bo(obj);
4128
4129         if (obj_priv->madv != I915_MADV_WILLNEED) {
4130                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
4131                 drm_gem_object_unreference(obj);
4132                 mutex_unlock(&dev->struct_mutex);
4133                 return -EINVAL;
4134         }
4135
4136         if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) {
4137                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
4138                           args->handle);
4139                 drm_gem_object_unreference(obj);
4140                 mutex_unlock(&dev->struct_mutex);
4141                 return -EINVAL;
4142         }
4143
4144         obj_priv->user_pin_count++;
4145         obj_priv->pin_filp = file_priv;
4146         if (obj_priv->user_pin_count == 1) {
4147                 ret = i915_gem_object_pin(obj, args->alignment);
4148                 if (ret != 0) {
4149                         drm_gem_object_unreference(obj);
4150                         mutex_unlock(&dev->struct_mutex);
4151                         return ret;
4152                 }
4153         }
4154
4155         /* XXX - flush the CPU caches for pinned objects
4156          * as the X server doesn't manage domains yet
4157          */
4158         i915_gem_object_flush_cpu_write_domain(obj);
4159         args->offset = obj_priv->gtt_offset;
4160         drm_gem_object_unreference(obj);
4161         mutex_unlock(&dev->struct_mutex);
4162
4163         return 0;
4164 }
4165
4166 int
4167 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
4168                      struct drm_file *file_priv)
4169 {
4170         struct drm_i915_gem_pin *args = data;
4171         struct drm_gem_object *obj;
4172         struct drm_i915_gem_object *obj_priv;
4173
4174         mutex_lock(&dev->struct_mutex);
4175
4176         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4177         if (obj == NULL) {
4178                 DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n",
4179                           args->handle);
4180                 mutex_unlock(&dev->struct_mutex);
4181                 return -ENOENT;
4182         }
4183
4184         obj_priv = to_intel_bo(obj);
4185         if (obj_priv->pin_filp != file_priv) {
4186                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
4187                           args->handle);
4188                 drm_gem_object_unreference(obj);
4189                 mutex_unlock(&dev->struct_mutex);
4190                 return -EINVAL;
4191         }
4192         obj_priv->user_pin_count--;
4193         if (obj_priv->user_pin_count == 0) {
4194                 obj_priv->pin_filp = NULL;
4195                 i915_gem_object_unpin(obj);
4196         }
4197
4198         drm_gem_object_unreference(obj);
4199         mutex_unlock(&dev->struct_mutex);
4200         return 0;
4201 }
4202
4203 int
4204 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4205                     struct drm_file *file_priv)
4206 {
4207         struct drm_i915_gem_busy *args = data;
4208         struct drm_gem_object *obj;
4209         struct drm_i915_gem_object *obj_priv;
4210
4211         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4212         if (obj == NULL) {
4213                 DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n",
4214                           args->handle);
4215                 return -ENOENT;
4216         }
4217
4218         mutex_lock(&dev->struct_mutex);
4219
4220         /* Count all active objects as busy, even if they are currently not used
4221          * by the gpu. Users of this interface expect objects to eventually
4222          * become non-busy without any further actions, therefore emit any
4223          * necessary flushes here.
4224          */
4225         obj_priv = to_intel_bo(obj);
4226         args->busy = obj_priv->active;
4227         if (args->busy) {
4228                 /* Unconditionally flush objects, even when the gpu still uses this
4229                  * object. Userspace calling this function indicates that it wants to
4230                  * use this buffer rather sooner than later, so issuing the required
4231                  * flush earlier is beneficial.
4232                  */
4233                 if (obj->write_domain) {
4234                         i915_gem_flush(dev, 0, obj->write_domain);
4235                         (void)i915_add_request(dev, file_priv, obj->write_domain, obj_priv->ring);
4236                 }
4237
4238                 /* Update the active list for the hardware's current position.
4239                  * Otherwise this only updates on a delayed timer or when irqs
4240                  * are actually unmasked, and our working set ends up being
4241                  * larger than required.
4242                  */
4243                 i915_gem_retire_requests_ring(dev, obj_priv->ring);
4244
4245                 args->busy = obj_priv->active;
4246         }
4247
4248         drm_gem_object_unreference(obj);
4249         mutex_unlock(&dev->struct_mutex);
4250         return 0;
4251 }
4252
4253 int
4254 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4255                         struct drm_file *file_priv)
4256 {
4257     return i915_gem_ring_throttle(dev, file_priv);
4258 }
4259
4260 int
4261 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4262                        struct drm_file *file_priv)
4263 {
4264         struct drm_i915_gem_madvise *args = data;
4265         struct drm_gem_object *obj;
4266         struct drm_i915_gem_object *obj_priv;
4267
4268         switch (args->madv) {
4269         case I915_MADV_DONTNEED:
4270         case I915_MADV_WILLNEED:
4271             break;
4272         default:
4273             return -EINVAL;
4274         }
4275
4276         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4277         if (obj == NULL) {
4278                 DRM_ERROR("Bad handle in i915_gem_madvise_ioctl(): %d\n",
4279                           args->handle);
4280                 return -ENOENT;
4281         }
4282
4283         mutex_lock(&dev->struct_mutex);
4284         obj_priv = to_intel_bo(obj);
4285
4286         if (obj_priv->pin_count) {
4287                 drm_gem_object_unreference(obj);
4288                 mutex_unlock(&dev->struct_mutex);
4289
4290                 DRM_ERROR("Attempted i915_gem_madvise_ioctl() on a pinned object\n");
4291                 return -EINVAL;
4292         }
4293
4294         if (obj_priv->madv != __I915_MADV_PURGED)
4295                 obj_priv->madv = args->madv;
4296
4297         /* if the object is no longer bound, discard its backing storage */
4298         if (i915_gem_object_is_purgeable(obj_priv) &&
4299             obj_priv->gtt_space == NULL)
4300                 i915_gem_object_truncate(obj);
4301
4302         args->retained = obj_priv->madv != __I915_MADV_PURGED;
4303
4304         drm_gem_object_unreference(obj);
4305         mutex_unlock(&dev->struct_mutex);
4306
4307         return 0;
4308 }
4309
4310 struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev,
4311                                               size_t size)
4312 {
4313         struct drm_i915_gem_object *obj;
4314
4315         obj = kzalloc(sizeof(*obj), GFP_KERNEL);
4316         if (obj == NULL)
4317                 return NULL;
4318
4319         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4320                 kfree(obj);
4321                 return NULL;
4322         }
4323
4324         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4325         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4326
4327         obj->agp_type = AGP_USER_MEMORY;
4328         obj->base.driver_private = NULL;
4329         obj->fence_reg = I915_FENCE_REG_NONE;
4330         INIT_LIST_HEAD(&obj->list);
4331         INIT_LIST_HEAD(&obj->gpu_write_list);
4332         obj->madv = I915_MADV_WILLNEED;
4333
4334         trace_i915_gem_object_create(&obj->base);
4335
4336         return &obj->base;
4337 }
4338
4339 int i915_gem_init_object(struct drm_gem_object *obj)
4340 {
4341         BUG();
4342
4343         return 0;
4344 }
4345
4346 static void i915_gem_free_object_tail(struct drm_gem_object *obj)
4347 {
4348         struct drm_device *dev = obj->dev;
4349         drm_i915_private_t *dev_priv = dev->dev_private;
4350         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4351         int ret;
4352
4353         ret = i915_gem_object_unbind(obj);
4354         if (ret == -ERESTARTSYS) {
4355                 list_move(&obj_priv->list,
4356                           &dev_priv->mm.deferred_free_list);
4357                 return;
4358         }
4359
4360         if (obj_priv->mmap_offset)
4361                 i915_gem_free_mmap_offset(obj);
4362
4363         drm_gem_object_release(obj);
4364
4365         kfree(obj_priv->page_cpu_valid);
4366         kfree(obj_priv->bit_17);
4367         kfree(obj_priv);
4368 }
4369
4370 void i915_gem_free_object(struct drm_gem_object *obj)
4371 {
4372         struct drm_device *dev = obj->dev;
4373         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4374
4375         trace_i915_gem_object_destroy(obj);
4376
4377         while (obj_priv->pin_count > 0)
4378                 i915_gem_object_unpin(obj);
4379
4380         if (obj_priv->phys_obj)
4381                 i915_gem_detach_phys_object(dev, obj);
4382
4383         i915_gem_free_object_tail(obj);
4384 }
4385
4386 int
4387 i915_gem_idle(struct drm_device *dev)
4388 {
4389         drm_i915_private_t *dev_priv = dev->dev_private;
4390         int ret;
4391
4392         mutex_lock(&dev->struct_mutex);
4393
4394         if (dev_priv->mm.suspended ||
4395                         (dev_priv->render_ring.gem_object == NULL) ||
4396                         (HAS_BSD(dev) &&
4397                          dev_priv->bsd_ring.gem_object == NULL)) {
4398                 mutex_unlock(&dev->struct_mutex);
4399                 return 0;
4400         }
4401
4402         ret = i915_gpu_idle(dev);
4403         if (ret) {
4404                 mutex_unlock(&dev->struct_mutex);
4405                 return ret;
4406         }
4407
4408         /* Under UMS, be paranoid and evict. */
4409         if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
4410                 ret = i915_gem_evict_inactive(dev);
4411                 if (ret) {
4412                         mutex_unlock(&dev->struct_mutex);
4413                         return ret;
4414                 }
4415         }
4416
4417         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
4418          * We need to replace this with a semaphore, or something.
4419          * And not confound mm.suspended!
4420          */
4421         dev_priv->mm.suspended = 1;
4422         del_timer(&dev_priv->hangcheck_timer);
4423
4424         i915_kernel_lost_context(dev);
4425         i915_gem_cleanup_ringbuffer(dev);
4426
4427         mutex_unlock(&dev->struct_mutex);
4428
4429         /* Cancel the retire work handler, which should be idle now. */
4430         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4431
4432         return 0;
4433 }
4434
4435 /*
4436  * 965+ support PIPE_CONTROL commands, which provide finer grained control
4437  * over cache flushing.
4438  */
4439 static int
4440 i915_gem_init_pipe_control(struct drm_device *dev)
4441 {
4442         drm_i915_private_t *dev_priv = dev->dev_private;
4443         struct drm_gem_object *obj;
4444         struct drm_i915_gem_object *obj_priv;
4445         int ret;
4446
4447         obj = i915_gem_alloc_object(dev, 4096);
4448         if (obj == NULL) {
4449                 DRM_ERROR("Failed to allocate seqno page\n");
4450                 ret = -ENOMEM;
4451                 goto err;
4452         }
4453         obj_priv = to_intel_bo(obj);
4454         obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
4455
4456         ret = i915_gem_object_pin(obj, 4096);
4457         if (ret)
4458                 goto err_unref;
4459
4460         dev_priv->seqno_gfx_addr = obj_priv->gtt_offset;
4461         dev_priv->seqno_page =  kmap(obj_priv->pages[0]);
4462         if (dev_priv->seqno_page == NULL)
4463                 goto err_unpin;
4464
4465         dev_priv->seqno_obj = obj;
4466         memset(dev_priv->seqno_page, 0, PAGE_SIZE);
4467
4468         return 0;
4469
4470 err_unpin:
4471         i915_gem_object_unpin(obj);
4472 err_unref:
4473         drm_gem_object_unreference(obj);
4474 err:
4475         return ret;
4476 }
4477
4478
4479 static void
4480 i915_gem_cleanup_pipe_control(struct drm_device *dev)
4481 {
4482         drm_i915_private_t *dev_priv = dev->dev_private;
4483         struct drm_gem_object *obj;
4484         struct drm_i915_gem_object *obj_priv;
4485
4486         obj = dev_priv->seqno_obj;
4487         obj_priv = to_intel_bo(obj);
4488         kunmap(obj_priv->pages[0]);
4489         i915_gem_object_unpin(obj);
4490         drm_gem_object_unreference(obj);
4491         dev_priv->seqno_obj = NULL;
4492
4493         dev_priv->seqno_page = NULL;
4494 }
4495
4496 int
4497 i915_gem_init_ringbuffer(struct drm_device *dev)
4498 {
4499         drm_i915_private_t *dev_priv = dev->dev_private;
4500         int ret;
4501
4502         dev_priv->render_ring = render_ring;
4503
4504         if (!I915_NEED_GFX_HWS(dev)) {
4505                 dev_priv->render_ring.status_page.page_addr
4506                         = dev_priv->status_page_dmah->vaddr;
4507                 memset(dev_priv->render_ring.status_page.page_addr,
4508                                 0, PAGE_SIZE);
4509         }
4510
4511         if (HAS_PIPE_CONTROL(dev)) {
4512                 ret = i915_gem_init_pipe_control(dev);
4513                 if (ret)
4514                         return ret;
4515         }
4516
4517         ret = intel_init_ring_buffer(dev, &dev_priv->render_ring);
4518         if (ret)
4519                 goto cleanup_pipe_control;
4520
4521         if (HAS_BSD(dev)) {
4522                 dev_priv->bsd_ring = bsd_ring;
4523                 ret = intel_init_ring_buffer(dev, &dev_priv->bsd_ring);
4524                 if (ret)
4525                         goto cleanup_render_ring;
4526         }
4527
4528         dev_priv->next_seqno = 1;
4529
4530         return 0;
4531
4532 cleanup_render_ring:
4533         intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
4534 cleanup_pipe_control:
4535         if (HAS_PIPE_CONTROL(dev))
4536                 i915_gem_cleanup_pipe_control(dev);
4537         return ret;
4538 }
4539
4540 void
4541 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4542 {
4543         drm_i915_private_t *dev_priv = dev->dev_private;
4544
4545         intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
4546         if (HAS_BSD(dev))
4547                 intel_cleanup_ring_buffer(dev, &dev_priv->bsd_ring);
4548         if (HAS_PIPE_CONTROL(dev))
4549                 i915_gem_cleanup_pipe_control(dev);
4550 }
4551
4552 int
4553 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4554                        struct drm_file *file_priv)
4555 {
4556         drm_i915_private_t *dev_priv = dev->dev_private;
4557         int ret;
4558
4559         if (drm_core_check_feature(dev, DRIVER_MODESET))
4560                 return 0;
4561
4562         if (atomic_read(&dev_priv->mm.wedged)) {
4563                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
4564                 atomic_set(&dev_priv->mm.wedged, 0);
4565         }
4566
4567         mutex_lock(&dev->struct_mutex);
4568         dev_priv->mm.suspended = 0;
4569
4570         ret = i915_gem_init_ringbuffer(dev);
4571         if (ret != 0) {
4572                 mutex_unlock(&dev->struct_mutex);
4573                 return ret;
4574         }
4575
4576         spin_lock(&dev_priv->mm.active_list_lock);
4577         BUG_ON(!list_empty(&dev_priv->render_ring.active_list));
4578         BUG_ON(HAS_BSD(dev) && !list_empty(&dev_priv->bsd_ring.active_list));
4579         spin_unlock(&dev_priv->mm.active_list_lock);
4580
4581         BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
4582         BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
4583         BUG_ON(!list_empty(&dev_priv->render_ring.request_list));
4584         BUG_ON(HAS_BSD(dev) && !list_empty(&dev_priv->bsd_ring.request_list));
4585         mutex_unlock(&dev->struct_mutex);
4586
4587         ret = drm_irq_install(dev);
4588         if (ret)
4589                 goto cleanup_ringbuffer;
4590
4591         return 0;
4592
4593 cleanup_ringbuffer:
4594         mutex_lock(&dev->struct_mutex);
4595         i915_gem_cleanup_ringbuffer(dev);
4596         dev_priv->mm.suspended = 1;
4597         mutex_unlock(&dev->struct_mutex);
4598
4599         return ret;
4600 }
4601
4602 int
4603 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
4604                        struct drm_file *file_priv)
4605 {
4606         if (drm_core_check_feature(dev, DRIVER_MODESET))
4607                 return 0;
4608
4609         drm_irq_uninstall(dev);
4610         return i915_gem_idle(dev);
4611 }
4612
4613 void
4614 i915_gem_lastclose(struct drm_device *dev)
4615 {
4616         int ret;
4617
4618         if (drm_core_check_feature(dev, DRIVER_MODESET))
4619                 return;
4620
4621         ret = i915_gem_idle(dev);
4622         if (ret)
4623                 DRM_ERROR("failed to idle hardware: %d\n", ret);
4624 }
4625
4626 void
4627 i915_gem_load(struct drm_device *dev)
4628 {
4629         int i;
4630         drm_i915_private_t *dev_priv = dev->dev_private;
4631
4632         spin_lock_init(&dev_priv->mm.active_list_lock);
4633         INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
4634         INIT_LIST_HEAD(&dev_priv->mm.gpu_write_list);
4635         INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
4636         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4637         INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
4638         INIT_LIST_HEAD(&dev_priv->render_ring.active_list);
4639         INIT_LIST_HEAD(&dev_priv->render_ring.request_list);
4640         if (HAS_BSD(dev)) {
4641                 INIT_LIST_HEAD(&dev_priv->bsd_ring.active_list);
4642                 INIT_LIST_HEAD(&dev_priv->bsd_ring.request_list);
4643         }
4644         for (i = 0; i < 16; i++)
4645                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
4646         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4647                           i915_gem_retire_work_handler);
4648         spin_lock(&shrink_list_lock);
4649         list_add(&dev_priv->mm.shrink_list, &shrink_list);
4650         spin_unlock(&shrink_list_lock);
4651
4652         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
4653         if (IS_GEN3(dev)) {
4654                 u32 tmp = I915_READ(MI_ARB_STATE);
4655                 if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) {
4656                         /* arb state is a masked write, so set bit + bit in mask */
4657                         tmp = MI_ARB_C3_LP_WRITE_ENABLE | (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT);
4658                         I915_WRITE(MI_ARB_STATE, tmp);
4659                 }
4660         }
4661
4662         /* Old X drivers will take 0-2 for front, back, depth buffers */
4663         if (!drm_core_check_feature(dev, DRIVER_MODESET))
4664                 dev_priv->fence_reg_start = 3;
4665
4666         if (IS_I965G(dev) || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4667                 dev_priv->num_fence_regs = 16;
4668         else
4669                 dev_priv->num_fence_regs = 8;
4670
4671         /* Initialize fence registers to zero */
4672         if (IS_I965G(dev)) {
4673                 for (i = 0; i < 16; i++)
4674                         I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0);
4675         } else {
4676                 for (i = 0; i < 8; i++)
4677                         I915_WRITE(FENCE_REG_830_0 + (i * 4), 0);
4678                 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4679                         for (i = 0; i < 8; i++)
4680                                 I915_WRITE(FENCE_REG_945_8 + (i * 4), 0);
4681         }
4682         i915_gem_detect_bit_6_swizzle(dev);
4683         init_waitqueue_head(&dev_priv->pending_flip_queue);
4684 }
4685
4686 /*
4687  * Create a physically contiguous memory object for this object
4688  * e.g. for cursor + overlay regs
4689  */
4690 int i915_gem_init_phys_object(struct drm_device *dev,
4691                               int id, int size, int align)
4692 {
4693         drm_i915_private_t *dev_priv = dev->dev_private;
4694         struct drm_i915_gem_phys_object *phys_obj;
4695         int ret;
4696
4697         if (dev_priv->mm.phys_objs[id - 1] || !size)
4698                 return 0;
4699
4700         phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
4701         if (!phys_obj)
4702                 return -ENOMEM;
4703
4704         phys_obj->id = id;
4705
4706         phys_obj->handle = drm_pci_alloc(dev, size, align);
4707         if (!phys_obj->handle) {
4708                 ret = -ENOMEM;
4709                 goto kfree_obj;
4710         }
4711 #ifdef CONFIG_X86
4712         set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4713 #endif
4714
4715         dev_priv->mm.phys_objs[id - 1] = phys_obj;
4716
4717         return 0;
4718 kfree_obj:
4719         kfree(phys_obj);
4720         return ret;
4721 }
4722
4723 void i915_gem_free_phys_object(struct drm_device *dev, int id)
4724 {
4725         drm_i915_private_t *dev_priv = dev->dev_private;
4726         struct drm_i915_gem_phys_object *phys_obj;
4727
4728         if (!dev_priv->mm.phys_objs[id - 1])
4729                 return;
4730
4731         phys_obj = dev_priv->mm.phys_objs[id - 1];
4732         if (phys_obj->cur_obj) {
4733                 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4734         }
4735
4736 #ifdef CONFIG_X86
4737         set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4738 #endif
4739         drm_pci_free(dev, phys_obj->handle);
4740         kfree(phys_obj);
4741         dev_priv->mm.phys_objs[id - 1] = NULL;
4742 }
4743
4744 void i915_gem_free_all_phys_object(struct drm_device *dev)
4745 {
4746         int i;
4747
4748         for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
4749                 i915_gem_free_phys_object(dev, i);
4750 }
4751
4752 void i915_gem_detach_phys_object(struct drm_device *dev,
4753                                  struct drm_gem_object *obj)
4754 {
4755         struct drm_i915_gem_object *obj_priv;
4756         int i;
4757         int ret;
4758         int page_count;
4759
4760         obj_priv = to_intel_bo(obj);
4761         if (!obj_priv->phys_obj)
4762                 return;
4763
4764         ret = i915_gem_object_get_pages(obj, 0);
4765         if (ret)
4766                 goto out;
4767
4768         page_count = obj->size / PAGE_SIZE;
4769
4770         for (i = 0; i < page_count; i++) {
4771                 char *dst = kmap_atomic(obj_priv->pages[i], KM_USER0);
4772                 char *src = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4773
4774                 memcpy(dst, src, PAGE_SIZE);
4775                 kunmap_atomic(dst, KM_USER0);
4776         }
4777         drm_clflush_pages(obj_priv->pages, page_count);
4778         drm_agp_chipset_flush(dev);
4779
4780         i915_gem_object_put_pages(obj);
4781 out:
4782         obj_priv->phys_obj->cur_obj = NULL;
4783         obj_priv->phys_obj = NULL;
4784 }
4785
4786 int
4787 i915_gem_attach_phys_object(struct drm_device *dev,
4788                             struct drm_gem_object *obj,
4789                             int id,
4790                             int align)
4791 {
4792         drm_i915_private_t *dev_priv = dev->dev_private;
4793         struct drm_i915_gem_object *obj_priv;
4794         int ret = 0;
4795         int page_count;
4796         int i;
4797
4798         if (id > I915_MAX_PHYS_OBJECT)
4799                 return -EINVAL;
4800
4801         obj_priv = to_intel_bo(obj);
4802
4803         if (obj_priv->phys_obj) {
4804                 if (obj_priv->phys_obj->id == id)
4805                         return 0;
4806                 i915_gem_detach_phys_object(dev, obj);
4807         }
4808
4809         /* create a new object */
4810         if (!dev_priv->mm.phys_objs[id - 1]) {
4811                 ret = i915_gem_init_phys_object(dev, id,
4812                                                 obj->size, align);
4813                 if (ret) {
4814                         DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size);
4815                         goto out;
4816                 }
4817         }
4818
4819         /* bind to the object */
4820         obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1];
4821         obj_priv->phys_obj->cur_obj = obj;
4822
4823         ret = i915_gem_object_get_pages(obj, 0);
4824         if (ret) {
4825                 DRM_ERROR("failed to get page list\n");
4826                 goto out;
4827         }
4828
4829         page_count = obj->size / PAGE_SIZE;
4830
4831         for (i = 0; i < page_count; i++) {
4832                 char *src = kmap_atomic(obj_priv->pages[i], KM_USER0);
4833                 char *dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4834
4835                 memcpy(dst, src, PAGE_SIZE);
4836                 kunmap_atomic(src, KM_USER0);
4837         }
4838
4839         i915_gem_object_put_pages(obj);
4840
4841         return 0;
4842 out:
4843         return ret;
4844 }
4845
4846 static int
4847 i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
4848                      struct drm_i915_gem_pwrite *args,
4849                      struct drm_file *file_priv)
4850 {
4851         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4852         void *obj_addr;
4853         int ret;
4854         char __user *user_data;
4855
4856         user_data = (char __user *) (uintptr_t) args->data_ptr;
4857         obj_addr = obj_priv->phys_obj->handle->vaddr + args->offset;
4858
4859         DRM_DEBUG_DRIVER("obj_addr %p, %lld\n", obj_addr, args->size);
4860         ret = copy_from_user(obj_addr, user_data, args->size);
4861         if (ret)
4862                 return -EFAULT;
4863
4864         drm_agp_chipset_flush(dev);
4865         return 0;
4866 }
4867
4868 void i915_gem_release(struct drm_device * dev, struct drm_file *file_priv)
4869 {
4870         struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
4871
4872         /* Clean up our request list when the client is going away, so that
4873          * later retire_requests won't dereference our soon-to-be-gone
4874          * file_priv.
4875          */
4876         mutex_lock(&dev->struct_mutex);
4877         while (!list_empty(&i915_file_priv->mm.request_list))
4878                 list_del_init(i915_file_priv->mm.request_list.next);
4879         mutex_unlock(&dev->struct_mutex);
4880 }
4881
4882 static int
4883 i915_gpu_is_active(struct drm_device *dev)
4884 {
4885         drm_i915_private_t *dev_priv = dev->dev_private;
4886         int lists_empty;
4887
4888         spin_lock(&dev_priv->mm.active_list_lock);
4889         lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
4890                       list_empty(&dev_priv->render_ring.active_list);
4891         if (HAS_BSD(dev))
4892                 lists_empty &= list_empty(&dev_priv->bsd_ring.active_list);
4893         spin_unlock(&dev_priv->mm.active_list_lock);
4894
4895         return !lists_empty;
4896 }
4897
4898 static int
4899 i915_gem_shrink(struct shrinker *shrink, int nr_to_scan, gfp_t gfp_mask)
4900 {
4901         drm_i915_private_t *dev_priv, *next_dev;
4902         struct drm_i915_gem_object *obj_priv, *next_obj;
4903         int cnt = 0;
4904         int would_deadlock = 1;
4905
4906         /* "fast-path" to count number of available objects */
4907         if (nr_to_scan == 0) {
4908                 spin_lock(&shrink_list_lock);
4909                 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
4910                         struct drm_device *dev = dev_priv->dev;
4911
4912                         if (mutex_trylock(&dev->struct_mutex)) {
4913                                 list_for_each_entry(obj_priv,
4914                                                     &dev_priv->mm.inactive_list,
4915                                                     list)
4916                                         cnt++;
4917                                 mutex_unlock(&dev->struct_mutex);
4918                         }
4919                 }
4920                 spin_unlock(&shrink_list_lock);
4921
4922                 return (cnt / 100) * sysctl_vfs_cache_pressure;
4923         }
4924
4925         spin_lock(&shrink_list_lock);
4926
4927 rescan:
4928         /* first scan for clean buffers */
4929         list_for_each_entry_safe(dev_priv, next_dev,
4930                                  &shrink_list, mm.shrink_list) {
4931                 struct drm_device *dev = dev_priv->dev;
4932
4933                 if (! mutex_trylock(&dev->struct_mutex))
4934                         continue;
4935
4936                 spin_unlock(&shrink_list_lock);
4937                 i915_gem_retire_requests(dev);
4938
4939                 list_for_each_entry_safe(obj_priv, next_obj,
4940                                          &dev_priv->mm.inactive_list,
4941                                          list) {
4942                         if (i915_gem_object_is_purgeable(obj_priv)) {
4943                                 i915_gem_object_unbind(&obj_priv->base);
4944                                 if (--nr_to_scan <= 0)
4945                                         break;
4946                         }
4947                 }
4948
4949                 spin_lock(&shrink_list_lock);
4950                 mutex_unlock(&dev->struct_mutex);
4951
4952                 would_deadlock = 0;
4953
4954                 if (nr_to_scan <= 0)
4955                         break;
4956         }
4957
4958         /* second pass, evict/count anything still on the inactive list */
4959         list_for_each_entry_safe(dev_priv, next_dev,
4960                                  &shrink_list, mm.shrink_list) {
4961                 struct drm_device *dev = dev_priv->dev;
4962
4963                 if (! mutex_trylock(&dev->struct_mutex))
4964                         continue;
4965
4966                 spin_unlock(&shrink_list_lock);
4967
4968                 list_for_each_entry_safe(obj_priv, next_obj,
4969                                          &dev_priv->mm.inactive_list,
4970                                          list) {
4971                         if (nr_to_scan > 0) {
4972                                 i915_gem_object_unbind(&obj_priv->base);
4973                                 nr_to_scan--;
4974                         } else
4975                                 cnt++;
4976                 }
4977
4978                 spin_lock(&shrink_list_lock);
4979                 mutex_unlock(&dev->struct_mutex);
4980
4981                 would_deadlock = 0;
4982         }
4983
4984         if (nr_to_scan) {
4985                 int active = 0;
4986
4987                 /*
4988                  * We are desperate for pages, so as a last resort, wait
4989                  * for the GPU to finish and discard whatever we can.
4990                  * This has a dramatic impact to reduce the number of
4991                  * OOM-killer events whilst running the GPU aggressively.
4992                  */
4993                 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
4994                         struct drm_device *dev = dev_priv->dev;
4995
4996                         if (!mutex_trylock(&dev->struct_mutex))
4997                                 continue;
4998
4999                         spin_unlock(&shrink_list_lock);
5000
5001                         if (i915_gpu_is_active(dev)) {
5002                                 i915_gpu_idle(dev);
5003                                 active++;
5004                         }
5005
5006                         spin_lock(&shrink_list_lock);
5007                         mutex_unlock(&dev->struct_mutex);
5008                 }
5009
5010                 if (active)
5011                         goto rescan;
5012         }
5013
5014         spin_unlock(&shrink_list_lock);
5015
5016         if (would_deadlock)
5017                 return -1;
5018         else if (cnt > 0)
5019                 return (cnt / 100) * sysctl_vfs_cache_pressure;
5020         else
5021                 return 0;
5022 }
5023
5024 static struct shrinker shrinker = {
5025         .shrink = i915_gem_shrink,
5026         .seeks = DEFAULT_SEEKS,
5027 };
5028
5029 __init void
5030 i915_gem_shrinker_init(void)
5031 {
5032     register_shrinker(&shrinker);
5033 }
5034
5035 __exit void
5036 i915_gem_shrinker_exit(void)
5037 {
5038     unregister_shrinker(&shrinker);
5039 }